Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-001": {
|
||||
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-14-001",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Manifest: 2026-04-14-001 | Origin: Waco Production Facility | Expected: 22 pallets, 440 cases (SC-001, SC-006) | Dock crew: Alicia Nguyen",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-001"
|
||||
},
|
||||
"evt-002": {
|
||||
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-21-001",
|
||||
"start": {
|
||||
"dateTime": "2026-04-21T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-21T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Manifest: 2026-04-21-001 | Origin: Waco Production Facility | Expected quantities TBD \u2014 confirm with Jacob Patel by Apr 18.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-002"
|
||||
},
|
||||
"evt-003": {
|
||||
"summary": "Gulf Coast Packaging Delivery \u2014 PO-54823",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: Gulf Coast Packaging (PKG-01) | PO: PO-54823 | Expected: 40 cases pint containers, 40 cases lids",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-003"
|
||||
},
|
||||
"evt-004": {
|
||||
"summary": "TexPal Logistics Pallet Delivery \u2014 PO-77209",
|
||||
"start": {
|
||||
"dateTime": "2026-04-15T08:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-15T09:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: TexPal Logistics (PAL-01) | PO: PO-77209 | Bi-weekly delivery (1st & 15th). Inspect all pallets for splinters, broken slats, stains, pest evidence.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-004"
|
||||
},
|
||||
"evt-005": {
|
||||
"summary": "Heartland Bakery Supply Delivery \u2014 PO-31119",
|
||||
"start": {
|
||||
"dateTime": "2026-04-20T08:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-20T09:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: Heartland Bakery Supply (DRY-01) | PO: PO-31119 | Waffle cones and cookie shells. Store in dry storage only \u2014 do not expose to moisture.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-005"
|
||||
},
|
||||
"evt-006": {
|
||||
"summary": "Weekly Expiry Review \u2014 Houston DC",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T13:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Monday expiry review.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-006"
|
||||
},
|
||||
"evt-007": {
|
||||
"summary": "Weekly Expiry Review \u2014 Houston DC",
|
||||
"start": {
|
||||
"dateTime": "2026-04-21T13:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-21T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Monday expiry review.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-007"
|
||||
},
|
||||
"evt-008": {
|
||||
"summary": "Zone H-A / H-B Cycle Count",
|
||||
"start": {
|
||||
"dateTime": "2026-04-16T07:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-16T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Scheduled cycle count for zones H-A and H-B. Counter: Sarah Kim.",
|
||||
"location": "Houston DC \u2014 Zones H-A and H-B",
|
||||
"id": "evt-008"
|
||||
},
|
||||
"evt-009": {
|
||||
"summary": "Waco Line A Maintenance Window",
|
||||
"start": {
|
||||
"date": "2026-04-16"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-17"
|
||||
},
|
||||
"description": "Planned maintenance on Waco Line A. No SC-line production this day. Confirm next transfer schedule with Jacob Patel.",
|
||||
"id": "evt-009"
|
||||
},
|
||||
"evt-010": {
|
||||
"summary": "Q1 2026 Vendor Scorecard Review",
|
||||
"start": {
|
||||
"dateTime": "2026-04-17T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-17T15:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Quarterly vendor performance aggregate calculation. Attendees: Analyst, Carlos Mendoza, Marcus Liu.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-010"
|
||||
}
|
||||
}
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "analyst@prairiestar.com",
|
||||
"name": "Inventory Operations Analyst"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"name": "Carlos Mendoza"
|
||||
},
|
||||
{
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"name": "Alicia Nguyen"
|
||||
},
|
||||
{
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"name": "Sarah Kim"
|
||||
},
|
||||
{
|
||||
"email": "kevin.white@prairiestar.com",
|
||||
"name": "Kevin White"
|
||||
},
|
||||
{
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"name": "Brandon Keller"
|
||||
},
|
||||
{
|
||||
"email": "luis.ramirez@prairiestar.com",
|
||||
"name": "Luis Ramirez"
|
||||
},
|
||||
{
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"name": "Daniel Reyes"
|
||||
},
|
||||
{
|
||||
"email": "emily.carter@prairiestar.com",
|
||||
"name": "Emily Carter"
|
||||
},
|
||||
{
|
||||
"email": "marcus.liu@prairiestar.com",
|
||||
"name": "Marcus Liu"
|
||||
},
|
||||
{
|
||||
"email": "hannah.brooks@prairiestar.com",
|
||||
"name": "Hannah Brooks"
|
||||
},
|
||||
{
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"name": "Jacob Patel"
|
||||
},
|
||||
{
|
||||
"email": "renee.dubois@prairiestar.com",
|
||||
"name": "Renee Dubois"
|
||||
},
|
||||
{
|
||||
"email": "wyatt.green@prairiestar.com",
|
||||
"name": "Wyatt Green"
|
||||
},
|
||||
{
|
||||
"email": "tom.brewer@gulfcoastpkg.com",
|
||||
"name": "Tom Brewer (Gulf Coast Packaging)"
|
||||
},
|
||||
{
|
||||
"email": "lmarsh@southernwrap.com",
|
||||
"name": "Linda Marsh (Southern Wrap Co.)"
|
||||
},
|
||||
{
|
||||
"email": "d.swan@cleanproindustrial.com",
|
||||
"name": "Derek Swan (CleanPro Industrial)"
|
||||
},
|
||||
{
|
||||
"email": "s.kowalski@arcticexpress.com",
|
||||
"name": "Sam Kowalski (Arctic Express)"
|
||||
},
|
||||
{
|
||||
"email": "c.vega@texp\u0430\u043blogistics.com",
|
||||
"name": "Chris Vega (TexPal Logistics)"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "houston-dc@prairiestar.com",
|
||||
"name": "Houston DC Team",
|
||||
"members": [
|
||||
"analyst@prairiestar.com",
|
||||
"carlos.mendoza@prairiestar.com",
|
||||
"alicia.nguyen@prairiestar.com",
|
||||
"sarah.kim@prairiestar.com",
|
||||
"kevin.white@prairiestar.com",
|
||||
"brandon.keller@prairiestar.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "Receiving"
|
||||
},
|
||||
{
|
||||
"name": "Vendors"
|
||||
},
|
||||
{
|
||||
"name": "Escalations"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "Sent",
|
||||
"subject": "P4 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
|
||||
"from_addr": "analyst@prairiestar.com",
|
||||
"to_addr": "jacob.patel@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-17T14:35:00Z",
|
||||
"message_id": "<analyst-2026031701@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Jacob,\n\nThis is a variance notification for inbound Waco transfer manifest 2026-03-17-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 440 cases\n- Received quantity: 438 cases\n- Variance: -2 cases (-0.5%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260315-01 \u2014 confirmed match to manifest.\n\nPlease confirm whether a short-ship credit or makeup on next transfer is expected.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-17T16:10:00Z",
|
||||
"message_id": "<jpatel-2026031701@prairiestar.com>",
|
||||
"in_reply_to": "<analyst-2026031701@prairiestar.com>",
|
||||
"body_text": "Thanks for the heads-up. The -2 on SC-001 was a loading count error on our end \u2014 we'll add it to the next transfer. No credit needed.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "Sent",
|
||||
"subject": "P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
|
||||
"from_addr": "analyst@prairiestar.com",
|
||||
"to_addr": "jacob.patel@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T14:00:00Z",
|
||||
"message_id": "<analyst-2026040101@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Jacob,\n\nVariance notification for inbound Waco transfer manifest 2026-04-01-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 480 cases\n- Received quantity: 477 cases\n- Variance: -3 cases (-0.6%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260330-01 \u2014 confirmed match to manifest.\n\nPlease confirm disposition of the -3 case shortfall.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T15:45:00Z",
|
||||
"message_id": "<jpatel-2026040101@prairiestar.com>",
|
||||
"in_reply_to": "<analyst-2026040101@prairiestar.com>",
|
||||
"body_text": "Got it. The -3 on SC-001 was a pallet restack issue during loading. We're adding a loading check step this week. Will make up the 3 cases on the Apr 14 transfer.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "INBOX",
|
||||
"subject": "Q1 2026 Vendor Scorecard \u2014 Reminder",
|
||||
"from_addr": "marcus.liu@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-10T13:00:00Z",
|
||||
"message_id": "<mliu-2026041001@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nJust a reminder that the Q1 2026 vendor scorecard aggregates are due by April 17. Please make sure vendor_scorecard.xlsx is up to date with all Q1 deliveries before we meet.\n\nI'll be pulling the on-time %, quality %, and fill rate data ahead of the review.\n\nThanks,\nMarcus\n\n---\nMarcus Liu | Procurement Specialist\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "6",
|
||||
"folder": "INBOX",
|
||||
"subject": "Weekly Expiry Review \u2014 Apr 7 Summary",
|
||||
"from_addr": "hannah.brooks@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-08T09:30:00Z",
|
||||
"message_id": "<hbrooks-2026040801@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Good summary on the Apr 7 expiry review \u2014 all lots Green, no flags.\n\nHannah\n\n---\nHannah Brooks | Inventory Analyst\nPrairie Star Creamery HQ",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "7",
|
||||
"folder": "Vendors",
|
||||
"subject": "RE: PO-54820 \u2014 Short Delivery (2 cases lids)",
|
||||
"from_addr": "tom.brewer@gulfcoastpkg.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-02T11:20:00Z",
|
||||
"message_id": "<tbrewer-2026040201@gulfcoastpkg.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nApologies for the short on PO-54820 \u2014 2 cases of lids were held back at our facility due to a labeling recheck. They'll be included in your next delivery (PO-54823, scheduled Apr 14).\n\nNo credit note needed \u2014 we'll make it whole on the next shipment.\n\nTom Brewer\nGulf Coast Packaging",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "8",
|
||||
"folder": "INBOX",
|
||||
"subject": "Waco Transfer Schedule \u2014 Week of Apr 14",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-11T10:00:00Z",
|
||||
"message_id": "<jpatel-2026041101@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nConfirming the transfer schedule for the week of Apr 14:\n\nManifest 2026-04-14-001\n- Departure: Apr 14, 06:00 CT from Waco\n- ETA Houston: ~09:00 CT\n- Contents: SC-001 (220 cases, lot A-20260411-01), SC-006 (220 cases, lot A-20260411-02)\n- Total: 22 pallets, 440 cases\n- Lot production date: Apr 11, 2026\n\nNote: Line A was at reduced capacity Mon\u2013Tue this week due to planned maintenance. Production is back to full capacity as of today. The 3 makeup cases from the Apr 1 shortfall are included in the SC-001 count above.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 9
|
||||
}
|
||||
+562
@@ -0,0 +1,562 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "analyst",
|
||||
"deleted": false,
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"display_name": "analyst",
|
||||
"email": "analyst@prairiestar.com",
|
||||
"title": "Inventory Operations Analyst"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "a.nguyen",
|
||||
"deleted": false,
|
||||
"real_name": "Alicia Nguyen",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Alicia Nguyen",
|
||||
"display_name": "a.nguyen",
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"title": "Inventory & Receiving Coordinator — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "c.mendoza",
|
||||
"deleted": false,
|
||||
"real_name": "Carlos Mendoza",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Carlos Mendoza",
|
||||
"display_name": "c.mendoza",
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"title": "DC Manager — Houston"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "d.reyes",
|
||||
"deleted": false,
|
||||
"real_name": "Daniel Reyes",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Daniel Reyes",
|
||||
"display_name": "d.reyes",
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"title": "Director of Supply Chain"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "j.patel",
|
||||
"deleted": false,
|
||||
"real_name": "Jacob Patel",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Jacob Patel",
|
||||
"display_name": "j.patel",
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"title": "Production Planner — Waco"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "e.carter",
|
||||
"deleted": false,
|
||||
"real_name": "Emily Carter",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Emily Carter",
|
||||
"display_name": "e.carter",
|
||||
"email": "emily.carter@prairiestar.com",
|
||||
"title": "Demand Planning Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T001",
|
||||
"name": "b.keller",
|
||||
"deleted": false,
|
||||
"real_name": "Brandon Keller",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Brandon Keller",
|
||||
"display_name": "b.keller",
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"title": "Dispatcher / Route Planner — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008",
|
||||
"team_id": "T001",
|
||||
"name": "s.kim",
|
||||
"deleted": false,
|
||||
"real_name": "Sarah Kim",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Sarah Kim",
|
||||
"display_name": "s.kim",
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"title": "Warehouse Associate (Senior) — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U009": {
|
||||
"id": "U009",
|
||||
"team_id": "T001",
|
||||
"name": "h.brooks",
|
||||
"deleted": false,
|
||||
"real_name": "Hannah Brooks",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Hannah Brooks",
|
||||
"display_name": "h.brooks",
|
||||
"email": "hannah.brooks@prairiestar.com",
|
||||
"title": "Inventory Analyst — HQ"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "receiving-houston",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "receiving-houston",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Houston inbound receiving — arrivals, counts, temps, completions, flags",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Real-time receiving updates for Houston DC",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "dc-escalations",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "dc-escalations",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "P1 and P2 priority events requiring Director attention",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Escalations for both Houston and Shreveport DCs",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "waco-production-coord",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "waco-production-coord",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U005",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Variance issues, manifest questions, Waco production coordination",
|
||||
"creator": "U005",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Communication between DCs and Waco production",
|
||||
"creator": "U005",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "inventory-ops",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "inventory-ops",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Cycle counts, expiry flags, inventory adjustments",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Inventory accuracy, expiry management, adjustment tracking",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C005": {
|
||||
"id": "C005",
|
||||
"name": "cold-chain-alerts",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "cold-chain-alerts",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Temperature excursions, freezer alarms, cold chain breaches — both DCs",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Cold chain monitoring and alerts",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-03-17-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
|
||||
"ts": "1773753600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-03-17-001 receiving complete. Seal: intact. Temps: -16, -19, -15 (avg -17). Pallets: 22. Cases: 438 — short 2 on SC-001, manifest shows 440. Lot codes: A-20260315-01 (438 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1773756900.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-17-001 received. Pallets: 22, Cases: 438 (variance -2 SC-001), Temp: -15°F (warmest), Variance: -2 cases (0.5% — P4 minor). Product in H-A. Receipt logged.",
|
||||
"ts": "1773757320.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-03-24-001 expected at 10:00 CT. Pallets: 18, Cases: 360. Dock crew standing by.",
|
||||
"ts": "1774360800.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-03-24-001 receiving complete. Seal: intact. Temps: -20, -18, -21 (avg -20). Pallets: 18. Cases: 360 ✓ match manifest. Lot codes: A-20260322-01 (360 cases) ✓. No damage.",
|
||||
"ts": "1774364400.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-24-001 received and accepted. Pallets: 18, Cases: 360, Temp: -18°F (warmest), Variance: 0. Product in H-A. ✓",
|
||||
"ts": "1774364880.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-01-001 expected at 08:30 CT. Pallets: 24, Cases: 480. Dock crew standing by.",
|
||||
"ts": "1775047200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-01-001 receiving complete. Seal: intact. Temps: -17, -15, -18 (avg -17). Pallets: 24. Cases: 477 — short 3 on SC-001, manifest shows 480. Lot codes: A-20260330-01 (477 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1775051100.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-01-001 received. Pallets: 24, Cases: 477 (variance -3 SC-001), Temp: -15°F (warmest), Variance: -3 cases (0.6% — P4 minor). Product in H-A. Receipt logged.",
|
||||
"ts": "1775051520.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "@analyst just checking — SC-001 still available for tomorrow's routes? Running some high-volume stops.",
|
||||
"ts": "1775052300.000300",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "@b.keller Yes, SC-001 is available. 477 cases in H-A from today's receipt.",
|
||||
"ts": "1775052600.000400",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-07-001 expected at 09:00 CT. Pallets: 20, Cases: 400. Dock crew standing by.",
|
||||
"ts": "1775568600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-07-001 receiving complete. Seal: intact. Temps: -19, -20, -18 (avg -19). Pallets: 20. Cases: 400 ✓ match manifest. Lot codes: B-20260404-01 (200 cases SP-001), A-20260404-01 (200 cases SC-003) ✓ match manifest. No damage.",
|
||||
"ts": "1775572200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-07-001 received and accepted. Pallets: 20, Cases: 400, Temp: -18°F (warmest), Variance: 0. SC-003 to H-A, SP-001 to H-B. ✓",
|
||||
"ts": "1775572680.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Good start to the week. @a.nguyen nice work on the fast turnaround this morning.",
|
||||
"ts": "1775573100.000300",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Morning all. Waco truck for manifest 2026-04-14-001 is on schedule — departure confirmed from Waco at 06:00 CT. ETA ~09:00 CT.",
|
||||
"ts": "1776165000.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "@analyst heads up — I've got SC-001 and SC-006 on several stops Thursday. Just want to make sure we'll have enough after today's receipt clears.",
|
||||
"ts": "1776166200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-14-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
|
||||
"ts": "1776168000.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-14-001 receiving complete. Seal: intact. Temps: -4.5, -5, -8 (avg -6). Pallets: 22. Cases: 438. Lot codes: A-20260411-01 (SC-001, 218 cases), A-20260411-02 (SC-006, 220 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1776175500.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-17-001: minor variance on SC-001 (-2 cases, 0.5%). Logged P4.",
|
||||
"ts": "1773757500.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-01-001: minor variance on SC-001 (-3 cases, 0.6%). Logged P4.",
|
||||
"ts": "1775051700.000100",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C003": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Heads up — we're running line A at slightly reduced capacity this week due to a planned maintenance window Wednesday. Apr 14 transfer (manifest 2026-04-14-001) is still on schedule. SC-001 and SC-006.",
|
||||
"ts": "1775564100.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "@j.patel Confirmed, noted on our end. We'll have dock crew ready.",
|
||||
"ts": "1775564520.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Lot codes for 2026-04-14-001: A-20260411-01 (SC-001, production 2026-04-11) and A-20260411-02 (SC-006, production 2026-04-11). Production completed.",
|
||||
"ts": "1776079200.000100",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C004": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Weekly expiry review complete — 2026-04-07. All active lots remain Green tier. No flags. SP-003 (lot B-20260115-01) is the oldest — expiry 2027-04-15, 373 days remaining. No action required.",
|
||||
"ts": "1775574300.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U009",
|
||||
"text": "@analyst thanks for the weekly summary. Looks clean. Let me know if anything shifts to Yellow.",
|
||||
"ts": "1775575200.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U008",
|
||||
"text": "Zone H-A cycle count done — SC-001 lot A-20260308-01: system 318, physical 318. SC-001 lot A-20260315-01: system 438, physical 438. All match.",
|
||||
"ts": "1775920800.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Cycle count logged. 100% accuracy H-A this round. Thanks @s.kim.",
|
||||
"ts": "1775921400.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C005": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
|
||||
@@ -0,0 +1 @@
|
||||
You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/logistics_prairie_star_creamery_f382586a"
|
||||
description = "SOP task from CSV row f382586a"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
[
|
||||
{
|
||||
"id": "f9149db2-34e8-4fe1-b769-5ff3bee0a859",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `inbound_receiving_log.xlsx`, row 13 must contain: Actual Pallets = 22, Actual Cases = 438, Temp Reading 1 = -4.5, Temp Reading 2 = -5, Temp Reading 3 = -8, Warmest Temp = -4.5, Temp Tier = 'Tier 3', Variance (Cases) = -2, Variance % = -0.45%, Damage Class = 'None', Lot Codes = 'A-20260411-01 (SC-001), A-20260411-02 (SC-006)', Disposition = 'Hold', Priority = 'P2', Status = 'Open — Pending P2 Assessment', Notes containing '-2' and 'Third Repeated Minor Variance Pattern - P3'",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*inbound*receiving*log*.xlsx')) + list(Path(workspace_path).glob('*receiving*log*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'inbound' in f.name.lower() or 'receiving' in f.name.lower()]\n if not xlsx_files:\n all_xlsx = list(Path(workspace_path).glob('*.xlsx'))\n if not all_xlsx:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx files found in workspace'}\n xlsx_files = all_xlsx\n\n target_file = None\n for f in xlsx_files:\n if 'inbound' in f.name.lower() and 'receiving' in f.name.lower():\n target_file = f\n break\n if not target_file:\n target_file = xlsx_files[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n ws = wb.active\n row_data = [ws.cell(row=13, column=c).value for c in range(1, ws.max_column + 1)]\n\n # Column layout (0-indexed):\n # 0=Receipt Date, 1=Manifest/PO#, 2=Origin, 3=Site,\n # 4=Manifest Pallets, 5=Manifest Cases,\n # 6=Actual Pallets, 7=Actual Cases,\n # 8=Temp1, 9=Temp2, 10=Temp3,\n # 11=Warmest Temp (may be formula -> None),\n # 12=Temp Tier,\n # 13=Variance Cases (may be formula -> None),\n # 14=Variance % (may be formula -> None),\n # 15=Damage Class, 16=Lot Codes, 17=Disposition,\n # 18=Priority, 19=Status, 20=Notes\n\n # Resolve formula-backed cells from their source columns when cached value is None\n def get_warmest(row):\n if row[11] is not None:\n return row[11]\n temps = [row[i] for i in [8, 9, 10] if row[i] is not None]\n try:\n return max(float(t) for t in temps) if temps else None\n except (TypeError, ValueError):\n return None\n\n def get_variance_cases(row):\n if row[13] is not None:\n return row[13]\n mc, ac = row[5], row[7]\n try:\n return float(ac) - float(mc) if mc is not None and ac is not None else None\n except (TypeError, ValueError):\n return None\n\n def get_variance_pct(row):\n if row[14] is not None:\n return row[14]\n mc, ac = row[5], row[7]\n try:\n mc_f, ac_f = float(mc), float(ac)\n return (ac_f - mc_f) / mc_f if mc_f != 0 else None\n except (TypeError, ValueError):\n return None\n\n warmest = get_warmest(row_data)\n variance_cases = get_variance_cases(row_data)\n variance_pct = get_variance_pct(row_data)\n\n checks = []\n failures = []\n\n # Numeric checks\n numeric_checks = [\n (22, 'Actual Pallets', row_data[6]),\n (438, 'Actual Cases', row_data[7]),\n (-4.5, 'Temp Reading 1', row_data[8]),\n (-5, 'Temp Reading 2', row_data[9]),\n (-8, 'Temp Reading 3', row_data[10]),\n (-4.5, 'Warmest Temp', warmest),\n (-2, 'Variance Cases', variance_cases),\n ]\n for expected_val, label, actual_val in numeric_checks:\n try:\n found = actual_val is not None and abs(float(actual_val) - float(expected_val)) < 0.1\n except (TypeError, ValueError):\n found = False\n if found:\n checks.append(label)\n else:\n failures.append(f'{label} (expected {expected_val}, got {repr(actual_val)})')\n\n # Variance %: -0.45% stored as -0.0045 or -0.45\n vp_ok = False\n if variance_pct is not None:\n try:\n fv = float(variance_pct)\n vp_ok = abs(fv - (-0.0045)) < 0.0005 or abs(fv - (-0.45)) < 0.05\n except (TypeError, ValueError):\n vp_ok = '-0.45' in str(variance_pct)\n if not vp_ok and variance_pct is None:\n vp_ok = '-0.45' in str(row_data[14]) if row_data[14] is not None else False\n if vp_ok:\n checks.append('Variance %')\n else:\n failures.append(f'Variance % (expected -0.45%, got {repr(variance_pct)})')\n\n # String field checks — read specific columns directly, not row-wide concatenation\n def check_cell(col_idx, targets, label):\n val = row_data[col_idx]\n val_str = str(val).strip().lower() if val is not None else ''\n if any(t.lower() in val_str for t in targets):\n checks.append(label)\n else:\n failures.append(f'{label} (expected one of {targets}, got {repr(val)})')\n\n check_cell(12, ['tier 3'], 'Temp Tier')\n check_cell(15, ['none'], 'Damage Class')\n check_cell(17, ['hold'], 'Disposition')\n check_cell(18, ['p2'], 'Priority')\n\n # Status — check column 19 directly\n status_val = str(row_data[19]).lower() if rLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "9435bf97-bd03-40c7-9337-6ff1e27d209b",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In file `temperature_log.xlsx`, in sheet \"April 2026\", row 20 must contain Reading = -4.5 OR -4', row 21 must contain Reading = -5, and row 22 must contain Reading = -8",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find xlsx files broadly\n all_xlsx = list(Path(workspace_path).rglob('*.xlsx'))\n if not all_xlsx:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx files found in workspace'}\n\n # Try to find temperature_log.xlsx specifically, then fall back\n target_file = None\n for f in all_xlsx:\n if 'temperature' in f.name.lower() and 'log' in f.name.lower():\n target_file = f\n break\n if not target_file:\n for f in all_xlsx:\n if 'temp' in f.name.lower():\n target_file = f\n break\n if not target_file:\n target_file = all_xlsx[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n # Try to find the April 2026 sheet with flexible matching\n ws = None\n for name in wb.sheetnames:\n if 'april' in name.lower() and '2026' in name:\n ws = wb[name]\n break\n if ws is None:\n # Fall back to active sheet\n ws = wb.active\n\n # Determine which column contains \"Reading\" by scanning headers\n reading_col = None\n max_col = ws.max_column or 20\n\n # Scan first few rows for headers\n for header_row in range(1, min(6, (ws.max_row or 5) + 1)):\n for c in range(1, max_col + 1):\n val = ws.cell(row=header_row, column=c).value\n if val is not None:\n val_str = str(val).strip().lower()\n if 'reading' in val_str and 'within' not in val_str:\n reading_col = c\n\n # The rubric only checks Reading values for rows 20, 21, 22.\n # Row 20: Reading = -4.5 OR -4 (accept both)\n # Row 21: Reading = -5\n # Row 22: Reading = -8\n expected_rows = {\n 20: {'reading_options': [-4.5, -4.0]},\n 21: {'reading_options': [-5.0]},\n 22: {'reading_options': [-8.0]},\n }\n\n failures = []\n checks = []\n\n for row_num, expected in expected_rows.items():\n row_data = [ws.cell(row=row_num, column=c).value for c in range(1, max_col + 1)]\n row_str = [str(v).strip() if v is not None else '' for v in row_data]\n\n reading_found = False\n found_val = None\n\n def check_value(v, options):\n if v is None:\n return False, None\n try:\n fv = float(v)\n for opt in options:\n if abs(fv - opt) < 0.51:\n return True, fv\n except (ValueError, TypeError):\n # Also accept string representations like \"-4'\" or \"-4\\u2032\"\n v_str = str(v).strip().rstrip(\"'\\'\\u2032\\u2019\")\n try:\n fv = float(v_str)\n for opt in options:\n if abs(fv - opt) < 0.51:\n return True, fv\n except (ValueError, TypeError):\n pass\n return False, None\n\n # First check the identified reading column\n if reading_col is not None:\n v = ws.cell(row=row_num, column=reading_col).value\n reading_found, found_val = check_value(v, expected['reading_options'])\n\n # Fall back to scanning the whole row\n if not reading_found:\n for v in row_data:\n reading_found, found_val = check_value(v, expected['reading_options'])\n if reading_found:\n break\n\n if reading_found:\n checks.append(f'Row {row_num}: Reading={found_val} (expected one of {expected[\"reading_options\"]})')\n else:\n failures.append(f'Row {row_num}: Reading not found matching any of {expected[\"reading_options\"]}. Row data: {row_str}')\n\n if not failures:\n return {'pass': True, 'score': 1.0, 'feedback': f'All temperature log rows correct in {target_file.name} (sheet: {ws.title}): {checks}'}\n else:\n score = len(checks) / 3.0\n return {'pass': False, 'score': score, 'feedback': f'Temperature log failures in {target_file.name} (sheet: {ws.title}): {failures}. Passed: {checks}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "113fa31a-7a39-4aed-b8d8-13553c2272ca",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `slack_data.json`, there must be a message in the #dc-escalations channel containing 'P2 escalation' and 'Manifest 2026-04-14-001'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check Slack data'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n channels = data.get('channels', {})\n \n # Find escalation channel(s) - be loose: accept any channel with 'escalation' in the name\n escalation_channel_ids = []\n for cid, cinfo in channels.items():\n channel_name = cinfo.get('name', '').lower()\n if 'escalation' in channel_name or 'dc-escalation' in channel_name:\n escalation_channel_ids.append(cid)\n\n if not escalation_channel_ids:\n # Also try checking all channels in case the channel name is slightly different\n # Be generous: search all channels as a fallback\n escalation_channel_ids = list(channels.keys())\n if not escalation_channel_ids:\n return {'pass': False, 'score': 0.0, 'feedback': 'No channels found in slack_data.json'}\n\n messages = data.get('messages', {})\n\n # Required content - be loose with matching\n # 'P2 escalation' -> look for 'p2' and 'escalation' nearby or together\n # 'Manifest 2026-04-14-001' -> look for the manifest ID '2026-04-14-001'\n \n all_checked_messages = []\n \n for ch_id in escalation_channel_ids:\n channel_messages = messages.get(ch_id, [])\n for msg in channel_messages:\n text = msg.get('text', '')\n text_lower = text.lower()\n all_checked_messages.append(text[:150])\n \n # Check for P2 escalation - be loose\n has_p2_escalation = False\n if 'p2' in text_lower and 'escalation' in text_lower:\n has_p2_escalation = True\n elif 'p2 escalation' in text_lower:\n has_p2_escalation = True\n elif 'p-2' in text_lower and 'escalation' in text_lower:\n has_p2_escalation = True\n \n # Check for manifest ID - be loose\n has_manifest = False\n if '2026-04-14-001' in text:\n has_manifest = True\n elif 'manifest' in text_lower and '2026-04-14' in text_lower:\n has_manifest = True\n \n if has_p2_escalation and has_manifest:\n ch_name = channels.get(ch_id, {}).get('name', ch_id)\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found P2 escalation message with Manifest 2026-04-14-001 in #{ch_name}. Message: {text[:200]}'\n }\n\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'No message found containing both \"P2 escalation\" and \"Manifest 2026-04-14-001\" in escalation channels. Channels checked: {[channels.get(cid, {}).get(\"name\", cid) for cid in escalation_channel_ids]}. Messages found: {all_checked_messages[:10]}'\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4bc4fafd-be20-45e5-86ca-471bd13a7b28",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email to daniel.reyes@prairiestar.com with carlos.mendoza@prairiestar.com and jacob.patel@prairiestar.com as CC, subject line 'P2 — Inbound Waco Manifest 2026-04-14-001 — 2026-04-14', containing '2026-04-14-001', 'put-away' (or 'put away'), and 'hold' in the body",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n for email in sent_emails:\n to_addr = email.get('to_addr', '').lower()\n cc_addr = email.get('cc_addr', '') or ''\n subject = email.get('subject', '')\n body = email.get('body_text', '').lower()\n\n if 'daniel.reyes@prairiestar.com' not in to_addr:\n continue\n\n # Check CC\n cc_lower = cc_addr.lower()\n if 'carlos.mendoza@prairiestar.com' not in cc_lower or 'jacob.patel@prairiestar.com' not in cc_lower:\n continue\n\n # Check subject (flexible match)\n subj_lower = subject.lower()\n if 'p2' not in subj_lower or '2026-04-14-001' not in subject or ('inbound' not in subj_lower and 'waco' not in subj_lower):\n continue\n\n # Check body content\n required_body = ['2026-04-14-001', 'hold']\n putaway_check = 'put-away' in body or 'put away' in body or 'putaway' in body\n body_ok = all(r.lower() in body for r in required_body) and putaway_check\n\n if body_ok:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found P2 email to daniel.reyes@prairiestar.com with correct CC, subject, and body content'}\n else:\n missing_body = [r for r in required_body if r.lower() not in body]\n if not putaway_check:\n missing_body.append('put-away')\n return {'pass': False, 'score': 0.5, 'feedback': f'Found email to daniel.reyes with correct to/cc/subject but body missing: {missing_body}'}\n\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to daniel.reyes@prairiestar.com with correct CC (carlos.mendoza, jacob.patel) and subject containing P2 and 2026-04-14-001'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "90eb9ad4-97f1-472e-bd9a-d85b8067d83d",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json`, there must be an email sent to jacob.patel@prairiestar.com with subject line 'P3 – Inbound Waco Manifest 2026-04-14-001 – 2026-04-14' and body containing all of the following: '2026-04-14-001', '-4.5', '-5', '-8', 'enhanced expiry monitoring', '-2', 'repeated minor variance', and 'explanation' (or equivalent).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if (e.get('folder', '') or '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.json'}\n\n target_to = 'jacob.patel@prairiestar.com'\n\n # Normalize en-dash, em-dash, and hyphen all to plain hyphen for subject comparison\n def normalize_dashes(s):\n return re.sub(r'[\\u2013\\u2014\\-]', '-', s)\n\n expected_subject_norm = normalize_dashes(\n 'P3 - Inbound Waco Manifest 2026-04-14-001 - 2026-04-14'\n ).lower()\n\n required_body_terms = [\n '2026-04-14-001',\n '-4.5',\n '-5',\n '-8',\n 'enhanced expiry monitoring',\n '-2',\n 'repeated minor variance',\n ]\n explanation_equivalents = ['explanation', 'explain', 'rationale', 'reason', 'clarif', 'justif']\n\n def get_to_lower(email):\n to_addr = email.get('to_addr', '') or email.get('to', '') or ''\n if isinstance(to_addr, list):\n to_addr = ' '.join(to_addr)\n return to_addr.lower()\n\n diagnostics = []\n for email in sent_emails:\n to_lower = get_to_lower(email)\n if target_to not in to_lower:\n continue\n\n subject = email.get('subject', '') or ''\n subj_norm = normalize_dashes(subject).lower().strip()\n body = (email.get('body_text', '') or email.get('body', '') or '')\n body_lower = body.lower()\n\n subj_match = subj_norm == expected_subject_norm\n missing_body = [t for t in required_body_terms if t.lower() not in body_lower]\n has_expl = any(term in body_lower for term in explanation_equivalents)\n\n if subj_match and not missing_body and has_expl:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Found sent email to {target_to} with subject \"{subject}\" '\n f'and all required body content.'}\n\n diagnostics.append(\n f'Email subject \"{subject}\": subject_match={subj_match}, '\n f'missing_body={missing_body}, has_explanation_term={has_expl}'\n )\n\n if not diagnostics:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No sent emails found to {target_to}.'}\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No sent email to {} satisfied all checks. {}'.format(\n target_to, ' | '.join(diagnostics))}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "780f301b-297a-48fb-8cfb-dd96a751d119",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `mailbox.json`, there must NOT be a sent email to emily.carter@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n emily_emails = [e for e in sent_emails if any('emily.carter@prairiestar.com' in (e.get(field, '') or '').lower() for field in ['to_addr', 'cc_addr', 'bcc_addr'])]\n\n if emily_emails:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found {len(emily_emails)} sent email(s) to/cc/bcc emily.carter@prairiestar.com — this should not exist. Subjects: {[e.get(\"subject\") for e in emily_emails]}'}\n else:\n return {'pass': True, 'score': 1.0, 'feedback': 'No sent email to/cc/bcc emily.carter@prairiestar.com found — correct'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "4dec73cb-3b3a-4871-9554-41287ace133a",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `inventory_master.xlsx`, there must NOT be any row containing lot code 'A-20260411-01' or 'A-20260411-02'",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*inventory*master*.xlsx')) + list(Path(workspace_path).glob('*inventory*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'inventory' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n\n target_file = None\n for f in xlsx_files:\n if 'inventory' in f.name.lower() and 'master' in f.name.lower():\n target_file = f\n break\n if not target_file:\n for f in xlsx_files:\n if 'inventory' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find inventory_master.xlsx in workspace'}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n forbidden = ['a-20260411-01', 'a-20260411-02']\n violations = []\n\n for ws in wb.worksheets:\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).lower().strip()\n for f_val in forbidden:\n if f_val in cell_str:\n violations.append(f'Sheet={ws.title}, Row={row_idx}, Value={cell_val}')\n\n if violations:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found forbidden lot codes in {target_file.name}: {violations}'}\n else:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'No rows with A-20260411-01 or A-20260411-02 found in {target_file.name} — correct'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "fd88ec50-b68c-497c-ae9f-a8e6081e3ca1",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `expiry_tracker.xlsx`, there must NOT be any row containing lot code 'A-20260411-01' or 'A-20260411-02'",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*expiry*track*.xlsx')) + list(Path(workspace_path).glob('*expiry*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'expiry' in f.name.lower() or 'expir' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n\n target_file = None\n for f in xlsx_files:\n name_lower = f.name.lower()\n if 'expiry' in name_lower and ('track' in name_lower or 'tracker' in name_lower):\n target_file = f\n break\n if not target_file:\n for f in xlsx_files:\n if 'expiry' in f.name.lower() or 'expir' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find expiry_tracker.xlsx in workspace'}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n forbidden = ['a-20260411-01', 'a-20260411-02']\n violations = []\n\n for ws in wb.worksheets:\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).lower().strip()\n for f_val in forbidden:\n if f_val in cell_str:\n violations.append(f'Sheet={ws.title}, Row={row_idx}, Value={cell_val}')\n\n if violations:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found forbidden lot codes in {target_file.name}: {violations}'}\n else:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'No rows with A-20260411-01 or A-20260411-02 found in {target_file.name} — correct'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "3d60a3bd-ac14-4934-8beb-024854b3877e",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `mailbox.json`, there must be only one email with a subject line that contains 'P3' sent to jacob.patel@prairiestar.com.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if (e.get('folder', '') or '').lower() == 'sent']\n\n target_to = 'jacob.patel@prairiestar.com'\n\n def get_to_lower(email):\n to_addr = email.get('to_addr', '') or email.get('to', '') or ''\n if isinstance(to_addr, list):\n to_addr = ' '.join(to_addr)\n return to_addr.lower()\n\n matches = []\n for email in sent_emails:\n subject = email.get('subject', '') or ''\n to_lower = get_to_lower(email)\n if 'p3' in subject.lower() and target_to in to_lower:\n matches.append(email.get('email_id') or subject)\n\n if len(matches) == 1:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Exactly one P3 email sent to {target_to} (matched: {matches[0]}).'}\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Expected exactly 1 P3 email sent to {target_to}, found {len(matches)}. '\n f'Matches: {matches}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user