Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,141 @@
{
"events": {
"evt-001": {
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-14-001",
"start": {
"dateTime": "2026-04-14T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-14T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Manifest: 2026-04-14-001 | Origin: Waco Production Facility | Expected: 22 pallets, 440 cases (SC-001, SC-006) | Dock crew: Alicia Nguyen",
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
"id": "evt-001"
},
"evt-002": {
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-21-001",
"start": {
"dateTime": "2026-04-21T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-21T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Manifest: 2026-04-21-001 | Origin: Waco Production Facility | Expected quantities TBD \u2014 confirm with Jacob Patel by Apr 18.",
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
"id": "evt-002"
},
"evt-003": {
"summary": "Gulf Coast Packaging Delivery \u2014 PO-54823",
"start": {
"dateTime": "2026-04-14T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-14T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Vendor: Gulf Coast Packaging (PKG-01) | PO: PO-54823 | Expected: 40 cases pint containers, 40 cases lids",
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
"id": "evt-003"
},
"evt-004": {
"summary": "TexPal Logistics Pallet Delivery \u2014 PO-77209",
"start": {
"dateTime": "2026-04-15T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-15T09:30:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Vendor: TexPal Logistics (PAL-01) | PO: PO-77209 | Bi-weekly delivery (1st & 15th). Inspect all pallets for splinters, broken slats, stains, pest evidence.",
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
"id": "evt-004"
},
"evt-005": {
"summary": "Heartland Bakery Supply Delivery \u2014 PO-31119",
"start": {
"dateTime": "2026-04-20T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-20T09:30:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Vendor: Heartland Bakery Supply (DRY-01) | PO: PO-31119 | Waffle cones and cookie shells. Store in dry storage only \u2014 do not expose to moisture.",
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
"id": "evt-005"
},
"evt-006": {
"summary": "Weekly Expiry Review \u2014 Houston DC",
"start": {
"dateTime": "2026-04-14T13:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-14T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Monday expiry review.",
"location": "Houston DC",
"id": "evt-006"
},
"evt-007": {
"summary": "Weekly Expiry Review \u2014 Houston DC",
"start": {
"dateTime": "2026-04-21T13:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-21T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Monday expiry review.",
"location": "Houston DC",
"id": "evt-007"
},
"evt-008": {
"summary": "Zone H-A / H-B Cycle Count",
"start": {
"dateTime": "2026-04-16T07:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-16T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Scheduled cycle count for zones H-A and H-B. Counter: Sarah Kim.",
"location": "Houston DC \u2014 Zones H-A and H-B",
"id": "evt-008"
},
"evt-009": {
"summary": "Waco Line A Maintenance Window",
"start": {
"date": "2026-04-16"
},
"end": {
"date": "2026-04-17"
},
"description": "Planned maintenance on Waco Line A. No SC-line production this day. Confirm next transfer schedule with Jacob Patel.",
"id": "evt-009"
},
"evt-010": {
"summary": "Q1 2026 Vendor Scorecard Review",
"start": {
"dateTime": "2026-04-17T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-17T15:00:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Quarterly vendor performance aggregate calculation. Attendees: Analyst, Carlos Mendoza, Marcus Liu.",
"location": "Houston DC",
"id": "evt-010"
}
}
}
@@ -0,0 +1,244 @@
{
"mailbox": {
"email": "analyst@prairiestar.com",
"name": "Inventory Operations Analyst"
},
"contacts": [
{
"email": "carlos.mendoza@prairiestar.com",
"name": "Carlos Mendoza"
},
{
"email": "alicia.nguyen@prairiestar.com",
"name": "Alicia Nguyen"
},
{
"email": "sarah.kim@prairiestar.com",
"name": "Sarah Kim"
},
{
"email": "kevin.white@prairiestar.com",
"name": "Kevin White"
},
{
"email": "brandon.keller@prairiestar.com",
"name": "Brandon Keller"
},
{
"email": "luis.ramirez@prairiestar.com",
"name": "Luis Ramirez"
},
{
"email": "daniel.reyes@prairiestar.com",
"name": "Daniel Reyes"
},
{
"email": "emily.carter@prairiestar.com",
"name": "Emily Carter"
},
{
"email": "marcus.liu@prairiestar.com",
"name": "Marcus Liu"
},
{
"email": "hannah.brooks@prairiestar.com",
"name": "Hannah Brooks"
},
{
"email": "jacob.patel@prairiestar.com",
"name": "Jacob Patel"
},
{
"email": "renee.dubois@prairiestar.com",
"name": "Renee Dubois"
},
{
"email": "wyatt.green@prairiestar.com",
"name": "Wyatt Green"
},
{
"email": "tom.brewer@gulfcoastpkg.com",
"name": "Tom Brewer (Gulf Coast Packaging)"
},
{
"email": "lmarsh@southernwrap.com",
"name": "Linda Marsh (Southern Wrap Co.)"
},
{
"email": "d.swan@cleanproindustrial.com",
"name": "Derek Swan (CleanPro Industrial)"
},
{
"email": "s.kowalski@arcticexpress.com",
"name": "Sam Kowalski (Arctic Express)"
},
{
"email": "c.vega@texp\u0430\u043blogistics.com",
"name": "Chris Vega (TexPal Logistics)"
}
],
"groups": [
{
"email": "houston-dc@prairiestar.com",
"name": "Houston DC Team",
"members": [
"analyst@prairiestar.com",
"carlos.mendoza@prairiestar.com",
"alicia.nguyen@prairiestar.com",
"sarah.kim@prairiestar.com",
"kevin.white@prairiestar.com",
"brandon.keller@prairiestar.com"
]
}
],
"folders": [
{
"name": "Receiving"
},
{
"name": "Vendors"
},
{
"name": "Escalations"
}
],
"emails": [
{
"email_id": "1",
"folder": "Sent",
"subject": "P4 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
"from_addr": "analyst@prairiestar.com",
"to_addr": "jacob.patel@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-03-17T14:35:00Z",
"message_id": "<analyst-2026031701@prairiestar.com>",
"in_reply_to": null,
"body_text": "Jacob,\n\nThis is a variance notification for inbound Waco transfer manifest 2026-03-17-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 440 cases\n- Received quantity: 438 cases\n- Variance: -2 cases (-0.5%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260315-01 \u2014 confirmed match to manifest.\n\nPlease confirm whether a short-ship credit or makeup on next transfer is expected.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
"from_addr": "jacob.patel@prairiestar.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-03-17T16:10:00Z",
"message_id": "<jpatel-2026031701@prairiestar.com>",
"in_reply_to": "<analyst-2026031701@prairiestar.com>",
"body_text": "Thanks for the heads-up. The -2 on SC-001 was a loading count error on our end \u2014 we'll add it to the next transfer. No credit needed.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "3",
"folder": "Sent",
"subject": "P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
"from_addr": "analyst@prairiestar.com",
"to_addr": "jacob.patel@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-04-01T14:00:00Z",
"message_id": "<analyst-2026040101@prairiestar.com>",
"in_reply_to": null,
"body_text": "Jacob,\n\nVariance notification for inbound Waco transfer manifest 2026-04-01-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 480 cases\n- Received quantity: 477 cases\n- Variance: -3 cases (-0.6%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260330-01 \u2014 confirmed match to manifest.\n\nPlease confirm disposition of the -3 case shortfall.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "4",
"folder": "INBOX",
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
"from_addr": "jacob.patel@prairiestar.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-04-01T15:45:00Z",
"message_id": "<jpatel-2026040101@prairiestar.com>",
"in_reply_to": "<analyst-2026040101@prairiestar.com>",
"body_text": "Got it. The -3 on SC-001 was a pallet restack issue during loading. We're adding a loading check step this week. Will make up the 3 cases on the Apr 14 transfer.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "5",
"folder": "INBOX",
"subject": "Q1 2026 Vendor Scorecard \u2014 Reminder",
"from_addr": "marcus.liu@prairiestar.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-04-10T13:00:00Z",
"message_id": "<mliu-2026041001@prairiestar.com>",
"in_reply_to": null,
"body_text": "Hi,\n\nJust a reminder that the Q1 2026 vendor scorecard aggregates are due by April 17. Please make sure vendor_scorecard.xlsx is up to date with all Q1 deliveries before we meet.\n\nI'll be pulling the on-time %, quality %, and fill rate data ahead of the review.\n\nThanks,\nMarcus\n\n---\nMarcus Liu | Procurement Specialist\nPrairie Star Creamery",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "6",
"folder": "INBOX",
"subject": "Weekly Expiry Review \u2014 Apr 7 Summary",
"from_addr": "hannah.brooks@prairiestar.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-08T09:30:00Z",
"message_id": "<hbrooks-2026040801@prairiestar.com>",
"in_reply_to": null,
"body_text": "Good summary on the Apr 7 expiry review \u2014 all lots Green, no flags.\n\nHannah\n\n---\nHannah Brooks | Inventory Analyst\nPrairie Star Creamery HQ",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "7",
"folder": "Vendors",
"subject": "RE: PO-54820 \u2014 Short Delivery (2 cases lids)",
"from_addr": "tom.brewer@gulfcoastpkg.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-02T11:20:00Z",
"message_id": "<tbrewer-2026040201@gulfcoastpkg.com>",
"in_reply_to": null,
"body_text": "Hi,\n\nApologies for the short on PO-54820 \u2014 2 cases of lids were held back at our facility due to a labeling recheck. They'll be included in your next delivery (PO-54823, scheduled Apr 14).\n\nNo credit note needed \u2014 we'll make it whole on the next shipment.\n\nTom Brewer\nGulf Coast Packaging",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "8",
"folder": "INBOX",
"subject": "Waco Transfer Schedule \u2014 Week of Apr 14",
"from_addr": "jacob.patel@prairiestar.com",
"to_addr": "analyst@prairiestar.com",
"cc_addr": "carlos.mendoza@prairiestar.com",
"bcc_addr": null,
"date": "2026-04-11T10:00:00Z",
"message_id": "<jpatel-2026041101@prairiestar.com>",
"in_reply_to": null,
"body_text": "Hi,\n\nConfirming the transfer schedule for the week of Apr 14:\n\nManifest 2026-04-14-001\n- Departure: Apr 14, 06:00 CT from Waco\n- ETA Houston: ~09:00 CT\n- Contents: SC-001 (220 cases, lot A-20260411-01), SC-006 (220 cases, lot A-20260411-02)\n- Total: 22 pallets, 440 cases\n- Lot production date: Apr 11, 2026\n\nNote: Line A was at reduced capacity Mon\u2013Tue this week due to planned maintenance. Production is back to full capacity as of today. The 3 makeup cases from the Apr 1 shortfall are included in the SC-001 count above.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
}
],
"next_email_id": 9
}
@@ -0,0 +1,562 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "analyst",
"deleted": false,
"real_name": "Inventory Operations Analyst",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Inventory Operations Analyst",
"display_name": "analyst",
"email": "analyst@prairiestar.com",
"title": "Inventory Operations Analyst"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "a.nguyen",
"deleted": false,
"real_name": "Alicia Nguyen",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Alicia Nguyen",
"display_name": "a.nguyen",
"email": "alicia.nguyen@prairiestar.com",
"title": "Inventory & Receiving Coordinator — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "c.mendoza",
"deleted": false,
"real_name": "Carlos Mendoza",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Carlos Mendoza",
"display_name": "c.mendoza",
"email": "carlos.mendoza@prairiestar.com",
"title": "DC Manager — Houston"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "d.reyes",
"deleted": false,
"real_name": "Daniel Reyes",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Daniel Reyes",
"display_name": "d.reyes",
"email": "daniel.reyes@prairiestar.com",
"title": "Director of Supply Chain"
},
"is_admin": true,
"is_owner": true,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "j.patel",
"deleted": false,
"real_name": "Jacob Patel",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Jacob Patel",
"display_name": "j.patel",
"email": "jacob.patel@prairiestar.com",
"title": "Production Planner — Waco"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "e.carter",
"deleted": false,
"real_name": "Emily Carter",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Emily Carter",
"display_name": "e.carter",
"email": "emily.carter@prairiestar.com",
"title": "Demand Planning Manager"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "b.keller",
"deleted": false,
"real_name": "Brandon Keller",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Brandon Keller",
"display_name": "b.keller",
"email": "brandon.keller@prairiestar.com",
"title": "Dispatcher / Route Planner — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "s.kim",
"deleted": false,
"real_name": "Sarah Kim",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Sarah Kim",
"display_name": "s.kim",
"email": "sarah.kim@prairiestar.com",
"title": "Warehouse Associate (Senior) — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "h.brooks",
"deleted": false,
"real_name": "Hannah Brooks",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Hannah Brooks",
"display_name": "h.brooks",
"email": "hannah.brooks@prairiestar.com",
"title": "Inventory Analyst — HQ"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "receiving-houston",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "receiving-houston",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Houston inbound receiving — arrivals, counts, temps, completions, flags",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Real-time receiving updates for Houston DC",
"creator": "U003",
"last_set": 1700000000
}
},
"C002": {
"id": "C002",
"name": "dc-escalations",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "dc-escalations",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U004",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "P1 and P2 priority events requiring Director attention",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Escalations for both Houston and Shreveport DCs",
"creator": "U004",
"last_set": 1700000000
}
},
"C003": {
"id": "C003",
"name": "waco-production-coord",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "waco-production-coord",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U005",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Variance issues, manifest questions, Waco production coordination",
"creator": "U005",
"last_set": 1700000000
},
"purpose": {
"value": "Communication between DCs and Waco production",
"creator": "U005",
"last_set": 1700000000
}
},
"C004": {
"id": "C004",
"name": "inventory-ops",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "inventory-ops",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Cycle counts, expiry flags, inventory adjustments",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Inventory accuracy, expiry management, adjustment tracking",
"creator": "U003",
"last_set": 1700000000
}
},
"C005": {
"id": "C005",
"name": "cold-chain-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "cold-chain-alerts",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U004",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Temperature excursions, freezer alarms, cold chain breaches — both DCs",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Cold chain monitoring and alerts",
"creator": "U004",
"last_set": 1700000000
}
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-03-17-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
"ts": "1773753600.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-03-17-001 receiving complete. Seal: intact. Temps: -16, -19, -15 (avg -17). Pallets: 22. Cases: 438 — short 2 on SC-001, manifest shows 440. Lot codes: A-20260315-01 (438 cases) ✓ match manifest. No damage.",
"ts": "1773756900.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-03-17-001 received. Pallets: 22, Cases: 438 (variance -2 SC-001), Temp: -15°F (warmest), Variance: -2 cases (0.5% — P4 minor). Product in H-A. Receipt logged.",
"ts": "1773757320.000200",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-03-24-001 expected at 10:00 CT. Pallets: 18, Cases: 360. Dock crew standing by.",
"ts": "1774360800.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-03-24-001 receiving complete. Seal: intact. Temps: -20, -18, -21 (avg -20). Pallets: 18. Cases: 360 ✓ match manifest. Lot codes: A-20260322-01 (360 cases) ✓. No damage.",
"ts": "1774364400.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-03-24-001 received and accepted. Pallets: 18, Cases: 360, Temp: -18°F (warmest), Variance: 0. Product in H-A. ✓",
"ts": "1774364880.000200",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-04-01-001 expected at 08:30 CT. Pallets: 24, Cases: 480. Dock crew standing by.",
"ts": "1775047200.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-04-01-001 receiving complete. Seal: intact. Temps: -17, -15, -18 (avg -17). Pallets: 24. Cases: 477 — short 3 on SC-001, manifest shows 480. Lot codes: A-20260330-01 (477 cases) ✓ match manifest. No damage.",
"ts": "1775051100.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-04-01-001 received. Pallets: 24, Cases: 477 (variance -3 SC-001), Temp: -15°F (warmest), Variance: -3 cases (0.6% — P4 minor). Product in H-A. Receipt logged.",
"ts": "1775051520.000200",
"team": "T001"
},
{
"type": "message",
"user": "U007",
"text": "@analyst just checking — SC-001 still available for tomorrow's routes? Running some high-volume stops.",
"ts": "1775052300.000300",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "@b.keller Yes, SC-001 is available. 477 cases in H-A from today's receipt.",
"ts": "1775052600.000400",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-04-07-001 expected at 09:00 CT. Pallets: 20, Cases: 400. Dock crew standing by.",
"ts": "1775568600.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-04-07-001 receiving complete. Seal: intact. Temps: -19, -20, -18 (avg -19). Pallets: 20. Cases: 400 ✓ match manifest. Lot codes: B-20260404-01 (200 cases SP-001), A-20260404-01 (200 cases SC-003) ✓ match manifest. No damage.",
"ts": "1775572200.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-04-07-001 received and accepted. Pallets: 20, Cases: 400, Temp: -18°F (warmest), Variance: 0. SC-003 to H-A, SP-001 to H-B. ✓",
"ts": "1775572680.000200",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "Good start to the week. @a.nguyen nice work on the fast turnaround this morning.",
"ts": "1775573100.000300",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "Morning all. Waco truck for manifest 2026-04-14-001 is on schedule — departure confirmed from Waco at 06:00 CT. ETA ~09:00 CT.",
"ts": "1776165000.000100",
"team": "T001"
},
{
"type": "message",
"user": "U007",
"text": "@analyst heads up — I've got SC-001 and SC-006 on several stops Thursday. Just want to make sure we'll have enough after today's receipt clears.",
"ts": "1776166200.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-04-14-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
"ts": "1776168000.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-04-14-001 receiving complete. Seal: intact. Temps: -4.5, -5, -8 (avg -6). Pallets: 22. Cases: 438. Lot codes: A-20260411-01 (SC-001, 218 cases), A-20260411-02 (SC-006, 220 cases) ✓ match manifest. No damage.",
"ts": "1776175500.000200",
"team": "T001"
}
],
"C002": [
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-03-17-001: minor variance on SC-001 (-2 cases, 0.5%). Logged P4.",
"ts": "1773757500.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Manifest 2026-04-01-001: minor variance on SC-001 (-3 cases, 0.6%). Logged P4.",
"ts": "1775051700.000100",
"team": "T001"
}
],
"C003": [
{
"type": "message",
"user": "U005",
"text": "Heads up — we're running line A at slightly reduced capacity this week due to a planned maintenance window Wednesday. Apr 14 transfer (manifest 2026-04-14-001) is still on schedule. SC-001 and SC-006.",
"ts": "1775564100.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "@j.patel Confirmed, noted on our end. We'll have dock crew ready.",
"ts": "1775564520.000200",
"team": "T001"
},
{
"type": "message",
"user": "U005",
"text": "Lot codes for 2026-04-14-001: A-20260411-01 (SC-001, production 2026-04-11) and A-20260411-02 (SC-006, production 2026-04-11). Production completed.",
"ts": "1776079200.000100",
"team": "T001"
}
],
"C004": [
{
"type": "message",
"user": "U001",
"text": "Weekly expiry review complete — 2026-04-07. All active lots remain Green tier. No flags. SP-003 (lot B-20260115-01) is the oldest — expiry 2027-04-15, 373 days remaining. No action required.",
"ts": "1775574300.000100",
"team": "T001"
},
{
"type": "message",
"user": "U009",
"text": "@analyst thanks for the weekly summary. Looks clean. Let me know if anything shifts to Yellow.",
"ts": "1775575200.000200",
"team": "T001"
},
{
"type": "message",
"user": "U008",
"text": "Zone H-A cycle count done — SC-001 lot A-20260308-01: system 318, physical 318. SC-001 lot A-20260315-01: system 438, physical 438. All match.",
"ts": "1775920800.000100",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "Cycle count logged. 100% accuracy H-A this round. Thanks @s.kim.",
"ts": "1775921400.000200",
"team": "T001"
}
],
"C005": []
},
"counters": {
"messageId": 1000000
}
}
@@ -0,0 +1 @@
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
@@ -0,0 +1 @@
You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/logistics_prairie_star_creamery_f382586a"
description = "SOP task from CSV row f382586a"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,65 @@
[
{
"id": "f9149db2-34e8-4fe1-b769-5ff3bee0a859",
"sort_order": 0,
"rubric_text": "In file `inbound_receiving_log.xlsx`, row 13 must contain: Actual Pallets = 22, Actual Cases = 438, Temp Reading 1 = -4.5, Temp Reading 2 = -5, Temp Reading 3 = -8, Warmest Temp = -4.5, Temp Tier = 'Tier 3', Variance (Cases) = -2, Variance % = -0.45%, Damage Class = 'None', Lot Codes = 'A-20260411-01 (SC-001), A-20260411-02 (SC-006)', Disposition = 'Hold', Priority = 'P2', Status = 'Open — Pending P2 Assessment', Notes containing '-2' and 'Third Repeated Minor Variance Pattern - P3'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*inbound*receiving*log*.xlsx')) + list(Path(workspace_path).glob('*receiving*log*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'inbound' in f.name.lower() or 'receiving' in f.name.lower()]\n if not xlsx_files:\n all_xlsx = list(Path(workspace_path).glob('*.xlsx'))\n if not all_xlsx:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx files found in workspace'}\n xlsx_files = all_xlsx\n\n target_file = None\n for f in xlsx_files:\n if 'inbound' in f.name.lower() and 'receiving' in f.name.lower():\n target_file = f\n break\n if not target_file:\n target_file = xlsx_files[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n ws = wb.active\n row_data = [ws.cell(row=13, column=c).value for c in range(1, ws.max_column + 1)]\n\n # Column layout (0-indexed):\n # 0=Receipt Date, 1=Manifest/PO#, 2=Origin, 3=Site,\n # 4=Manifest Pallets, 5=Manifest Cases,\n # 6=Actual Pallets, 7=Actual Cases,\n # 8=Temp1, 9=Temp2, 10=Temp3,\n # 11=Warmest Temp (may be formula -> None),\n # 12=Temp Tier,\n # 13=Variance Cases (may be formula -> None),\n # 14=Variance % (may be formula -> None),\n # 15=Damage Class, 16=Lot Codes, 17=Disposition,\n # 18=Priority, 19=Status, 20=Notes\n\n # Resolve formula-backed cells from their source columns when cached value is None\n def get_warmest(row):\n if row[11] is not None:\n return row[11]\n temps = [row[i] for i in [8, 9, 10] if row[i] is not None]\n try:\n return max(float(t) for t in temps) if temps else None\n except (TypeError, ValueError):\n return None\n\n def get_variance_cases(row):\n if row[13] is not None:\n return row[13]\n mc, ac = row[5], row[7]\n try:\n return float(ac) - float(mc) if mc is not None and ac is not None else None\n except (TypeError, ValueError):\n return None\n\n def get_variance_pct(row):\n if row[14] is not None:\n return row[14]\n mc, ac = row[5], row[7]\n try:\n mc_f, ac_f = float(mc), float(ac)\n return (ac_f - mc_f) / mc_f if mc_f != 0 else None\n except (TypeError, ValueError):\n return None\n\n warmest = get_warmest(row_data)\n variance_cases = get_variance_cases(row_data)\n variance_pct = get_variance_pct(row_data)\n\n checks = []\n failures = []\n\n # Numeric checks\n numeric_checks = [\n (22, 'Actual Pallets', row_data[6]),\n (438, 'Actual Cases', row_data[7]),\n (-4.5, 'Temp Reading 1', row_data[8]),\n (-5, 'Temp Reading 2', row_data[9]),\n (-8, 'Temp Reading 3', row_data[10]),\n (-4.5, 'Warmest Temp', warmest),\n (-2, 'Variance Cases', variance_cases),\n ]\n for expected_val, label, actual_val in numeric_checks:\n try:\n found = actual_val is not None and abs(float(actual_val) - float(expected_val)) < 0.1\n except (TypeError, ValueError):\n found = False\n if found:\n checks.append(label)\n else:\n failures.append(f'{label} (expected {expected_val}, got {repr(actual_val)})')\n\n # Variance %: -0.45% stored as -0.0045 or -0.45\n vp_ok = False\n if variance_pct is not None:\n try:\n fv = float(variance_pct)\n vp_ok = abs(fv - (-0.0045)) < 0.0005 or abs(fv - (-0.45)) < 0.05\n except (TypeError, ValueError):\n vp_ok = '-0.45' in str(variance_pct)\n if not vp_ok and variance_pct is None:\n vp_ok = '-0.45' in str(row_data[14]) if row_data[14] is not None else False\n if vp_ok:\n checks.append('Variance %')\n else:\n failures.append(f'Variance % (expected -0.45%, got {repr(variance_pct)})')\n\n # String field checks — read specific columns directly, not row-wide concatenation\n def check_cell(col_idx, targets, label):\n val = row_data[col_idx]\n val_str = str(val).strip().lower() if val is not None else ''\n if any(t.lower() in val_str for t in targets):\n checks.append(label)\n else:\n failures.append(f'{label} (expected one of {targets}, got {repr(val)})')\n\n check_cell(12, ['tier 3'], 'Temp Tier')\n check_cell(15, ['none'], 'Damage Class')\n check_cell(17, ['hold'], 'Disposition')\n check_cell(18, ['p2'], 'Priority')\n\n # Status — check column 19 directly\n status_val = str(row_data[19]).lower() if rLine truncated
"criterion_type": "expected_output"
},
{
"id": "9435bf97-bd03-40c7-9337-6ff1e27d209b",
"sort_order": 1,
"rubric_text": "In file `temperature_log.xlsx`, in sheet \"April 2026\", row 20 must contain Reading = -4.5 OR -4', row 21 must contain Reading = -5, and row 22 must contain Reading = -8",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find xlsx files broadly\n all_xlsx = list(Path(workspace_path).rglob('*.xlsx'))\n if not all_xlsx:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx files found in workspace'}\n\n # Try to find temperature_log.xlsx specifically, then fall back\n target_file = None\n for f in all_xlsx:\n if 'temperature' in f.name.lower() and 'log' in f.name.lower():\n target_file = f\n break\n if not target_file:\n for f in all_xlsx:\n if 'temp' in f.name.lower():\n target_file = f\n break\n if not target_file:\n target_file = all_xlsx[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n # Try to find the April 2026 sheet with flexible matching\n ws = None\n for name in wb.sheetnames:\n if 'april' in name.lower() and '2026' in name:\n ws = wb[name]\n break\n if ws is None:\n # Fall back to active sheet\n ws = wb.active\n\n # Determine which column contains \"Reading\" by scanning headers\n reading_col = None\n max_col = ws.max_column or 20\n\n # Scan first few rows for headers\n for header_row in range(1, min(6, (ws.max_row or 5) + 1)):\n for c in range(1, max_col + 1):\n val = ws.cell(row=header_row, column=c).value\n if val is not None:\n val_str = str(val).strip().lower()\n if 'reading' in val_str and 'within' not in val_str:\n reading_col = c\n\n # The rubric only checks Reading values for rows 20, 21, 22.\n # Row 20: Reading = -4.5 OR -4 (accept both)\n # Row 21: Reading = -5\n # Row 22: Reading = -8\n expected_rows = {\n 20: {'reading_options': [-4.5, -4.0]},\n 21: {'reading_options': [-5.0]},\n 22: {'reading_options': [-8.0]},\n }\n\n failures = []\n checks = []\n\n for row_num, expected in expected_rows.items():\n row_data = [ws.cell(row=row_num, column=c).value for c in range(1, max_col + 1)]\n row_str = [str(v).strip() if v is not None else '' for v in row_data]\n\n reading_found = False\n found_val = None\n\n def check_value(v, options):\n if v is None:\n return False, None\n try:\n fv = float(v)\n for opt in options:\n if abs(fv - opt) < 0.51:\n return True, fv\n except (ValueError, TypeError):\n # Also accept string representations like \"-4'\" or \"-4\\u2032\"\n v_str = str(v).strip().rstrip(\"'\\'\\u2032\\u2019\")\n try:\n fv = float(v_str)\n for opt in options:\n if abs(fv - opt) < 0.51:\n return True, fv\n except (ValueError, TypeError):\n pass\n return False, None\n\n # First check the identified reading column\n if reading_col is not None:\n v = ws.cell(row=row_num, column=reading_col).value\n reading_found, found_val = check_value(v, expected['reading_options'])\n\n # Fall back to scanning the whole row\n if not reading_found:\n for v in row_data:\n reading_found, found_val = check_value(v, expected['reading_options'])\n if reading_found:\n break\n\n if reading_found:\n checks.append(f'Row {row_num}: Reading={found_val} (expected one of {expected[\"reading_options\"]})')\n else:\n failures.append(f'Row {row_num}: Reading not found matching any of {expected[\"reading_options\"]}. Row data: {row_str}')\n\n if not failures:\n return {'pass': True, 'score': 1.0, 'feedback': f'All temperature log rows correct in {target_file.name} (sheet: {ws.title}): {checks}'}\n else:\n score = len(checks) / 3.0\n return {'pass': False, 'score': score, 'feedback': f'Temperature log failures in {target_file.name} (sheet: {ws.title}): {failures}. Passed: {checks}'}\n",
"criterion_type": "expected_output"
},
{
"id": "113fa31a-7a39-4aed-b8d8-13553c2272ca",
"sort_order": 2,
"rubric_text": "In `slack_data.json`, there must be a message in the #dc-escalations channel containing 'P2 escalation' and 'Manifest 2026-04-14-001'",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check Slack data'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n channels = data.get('channels', {})\n \n # Find escalation channel(s) - be loose: accept any channel with 'escalation' in the name\n escalation_channel_ids = []\n for cid, cinfo in channels.items():\n channel_name = cinfo.get('name', '').lower()\n if 'escalation' in channel_name or 'dc-escalation' in channel_name:\n escalation_channel_ids.append(cid)\n\n if not escalation_channel_ids:\n # Also try checking all channels in case the channel name is slightly different\n # Be generous: search all channels as a fallback\n escalation_channel_ids = list(channels.keys())\n if not escalation_channel_ids:\n return {'pass': False, 'score': 0.0, 'feedback': 'No channels found in slack_data.json'}\n\n messages = data.get('messages', {})\n\n # Required content - be loose with matching\n # 'P2 escalation' -> look for 'p2' and 'escalation' nearby or together\n # 'Manifest 2026-04-14-001' -> look for the manifest ID '2026-04-14-001'\n \n all_checked_messages = []\n \n for ch_id in escalation_channel_ids:\n channel_messages = messages.get(ch_id, [])\n for msg in channel_messages:\n text = msg.get('text', '')\n text_lower = text.lower()\n all_checked_messages.append(text[:150])\n \n # Check for P2 escalation - be loose\n has_p2_escalation = False\n if 'p2' in text_lower and 'escalation' in text_lower:\n has_p2_escalation = True\n elif 'p2 escalation' in text_lower:\n has_p2_escalation = True\n elif 'p-2' in text_lower and 'escalation' in text_lower:\n has_p2_escalation = True\n \n # Check for manifest ID - be loose\n has_manifest = False\n if '2026-04-14-001' in text:\n has_manifest = True\n elif 'manifest' in text_lower and '2026-04-14' in text_lower:\n has_manifest = True\n \n if has_p2_escalation and has_manifest:\n ch_name = channels.get(ch_id, {}).get('name', ch_id)\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found P2 escalation message with Manifest 2026-04-14-001 in #{ch_name}. Message: {text[:200]}'\n }\n\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'No message found containing both \"P2 escalation\" and \"Manifest 2026-04-14-001\" in escalation channels. Channels checked: {[channels.get(cid, {}).get(\"name\", cid) for cid in escalation_channel_ids]}. Messages found: {all_checked_messages[:10]}'\n }\n",
"criterion_type": "expected_output"
},
{
"id": "4bc4fafd-be20-45e5-86ca-471bd13a7b28",
"sort_order": 3,
"rubric_text": "In `mailbox.json`, there must be a sent email to daniel.reyes@prairiestar.com with carlos.mendoza@prairiestar.com and jacob.patel@prairiestar.com as CC, subject line 'P2 — Inbound Waco Manifest 2026-04-14-001 — 2026-04-14', containing '2026-04-14-001', 'put-away' (or 'put away'), and 'hold' in the body",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n for email in sent_emails:\n to_addr = email.get('to_addr', '').lower()\n cc_addr = email.get('cc_addr', '') or ''\n subject = email.get('subject', '')\n body = email.get('body_text', '').lower()\n\n if 'daniel.reyes@prairiestar.com' not in to_addr:\n continue\n\n # Check CC\n cc_lower = cc_addr.lower()\n if 'carlos.mendoza@prairiestar.com' not in cc_lower or 'jacob.patel@prairiestar.com' not in cc_lower:\n continue\n\n # Check subject (flexible match)\n subj_lower = subject.lower()\n if 'p2' not in subj_lower or '2026-04-14-001' not in subject or ('inbound' not in subj_lower and 'waco' not in subj_lower):\n continue\n\n # Check body content\n required_body = ['2026-04-14-001', 'hold']\n putaway_check = 'put-away' in body or 'put away' in body or 'putaway' in body\n body_ok = all(r.lower() in body for r in required_body) and putaway_check\n\n if body_ok:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found P2 email to daniel.reyes@prairiestar.com with correct CC, subject, and body content'}\n else:\n missing_body = [r for r in required_body if r.lower() not in body]\n if not putaway_check:\n missing_body.append('put-away')\n return {'pass': False, 'score': 0.5, 'feedback': f'Found email to daniel.reyes with correct to/cc/subject but body missing: {missing_body}'}\n\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to daniel.reyes@prairiestar.com with correct CC (carlos.mendoza, jacob.patel) and subject containing P2 and 2026-04-14-001'}\n",
"criterion_type": "expected_output"
},
{
"id": "90eb9ad4-97f1-472e-bd9a-d85b8067d83d",
"sort_order": 4,
"rubric_text": "In `mailbox.json`, there must be an email sent to jacob.patel@prairiestar.com with subject line 'P3 – Inbound Waco Manifest 2026-04-14-001 – 2026-04-14' and body containing all of the following: '2026-04-14-001', '-4.5', '-5', '-8', 'enhanced expiry monitoring', '-2', 'repeated minor variance', and 'explanation' (or equivalent).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if (e.get('folder', '') or '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.json'}\n\n target_to = 'jacob.patel@prairiestar.com'\n\n # Normalize en-dash, em-dash, and hyphen all to plain hyphen for subject comparison\n def normalize_dashes(s):\n return re.sub(r'[\\u2013\\u2014\\-]', '-', s)\n\n expected_subject_norm = normalize_dashes(\n 'P3 - Inbound Waco Manifest 2026-04-14-001 - 2026-04-14'\n ).lower()\n\n required_body_terms = [\n '2026-04-14-001',\n '-4.5',\n '-5',\n '-8',\n 'enhanced expiry monitoring',\n '-2',\n 'repeated minor variance',\n ]\n explanation_equivalents = ['explanation', 'explain', 'rationale', 'reason', 'clarif', 'justif']\n\n def get_to_lower(email):\n to_addr = email.get('to_addr', '') or email.get('to', '') or ''\n if isinstance(to_addr, list):\n to_addr = ' '.join(to_addr)\n return to_addr.lower()\n\n diagnostics = []\n for email in sent_emails:\n to_lower = get_to_lower(email)\n if target_to not in to_lower:\n continue\n\n subject = email.get('subject', '') or ''\n subj_norm = normalize_dashes(subject).lower().strip()\n body = (email.get('body_text', '') or email.get('body', '') or '')\n body_lower = body.lower()\n\n subj_match = subj_norm == expected_subject_norm\n missing_body = [t for t in required_body_terms if t.lower() not in body_lower]\n has_expl = any(term in body_lower for term in explanation_equivalents)\n\n if subj_match and not missing_body and has_expl:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Found sent email to {target_to} with subject \"{subject}\" '\n f'and all required body content.'}\n\n diagnostics.append(\n f'Email subject \"{subject}\": subject_match={subj_match}, '\n f'missing_body={missing_body}, has_explanation_term={has_expl}'\n )\n\n if not diagnostics:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No sent emails found to {target_to}.'}\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No sent email to {} satisfied all checks. {}'.format(\n target_to, ' | '.join(diagnostics))}\n",
"criterion_type": "expected_output"
},
{
"id": "780f301b-297a-48fb-8cfb-dd96a751d119",
"sort_order": 5,
"rubric_text": "In `mailbox.json`, there must NOT be a sent email to emily.carter@prairiestar.com",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n emily_emails = [e for e in sent_emails if any('emily.carter@prairiestar.com' in (e.get(field, '') or '').lower() for field in ['to_addr', 'cc_addr', 'bcc_addr'])]\n\n if emily_emails:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found {len(emily_emails)} sent email(s) to/cc/bcc emily.carter@prairiestar.com — this should not exist. Subjects: {[e.get(\"subject\") for e in emily_emails]}'}\n else:\n return {'pass': True, 'score': 1.0, 'feedback': 'No sent email to/cc/bcc emily.carter@prairiestar.com found — correct'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "4dec73cb-3b3a-4871-9554-41287ace133a",
"sort_order": 6,
"rubric_text": "In `inventory_master.xlsx`, there must NOT be any row containing lot code 'A-20260411-01' or 'A-20260411-02'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*inventory*master*.xlsx')) + list(Path(workspace_path).glob('*inventory*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'inventory' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n\n target_file = None\n for f in xlsx_files:\n if 'inventory' in f.name.lower() and 'master' in f.name.lower():\n target_file = f\n break\n if not target_file:\n for f in xlsx_files:\n if 'inventory' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find inventory_master.xlsx in workspace'}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n forbidden = ['a-20260411-01', 'a-20260411-02']\n violations = []\n\n for ws in wb.worksheets:\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).lower().strip()\n for f_val in forbidden:\n if f_val in cell_str:\n violations.append(f'Sheet={ws.title}, Row={row_idx}, Value={cell_val}')\n\n if violations:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found forbidden lot codes in {target_file.name}: {violations}'}\n else:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'No rows with A-20260411-01 or A-20260411-02 found in {target_file.name} — correct'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "fd88ec50-b68c-497c-ae9f-a8e6081e3ca1",
"sort_order": 7,
"rubric_text": "In `expiry_tracker.xlsx`, there must NOT be any row containing lot code 'A-20260411-01' or 'A-20260411-02'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*expiry*track*.xlsx')) + list(Path(workspace_path).glob('*expiry*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'expiry' in f.name.lower() or 'expir' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n\n target_file = None\n for f in xlsx_files:\n name_lower = f.name.lower()\n if 'expiry' in name_lower and ('track' in name_lower or 'tracker' in name_lower):\n target_file = f\n break\n if not target_file:\n for f in xlsx_files:\n if 'expiry' in f.name.lower() or 'expir' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find expiry_tracker.xlsx in workspace'}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {target_file.name}: {e}'}\n\n forbidden = ['a-20260411-01', 'a-20260411-02']\n violations = []\n\n for ws in wb.worksheets:\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).lower().strip()\n for f_val in forbidden:\n if f_val in cell_str:\n violations.append(f'Sheet={ws.title}, Row={row_idx}, Value={cell_val}')\n\n if violations:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Found forbidden lot codes in {target_file.name}: {violations}'}\n else:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'No rows with A-20260411-01 or A-20260411-02 found in {target_file.name} — correct'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "3d60a3bd-ac14-4934-8beb-024854b3877e",
"sort_order": 8,
"rubric_text": "In `mailbox.json`, there must be only one email with a subject line that contains 'P3' sent to jacob.patel@prairiestar.com.",
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if (e.get('folder', '') or '').lower() == 'sent']\n\n target_to = 'jacob.patel@prairiestar.com'\n\n def get_to_lower(email):\n to_addr = email.get('to_addr', '') or email.get('to', '') or ''\n if isinstance(to_addr, list):\n to_addr = ' '.join(to_addr)\n return to_addr.lower()\n\n matches = []\n for email in sent_emails:\n subject = email.get('subject', '') or ''\n to_lower = get_to_lower(email)\n if 'p3' in subject.lower() and target_to in to_lower:\n matches.append(email.get('email_id') or subject)\n\n if len(matches) == 1:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Exactly one P3 email sent to {target_to} (matched: {matches[0]}).'}\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Expected exactly 1 P3 email sent to {target_to}, found {len(matches)}. '\n f'Matches: {matches}'}\n",
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py