Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-001": {
|
||||
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-14-001",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Manifest: 2026-04-14-001 | Origin: Waco Production Facility | Expected: 22 pallets, 440 cases (SC-001, SC-006) | Dock crew: Alicia Nguyen",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-001"
|
||||
},
|
||||
"evt-002": {
|
||||
"summary": "Waco Transfer Arrival \u2014 Manifest 2026-04-21-001",
|
||||
"start": {
|
||||
"dateTime": "2026-04-21T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-21T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Manifest: 2026-04-21-001 | Origin: Waco Production Facility | Expected quantities TBD \u2014 confirm with Jacob Patel by Apr 18.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-002"
|
||||
},
|
||||
"evt-003": {
|
||||
"summary": "Gulf Coast Packaging Delivery \u2014 PO-54823",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T11:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: Gulf Coast Packaging (PKG-01) | PO: PO-54823 | Expected: 40 cases pint containers, 40 cases lids",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-003"
|
||||
},
|
||||
"evt-004": {
|
||||
"summary": "TexPal Logistics Pallet Delivery \u2014 PO-77209",
|
||||
"start": {
|
||||
"dateTime": "2026-04-15T08:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-15T09:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: TexPal Logistics (PAL-01) | PO: PO-77209 | Bi-weekly delivery (1st & 15th). Inspect all pallets for splinters, broken slats, stains, pest evidence.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-004"
|
||||
},
|
||||
"evt-005": {
|
||||
"summary": "Heartland Bakery Supply Delivery \u2014 PO-31119",
|
||||
"start": {
|
||||
"dateTime": "2026-04-20T08:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-20T09:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Vendor: Heartland Bakery Supply (DRY-01) | PO: PO-31119 | Waffle cones and cookie shells. Store in dry storage only \u2014 do not expose to moisture.",
|
||||
"location": "Houston DC \u2014 Receiving Dock, 2847 Eastgate Industrial Blvd, Houston, TX 77015",
|
||||
"id": "evt-005"
|
||||
},
|
||||
"evt-006": {
|
||||
"summary": "Weekly Expiry Review \u2014 Houston DC",
|
||||
"start": {
|
||||
"dateTime": "2026-04-14T13:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-14T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Monday expiry review.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-006"
|
||||
},
|
||||
"evt-007": {
|
||||
"summary": "Weekly Expiry Review \u2014 Houston DC",
|
||||
"start": {
|
||||
"dateTime": "2026-04-21T13:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-21T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Monday expiry review.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-007"
|
||||
},
|
||||
"evt-008": {
|
||||
"summary": "Zone H-A / H-B Cycle Count",
|
||||
"start": {
|
||||
"dateTime": "2026-04-16T07:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-16T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Scheduled cycle count for zones H-A and H-B. Counter: Sarah Kim.",
|
||||
"location": "Houston DC \u2014 Zones H-A and H-B",
|
||||
"id": "evt-008"
|
||||
},
|
||||
"evt-009": {
|
||||
"summary": "Waco Line A Maintenance Window",
|
||||
"start": {
|
||||
"date": "2026-04-16"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-17"
|
||||
},
|
||||
"description": "Planned maintenance on Waco Line A. No SC-line production this day. Confirm next transfer schedule with Jacob Patel.",
|
||||
"id": "evt-009"
|
||||
},
|
||||
"evt-010": {
|
||||
"summary": "Q1 2026 Vendor Scorecard Review",
|
||||
"start": {
|
||||
"dateTime": "2026-04-17T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-17T15:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Quarterly vendor performance aggregate calculation. Attendees: Analyst, Carlos Mendoza, Marcus Liu.",
|
||||
"location": "Houston DC",
|
||||
"id": "evt-010"
|
||||
}
|
||||
}
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "analyst@prairiestar.com",
|
||||
"name": "Inventory Operations Analyst"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"name": "Carlos Mendoza"
|
||||
},
|
||||
{
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"name": "Alicia Nguyen"
|
||||
},
|
||||
{
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"name": "Sarah Kim"
|
||||
},
|
||||
{
|
||||
"email": "kevin.white@prairiestar.com",
|
||||
"name": "Kevin White"
|
||||
},
|
||||
{
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"name": "Brandon Keller"
|
||||
},
|
||||
{
|
||||
"email": "luis.ramirez@prairiestar.com",
|
||||
"name": "Luis Ramirez"
|
||||
},
|
||||
{
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"name": "Daniel Reyes"
|
||||
},
|
||||
{
|
||||
"email": "emily.carter@prairiestar.com",
|
||||
"name": "Emily Carter"
|
||||
},
|
||||
{
|
||||
"email": "marcus.liu@prairiestar.com",
|
||||
"name": "Marcus Liu"
|
||||
},
|
||||
{
|
||||
"email": "hannah.brooks@prairiestar.com",
|
||||
"name": "Hannah Brooks"
|
||||
},
|
||||
{
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"name": "Jacob Patel"
|
||||
},
|
||||
{
|
||||
"email": "renee.dubois@prairiestar.com",
|
||||
"name": "Renee Dubois"
|
||||
},
|
||||
{
|
||||
"email": "wyatt.green@prairiestar.com",
|
||||
"name": "Wyatt Green"
|
||||
},
|
||||
{
|
||||
"email": "tom.brewer@gulfcoastpkg.com",
|
||||
"name": "Tom Brewer (Gulf Coast Packaging)"
|
||||
},
|
||||
{
|
||||
"email": "lmarsh@southernwrap.com",
|
||||
"name": "Linda Marsh (Southern Wrap Co.)"
|
||||
},
|
||||
{
|
||||
"email": "d.swan@cleanproindustrial.com",
|
||||
"name": "Derek Swan (CleanPro Industrial)"
|
||||
},
|
||||
{
|
||||
"email": "s.kowalski@arcticexpress.com",
|
||||
"name": "Sam Kowalski (Arctic Express)"
|
||||
},
|
||||
{
|
||||
"email": "c.vega@texp\u0430\u043blogistics.com",
|
||||
"name": "Chris Vega (TexPal Logistics)"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "houston-dc@prairiestar.com",
|
||||
"name": "Houston DC Team",
|
||||
"members": [
|
||||
"analyst@prairiestar.com",
|
||||
"carlos.mendoza@prairiestar.com",
|
||||
"alicia.nguyen@prairiestar.com",
|
||||
"sarah.kim@prairiestar.com",
|
||||
"kevin.white@prairiestar.com",
|
||||
"brandon.keller@prairiestar.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "Receiving"
|
||||
},
|
||||
{
|
||||
"name": "Vendors"
|
||||
},
|
||||
{
|
||||
"name": "Escalations"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "Sent",
|
||||
"subject": "P4 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
|
||||
"from_addr": "analyst@prairiestar.com",
|
||||
"to_addr": "jacob.patel@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-17T14:35:00Z",
|
||||
"message_id": "<analyst-2026031701@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Jacob,\n\nThis is a variance notification for inbound Waco transfer manifest 2026-03-17-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 440 cases\n- Received quantity: 438 cases\n- Variance: -2 cases (-0.5%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260315-01 \u2014 confirmed match to manifest.\n\nPlease confirm whether a short-ship credit or makeup on next transfer is expected.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-03-17-001 \u2014 2026-03-17",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-17T16:10:00Z",
|
||||
"message_id": "<jpatel-2026031701@prairiestar.com>",
|
||||
"in_reply_to": "<analyst-2026031701@prairiestar.com>",
|
||||
"body_text": "Thanks for the heads-up. The -2 on SC-001 was a loading count error on our end \u2014 we'll add it to the next transfer. No credit needed.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "Sent",
|
||||
"subject": "P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
|
||||
"from_addr": "analyst@prairiestar.com",
|
||||
"to_addr": "jacob.patel@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T14:00:00Z",
|
||||
"message_id": "<analyst-2026040101@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Jacob,\n\nVariance notification for inbound Waco transfer manifest 2026-04-01-001, received today at Houston DC.\n\nVariance detail:\n- SKU: SC-001 (Sweet Cream Prairie)\n- Manifest quantity: 480 cases\n- Received quantity: 477 cases\n- Variance: -3 cases (-0.6%)\n- Classification: Minor (P4 threshold)\n\nTemperature: Warmest reading -15\u00b0F. Tier 1 \u2014 accepted.\nSeal: Intact.\nLot codes: A-20260330-01 \u2014 confirmed match to manifest.\n\nPlease confirm disposition of the -3 case shortfall.\n\nRegards,\nInventory Operations Analyst\nHouston DC",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Re: P4 \u2014 Inbound Waco Manifest 2026-04-01-001 \u2014 2026-04-01",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T15:45:00Z",
|
||||
"message_id": "<jpatel-2026040101@prairiestar.com>",
|
||||
"in_reply_to": "<analyst-2026040101@prairiestar.com>",
|
||||
"body_text": "Got it. The -3 on SC-001 was a pallet restack issue during loading. We're adding a loading check step this week. Will make up the 3 cases on the Apr 14 transfer.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "INBOX",
|
||||
"subject": "Q1 2026 Vendor Scorecard \u2014 Reminder",
|
||||
"from_addr": "marcus.liu@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-10T13:00:00Z",
|
||||
"message_id": "<mliu-2026041001@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nJust a reminder that the Q1 2026 vendor scorecard aggregates are due by April 17. Please make sure vendor_scorecard.xlsx is up to date with all Q1 deliveries before we meet.\n\nI'll be pulling the on-time %, quality %, and fill rate data ahead of the review.\n\nThanks,\nMarcus\n\n---\nMarcus Liu | Procurement Specialist\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "6",
|
||||
"folder": "INBOX",
|
||||
"subject": "Weekly Expiry Review \u2014 Apr 7 Summary",
|
||||
"from_addr": "hannah.brooks@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-08T09:30:00Z",
|
||||
"message_id": "<hbrooks-2026040801@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Good summary on the Apr 7 expiry review \u2014 all lots Green, no flags.\n\nHannah\n\n---\nHannah Brooks | Inventory Analyst\nPrairie Star Creamery HQ",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "7",
|
||||
"folder": "Vendors",
|
||||
"subject": "RE: PO-54820 \u2014 Short Delivery (2 cases lids)",
|
||||
"from_addr": "tom.brewer@gulfcoastpkg.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-02T11:20:00Z",
|
||||
"message_id": "<tbrewer-2026040201@gulfcoastpkg.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nApologies for the short on PO-54820 \u2014 2 cases of lids were held back at our facility due to a labeling recheck. They'll be included in your next delivery (PO-54823, scheduled Apr 14).\n\nNo credit note needed \u2014 we'll make it whole on the next shipment.\n\nTom Brewer\nGulf Coast Packaging",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "8",
|
||||
"folder": "INBOX",
|
||||
"subject": "Waco Transfer Schedule \u2014 Week of Apr 14",
|
||||
"from_addr": "jacob.patel@prairiestar.com",
|
||||
"to_addr": "analyst@prairiestar.com",
|
||||
"cc_addr": "carlos.mendoza@prairiestar.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-11T10:00:00Z",
|
||||
"message_id": "<jpatel-2026041101@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nConfirming the transfer schedule for the week of Apr 14:\n\nManifest 2026-04-14-001\n- Departure: Apr 14, 06:00 CT from Waco\n- ETA Houston: ~09:00 CT\n- Contents: SC-001 (220 cases, lot A-20260411-01), SC-006 (220 cases, lot A-20260411-02)\n- Total: 22 pallets, 440 cases\n- Lot production date: Apr 11, 2026\n\nNote: Line A was at reduced capacity Mon\u2013Tue this week due to planned maintenance. Production is back to full capacity as of today. The 3 makeup cases from the Apr 1 shortfall are included in the SC-001 count above.\n\nJacob\n\n---\nJacob Patel | Production Planner \u2014 Waco\nPrairie Star Creamery",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 9
|
||||
}
|
||||
+562
@@ -0,0 +1,562 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "analyst",
|
||||
"deleted": false,
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"display_name": "analyst",
|
||||
"email": "analyst@prairiestar.com",
|
||||
"title": "Inventory Operations Analyst"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "a.nguyen",
|
||||
"deleted": false,
|
||||
"real_name": "Alicia Nguyen",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Alicia Nguyen",
|
||||
"display_name": "a.nguyen",
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"title": "Inventory & Receiving Coordinator — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "c.mendoza",
|
||||
"deleted": false,
|
||||
"real_name": "Carlos Mendoza",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Carlos Mendoza",
|
||||
"display_name": "c.mendoza",
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"title": "DC Manager — Houston"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "d.reyes",
|
||||
"deleted": false,
|
||||
"real_name": "Daniel Reyes",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Daniel Reyes",
|
||||
"display_name": "d.reyes",
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"title": "Director of Supply Chain"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "j.patel",
|
||||
"deleted": false,
|
||||
"real_name": "Jacob Patel",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Jacob Patel",
|
||||
"display_name": "j.patel",
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"title": "Production Planner — Waco"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "e.carter",
|
||||
"deleted": false,
|
||||
"real_name": "Emily Carter",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Emily Carter",
|
||||
"display_name": "e.carter",
|
||||
"email": "emily.carter@prairiestar.com",
|
||||
"title": "Demand Planning Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T001",
|
||||
"name": "b.keller",
|
||||
"deleted": false,
|
||||
"real_name": "Brandon Keller",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Brandon Keller",
|
||||
"display_name": "b.keller",
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"title": "Dispatcher / Route Planner — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008",
|
||||
"team_id": "T001",
|
||||
"name": "s.kim",
|
||||
"deleted": false,
|
||||
"real_name": "Sarah Kim",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Sarah Kim",
|
||||
"display_name": "s.kim",
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"title": "Warehouse Associate (Senior) — Houston"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U009": {
|
||||
"id": "U009",
|
||||
"team_id": "T001",
|
||||
"name": "h.brooks",
|
||||
"deleted": false,
|
||||
"real_name": "Hannah Brooks",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Hannah Brooks",
|
||||
"display_name": "h.brooks",
|
||||
"email": "hannah.brooks@prairiestar.com",
|
||||
"title": "Inventory Analyst — HQ"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "receiving-houston",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "receiving-houston",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Houston inbound receiving — arrivals, counts, temps, completions, flags",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Real-time receiving updates for Houston DC",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "dc-escalations",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "dc-escalations",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "P1 and P2 priority events requiring Director attention",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Escalations for both Houston and Shreveport DCs",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "waco-production-coord",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "waco-production-coord",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U005",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Variance issues, manifest questions, Waco production coordination",
|
||||
"creator": "U005",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Communication between DCs and Waco production",
|
||||
"creator": "U005",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "inventory-ops",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "inventory-ops",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Cycle counts, expiry flags, inventory adjustments",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Inventory accuracy, expiry management, adjustment tracking",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C005": {
|
||||
"id": "C005",
|
||||
"name": "cold-chain-alerts",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "cold-chain-alerts",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Temperature excursions, freezer alarms, cold chain breaches — both DCs",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Cold chain monitoring and alerts",
|
||||
"creator": "U004",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-03-17-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
|
||||
"ts": "1773753600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-03-17-001 receiving complete. Seal: intact. Temps: -16, -19, -15 (avg -17). Pallets: 22. Cases: 438 — short 2 on SC-001, manifest shows 440. Lot codes: A-20260315-01 (438 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1773756900.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-17-001 received. Pallets: 22, Cases: 438 (variance -2 SC-001), Temp: -15°F (warmest), Variance: -2 cases (0.5% — P4 minor). Product in H-A. Receipt logged.",
|
||||
"ts": "1773757320.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-03-24-001 expected at 10:00 CT. Pallets: 18, Cases: 360. Dock crew standing by.",
|
||||
"ts": "1774360800.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-03-24-001 receiving complete. Seal: intact. Temps: -20, -18, -21 (avg -20). Pallets: 18. Cases: 360 ✓ match manifest. Lot codes: A-20260322-01 (360 cases) ✓. No damage.",
|
||||
"ts": "1774364400.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-24-001 received and accepted. Pallets: 18, Cases: 360, Temp: -18°F (warmest), Variance: 0. Product in H-A. ✓",
|
||||
"ts": "1774364880.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-01-001 expected at 08:30 CT. Pallets: 24, Cases: 480. Dock crew standing by.",
|
||||
"ts": "1775047200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-01-001 receiving complete. Seal: intact. Temps: -17, -15, -18 (avg -17). Pallets: 24. Cases: 477 — short 3 on SC-001, manifest shows 480. Lot codes: A-20260330-01 (477 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1775051100.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-01-001 received. Pallets: 24, Cases: 477 (variance -3 SC-001), Temp: -15°F (warmest), Variance: -3 cases (0.6% — P4 minor). Product in H-A. Receipt logged.",
|
||||
"ts": "1775051520.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "@analyst just checking — SC-001 still available for tomorrow's routes? Running some high-volume stops.",
|
||||
"ts": "1775052300.000300",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "@b.keller Yes, SC-001 is available. 477 cases in H-A from today's receipt.",
|
||||
"ts": "1775052600.000400",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-07-001 expected at 09:00 CT. Pallets: 20, Cases: 400. Dock crew standing by.",
|
||||
"ts": "1775568600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-07-001 receiving complete. Seal: intact. Temps: -19, -20, -18 (avg -19). Pallets: 20. Cases: 400 ✓ match manifest. Lot codes: B-20260404-01 (200 cases SP-001), A-20260404-01 (200 cases SC-003) ✓ match manifest. No damage.",
|
||||
"ts": "1775572200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-07-001 received and accepted. Pallets: 20, Cases: 400, Temp: -18°F (warmest), Variance: 0. SC-003 to H-A, SP-001 to H-B. ✓",
|
||||
"ts": "1775572680.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Good start to the week. @a.nguyen nice work on the fast turnaround this morning.",
|
||||
"ts": "1775573100.000300",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Morning all. Waco truck for manifest 2026-04-14-001 is on schedule — departure confirmed from Waco at 06:00 CT. ETA ~09:00 CT.",
|
||||
"ts": "1776165000.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "@analyst heads up — I've got SC-001 and SC-006 on several stops Thursday. Just want to make sure we'll have enough after today's receipt clears.",
|
||||
"ts": "1776166200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-04-14-001 expected at 09:00 CT. Pallets: 22, Cases: 440. Dock crew standing by.",
|
||||
"ts": "1776168000.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-04-14-001 receiving complete. Seal: intact. Temps: -4.5, -5, -8 (avg -6). Pallets: 22. Cases: 438. Lot codes: A-20260411-01 (SC-001, 218 cases), A-20260411-02 (SC-006, 220 cases) ✓ match manifest. No damage.",
|
||||
"ts": "1776175500.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-03-17-001: minor variance on SC-001 (-2 cases, 0.5%). Logged P4.",
|
||||
"ts": "1773757500.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Manifest 2026-04-01-001: minor variance on SC-001 (-3 cases, 0.6%). Logged P4.",
|
||||
"ts": "1775051700.000100",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C003": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Heads up — we're running line A at slightly reduced capacity this week due to a planned maintenance window Wednesday. Apr 14 transfer (manifest 2026-04-14-001) is still on schedule. SC-001 and SC-006.",
|
||||
"ts": "1775564100.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "@j.patel Confirmed, noted on our end. We'll have dock crew ready.",
|
||||
"ts": "1775564520.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Lot codes for 2026-04-14-001: A-20260411-01 (SC-001, production 2026-04-11) and A-20260411-02 (SC-006, production 2026-04-11). Production completed.",
|
||||
"ts": "1776079200.000100",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C004": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Weekly expiry review complete — 2026-04-07. All active lots remain Green tier. No flags. SP-003 (lot B-20260115-01) is the oldest — expiry 2027-04-15, 373 days remaining. No action required.",
|
||||
"ts": "1775574300.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U009",
|
||||
"text": "@analyst thanks for the weekly summary. Looks clean. Let me know if anything shifts to Yellow.",
|
||||
"ts": "1775575200.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U008",
|
||||
"text": "Zone H-A cycle count done — SC-001 lot A-20260308-01: system 318, physical 318. SC-001 lot A-20260315-01: system 438, physical 438. All match.",
|
||||
"ts": "1775920800.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Cycle count logged. 100% accuracy H-A this round. Thanks @s.kim.",
|
||||
"ts": "1775921400.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C005": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
|
||||
@@ -0,0 +1 @@
|
||||
You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/logistics_prairie_star_creamery_f382586a"
|
||||
description = "SOP task from CSV row f382586a"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Alicia has sent a new report today. I need you to process it in accordance with the instructions written in the SOP. It's her most recent report.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user