Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,316 @@
{
"events": {
"evt-001": {
"summary": "Waco Transfer \u2014 Manifest 2026-04-03-001",
"start": {
"dateTime": "2026-04-03T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-03T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-001"
},
"evt-002": {
"summary": "Weekly Expiry Review",
"start": {
"dateTime": "2026-04-03T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-03T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-002"
},
"evt-003": {
"summary": "PKG-01 Gulf Coast Packaging \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-07T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-07T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-003"
},
"evt-004": {
"summary": "Waco Transfer \u2014 Manifest 2026-04-08-001",
"start": {
"dateTime": "2026-04-08T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-08T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-004"
},
"evt-005": {
"summary": "Weekly Claims Review",
"start": {
"dateTime": "2026-04-10T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-10T14:30:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-005"
},
"evt-006": {
"summary": "Waco Transfer \u2014 Manifest 2026-04-10-001",
"start": {
"dateTime": "2026-04-10T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-10T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-006"
},
"evt-007": {
"summary": "Weekly Expiry Review",
"start": {
"dateTime": "2026-04-10T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-10T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-007"
},
"evt-008": {
"summary": "Waco Transfer \u2014 Manifest 2026-04-13-001",
"start": {
"dateTime": "2026-04-13T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-13T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-008"
},
"evt-009": {
"summary": "PKG-01 Gulf Coast Packaging \u2014 PO-54780",
"start": {
"dateTime": "2026-04-14T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-14T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-009"
},
"evt-010": {
"summary": "DRY-02 Lone Star Toppings \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-15T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-15T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-010"
},
"evt-011": {
"summary": "PAL-01 TexPal Logistics \u2014 Bi-Weekly Delivery",
"start": {
"dateTime": "2026-04-15T13:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-15T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-011"
},
"evt-012": {
"summary": "PKG-02 Southern Wrap Co. \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-16T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-16T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-012"
},
"evt-013": {
"summary": "CLN-01 CleanPro Industrial \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-17T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-17T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-013"
},
"evt-014": {
"summary": "Weekly Claims Review",
"start": {
"dateTime": "2026-04-17T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-17T14:30:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-014"
},
"evt-015": {
"summary": "Weekly Expiry Review",
"start": {
"dateTime": "2026-04-17T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-17T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-015"
},
"evt-016": {
"summary": "DRY-01 Heartland Bakery Supply \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-20T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-20T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-016"
},
"evt-017": {
"summary": "PKG-01 Gulf Coast Packaging \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-21T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-21T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-017"
},
"evt-018": {
"summary": "DRY-02 Lone Star Toppings \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-22T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-22T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-018"
},
"evt-019": {
"summary": "PKG-02 Southern Wrap Co. \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-23T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-23T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-019"
},
"evt-020": {
"summary": "CLN-01 CleanPro Industrial \u2014 Scheduled Delivery",
"start": {
"dateTime": "2026-04-24T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-24T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-020"
},
"evt-021": {
"summary": "Weekly Claims Review",
"start": {
"dateTime": "2026-04-24T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-24T14:30:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-021"
},
"evt-022": {
"summary": "PKG-01 Gulf Coast Packaging \u2014 PO-54750",
"start": {
"dateTime": "2026-03-31T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-03-31T11:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-022"
},
"evt-023": {
"summary": "Weekly Expiry Review",
"start": {
"dateTime": "2026-03-27T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-03-27T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-023"
},
"evt-024": {
"summary": "Weekly Claims Review",
"start": {
"dateTime": "2026-04-03T14:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-03T14:30:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-024"
},
"evt-025": {
"summary": "Waco Transfer \u2014 2026-04-15-001",
"start": {
"dateTime": "2026-04-15T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-15T10:00:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-025"
},
"evt-026": {
"summary": "Waco Transfer \u2014 2026-04-15-002",
"start": {
"dateTime": "2026-04-15T10:30:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-04-15T12:30:00-05:00",
"timeZone": "America/Chicago"
},
"id": "evt-026"
}
}
}
@@ -0,0 +1,33 @@
{
"mailbox": {
"email": "michael.andrews@prairiestar.com",
"name": "Michael Andrews"
},
"contacts": [
{"email": "daniel.reyes@prairiestar.com", "name": "Daniel Reyes"},
{"email": "emily.carter@prairiestar.com", "name": "Emily Carter"},
{"email": "marcus.liu@prairiestar.com", "name": "Marcus Liu"},
{"email": "hannah.brooks@prairiestar.com", "name": "Hannah Brooks"},
{"email": "jacob.patel@prairiestar.com", "name": "Jacob Patel"},
{"email": "carlos.mendoza@prairiestar.com", "name": "Carlos Mendoza"},
{"email": "alicia.nguyen@prairiestar.com", "name": "Alicia Nguyen"},
{"email": "sarah.kim@prairiestar.com", "name": "Sarah Kim"},
{"email": "kevin.white@prairiestar.com", "name": "Kevin White"},
{"email": "brandon.keller@prairiestar.com", "name": "Brandon Keller"},
{"email": "luis.ramirez@prairiestar.com", "name": "Luis Ramirez"},
{"email": "renee.dubois@prairiestar.com", "name": "Renee Dubois"},
{"email": "wyatt.green@prairiestar.com", "name": "Wyatt Green"},
{"email": "olivia.bennett@prairiestar.com", "name": "Olivia Bennett"},
{"email": "dylan.harper@prairiestar.com", "name": "Dylan Harper"},
{"email": "mtorres@gulfcoastpkg.com", "name": "Maria Torres"},
{"email": "dchen@southernwrap.com", "name": "David Chen"},
{"email": "padams@heartlandbakery.com", "name": "Peggy Adams"},
{"email": "fgutierrez@lonestartoppings.com", "name": "Frank Gutierrez"},
{"email": "jsimmons@cleanproindustrial.com", "name": "Janet Simmons"},
{"email": "treddick@arcticexpress.com", "name": "Tom Reddick"},
{"email": "bkowalski@texpal.com", "name": "Brenda Kowalski"}
],
"folders": [],
"emails": [],
"next_email_id": 1
}
@@ -0,0 +1,556 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "m.andrews",
"deleted": false,
"real_name": "Michael Andrews",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Michael Andrews",
"display_name": "m.andrews",
"email": "michael.andrews@prairiestar.com",
"title": "Inventory Operations Analyst"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "d.reyes",
"deleted": false,
"real_name": "Daniel Reyes",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Daniel Reyes",
"display_name": "d.reyes",
"email": "daniel.reyes@prairiestar.com",
"title": "Director of Supply Chain"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "e.carter",
"deleted": false,
"real_name": "Emily Carter",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Emily Carter",
"display_name": "e.carter",
"email": "emily.carter@prairiestar.com",
"title": "Demand Planning Manager"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "m.liu",
"deleted": false,
"real_name": "Marcus Liu",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Marcus Liu",
"display_name": "m.liu",
"email": "marcus.liu@prairiestar.com",
"title": "Procurement Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "h.brooks",
"deleted": false,
"real_name": "Hannah Brooks",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Hannah Brooks",
"display_name": "h.brooks",
"email": "hannah.brooks@prairiestar.com",
"title": "Inventory Analyst"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "j.patel",
"deleted": false,
"real_name": "Jacob Patel",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Jacob Patel",
"display_name": "j.patel",
"email": "jacob.patel@prairiestar.com",
"title": "Production Planner"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "c.mendoza",
"deleted": false,
"real_name": "Carlos Mendoza",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Carlos Mendoza",
"display_name": "c.mendoza",
"email": "carlos.mendoza@prairiestar.com",
"title": "DC Manager — Houston"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "a.nguyen",
"deleted": false,
"real_name": "Alicia Nguyen",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Alicia Nguyen",
"display_name": "a.nguyen",
"email": "alicia.nguyen@prairiestar.com",
"title": "Inventory & Receiving Coordinator — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "s.kim",
"deleted": false,
"real_name": "Sarah Kim",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Sarah Kim",
"display_name": "s.kim",
"email": "sarah.kim@prairiestar.com",
"title": "Warehouse Associate (Senior) — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U010": {
"id": "U010",
"team_id": "T001",
"name": "k.white",
"deleted": false,
"real_name": "Kevin White",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Kevin White",
"display_name": "k.white",
"email": "kevin.white@prairiestar.com",
"title": "Warehouse Associate — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U011": {
"id": "U011",
"team_id": "T001",
"name": "b.keller",
"deleted": false,
"real_name": "Brandon Keller",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Brandon Keller",
"display_name": "b.keller",
"email": "brandon.keller@prairiestar.com",
"title": "Dispatcher / Route Planner — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U012": {
"id": "U012",
"team_id": "T001",
"name": "l.ramirez",
"deleted": false,
"real_name": "Luis Ramirez",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Luis Ramirez",
"display_name": "l.ramirez",
"email": "luis.ramirez@prairiestar.com",
"title": "Lead Driver — Houston"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U013": {
"id": "U013",
"team_id": "T001",
"name": "r.dubois",
"deleted": false,
"real_name": "Renee Dubois",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Renee Dubois",
"display_name": "r.dubois",
"email": "renee.dubois@prairiestar.com",
"title": "DC Manager — Shreveport"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U014": {
"id": "U014",
"team_id": "T001",
"name": "w.green",
"deleted": false,
"real_name": "Wyatt Green",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Wyatt Green",
"display_name": "w.green",
"email": "wyatt.green@prairiestar.com",
"title": "Inventory & Receiving Coordinator — Shreveport"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U015": {
"id": "U015",
"team_id": "T001",
"name": "o.bennett",
"deleted": false,
"real_name": "Olivia Bennett",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Olivia Bennett",
"display_name": "o.bennett",
"email": "olivia.bennett@prairiestar.com",
"title": "Warehouse Associate (Senior) — Shreveport"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U016": {
"id": "U016",
"team_id": "T001",
"name": "d.harper",
"deleted": false,
"real_name": "Dylan Harper",
"tz": "America/Chicago",
"tz_label": "Central Daylight Time",
"tz_offset": -18000,
"profile": {
"real_name": "Dylan Harper",
"display_name": "d.harper",
"email": "dylan.harper@prairiestar.com",
"title": "Dispatcher / Route Planner — Shreveport"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "receiving-houston",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "receiving-houston",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U007",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Houston inbound receiving: arrivals, counts, temperatures, completions, flags",
"creator": "U007",
"last_set": 1704067200
},
"purpose": {
"value": "Real-time receiving updates for Houston DC",
"creator": "U007",
"last_set": 1704067200
}
},
"C002": {
"id": "C002",
"name": "receiving-shreveport",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "receiving-shreveport",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U013",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Shreveport inbound receiving: arrivals, counts, temperatures, completions, flags",
"creator": "U013",
"last_set": 1704067200
},
"purpose": {
"value": "Real-time receiving updates for Shreveport DC",
"creator": "U013",
"last_set": 1704067200
}
},
"C003": {
"id": "C003",
"name": "cold-chain-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "cold-chain-alerts",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Temperature excursions, freezer zone alarms, cold chain breaches",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "Cold chain alerts for both DCs",
"creator": "U002",
"last_set": 1704067200
}
},
"C004": {
"id": "C004",
"name": "waco-production-coord",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "waco-production-coord",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U006",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Variance issues, manifest questions, Waco production coordination",
"creator": "U006",
"last_set": 1704067200
},
"purpose": {
"value": "Communication with Waco production team",
"creator": "U006",
"last_set": 1704067200
}
},
"C005": {
"id": "C005",
"name": "inventory-ops",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "inventory-ops",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Cycle count results, expiry flags, inventory adjustments",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "Inventory operations coordination",
"creator": "U002",
"last_set": 1704067200
}
},
"C006": {
"id": "C006",
"name": "dc-escalations",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1704067200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "dc-escalations",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "P1 and P2 priority events, escalations requiring Director attention",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "Escalation channel for critical DC events",
"creator": "U002",
"last_set": 1704067200
}
}
},
"messages": {
"C001": [
{
"client_msg_id": "msg-1000000",
"type": "message",
"text": "Waco transfer 2026-04-15-001 expected at 08:00. Pallets: 10, Cases: 210. Dock crew standing by.",
"user": "U001",
"ts": "1776254400.000000",
"team": "T001",
"channel": "C001"
},
{
"client_msg_id": "msg-1000001",
"type": "message",
"text": "Waco transfer 2026-04-15-002 expected at 10:30. Pallets: 6, Cases: 80. Dock crew standing by.",
"user": "U001",
"ts": "1776254520.000000",
"team": "T001",
"channel": "C001"
},
{
"client_msg_id": "msg-1000002",
"type": "message",
"text": "Manifest 2026-04-15-001 dock 3 receiving complete. Seal: intact. Temps: -10.2, -10.2, -18.7 (avg -12.37). Pallets: 10. Lot codes: A-20260410-01 (100 cases), A-20260411-01 (80 cases), B-20260408-01 (30 cases). No damage. Wrap and pallets intact.",
"user": "U008",
"ts": "1776264300.000000",
"team": "T001",
"channel": "C001"
}
]
},
"counters": {
"messageId": 1000000
}
}
@@ -0,0 +1,111 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 5 0 R /F5 7 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/BaseFont /Helvetica-Oblique /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
>>
endobj
5 0 obj
<<
/BaseFont /Courier-Bold /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
>>
endobj
6 0 obj
<<
/Contents 12 0 R /MediaBox [ 0 0 612 792 ] /Parent 11 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
7 0 obj
<<
/BaseFont /Helvetica-BoldOblique /Encoding /WinAnsiEncoding /Name /F5 /Subtype /Type1 /Type /Font
>>
endobj
8 0 obj
<<
/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 11 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
9 0 obj
<<
/PageMode /UseNone /Pages 11 0 R /Type /Catalog
>>
endobj
10 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260418124525+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260418124525+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
11 0 obj
<<
/Count 2 /Kids [ 6 0 R 8 0 R ] /Type /Pages
>>
endobj
12 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 2042
>>
stream
Gatm;d;mu"&:F4Rge8dSA1X%B85[N&X#`l=9N!s817_1hl+eQT+X$k$+>)pJ])X;%U2@nFD0nqa!HGTXH[A:K5/%-n4"Z8;RF@48!t%A`0.;XhM!"/?1-bX0-tmcN6V@7fS$D)ZIP;ZCe:-\`,9.EacL0Z$!XFK?dN'=]@ui:?j0S"'KP'iGPYS-?nV>6Yc^-Kg6M"8:kXU"B"<Iu,:#Nb_U@to6\;:SYriN\Cns[a"kma2e\7VR6da7/29^#09>YCC3n#@'Nn'8leTtQnRl3p$`A,Y59-(=gZ,i*5>7I^B(S7_Ku)%>s:#q6o)l6*fFX$O3l*g#M_S32e:4_(rO2a;fj?qoYEH\8]h)<X$99@4U#CeWN$op_sTW?f/;`TKNRZ_cg9I&1qAV$F@!WBY]h9"^^dJ*<uQGD9h^iYMA$9X60TTI_,A_VdGi]c#-DS`%4<9Uk72(-*r)KL.mCFA_`OTVdp^h1RorUtRckm.[**eFWXs:<8K?gH6P6D0*<MnDYdoG\\Z/qit"Tq)CHmPGCCI]cN/VOWFfnkjOBKgql;QoZOkgl25\kUnXLiX\t`jRa\07`_,)En+eim=iKV7cNX='cs)duC>Jco+so+YIAn]aD/4>XjXD"t6=/&dF[`VLH^N*3ENRXMdS]RGA?k$$2R^[.c]:Ftl1l77PuKb#)'O9X*(VZ[>k1YIU=o>G#t&F'2)aP8]&XXT9(NeH36NLQ$b5*UDY!Q\?JgZq/hR^%>$P)dY`LK1j@n$s.T'UbdG8!BR4LeC:Y_k%X)&fTjoY-+>r$[/1(]fI@_pM+1`QW-]\tP>m(>eq;%u,+!1[si]#ZlfS2=jr!W\=`7A&SD7?mtA"i=:jr.P9X<Cl1VV0+YWpdS8P1+'56Xn5[uBN6#T5;#sU2Er%a70Wh,#2NTEd6YBEL/!T5Sr[ZP.!hV(mPg]Pl]?E/T)NoQ5;msT^/)1D0M7M(<mTG#9Mq\8YMd^NaX&ai)OL58qhn\SJPc_,>_4/0<R&U\9;j1]RWE=;fbTJ2jn4dhArW@2YZ)Y4(a=E:04p5Cc(nKMUV^>Z9N<!nEHI;dY[Wo-AVXK<VVtQB1s6cBblmSf+_tRA!NB&j1\9[-[6`r?Z-#Q66iH@?&+L@8I+\73XTs;Rld9*"5kOa`[?d-+hLM>L8.k8p:us3OcK"[+nS;5?eDCo"mafF/g](!p3#AQ%2>Fp?lG\@m0j`L9?(BB%^M+.[p.PH$rFL?Rn(KE\-0g;q_fAO"F(F`5ZmOV(;s$-&;C*,G(M$-=<lRUtbf;s,N2X'biDu<XeN\7I6a?kVE]Eshe^6F32$F$8JI,IsC="kSA,0JrNU/hNqh*M7O+.uL1&dipq<+(c)Fn_BB+KR]5_HJSIa>_5nNof_>-L+ZW!*A:\^sW#-)b3@c@!?*fp,L*XWFKA\I\Bb.n_s2Cjr,>`R\]GaIZT-l82@I,u]p6kO?Q:`gs_mB<R@q3sumamZ-u?j]19<s3r39^KZEKG4O)?`-&B@O'QGb<97+'Bd-3n72Of7MRM3o@p#E0@&)(/6u?(R$[W^!jJ5<\(hhOVlI]JuEg\hLVohEL0#:;>KiBQ)QW)uYs",qlO_>Y,FNsBVR>4\B>E^a7_V!P!k5SIK2q)uQ&DC1$q4r@K?d[oR/-"#O6N31L/+4fD2pKW"b2_;G]fe.<8?AX;q+]>eX7FWuPl0ZG6o4;-&Q8T!3%ZriJ#p21_[:1m\DD[eCd3C>II"VXAVZO^b"\p9o5ujpN#Q1XR1ls'D/k$B;I$([*Bk<(0s,CrQDf5JDo-^QMt]1)Vn"'\f>ftfY;,Y8KM5A[9V_1hX&</7`_EJ,NnTAZpX>WMa?<V(o;E>S)NYShpA$#nnB+]]pH[ug*U?@lSZh](=g*sYXgDs::8nEqs!b?Y,;in1b(sulm"=5aPkGDP<K_9=n@*<Gke>%QPfu'6FNLWWWHo[Df6B5<GAfDDlP/CC\[4=k;ndiG\q26G4XnWlBg(nfMcs\lmjZ^T$>HZ%<@#49_j-fo'o(q@TfVC2@8(fF-?Sl%[d680~>endstream
endobj
13 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 864
>>
stream
Gatn#d<:sD&;KY%MZ;,u-t*]a0AE;2l"LeE^$NuZ#2Kt/Z7NDtI:kuC@]`AN;8>3-0]1/?1FqDT+ZB?l!PG&b@NlorbVP(%$Sh[5&+e(V7r1>D,7OO$Sb\[$HngWb%]rlU37.)O^^#WF3,f/^DZU.rD"k@^r<DiubhkJ$*?QponL3+3Js_$T*83V:@ET6Ec=Zn.HV-h/(90jG8FIf[*h=Ud]1*^<XjWH*`'rZ:.8EU*2MoD9ne=^V;u[?(_2Jh9S1;4&XD=j%K![Zm^`Yat!;H+NM>]s<@X+4*-V^2EF=V(,Ou^a@Xif+hN^e"]ItllskR_4]!aeV*(cr5R6$o%(\lU2"C!kF`bRjG!rNc2INKOcSHaqA*/fR>KBQW(7=Y3XZA=pLXaK!AteNK^maX4L$,rWrZQp`OnBe]5gkX+PK8^<Q:NKpeBLG66Lgh#.:.^`NC3M#_[m-SC7_MV]KmWbM/jp1VZ@n1IO/.b9*gDmP0&VE"$iU\bUP?ud1^+8Zqi^U'T_Mc;ci)7g$kg#O5W[CH=(KA;p*sB_AXu00U:s'NGI-lL@cr;IOV3p(6H`TGG*rigFP]%7[Mpb12Wm\*^V!FT\G"Fd_r!cAL07WQI)Hgb]DQCpE$)LnQI>O<=3Tr=A)lbjIGJhIP:lq1GJpKs[D'cB)>;*Xh:1A<M<BpJ$<=_LRDnW#bfBmn>^2RgUiE2V*4>g"S7@74lE+qiYAC`-M__^EPXthKi/Yg@W(;qS_dGUu$@2'[k:V7^:"8f[g$,cNTA\S>fB[=Top:K8H.6U^G0FD+KVLK0qhh#6lCMh1,mcL"eWe>>>ds#j)I:'>k04emj9t<WYbX3Rt](MZt[j%R<)nHsh$3~>endstream
endobj
xref
0 14
0000000000 65535 f
0000000061 00000 n
0000000132 00000 n
0000000239 00000 n
0000000351 00000 n
0000000466 00000 n
0000000576 00000 n
0000000771 00000 n
0000000890 00000 n
0000001085 00000 n
0000001154 00000 n
0000001435 00000 n
0000001501 00000 n
0000003635 00000 n
trailer
<<
/ID
[<1aa07c40c21187ab5d698a6eaba1f1cf><1aa07c40c21187ab5d698a6eaba1f1cf>]
% ReportLab generated PDF document -- digest (opensource)
/Info 10 0 R
/Root 9 0 R
/Size 14
>>
startxref
4590
%%EOF
@@ -0,0 +1 @@
I'm way behind. i ended up getting tied up for a few hours while i was in the middle of processing this mornings expected arrivals. can you finish processing anything remaining for today and handle any msgs that came in while i was away? thanks
@@ -0,0 +1 @@
Today's date is, April, 15, 1pm. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/logistics_prairie_star_creamery_4a2f4b1a"
description = "SOP task from CSV row 4a2f4b1a"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,79 @@
[
{
"id": "562add5f-6c5a-4847-867b-95a7857b0ee0",
"sort_order": 0,
"rubric_text": "In file `Inbound_Receiving_Log.xlsx`, a row with manifest 2026-04-15-001 must exist with columns F-N containing (in order): 210, -10.2, -10.2, -18.7, -13.03, Tier 1, 0, 0, Exact Match; column P must be 'none'; column R must be 'P3'; column T must contain 'Open' (and may have other text as well in it); and the Lot Codes column must contain text referencing 'A-20260411-01 -- MISMATCH – manifest A-20260412-01 vs received A-20260411-01', 'A-20260410-01', and 'B-20260408-01'. Column Q must contain 'No damage'. Column S must contain 'Open'. Rounded whole-degree values are acceptable.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the Inbound Receiving Log file broadly\n xlsx_files = list(Path(workspace_path).glob('**/*nbound*eceiving*og*.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('**/Inbound_Receiving_Log.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('**/inbound_receiving_log.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'inbound' in f.name.lower() or 'receiving' in f.name.lower()]\n if not xlsx_files:\n # Try any xlsx\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Inbound_Receiving_Log.xlsx file found in workspace'}\n\n # Try each xlsx file to find the one with our manifest\n target_row = None\n row_num = None\n ws = None\n wb = None\n found_file = None\n\n for xlsx_file in xlsx_files:\n try:\n wb_candidate = openpyxl.load_workbook(xlsx_file, data_only=True)\n ws_candidate = wb_candidate.active\n for row in ws_candidate.iter_rows():\n for cell in row:\n if cell.value and '2026-04-15-001' in str(cell.value):\n target_row = row\n row_num = cell.row\n ws = ws_candidate\n wb = wb_candidate\n found_file = xlsx_file\n break\n if target_row:\n break\n if target_row:\n break\n except Exception:\n continue\n\n if not target_row:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row found with manifest 2026-04-15-001 in any xlsx file'}\n\n def get_cell_val(col_idx):\n return ws.cell(row=row_num, column=col_idx).value\n\n # Build header map to find columns by name\n header_map = {}\n for cell in ws[1]:\n if cell.value:\n header_map[str(cell.value).strip().lower()] = cell.column\n\n # Standard column mapping: F=6, G=7, H=8, I=9, J=10, K=11, L=12, M=13, N=14, O=15, P=16, Q=17, R=18, S=19, T=20\n col_f = get_cell_val(6)\n col_g = get_cell_val(7)\n col_h = get_cell_val(8)\n col_i = get_cell_val(9)\n col_j = get_cell_val(10)\n col_k = get_cell_val(11)\n col_l = get_cell_val(12)\n col_m = get_cell_val(13)\n col_n = get_cell_val(14)\n col_p = get_cell_val(16)\n col_q = get_cell_val(17)\n col_r = get_cell_val(18)\n col_s = get_cell_val(19)\n col_t = get_cell_val(20)\n\n issues = []\n\n # Helper for numeric checks with generous tolerance\n def check_numeric(val, expected, col_name, tolerance=None):\n if tolerance is None:\n tolerance = max(abs(expected) * 0.05, 1.0)\n try:\n # Handle various string formats\n val_str = str(val).replace(',', '').strip() if val is not None else ''\n # Remove degree symbols, F/C suffixes\n val_str = re.sub(r'[°ºFCfc]', '', val_str).strip()\n if val_str == '' or val_str.lower() == 'none':\n issues.append(f'{col_name} expected ~{expected}, got {val}')\n return\n num = float(val_str)\n if abs(num - expected) > tolerance:\n issues.append(f'{col_name} expected ~{expected}, got {val} (parsed as {num})')\n except (ValueError, TypeError):\n issues.append(f'{col_name} expected ~{expected}, got {val}')\n\n # Check column F = 210 (generous: ±10)\n check_numeric(col_f, 210, 'Column F', tolerance=10)\n\n # Check column G = -10.2 (allow whole-degree rounding: -10 is fine)\n check_numeric(col_g, -10.2, 'Column G', tolerance=1.5)\n\n # Check column H = -10.2\n check_numeric(col_h, -10.2, 'Column H', tolerance=1.5)\n\n # Check column I = -18.7 (allow -19 for rounding)\n check_numeric(col_i, -18.7, 'Column I', tolerance=2.0)\n\n # Check column J = -13.03 (allow -13 for rounding)\n check_numeric(col_j, -13.03, 'Column J', tolerance=2.0)\n\n # Check column K = Tier 1\n k_str = str(col_k).lower().strip() if col_k is not None else ''\n if 'tier' not in k_str or '1' not in str(col_k):\n issues.append(f'Column K expected Tier 1, got {col_k}')\n\n # Check column L = 0\n check_numeric(col_l, 0, 'Column L', tolerance=0.5)\n\n # Check column M = 0\n check_numeric(col_m, 0, 'Column M', tolerance=0.5)\n\n # Check column N = Exact Match\n n_str = str(col_n).lower().strip() if col_n is not None else ''\n if 'exact' not in n_str or 'match' not in n_str:\n issues.append(f'Column N expected Exact Match, got {col_n}')\n\n # Check column P = none (accept none, None, n/a, empty, 0, dash, etc.)\n p_sLine truncated
"criterion_type": "expected_output"
},
{
"id": "e78eb8ea-f315-42ab-aaa8-d21065c3f408",
"sort_order": 1,
"rubric_text": "In `mailbox.json`, a sent email to jacob.patel@prairiestar.com with cc to carlos.mendoza@prairiestar.com must exist containing text about 'LOT CODE MISMATCH', manifest '2026-04-15-001', lot codes 'A-20260412-01' and 'A-20260411-01', and 'Hold receipt open pending clarification'",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check email'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n \n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n cc_addr = str(email.get('cc_addr', '') or '').lower()\n body = str(email.get('body_text', '') or '').lower()\n subject = str(email.get('subject', '') or '').lower()\n combined = subject + ' ' + body\n \n if 'jacob.patel@prairiestar.com' not in to_addr:\n continue\n \n if 'carlos.mendoza@prairiestar.com' not in cc_addr:\n continue\n \n # Check for \"LOT CODE MISMATCH\" as a phrase in subject or body\n # Accept minor variations: \"lot code mismatch\", \"lot-code mismatch\", etc.\n has_lc_mismatch_phrase = bool(re.search(r'lot[\\s\\-_]*code[\\s\\-_]*mismatch', combined))\n \n has_manifest = '2026-04-15-001' in combined\n has_lot_manifest = 'a-20260412-01' in combined\n has_lot_received = 'a-20260411-01' in combined\n \n # Check for \"hold receipt open\" as a connected phrase (allow words in between)\n has_hold_open = bool(re.search(r'hold\\s+receipt\\s+open', combined))\n \n # Also accept \"hold .* open pending clarification\" as a looser but still structured match\n if not has_hold_open:\n has_hold_open = bool(re.search(r'hold\\b.*?\\bopen\\b.*?\\bpending\\b.*?\\bclarification\\b', combined))\n \n missing = []\n if not has_lc_mismatch_phrase:\n missing.append('\"LOT CODE MISMATCH\" phrase (found only scattered keyword \"mismatch\")' if 'mismatch' in combined else '\"LOT CODE MISMATCH\" phrase')\n if not has_manifest:\n missing.append('manifest 2026-04-15-001')\n if not has_lot_manifest:\n missing.append('lot A-20260412-01')\n if not has_lot_received:\n missing.append('lot A-20260411-01')\n if not has_hold_open:\n missing.append('\"Hold receipt open pending clarification\" phrase')\n \n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found sent email to jacob.patel@prairiestar.com with cc to carlos.mendoza@prairiestar.com containing LOT CODE MISMATCH template for manifest 2026-04-15-001'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found email to jacob.patel with cc carlos.mendoza but missing: {missing}'}\n \n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to jacob.patel@prairiestar.com with cc carlos.mendoza@prairiestar.com about lot code mismatch. Checked {len(sent_emails)} sent emails.'}",
"criterion_type": "expected_output"
},
{
"id": "f14469ac-f9fd-44fe-b08b-d15bbb125abc",
"sort_order": 2,
"rubric_text": "In `slack_data.json`, a message must exist in channel #receiving-houston or #waco-production-coord referencing 'P3', manifest '2026-04-15-001', and a lot code mismatch",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check Slack'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'slack_data.json not found at {slack_path}'}\n \n with open(slack_path, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n messages = data.get('messages', {})\n \n # Find target channel IDs\n target_channel_names = ['receiving-houston', 'waco-production-coord']\n target_channel_ids = []\n for ch_id, ch_info in channels.items():\n ch_name = str(ch_info.get('name', '')).lower()\n for target in target_channel_names:\n if target.lower() in ch_name or ch_name in target.lower():\n target_channel_ids.append(ch_id)\n break\n \n if not target_channel_ids:\n # Try to search all channels\n target_channel_ids = list(messages.keys())\n \n for ch_id in target_channel_ids:\n ch_msgs = messages.get(ch_id, [])\n for msg in ch_msgs:\n text = str(msg.get('text', '')).lower()\n has_p3 = 'p3' in text\n has_manifest = '2026-04-15-001' in text\n has_mismatch = 'mismatch' in text or 'lot code' in text\n \n if has_p3 and has_manifest and has_mismatch:\n ch_name = channels.get(ch_id, {}).get('name', ch_id)\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Slack message in channel #{ch_name} mentioning P3, manifest 2026-04-15-001, and lot code mismatch'}\n \n # Collect what was found for feedback\n found_msgs = []\n for ch_id in list(messages.keys()):\n ch_msgs = messages.get(ch_id, [])\n for msg in ch_msgs:\n text = str(msg.get('text', '')).lower()\n if '2026-04-15-001' in text or 'p3' in text:\n found_msgs.append(f'channel {ch_id}: {msg.get(\"text\", \"\")[:100]}')\n \n return {'pass': False, 'score': 0.0, 'feedback': f'No Slack message found in #receiving-houston or #waco-production-coord with P3, 2026-04-15-001, and mismatch. Target channels found: {target_channel_ids}. Related messages: {found_msgs[:3]}'}\n",
"criterion_type": "expected_output"
},
{
"id": "b94328d5-d410-4aff-b3b8-f0aa44ec29b9",
"sort_order": 3,
"rubric_text": "In file `expiry_tracker.xlsx`, a row must exist for Lot Code 'A-20260410-01' with columns A-L containing: SC-001, Sweet Cream Prairie, A-20260410-01, H-A, April 15 2026 (any date format), 2026-04-10 (any date format), 18, 2027-10-10 (any date format), 542 or 543, Green, 100, Monitored. Check the actual output of the cell, NOT the formula in the cell.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the expiry tracker file (flexible naming)\n xlsx_files = list(Path(workspace_path).glob('**/expiry_tracker.xlsx')) + list(Path(workspace_path).glob('**/Expiry_Tracker.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'expiry' in f.name.lower() and 'tracker' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'expiry' in f.name.lower() or 'tracker' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No expiry_tracker.xlsx file found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n\n # Search across all sheets for the lot code\n target_row_num = None\n lot_col = None\n target_ws = None\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows():\n for cell in row:\n if cell.value and 'a-20260410-01' in str(cell.value).lower().strip():\n target_row_num = cell.row\n lot_col = cell.column\n target_ws = ws\n break\n if target_row_num:\n break\n if target_row_num:\n break\n\n if not target_row_num or not target_ws:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row found with Lot Code A-20260410-01 in expiry_tracker.xlsx'}\n\n ws = target_ws\n\n def normalize_date(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.strftime('%Y-%m-%d')\n s = str(val).strip()\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', '%B %d %Y', '%b %d %Y', '%B %d, %Y', '%b %d, %Y',\n '%Y/%m/%d', '%d-%b-%Y', '%d-%B-%Y', '%d %B %Y', '%d %b %Y', '%d/%m/%Y',\n '%m-%d-%Y', '%Y%m%d', '%b %d, %y', '%B %d %Y', '%d-%m-%Y',\n '%b-%d-%Y', '%B-%d-%Y', '%m.%d.%Y', '%d.%m.%Y', '%Y.%m.%d']:\n try:\n return datetime.strptime(s, fmt).strftime('%Y-%m-%d')\n except:\n pass\n return s.lower()\n\n # Rubric says columns A-L contain 12 items:\n # A=1: SC-001\n # B=2: Sweet Cream Prairie\n # C=3: A-20260410-01\n # D=4: H-A\n # E=5: April 15 2026\n # F=6: 2026-04-10\n # G=7: 18\n # H=8: 2027-10-10\n # I=9: 542 or 543\n # J=10: Green\n # K=11: 100\n # L=12: Monitored\n #\n # The lot code (A-20260410-01) is the 3rd item, expected in column C (col index 3).\n offset = lot_col - 3\n\n def get_val(expected_col):\n return ws.cell(row=target_row_num, column=expected_col + offset).value\n\n issues = []\n\n # A=1: SC-001\n a = get_val(1)\n if a is None or 'sc-001' not in str(a).lower().replace(' ', ''):\n issues.append(f'Col A expected SC-001, got {a}')\n\n # B=2: Sweet Cream Prairie\n b = get_val(2)\n if b is None or 'sweet cream prairie' not in str(b).lower():\n issues.append(f'Col B expected Sweet Cream Prairie, got {b}')\n\n # C=3: A-20260410-01 (already confirmed by search)\n c = get_val(3)\n if c is None or 'a-20260410-01' not in str(c).lower():\n issues.append(f'Col C expected A-20260410-01, got {c}')\n\n # D=4: H-A\n d = get_val(4)\n if d is None or 'h-a' not in str(d).lower().replace(' ', ''):\n issues.append(f'Col D expected H-A, got {d}')\n\n # E=5: April 15 2026 (any date format)\n e = get_val(5)\n e_norm = normalize_date(e)\n if e_norm is None or '2026-04-15' not in e_norm:\n issues.append(f'Col E expected April 15 2026, got {e} (normalized: {e_norm})')\n\n # F=6: 2026-04-10 (any date format)\n f_val = get_val(6)\n f_norm = normalize_date(f_val)\n if f_norm is None or '2026-04-10' not in f_norm:\n issues.append(f'Col F expected 2026-04-10, got {f_val} (normalized: {f_norm})')\n\n # G=7: 18\n g = get_val(7)\n try:\n g_num = float(str(g))\n if abs(g_num - 18) > 1:\n issues.append(f'Col G expected 18, got {g}')\n except:\n issues.append(f'Col G expected 18, got {g}')\n\n # H=8: 2027-10-10 (any date format)\n h = get_val(8)\n h_norm = normalize_date(h)\n if h_norm is None or '2027-10-10' not in h_norm:\n issues.append(f'Col H expected 2027-10-10, got {h} (normalized: {h_norm})')\n\n # I=9: 542 or 543\n i_val = get_val(9)\n i_ok = False\n try:\n i_num = int(round(float(str(i_val))))\n if i_num in [542, 543]:\n i_ok = True\n except:\n pass\n if not i_ok:\n if i_val is not None and str(i_val).strip() in ['542', '543']:\n i_ok = True\n if not i_ok:\n issues.append(f'CoLine truncated
"criterion_type": "expected_output"
},
{
"id": "999b0abb-f444-47db-863c-9623d25639be",
"sort_order": 4,
"rubric_text": "In file `expiry_tracker.xlsx`, a row must exist for Lot Code B-20260408-01 with columns A-L containing: SP-001, Lavender Cream Moonrise, B-20260408-01, H-B, April 15 2026 (any format), April 8 2026 (any format), 15, July 8 2027 (any format), 448 or 449, Green, 30, Monitored",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the expiry tracker file - be lenient with naming\n xlsx_files = []\n for f in Path(workspace_path).glob('**/*.xlsx'):\n if 'expiry' in f.name.lower() or 'tracker' in f.name.lower():\n xlsx_files.append(f)\n \n if not xlsx_files:\n # Last resort: check all xlsx files\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n \n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx files found in workspace'}\n \n all_feedback = []\n \n for xlsx_file in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception as e:\n all_feedback.append(f'Could not open {xlsx_file.name}: {e}')\n continue\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Search for the lot code B-20260408-01 anywhere in the sheet\n target_rows = []\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row or 1):\n for cell in row:\n if cell.value is not None:\n val_str = str(cell.value).strip().lower().replace(' ', '')\n if 'b-20260408-01' in val_str or 'b20260408-01' in val_str or 'b-2026040801' in val_str or '20260408' in val_str:\n target_rows.append(cell.row)\n \n # Also search for SP-001\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row or 1):\n for cell in row:\n if cell.value is not None:\n val_str = str(cell.value).strip().lower().replace(' ', '')\n if 'sp-001' in val_str or 'sp001' in val_str:\n if cell.row not in target_rows:\n target_rows.append(cell.row)\n \n target_rows = sorted(set(target_rows))\n \n for row_num in target_rows:\n # Collect all values in this row\n row_vals = {}\n for c in range(1, 21):\n v = ws.cell(row=row_num, column=c).value\n row_vals[c] = v\n \n def normalize_date(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.strftime('%Y-%m-%d')\n s = str(val).strip()\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', '%B %d %Y', '%b %d %Y',\n '%B %d, %Y', '%b %d, %Y', '%d-%b-%Y', '%d-%B-%Y',\n '%d %B %Y', '%d %b %Y', '%Y/%m/%d', '%m-%d-%Y',\n '%d/%m/%Y', '%d/%m/%y', '%Y-%m-%dT%H:%M:%S',\n '%m/%d/%Y %H:%M', '%m/%d/%Y %H:%M:%S',\n '%B %d, %Y', '%b %d, %Y', '%d-%m-%Y',\n '%Y%m%d']:\n try:\n return datetime.strptime(s, fmt).strftime('%Y-%m-%d')\n except:\n pass\n return s.lower()\n \n def check_date(val, expected_date_str):\n if val is None:\n return False\n norm = normalize_date(val)\n if norm and expected_date_str == norm:\n return True\n raw = str(val).lower()\n parts = expected_date_str.split('-')\n year, month, day = parts[0], parts[1], parts[2]\n day_int = str(int(day))\n month_int = str(int(month))\n month_names = {1:'jan',2:'feb',3:'mar',4:'apr',5:'may',6:'jun',\n 7:'jul',8:'aug',9:'sep',10:'oct',11:'nov',12:'dec'}\n month_name = month_names.get(int(month), '')\n if year in raw and (day in raw or day_int in raw):\n if month in raw or month_int in raw or month_name in raw:\n return True\n return False\n \n def find_in_row(check_fn, start_col=1, end_col=20):\n for c in range(start_col, end_col+1):\n if check_fn(row_vals.get(c)):\n return c, row_vals.get(c)\n return None, None\n \n issues = []\n \n # Check A: SP-001\n a_ok = False\n a_val = row_vals.get(1)\n if a_val is not None and 'sp-001' in Line truncated
"criterion_type": "expected_output"
},
{
"id": "8e1be04c-7887-4d8d-b33b-66937b1f65da",
"sort_order": 5,
"rubric_text": "In file `expiry_tracker.xlsx`, there must be exactly 9 rows including the header row, and no row must have a lot code of 'A-20260411-01'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/expiry_tracker.xlsx')) + list(Path(workspace_path).glob('**/Expiry_Tracker.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'expiry' in f.name.lower() or 'tracker' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No expiry_tracker.xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n ws = wb.active\n \n # Count non-empty rows\n row_count = 0\n for row in ws.iter_rows():\n if any(cell.value is not None for cell in row):\n row_count += 1\n \n # Check for A-20260411-01 lot code\n bad_lot_rows = []\n lot_col = None\n \n # Find lot code column\n for row in ws.iter_rows():\n for cell in row:\n if cell.value and 'lot' in str(cell.value).lower():\n lot_col = cell.column\n break\n if lot_col:\n break\n \n if lot_col:\n for row in ws.iter_rows(min_row=2):\n cell = ws.cell(row=row[0].row, column=lot_col)\n if cell.value and 'a-20260411-01' in str(cell.value).lower():\n bad_lot_rows.append(row[0].row)\n else:\n # Search all cells\n for row in ws.iter_rows(min_row=2):\n for cell in row:\n if cell.value and 'a-20260411-01' in str(cell.value).lower():\n bad_lot_rows.append(row[0].row)\n break\n \n issues = []\n if row_count != 9:\n issues.append(f'Expected exactly 9 rows (including header), found {row_count}')\n if bad_lot_rows:\n issues.append(f'Found rows with lot code A-20260411-01 (should not exist): rows {bad_lot_rows}')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': f'expiry_tracker.xlsx issues: {chr(10).join(issues)}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'expiry_tracker.xlsx has exactly 9 rows and no row with lot code A-20260411-01'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "84fdd603-215e-477f-9ca5-c262da309356",
"sort_order": 6,
"rubric_text": "In file Inbound_Receiving_Log.xlsx, a row with manifest/PO# 11587 must exist with receipt date 4/15/26, origin TexPal Logistics, Total Cases (Manifest) 50, Status 'Pre-arrival', Notes about inspecting for splinters/broken slats/stains/pest evidence and rejecting pallets unable to hold 1500 lbs; columns D, G-K, and O set to 'X' or 'N/A' (N/A may have text after as long as it is present); columns F, L-N, P-R set to 'Pending', and S set to ‘Open’.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the file with loose matching\n xlsx_files = list(Path(workspace_path).glob('**/Inbound_Receiving_Log.xlsx')) + list(Path(workspace_path).glob('**/inbound_receiving_log.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'inbound' in f.name.lower() or 'receiving' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Inbound_Receiving_Log.xlsx found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n ws = wb.active\n\n # Find row with 11587\n target_row_num = None\n for row in ws.iter_rows():\n for cell in row:\n if cell.value is not None:\n val = str(cell.value).strip()\n if val == '11587' or val == '11587.0':\n target_row_num = cell.row\n break\n if target_row_num:\n break\n\n if not target_row_num:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row found with manifest/PO# 11587 in Inbound_Receiving_Log.xlsx'}\n\n row_num = target_row_num\n\n def get_val(col_idx):\n return ws.cell(row=row_num, column=col_idx).value\n\n def normalize_date(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.strftime('%Y-%m-%d')\n s = str(val).strip()\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', '%m-%d-%Y', '%m-%d-%y']:\n try:\n return datetime.strptime(s, fmt).strftime('%Y-%m-%d')\n except:\n pass\n return s\n\n def is_x_or_na(val):\n if val is None:\n return True # empty is acceptable (could be N/A)\n s = str(val).strip().lower()\n if s in ['x', 'n/a', 'na', '']:\n return True\n # N/A may have text after as long as it is present\n if s.startswith('n/a') or s.startswith('na '):\n return True\n return False\n\n def is_pending(val):\n if val is None:\n return False\n return 'pending' in str(val).strip().lower()\n\n def is_open(val):\n if val is None:\n return False\n return str(val).strip().lower() == 'open'\n\n # Gather all text from row for loose text matching\n max_col = ws.max_column or 30\n all_row_vals = []\n for c in range(1, max_col + 1):\n v = get_val(c)\n all_row_vals.append(str(v or '').lower())\n all_row_text = ' '.join(all_row_vals)\n\n issues = []\n\n # Check receipt date = 4/15/26 (i.e. April 15, 2026)\n date_found = False\n for c in range(1, max_col + 1):\n v = get_val(c)\n norm = normalize_date(v)\n if norm and norm == '2026-04-15':\n date_found = True\n break\n if not date_found:\n # Also check text representations\n if '4/15/26' in all_row_text or '4/15/2026' in all_row_text or '2026-04-15' in all_row_text or 'april 15' in all_row_text:\n date_found = True\n if not date_found:\n issues.append('Receipt date 4/15/26 not found in row')\n\n # Check origin TexPal Logistics\n if 'texpal' not in all_row_text:\n issues.append('Origin TexPal Logistics not found in row')\n\n # Check Total Cases (Manifest) = 50\n found_50 = False\n for c in range(1, max_col + 1):\n v = get_val(c)\n if v is not None:\n try:\n if abs(float(str(v)) - 50) < 1:\n found_50 = True\n break\n except:\n pass\n if not found_50:\n issues.append('Total Cases (Manifest) 50 not found in row')\n\n # Check Status = Pre-arrival\n if 'pre-arrival' not in all_row_text and 'pre arrival' not in all_row_text and 'prearrival' not in all_row_text:\n issues.append('Status Pre-arrival not found in row')\n\n # Check Notes about inspecting for splinters/broken slats/stains/pest evidence\n notes_keywords_found = 0\n for keyword in ['splinter', 'slat', 'stain', 'pest']:\n if keyword in all_row_text:\n notes_keywords_found += 1\n if notes_keywords_found < 2:\n issues.append(f'Notes about inspecting for splinters/broken slats/stains/pest evidence not sufficiently found in row (found {notes_keywords_found}/4 keywords)')\n\n # Check Notes about rejecting pallets unable to hold 1500 lbs\n if '1,500' not in all_row_text and '1500' not in all_row_text:\n issues.append('Notes about 1500 lbs not found in row')\n\n # Check columns D(4), G(7)-K(11), O(15) are X or N/A\n x_na_cols = [4, 7, 8, 9, 10, 11, 15]\n for col in x_na_cols:\n val = get_val(col)\n if not is_x_or_na(val):\n col_letter = openpyxl.utilsLine truncated
"criterion_type": "expected_output"
},
{
"id": "b369268f-dbfb-437c-bca4-c28ad2901138",
"sort_order": 7,
"rubric_text": "In file Inbound_Receiving_Log.xlsx, a row with manifest/PO# 67204 must exist with receipt date 4/15/26, origin Lone Star Toppings Co., Total Cases (Manifest) 45, Status 'Pre-arrival', Notes about allergen alert with tree nuts and peanuts and verifying allergen labeling; columns D, G-K, and O set to 'X' or 'N/A' (N/A may have text after as long as it is present); columns F, L-N, P-R set to 'Pending', and S set to ‘Open’",
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the file with loose matching\n xlsx_files = list(Path(workspace_path).glob('**/Inbound_Receiving_Log.xlsx')) + \\\n list(Path(workspace_path).glob('**/inbound_receiving_log.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'inbound' in f.name.lower() or 'receiving' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Inbound_Receiving_Log.xlsx found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n ws = wb.active\n \n # Find row containing 67204\n target_row_num = None\n for row in ws.iter_rows():\n for cell in row:\n if cell.value is not None:\n val = str(cell.value).strip()\n if val == '67204' or val == '67204.0':\n target_row_num = cell.row\n break\n if target_row_num:\n break\n \n if not target_row_num:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row found with manifest/PO# 67204 in Inbound_Receiving_Log.xlsx'}\n \n row_num = target_row_num\n \n def get_val(col_idx):\n return ws.cell(row=row_num, column=col_idx).value\n \n # Collect all text in the row for broad checks\n max_col = ws.max_column or 30\n all_row_values = []\n for c in range(1, max_col + 1):\n v = get_val(c)\n all_row_values.append(v)\n all_row_text = ' '.join([str(v or '').lower() for v in all_row_values])\n \n def normalize_date(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.strftime('%Y-%m-%d')\n s = str(val).strip()\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', '%m-%d-%Y', '%m-%d-%y']:\n try:\n return datetime.strptime(s, fmt).strftime('%Y-%m-%d')\n except:\n pass\n return s\n \n def is_x_or_na(val):\n \"\"\"Accept X, N/A (possibly with text after), empty/None, or NA.\"\"\"\n if val is None:\n return True\n s = str(val).strip().lower()\n if s == '':\n return True\n if s == 'x':\n return True\n if s.startswith('n/a') or s.startswith('na'):\n return True\n return False\n \n def is_pending(val):\n if val is None:\n return False\n return 'pending' in str(val).strip().lower()\n \n def is_open(val):\n if val is None:\n return False\n return str(val).strip().lower() == 'open'\n \n issues = []\n \n # Check receipt date = 4/15/26 (i.e., 2026-04-15)\n date_found = False\n date_checks = ['2026-04-15', '4/15/26', '4/15/2026', 'april 15']\n for dc in date_checks:\n if dc in all_row_text:\n date_found = True\n break\n if not date_found:\n for c in range(1, max_col + 1):\n norm = normalize_date(get_val(c))\n if norm and '2026-04-15' in norm:\n date_found = True\n break\n if not date_found:\n issues.append('Receipt date 4/15/26 not found in row')\n \n # Check origin Lone Star Toppings\n if 'lone star' not in all_row_text:\n issues.append('Origin Lone Star Toppings Co. not found in row')\n \n # Check Total Cases = 45\n found_45 = False\n for c in range(1, max_col + 1):\n v = get_val(c)\n if v is not None:\n try:\n if abs(float(str(v)) - 45) < 1:\n found_45 = True\n break\n except:\n pass\n if not found_45:\n issues.append('Total Cases (Manifest) 45 not found in row')\n \n # Check Status = Pre-arrival\n if 'pre-arrival' not in all_row_text and 'pre arrival' not in all_row_text and 'prearrival' not in all_row_text:\n issues.append('Status Pre-arrival not found in row')\n \n # Check Notes content - allergen\n if 'allergen' not in all_row_text:\n issues.append('Notes about allergen alert not found in row')\n if 'tree nut' not in all_row_text and 'treenut' not in all_row_text and 'peanut' not in all_row_text:\n issues.append('Notes about tree nuts/peanuts not found in row')\n \n # Check columns D(4), G(7)-K(11), O(15) are X or N/A\n x_na_cols = [4, 7, 8, 9, 10, 11, 15]\n for col in x_na_cols:\n val = get_val(col)\n if not is_x_or_na(val):\n issues.append(f'Column {col} (letter {chr(64+col)}) expected X or N/A, got \"{val}\"')\n \n # Check columns F(6), L(12)-N(14), P(16)-R(18) are Pending\n pending_cols = [6, 12, 13, 14, 16, 17, 18]\n for col in pending_cols:\n val = get_valLine truncated
"criterion_type": "expected_output"
},
{
"id": "f2e1929e-2228-4d11-9fd6-8d79cbb6d35a",
"sort_order": 8,
"rubric_text": "In file `temperature_log.xlsx`, there must be exactly 23 rows including the header row",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/temperature_log.xlsx')) + list(Path(workspace_path).glob('**/Temperature_Log.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'temperature' in f.name.lower() or 'temp_log' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No temperature_log.xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n ws = wb.active\n \n row_count = 0\n for row in ws.iter_rows():\n if any(cell.value is not None for cell in row):\n row_count += 1\n \n if row_count != 23:\n return {'pass': False, 'score': 0.0, 'feedback': f'temperature_log.xlsx has {row_count} non-empty rows, expected exactly 23 (including header)'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'temperature_log.xlsx has exactly 23 rows including header'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "c3753422-0cdb-427c-9bbf-4dcd10e2c1c6",
"sort_order": 9,
"rubric_text": "In file `inventory_master.xlsx`, there must be exactly 7 rows including the header row",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/inventory_master.xlsx')) + list(Path(workspace_path).glob('**/Inventory_Master.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('**/*.xlsx') if 'inventory' in f.name.lower() and 'master' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No inventory_master.xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n ws = wb.active\n \n row_count = 0\n for row in ws.iter_rows():\n if any(cell.value is not None for cell in row):\n row_count += 1\n \n if row_count != 7:\n return {'pass': False, 'score': 0.0, 'feedback': f'inventory_master.xlsx has {row_count} non-empty rows, expected exactly 7 (including header)'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'inventory_master.xlsx has exactly 7 rows including header'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776513856839",
"sort_order": 10,
"rubric_text": "There should be no Slack DM sent to Alicia Nguyen.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check external services (Slack) to verify no DM was sent to Alicia Nguyen.\n # Slack messages might be stored in external_services_path.\n \n if external_services_path is None:\n # If no external services path is provided, we can't confirm Slack messages.\n # But since all 3 examples failed, we should check workspace for any Slack-related logs too.\n # Be generous: if we can't check, we pass (prefer false positives).\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external_services_path provided to check Slack messages. Cannot confirm DMs were sent, so passing by default.\"}\n \n ext_path = Path(external_services_path)\n \n # Check for Slack data in various possible file locations\n slack_files = [\n ext_path / \"slack_data.json\",\n ext_path / \"slack_messages.json\",\n ext_path / \"slack.json\",\n ]\n \n found_slack_file = None\n slack_data = None\n \n for sf in slack_files:\n if sf.exists():\n found_slack_file = sf\n try:\n with open(sf, 'r') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading Slack data file {sf}: {e}\"}\n break\n \n if found_slack_file is None:\n # Also try to find any json file that might contain slack data\n for jf in ext_path.glob(\"*.json\"):\n try:\n with open(jf, 'r') as f:\n data = json.load(f)\n # Check if this looks like slack data\n if isinstance(data, dict) and any(k in data for k in ['messages', 'channels', 'dms', 'direct_messages', 'ims']):\n slack_data = data\n found_slack_file = jf\n break\n except:\n continue\n \n if slack_data is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Slack data file found in external services. Cannot confirm any DMs were sent, passing by default.\"}\n \n # Now search for DMs to Alicia Nguyen\n alicia_variants = [\"alicia nguyen\", \"alicia_nguyen\", \"alicia.nguyen\", \"nguyen, alicia\", \"alicianguyen\"]\n \n def is_alicia(text):\n if text is None:\n return False\n text_lower = text.lower().strip()\n for variant in alicia_variants:\n if variant in text_lower:\n return True\n return False\n \n dm_found = False\n dm_details = []\n \n # Check various possible structures for Slack DMs\n def search_messages(obj, path=\"\"):\n nonlocal dm_found, dm_details\n if isinstance(obj, dict):\n # Check if this dict represents a message/DM to Alicia\n to_field = obj.get('to') or obj.get('recipient') or obj.get('to_user') or obj.get('recipient_name') or obj.get('channel_name') or obj.get('dm_to') or ''\n user_field = obj.get('user') or obj.get('username') or obj.get('user_name') or ''\n channel_field = obj.get('channel') or obj.get('channel_name') or ''\n \n # Check if it's a DM context directed at Alicia\n if is_alicia(str(to_field)) or (is_alicia(str(channel_field)) and ('dm' in path.lower() or 'direct' in path.lower() or 'im' in path.lower())):\n dm_found = True\n dm_details.append(f\"Found DM to Alicia Nguyen at path={path}: {json.dumps(obj)[:300]}\")\n \n for k, v in obj.items():\n search_messages(v, f\"{path}.{k}\")\n elif isinstance(obj, list):\n for i, item in enumerate(obj):\n search_messages(item, f\"{path}[{i}]\")\n \n search_messages(slack_data)\n \n if dm_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Found Slack DM(s) sent to Alicia Nguyen. Details: {'; '.join(dm_details)}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS: No Slack DMs to Alicia Nguyen found in Slack data from {found_slack_file}.\"}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
today is april 15, it's 1pm and now i'm way behind. i ended up getting tied up for a few hours while i was in the middle of processing this mornings expected arrivals. can you finish processing anything remaining for today and handle any msgs that came in while i was away? thanks
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py