Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,98 @@
{
"mailbox": {
"email": "receiving.meridian@gearsystems.com",
"name": "Inbound Materials Coordinator — Meridian"
},
"contacts": [
{
"email": "darnell.hooks@gearsystems.com",
"name": "Darnell Hooks"
},
{
"email": "elizabeth.velasquez@gearsystems.com",
"name": "Elizabeth Velasquez"
},
{
"email": "walter.finch@gearsystems.com",
"name": "Walter Finch"
},
{
"email": "theo.brandt@gearsystems.com",
"name": "Theo Brandt"
},
{
"email": "joonho.park@gearsystems.com",
"name": "Joon-ho Park"
},
{
"email": "maribel.ortiz@gearsystems.com",
"name": "Maribel Ortiz"
},
{
"email": "aditi.krishnamurthy@gearsystems.com",
"name": "Aditi Krishnamurthy"
},
{
"email": "reggie.lemoine@gearsystems.com",
"name": "Reggie Lemoine"
},
{
"email": "chris.wojciechowski@gearsystems.com",
"name": "Chris Wojciechowski"
},
{
"email": "yuki.tanaka@gearsystems.com",
"name": "Yuki Tanaka"
},
{
"email": "curtis.yablonski@gearsystems.com",
"name": "Curtis Yablonski"
},
{
"email": "s.muth@vantagelube.com",
"name": "Sandra Muth"
},
{
"email": "b.kowalski@vantagelube.com",
"name": "Brian Kowalski"
}
],
"folders": [],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "BOL scan — MER-PO-204871 — Vantage Lubricants — APPT-20261124-003",
"from_addr": "curtis.yablonski@gearsystems.com",
"to_addr": "receiving.meridian@gearsystems.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-11-24T13:30:00-06:00",
"message_id": "<cyablonski-20261124-204871-bol@gearsystems.com>",
"in_reply_to": null,
"body_text": "BOL for Vantage Lubricants, Dock 2, APPT-20261124-003.\n\nBILL OF LADING\n\nBOL Number: XPO-2026-884417\nDate: 11/24/2026\nCarrier: XPO Logistics\n\nShipper:\nVantage Lubricants Inc.\n4200 Industrial Parkway\nIndianapolis, IN 46254\n\nConsignee:\nGear Systems Inc. — Meridian Plant\nReceiving Dock 2\nMeridian, MS\n\nPO Number: MER-PO-204871\n\nLine Items:\n 1. Gear oil, ISO VG 220, 20L drum (FL-08814-A1) — 24 drums\n\nTotal Weight: 1,056 lb\nFreight Terms: Prepaid\nHazmat: None\n\nCarrier signature obtained. No exceptions noted on BOL.",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "Packing slip scan — MER-PO-204871 — Vantage Lubricants — APPT-20261124-003",
"from_addr": "curtis.yablonski@gearsystems.com",
"to_addr": "receiving.meridian@gearsystems.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-11-24T13:30:00-06:00",
"message_id": "<cyablonski-20261124-204871-packingslip@gearsystems.com>",
"in_reply_to": null,
"body_text": "Packing slip for Vantage Lubricants, Dock 2, APPT-20261124-003.\n\nPACKING SLIP\n\nVantage Lubricants Inc.\n4200 Industrial Parkway\nIndianapolis, IN 46254\n\nShip To:\nGear Systems Inc. — Meridian Plant\nMeridian, MS\n\nPO Number: MER-PO-204871\nPacking Slip Number: VLI-PS-61042\nDate: 11/21/2026\n\nLine Items:\n Line 001 — SKU: FL-08814-A1\n Description: Gear oil, ISO VG 220, 20L drum\n Qty Ordered: 24\n Qty Shipped: 24\n\nTotal Packages: 24 drums on 2 pallets\nTotal Weight: 1,056 lb\nHazmat: None",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": []
}
],
"next_email_id": 3
}
@@ -0,0 +1,800 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "receiving.mer",
"deleted": false,
"real_name": "Inbound Materials Coordinator",
"profile": {
"real_name": "Inbound Materials Coordinator",
"display_name": "receiving-mer",
"email": "receiving.meridian@gearsystems.com",
"title": "Inbound Materials Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "dhooks",
"deleted": false,
"real_name": "Darnell Hooks",
"profile": {
"real_name": "Darnell Hooks",
"display_name": "dhooks",
"email": "darnell.hooks@gearsystems.com",
"title": "Receiving Supervisor"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "evelasquez",
"deleted": false,
"real_name": "Elizabeth Velasquez",
"profile": {
"real_name": "Elizabeth Velasquez",
"display_name": "evelasquez",
"email": "elizabeth.velasquez@gearsystems.com",
"title": "Plant Logistics Manager"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "wfinch",
"deleted": false,
"real_name": "Walter Finch",
"profile": {
"real_name": "Walter Finch",
"display_name": "wfinch",
"email": "walter.finch@gearsystems.com",
"title": "Plant Manager"
},
"is_admin": true,
"is_owner": true,
"is_primary_owner": true,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "cyablonski",
"deleted": false,
"real_name": "Curtis Yablonski",
"profile": {
"real_name": "Curtis Yablonski",
"display_name": "cyablonski",
"email": "curtis.yablonski@gearsystems.com",
"title": "Dock Lead"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "tbrandt",
"deleted": false,
"real_name": "Theo Brandt",
"profile": {
"real_name": "Theo Brandt",
"display_name": "tbrandt",
"email": "theo.brandt@gearsystems.com",
"title": "Quality Manager"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "jpark",
"deleted": false,
"real_name": "Joon-ho Park",
"profile": {
"real_name": "Joon-ho Park",
"display_name": "jpark",
"email": "joonho.park@gearsystems.com",
"title": "Procurement Lead, Meridian"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "mortiz",
"deleted": false,
"real_name": "Maribel Ortiz",
"profile": {
"real_name": "Maribel Ortiz",
"display_name": "mortiz",
"email": "maribel.ortiz@gearsystems.com",
"title": "Inventory Control Lead"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "akrishnamurthy",
"deleted": false,
"real_name": "Aditi Krishnamurthy",
"profile": {
"real_name": "Aditi Krishnamurthy",
"display_name": "akrishnamurthy",
"email": "aditi.krishnamurthy@gearsystems.com",
"title": "Production Planning Lead"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U010": {
"id": "U010",
"team_id": "T001",
"name": "rlemoine",
"deleted": false,
"real_name": "Reggie Lemoine",
"profile": {
"real_name": "Reggie Lemoine",
"display_name": "rlemoine",
"email": "reggie.lemoine@gearsystems.com",
"title": "Quality Inspector, 1st Shift"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U011": {
"id": "U011",
"team_id": "T001",
"name": "cwoj",
"deleted": false,
"real_name": "Chris Wojciechowski",
"profile": {
"real_name": "Chris Wojciechowski",
"display_name": "cwoj",
"email": "chris.wojciechowski@gearsystems.com",
"title": "Plant EHS Contact"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U012": {
"id": "U012",
"team_id": "T001",
"name": "ytanaka",
"deleted": false,
"real_name": "Yuki Tanaka",
"profile": {
"real_name": "Yuki Tanaka",
"display_name": "ytanaka",
"email": "yuki.tanaka@gearsystems.com",
"title": "RMA Coordinator, Quality"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U013": {
"id": "U013",
"team_id": "T001",
"name": "solds",
"deleted": false,
"real_name": "Sherri Olds",
"profile": {
"real_name": "Sherri Olds",
"display_name": "solds",
"email": "sherri.olds@gearsystems.com",
"title": "Plant Logistics Manager, Birmingham"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
},
"U014": {
"id": "U014",
"team_id": "T001",
"name": "akrishnan",
"deleted": false,
"real_name": "Anand Krishnan",
"profile": {
"real_name": "Anand Krishnan",
"display_name": "akrishnan",
"email": "anand.krishnan@gearsystems.com",
"title": "Plant Logistics Manager, Pittsburgh"
},
"is_admin": false,
"is_owner": false,
"is_primary_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "receiving-dock",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"name_normalized": "receiving-dock",
"is_shared": false,
"is_org_shared": false,
"context_team_id": "T001",
"creator": "U004",
"is_member": true,
"topic": {
"value": "Dock activity \u2014 arrivals, check-ins, unload complete, putaway authorization, handoffs",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Primary operational channel for the receiving dock. All dock crew posts, Coordinator instructions, and shift handoffs go here.",
"creator": "U004",
"last_set": 1700000000
},
"unlinked": 0,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"updated": 1700000000,
"shared_team_ids": [
"T001"
]
},
"C002": {
"id": "C002",
"name": "quality-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"name_normalized": "quality-alerts",
"is_shared": false,
"is_org_shared": false,
"context_team_id": "T001",
"creator": "U004",
"is_member": true,
"topic": {
"value": "Quality findings \u2014 damage reports, inspection results, quarantine notices, NCR alerts",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Quality-related alerts from the dock and Coordinator. Reggie Lemoine posts inspection findings here. Do not delete or modify posts \u2014 this channel is part of the audit trail.",
"creator": "U004",
"last_set": 1700000000
},
"unlinked": 0,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"updated": 1700000000,
"shared_team_ids": [
"T001"
]
},
"C003": {
"id": "C003",
"name": "escalations-meridian",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"name_normalized": "escalations-meridian",
"is_shared": false,
"is_org_shared": false,
"context_team_id": "T001",
"creator": "U004",
"is_member": true,
"topic": {
"value": "Plant-level escalations \u2014 suspended suppliers, dollar-value overrides, line-down risk, critical NCRs",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Escalations requiring Plant Logistics Manager or Plant Manager involvement. Posts here are never deleted \u2014 correction replies only.",
"creator": "U004",
"last_set": 1700000000
},
"unlinked": 0,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"updated": 1700000000,
"shared_team_ids": [
"T001"
]
},
"C004": {
"id": "C004",
"name": "planning-meridian",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"name_normalized": "planning-meridian",
"is_shared": false,
"is_org_shared": false,
"context_team_id": "T001",
"creator": "U004",
"is_member": true,
"topic": {
"value": "Production planning notifications \u2014 short ships, quarantines with line-down risk, schedule impacts",
"creator": "U004",
"last_set": 1700000000
},
"purpose": {
"value": "Channel for notifying Production Planning (Aditi Krishnamurthy) of receiving events that may affect the production schedule.",
"creator": "U004",
"last_set": 1700000000
},
"unlinked": 0,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"updated": 1700000000,
"shared_team_ids": [
"T001"
]
},
"D001": {
"id": "D001",
"name": "dhooks",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U002",
"members": [
"U001",
"U002"
]
},
"D002": {
"id": "D002",
"name": "evelasquez",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U003",
"members": [
"U001",
"U003"
]
},
"D003": {
"id": "D003",
"name": "wfinch",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U004",
"members": [
"U001",
"U004"
]
},
"D004": {
"id": "D004",
"name": "cyablonski",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U005",
"members": [
"U001",
"U005"
]
},
"D005": {
"id": "D005",
"name": "tbrandt",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U006",
"members": [
"U001",
"U006"
]
},
"D006": {
"id": "D006",
"name": "jpark",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U007",
"members": [
"U001",
"U007"
]
},
"D007": {
"id": "D007",
"name": "mortiz",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U008",
"members": [
"U001",
"U008"
]
},
"D008": {
"id": "D008",
"name": "akrishnamurthy",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U009",
"members": [
"U001",
"U009"
]
},
"D009": {
"id": "D009",
"name": "rlemoine",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U010",
"members": [
"U001",
"U010"
]
},
"D010": {
"id": "D010",
"name": "cwoj",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U011",
"members": [
"U001",
"U011"
]
},
"D011": {
"id": "D011",
"name": "ytanaka",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"user": "U012",
"members": [
"U001",
"U012"
]
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "DAILY DOCK PREVIEW 11/24/2026: 3 appointments scheduled. Hazmat: APPT-20261124-001, APPT-20261124-002. Oversize/special: none. Largest expected shipment: Vantage Lubricants Inc., 2,640 lb at 10:00.",
"ts": "1795521600.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "ARRIVAL ON TIME: APPT-20261124-001, Vantage Lubricants Inc., Dock 1, PO #MER-PO-204869. Cleared to unload.",
"ts": "1795547542.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "ARRIVAL ON TIME: APPT-20261124-002, Vantage Lubricants Inc., Dock 3, PO #MER-PO-204870. Cleared to unload.",
"ts": "1795548262.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "ARRIVAL ON TIME: APPT-20261124-003, Vantage Lubricants Inc., Dock 2, PO #MER-PO-204871. Cleared to unload.",
"ts": "1795548600.000000",
"team": "T001"
},
{
"type": "message",
"user": "U005",
"text": "UNLOAD COMPLETE: REC-20261124-003, Vantage Lubricants Inc., PO #MER-PO-204871. Total pallets: 2. Count match: Y. Condition: Clean. Photos: N.",
"ts": "1795551600.000000",
"team": "T001"
}
],
"C002": [],
"C003": [],
"C004": [],
"D001": [],
"D002": [],
"D003": [],
"D004": [],
"D005": [],
"D006": [],
"D007": [],
"D008": [],
"D009": [],
"D010": [],
"D011": []
},
"counters": {
"messageId": 1000000
}
}
@@ -0,0 +1,80 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R /F3 4 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/BaseFont /Helvetica-Oblique /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
>>
endobj
5 0 obj
<<
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
6 0 obj
<<
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
>>
endobj
7 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260415032415+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260415032415+00'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
8 0 obj
<<
/Count 1 /Kids [ 5 0 R ] /Type /Pages
>>
endobj
9 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1732
>>
stream
Gatm<>Bed\&:Vs/Qq,jD*(cH7qaLoaBaM$%G5!0Clu>:eDLA.ddqna0n&SRXK8$W<#t>h]o4^&X1kGeF?^guOp].c]gtYc"%.%mt\\c1S_8O(aGsudISUdCH`C+c`-#%/%]>B-+`#@Vh++Y8*4U#:-L7/PJln&W`Vm@6lmU.pk:ZqK@_#=B.HPi%QkjWfoXZJs08"HZJ:DKtR,pSjfE&Di$?d+;sao0)JNc&KndV[7-dkr&BR;8g9n;;XdEY>GMWuui[JQn;N#O<^ejK32D`$'IFU'&!jr*W,X0LO-"E4_LRr5F-ud;K3QbYg&nmr23(=Fou)Ee2\MqTq&ZVa_?]#RRBM[E`4#:*RuWTJBOWVY#W5)p.(JpGui;5<)O$?82D"qCi;$DOe5+q!?+8(s=>??kK[M</[tc/V]RfEmEkLb0MnU)Hj!04Cq^`C9\AnA5kBP/Y,6"rP,5-VJ@KT"ka8><0#L'i$Lf7(bu;XJ8alq)2r;rC=>D_#)s\YZDC6^'u45UEOoCM<q&)YMaII6h*IRrbS`?YKeMrb@52+bn)ol@qQRj^*ftjU?Gg<N"O413oMatj4Uf%c6/q56b[-kbGCpuHQj+KUFX"L-3$SQq[P)V5\W-nf-[L5ahC8WoF.L,"LRoGl_fQAnB,WuLL\$qj$R^Q!>FYs`is:\S2sZ*P>Y+p]*jVt_Xi*]=^YE3%j)FSuXQh6(>f4Gt<(eOEKBsu`qbbIIR_*isq)AV:F21;6TLI"8NMP,p.\i`M27i_LkGd2j,jQSNTF/"0;TEt2[!W6=gGpdm<]`+:ROq7:6.ODV?lb,!WA+L"R&l%ARWHs!_E<AJ[h=!@g@%Bf8:2!p/^2U_BrC+rV2Pok;:!$PF($&CD5IP@d^cnB2Ce$Q)*:]'W5Z3R)T)"uRG0EefV@\Ih<W`:\A[/V9?AtI./n8;j7kC.U<Dhp;tWsa0S=^O?c+Zqp7gcsgd'B8Mt1OXe$Cd<>CZY#D+66M8uo4Q2j(^Jgb@pm,t'Jh"5Ka5Gp8)lX>+@E8rWpo(H6W#25dH@3Brnd+2Zi^:JaKnVj`M*kJGUP'c<_n(<uUg<d1kn0hofGUE41gVSY=C[F"SC44al8H4PssCTI\PY`2ue*cI8l<&I8Vee^'"YGkT84N6ZF(rH\af;m$ACSDi85@(kAF'[k+lD."mlPc-J2Kc8]._7]2Z`9:Ug*cV\.Xgl]1o;O6Q1&9/ZO5p_WU*G]1Y9b8WTp^hHpm\H8Y:NRUK\/:AmHOPG*k6q99L.YZi'k_W:[;iZQ5[eFTLsq%,JORZK/J`jMriBb=Lt?GC'PN8*:uOBhS%[XQ`5:C9#f)2-^@YJ#j7pOd]QWQrB`<L"=?EZC:F/1"Mq@9iUX\fm,qG;I=IkN/l,DQd2Vt@El)jrS?;u"]dhT/oZu?8<bi)%F^3s&ZZpJ33h,Qe(]ulO^e-FR<&[or(;$nL+KotL6H#VDOt5/8(aaK6cgrj'*^0(&`O3_>N+ONF=]Tq]!h$`](B$Scf(qb`QrF%A"G6Pe&s2]>:+4D1-=+S!b*Y``kpIjgTY_Q^0@2L-==(ef_j:k;j"!nppJhi8[LLjX)pQM+6B2M7jBe?8K)!MG!b;mc"!f].f7s#eRq!6(es&@0eDc;VZBH62I?&Zf)0d6R@V7d/ohl7Zu"`ES:U7sK!%2M8it"sFjHnlHgX!!nb5MKj^JWaR,GC^"-?k;iU0EF?U4B<<u%R:lu;VN>KqWU~>endstream
endobj
xref
0 10
0000000000 65535 f
0000000061 00000 n
0000000112 00000 n
0000000219 00000 n
0000000331 00000 n
0000000446 00000 n
0000000639 00000 n
0000000707 00000 n
0000000987 00000 n
0000001046 00000 n
trailer
<<
/ID
[<5d66a97be850eb95aa8990225dfdd67b><5d66a97be850eb95aa8990225dfdd67b>]
% ReportLab generated PDF document -- digest (opensource)
/Info 7 0 R
/Root 6 0 R
/Size 10
>>
startxref
2869
%%EOF
@@ -0,0 +1,80 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R /F3 4 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/BaseFont /Helvetica-Oblique /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
>>
endobj
5 0 obj
<<
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
6 0 obj
<<
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
>>
endobj
7 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260415032415+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260415032415+00'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
8 0 obj
<<
/Count 1 /Kids [ 5 0 R ] /Type /Pages
>>
endobj
9 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1652
>>
stream
Gatm;=``R^&:Vs/kXRNGHR)b.o!A]W$o;.@Sa:7E^)3UgSq*Ba-E0]+O5H7A^ec13`FW6/T0B[ZO=1ZfdHoWF!8P0Tkq`Lc_#4:uWuTn,o\GGg$X.3^"AFfR(94-oAC>^FJVLPK;CK$o"CaX(%#@IlWp?ZsE#a(gd(-oZ\2T%(N:(TH__1q.?o[DNIt7D0ge3<6C*SB"ptB:Fc-n4g:4e)Ji*]BHfrX,I)'L[BQeNdF")eW)07J<AhQEY*n3ZRm=pGB5RfGC7c>.;lPZD[TIt4#+WuA5MD@?+lHM,k2A+8Z>(T-%]XLAS2qU#NHK0;fDBaokV@YSHY`,KYdpAm3Qce3#?b/]C3AZDW((]5m=0>[J)EVj6Wd%Bsn2f?A8]kR4Ygn2+G04Q?aO+o?Zp\JqJhb8!mZ]X#I?F\,X0l#uJoN(6(q`&_XWj^/Em_-^!DFBl/Y]h[%.PjRL.,bS?3tS*A]i6PdJIuKBPU[*8X_Z(_2%@-WMb;GGW>!8&OM(?\Zq9ucW*\'#bcU1dE6n)t;<7Y4@pb.mR1CpelsTuY$b%uQ/HFQW!KNs#YL:g3DPsVrRL>J+DS"_[P6!Y<iN0:3%@'u^=u#]1>fOKq[ZZe!m$\#c2OGDu:!'auoDHWO>_03kSmW21f-R1s?foU1&=[*5'KF@>h1mu+KB]\U'c9)mKlcD.(orWFrP\2bEHK4m#-brR;j_A[$PB#ErW6K_T.@7I_?dVEHF<n_OKKD?_0I#@*u+Euj"a*O.S/"':X3uJDLX_gIFJ]@!@:BMbMo%\BDWiBE?6eL58FYT@c9>jWP,C]NOGpOEkSK'AYDqC6>_@lrpP2+LWi\UN!e$gr/I[*D%/KTh5E"3Z%*p_H`;te"L>B)"ub];*W;$Vqd7D>17WnWD7F5>0^(OZ%MZ5p<XquLe8eUR%"oFLo9Q1]P@.ble\!i#73'Ai&$n,omVGKUB4iiE&\Q3EK2(>;=?TPc>f#%nc.3/;+o2p<5O7U*D,]E::#SfoP^?9p:*q5J1/]Wf_gBrd;8^X&3qhZiG5<baQ$^maj3D<tl,+&o[<$^WZM7k8MBD3AjeiA-:9"jLXW`>ud-T5nfqW$`aW#P='SKpSn#@)NZ;OWb3T'8YeL7"W/$5C6r&ApnK.&h`8e';r9p>&1)bf:gLrJq@M1oS/_pim)ACYocB#8\f'!9"L/e$1Gl>k<?V(!S[KRZUrcQe-G(?N]`&T-d0;E;pj-s5L1fU9VT!#Qq^GUOKH1>Zo/c%_67H%j*0Oei^Rlb#^u/$M=AHC]C]\)eS_EfSaS_oXu\Y#=!"\V;uBV6#n:9=6PBm%I=S,aZ[28uN>+2)91N<"GA/Qns5la_8-*E\c;]-V"OEV<nG:P4C$p>M99^%%EQ=LK.Vdq*oKiW@P4mV:cm4J&Ht[P./i=HZjr0/?TVAd&$6mXk+cAcO1aQr_q'?_#G^4eO+W_:9TIk^j9u)hA>Y4`]HSW6p$[tco3:gD;H0Po7(EE7RYKb?.e7Qs593rE?Bc8QK8S>eOs:M??s?@V@bFAnBq,OY]tfm6,7nt[=Q_K&`ZEL-@$nj0j]uD,(muV/9p+dTg]#kP[_=Uh?I*S4a96fCE0.LW9S4I$!*adDI]BWk9u)JX[ltc:;Dp03^8>Y(LVC3s)@Km:l*I\~>endstream
endobj
xref
0 10
0000000000 65535 f
0000000061 00000 n
0000000112 00000 n
0000000219 00000 n
0000000331 00000 n
0000000446 00000 n
0000000639 00000 n
0000000707 00000 n
0000000987 00000 n
0000001046 00000 n
trailer
<<
/ID
[<c89a49ab72d1c74d78061a021f991961><c89a49ab72d1c74d78061a021f991961>]
% ReportLab generated PDF document -- digest (opensource)
/Info 7 0 R
/Root 6 0 R
/Size 10
>>
startxref
2789
%%EOF
@@ -0,0 +1,3 @@
It is James Reed, Coordonator. We’ve just had 3 transports arrive together, I want to make sure we can process, unload them and store before the end of the shift. Please take care of the standard one while I deal with the two hazard ones. Follow the procedures outline in Gear_Systems_Logistics_SOP. We’ve been waiting for the gear oil for a couple of weeks so I don’t want any unnecessary delays.
Use my name or contact info as needed, and if you need to fill in any free text fields that the SOP doesn’t already specify exact wording, write “Actioned per SOP protocol”. Also, if you are able to make the disposition decision, use exactly the current time (as of right now) for the corresponding timestamp.
@@ -0,0 +1 @@
Today's date is, November, 24, 2026, 14:21 CST. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/logistics_gear_sytems_inc_dba0375e"
description = "SOP task from CSV row dba0375e"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,93 @@
[
{
"id": "72b5011c-bafc-42bb-b6bb-200cc6c0df58",
"sort_order": 0,
"rubric_text": "file `receiving_log.xlsx`, has 5 rows (2 headers, and 3 data)",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n fallback = [f for f in Path(workspace_path).glob(\"*.xlsx\") if \"receiving\" in f.name.lower()]\n if not fallback:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n file_path = fallback[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {file_path.name}: {e}\"}\n\n ws = wb.active\n non_empty_rows = [\n row_idx + 1\n for row_idx, row in enumerate(ws.iter_rows(values_only=True))\n if any(cell is not None for cell in row)\n ]\n total = len(non_empty_rows)\n\n if total == 5:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"{file_path.name} has 5 non-empty rows (2 header + 3 data) as expected.\"}\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n f\"{file_path.name} has {total} non-empty rows, expected exactly 5 (2 header + 3 data). \"\n f\"Non-empty row numbers: {non_empty_rows}.\"\n ),\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776277812496",
"sort_order": 1,
"rubric_text": "`receiving_log.xlsx`: Row 3 must contain the following values in order: REC-20261124-001 | MER-PO-204869 | 001 | LB-03341-B2 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:12 CST | | Dock 1 | 12 | 12 | | | | | | | | | | | | | | | | JR | | In Process | |",
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 3 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-001\", # A\n \"MER-PO-204869\", # B\n \"001\", # C\n \"LB-03341-B2\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:12 CST\", # H\n None, # I\n \"Dock 1\", # J\n 12, # K\n 12, # L\n None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, # M–AA\n \"JR\", # AB\n None, # AC\n \"In Process\", # AD\n None, # AE\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n # Case-insensitive exact match\n if exp_str.lower() == act_str.lower():\n return True\n\n # datetime objects written by openpyxl (date/time columns)\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n # Numeric match for integer-like values (e.g. \"001\" stored as 1, 12 stored as \"12\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=3, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 3 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 3 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277813709",
"sort_order": 2,
"rubric_text": "`receiving_log.xlsx`: Row 4 must contain the following values in order: REC-20261124-002 | MER-PO-204870 | 001 | LB-07752-A3 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:24 CST | | Dock 3 | 40 | 40 | | | | | | | | | | | | | | | | JR | | In Process | |",
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 4 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-002\", # A\n \"MER-PO-204870\", # B\n \"001\", # C\n \"LB-07752-A3\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:24 CST\", # H\n None, # I\n \"Dock 3\", # J\n 40, # K\n 40, # L\n None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, # M–AA\n \"JR\", # AB\n None, # AC\n \"In Process\", # AD\n None, # AE\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n if exp_str.lower() == act_str.lower():\n return True\n\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=4, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 4 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 4 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277827542",
"sort_order": 3,
"rubric_text": "receiving_log.xlsx: Row 5 must contain the following values in order: REC-20261124-003 | MER-PO-204871 | 001 | FL-08814-A1 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:30 CST | 11/24/2026 14:21 CST | Dock 2 | 24 | 24 | 24 | 0 | 0.0% | Exact | Accepted | Tier 0 | Accepted | N/A | Verified | Accepted | | | | ACCEPTED CLEAN | | JR | Actioned per SOP protocol | Posted | |",
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 5 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-003\", # A\n \"MER-PO-204871\", # B\n \"001\", # C\n \"FL-08814-A1\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:30 CST\", # H\n \"11/24/2026 14:21 CST\", # I\n \"Dock 2\", # J\n 24, # K\n 24, # L\n 24, # M\n 0, # N\n \"0.0%\", # O\n \"Exact\", # P\n \"Accepted\", # Q\n \"Tier 0\", # R\n \"Accepted\", # S\n \"N/A\", # T\n \"Verified\", # U\n \"Accepted\", # V\n None, # W (Heat_Numbers)\n None, # X (COC_Filename)\n None, # Y (MTR_Filename)\n \"ACCEPTED CLEAN\", # Z\n None, # AA (NCR_Number)\n \"JR\", # AB\n \"Actioned per SOP protocol\", # AC\n \"Posted\", # AD\n None, # AE (Handoff_Notes)\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n # Case-insensitive exact match\n if exp_str.lower() == act_str.lower():\n return True\n\n # Percentage field (col O): \"0.0%\" — accept \"0%\", \"0.00%\", or numeric 0\n if \"%\" in exp_str:\n exp_num = float(exp_str.replace(\"%\", \"\").strip())\n try:\n act_num = float(act_str.replace(\"%\", \"\").strip())\n return exp_num == act_num\n except (ValueError, TypeError):\n pass\n try:\n act_float = float(str(actual))\n # stored as decimal fraction (0.0) or as percentage number (0.0)\n return exp_num == act_float or exp_num == act_float * 100\n except (ValueError, TypeError):\n pass\n return False\n\n # datetime objects written by openpyxl\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n # Numeric exact match (e.g. \"001\" stored as 1, 24 stored as \"24\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=5, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 5 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 5 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776277829339",
"sort_order": 4,
"rubric_text": "`inventory_master.xlsx` does not include the value `FL-08814-A1`.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"inventory_master.xlsx\"\n\n if not target_file.exists():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'inventory_master.xlsx' not found in workspace: {workspace_path}\"\n }\n\n search_value = \"FL-08814-A1\"\n search_lower = search_value.strip().lower()\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not open 'inventory_master.xlsx': {e}\"\n }\n\n found_locations = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(values_only=False), start=1):\n for cell in row:\n if cell.value is not None:\n cell_str = str(cell.value).strip().lower()\n if cell_str == search_lower:\n found_locations.append(\n f\"Sheet '{sheet_name}', cell {cell.coordinate}, value='{cell.value}'\"\n )\n\n wb.close()\n\n if len(found_locations) == 0:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: 'FL-08814-A1' was NOT found anywhere in 'inventory_master.xlsx', which is the expected result.\"\n }\n else:\n locations_str = \"; \".join(found_locations)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: 'FL-08814-A1' was found in 'inventory_master.xlsx' at the following location(s): {locations_str}. The rubric requires it NOT to be present.\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277830559",
"sort_order": 5,
"rubric_text": "`inventory_master.xlsx`: has 67 rows (2 header rows, 65 data rows).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"inventory_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'inventory_master.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'inventory_master.xlsx': {e}\"}\n \n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n total_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n # Also get ws.max_row for reference\n max_row = ws.max_row\n \n expected_total = 67\n # Be generous: accept 67 total rows exactly, or allow a small tolerance\n # The rubric says 67 rows total (2 header + 65 data)\n # We check the total non-empty row count\n \n feedback_details = f\"Found {total_rows} non-empty rows (ws.max_row={max_row}) in 'inventory_master.xlsx'. Expected {expected_total} total rows (2 header + 65 data).\"\n \n if total_rows == expected_total:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS: {feedback_details}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: {feedback_details}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277841819",
"sort_order": 6,
"rubric_text": "In `inventory_master.xlsx`, row 67 must contain the following values: FL-08814-A0 | D-105 | 0 | AVAILABLE | 10/14/2026 | REC-20261014-003 | (remaining cells empty).",
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"inventory_master.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"inventory_master.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open inventory_master.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 67 (columns 1–11).\n # None = cell must be empty.\n expected = [\n \"FL-08814-A0\", # 1\n \"D-105\", # 2\n 0, # 3 (Quantity)\n \"AVAILABLE\", # 4\n \"10/14/2026\", # 5 (Date)\n \"REC-20261014-003\", # 6\n None, # 7\n None, # 8\n None, # 9\n None, # 10\n None, # 11\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n if exp_str.lower() == act_str.lower():\n return True\n\n # datetime objects from openpyxl (date-only column)\n if isinstance(actual, datetime):\n return actual.month == 10 and actual.day == 14 and actual.year == 2026\n\n # Numeric exact match (e.g. quantity 0 stored as 0 or \"0\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=67, column=i + 1).value\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {i + 1}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 67 matches all 11 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 67 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277870617",
"sort_order": 7,
"rubric_text": "`slack_data.json`: Channels C002, C003, C004, D001, D002, D003, D004, D005, D006, D008, D009, D010, and D011 have 0 messages.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages = data.get(\"messages\")\n if not isinstance(messages, dict):\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected data['messages'] to be a dict keyed by channel ID. Got: {type(messages).__name__}. Top-level keys: {list(data.keys()) if isinstance(data, dict) else 'N/A'}\",\n }\n\n required_zero = [\n \"C002\", \"C003\", \"C004\",\n \"D001\", \"D002\", \"D003\", \"D004\", \"D005\", \"D006\",\n \"D008\", \"D009\", \"D010\", \"D011\",\n ]\n\n failures = []\n for ch_id in required_zero:\n msgs = messages.get(ch_id, [])\n count = len(msgs) if isinstance(msgs, list) else int(msgs)\n if count != 0:\n failures.append(f\"{ch_id}: expected 0 messages, found {count}\")\n\n total = len(required_zero)\n score = round((total - len(failures)) / total, 4)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All 13 channels have 0 messages as expected.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"{len(failures)}/{total} channel(s) have unexpected messages:\\n\" + \"\\n\".join(f\" {f}\" for f in failures),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277884627",
"sort_order": 8,
"rubric_text": "`slack_data.json`: channel C001 has 5 messages.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages = data.get(\"messages\")\n if not isinstance(messages, dict):\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected data['messages'] to be a dict keyed by channel ID. Got: {type(messages).__name__}.\",\n }\n\n c001 = messages.get(\"C001\", [])\n if not isinstance(c001, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"C001 messages is not a list; got {type(c001).__name__}.\"}\n\n count = len(c001)\n\n if count == 5:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel C001 has exactly 5 messages as expected.\"}\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Channel C001 has {count} message(s); expected exactly 5.\",\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277884883",
"sort_order": 9,
"rubric_text": "`mailbox.json`: Has 0 emails in Sent.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for mailbox.json\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n # Also check workspace_path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).strip().lower() == \"sent\"]\n count = len(sent_emails)\n \n if count == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json has 0 emails in Sent as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 0 emails in Sent, but found {count}. Subjects: {[e.get('subject','') for e in sent_emails]}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776277917163",
"sort_order": 10,
"rubric_text": "`slack_data.json`: Channel D007 has exactly 1 message with the following content:\n\nHi Maribel,\n\nA new SKU was received today that does not have an existing entry in inventory_master.xlsx. Please create the master record so I can complete the receipt.\n\nReceipt details:\n\n- Receipt ID: REC-20261124-003\n- PO Number: MER-PO-204871\n- Supplier: Vantage Lubricants Inc.\n- SKU: FL-08814-A1\n- SKU description (per packing slip): Gear oil, ISO VG 220, 20L drum\n- Quantity received: 24\n- Safety-Critical: N\n- Suggested location: C-RCV-01\n\nMaterial is currently held in the receiving area pending master record creation. Please confirm when the row has been added.\n\nThanks,\nJR",
"verifier_code": "from pathlib import Path\nimport re\nimport json\nfrom collections import Counter\n\nEXPECTED_TEXT = \"\"\"\\\nHi Maribel,\n\nA new SKU was received today that does not have an existing entry in inventory_master.xlsx. Please create the master record so I can complete the receipt.\n\nReceipt details:\n\n- Receipt ID: REC-20261124-003\n- PO Number: MER-PO-204871\n- Supplier: Vantage Lubricants Inc.\n- SKU: FL-08814-A1\n- SKU description (per packing slip): Gear oil, ISO VG 220, 20L drum\n- Quantity received: 24\n- Safety-Critical: N\n- Suggested location: C-RCV-01\n\nMaterial is currently held in the receiving area pending master record creation. Please confirm when the row has been added.\n\nThanks,\nJR\"\"\"\n\nKEY_PHRASES = [\n \"rec 20261124 003\",\n \"mer po 204871\",\n \"vantage lubricants inc\",\n \"fl 08814 a1\",\n \"gear oil iso vg 220 20l drum\",\n \"quantity received 24\",\n \"safety critical n\",\n \"c rcv 01\",\n \"maribel\",\n \"jr\",\n]\n\ndef normalize(text):\n text = text.lower()\n text = re.sub(r\"[^a-z0-9 ]\", \" \", text)\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip()\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages_section = data.get(\"messages\")\n if not isinstance(messages_section, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"data['messages'] is missing or not a dict.\"}\n\n d007_msgs = messages_section.get(\"D007\", [])\n if not isinstance(d007_msgs, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"D007 messages is not a list; got {type(d007_msgs).__name__}.\"}\n\n count = len(d007_msgs)\n if count == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"D007 has 0 messages; expected exactly 1.\"}\n if count > 1:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"D007 has {count} messages; expected exactly 1. Duplicate sends are not permitted.\",\n }\n\n msg = d007_msgs[0]\n actual_text = msg.get(\"text\", msg.get(\"message\", msg.get(\"body\", \"\"))) if isinstance(msg, dict) else str(msg)\n\n exp_norm = normalize(EXPECTED_TEXT)\n act_norm = normalize(actual_text)\n\n # Pass if the full expected content is present in the actual message. This allows\n # legitimate preamble such as a \"Subject:\" line (Template 18 in the SOP is itself\n # drafted with a Subject line) and any trailing signature, while still requiring\n # that every word of the template body appear in order.\n if exp_norm in act_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"D007 message contains all expected Template 18 content (after normalisation).\"}\n\n # Partial score: token-level recall\n exp_tokens = Counter(exp_norm.split())\n act_tokens = Counter(act_norm.split())\n matched_tokens = sum((exp_tokens & act_tokens).values())\n score = round(matched_tokens / sum(exp_tokens.values()), 4)\n\n missing_phrases = [p for p in KEY_PHRASES if p not in act_norm]\n extra_note = f\"Missing key fields: {missing_phrases}\" if missing_phrases else \"All key fields present — mismatch is in boilerplate wording.\"\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": (\n f\"D007 message does not contain expected content (token recall: {score:.1%}).\\n\"\n f\"{extra_note}\\n\"\n f\"Expected (normalised):\\n {exp_norm}\\n\"\n f\"Actual (normalised):\\n {act_norm}\"\n ),\n }",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776720465519",
"sort_order": 11,
"rubric_text": "In `dock_schedule.xlsx`, row 5 must contain the following values in order: APPT-20261124-003 | 11/24/2026 | 12:00 CST | 13:30 CST | Dock 2 | XPO Logistics | SUP-0147 | Vantage Lubricants Inc. | MER-PO-204871 | 1056.0 | N | (empty) | 13:30 CST | (empty) | Arrived",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"dock_schedule.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File dock_schedule.xlsx not found at {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening dock_schedule.xlsx: {e}\"}\n\n # Row 5\n row_idx = 5\n row_values = []\n for col in range(1, ws.max_column + 1):\n cell = ws.cell(row=row_idx, column=col)\n row_values.append(cell.value)\n\n if not row_values or len(row_values) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 5 is empty or does not exist.\"}\n\n # Expected values in order (15 columns)\n # Index: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14\n # Values: APPT-20261124-003 | 11/24/2026 | 12:00 CST | 13:30 CST | Dock 2| XPO Logistics | SUP-0147| Vantage Lubricants Inc.| MER-PO-204871 | 1056.0 | N | '' | 13:30 CST | '' | Arrived\n\n expected = [\n \"APPT-20261124-003\",\n \"11/24/2026\",\n \"12:00 CST\",\n \"13:30 CST\",\n \"Dock 2\",\n \"XPO Logistics\",\n \"SUP-0147\",\n \"Vantage Lubricants Inc.\",\n \"MER-PO-204871\",\n 1056.0,\n \"N\",\n None, # empty\n \"13:30 CST\",\n None, # empty\n \"Arrived\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n return s\n\n def is_empty(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == \"\" or s.lower() == \"none\"\n\n def nums_close(a, b, tol=0.05):\n try:\n fa = float(a)\n fb = float(b)\n if fb == 0:\n return abs(fa) < 0.01\n return abs(fa - fb) / max(abs(fb), 1e-9) <= tol\n except:\n return False\n\n def match_date(actual, expected_str):\n # expected_str is \"11/24/2026\"\n # actual could be a datetime object or string\n import datetime\n norm_a = normalize(actual)\n if norm_a == expected_str:\n return True\n # Try to parse as date\n try:\n if hasattr(actual, 'strftime'):\n formatted = actual.strftime('%m/%d/%Y')\n if formatted == expected_str:\n return True\n formatted2 = actual.strftime('%-m/%-d/%Y')\n if formatted2 == expected_str:\n return True\n except:\n pass\n # Try flexible string matching\n # Accept 2026-11-24 etc.\n if \"2026\" in norm_a and \"11\" in norm_a and \"24\" in norm_a:\n return True\n return False\n\n def match_str(actual, expected_str):\n na = normalize(actual).lower()\n ne = expected_str.lower().strip()\n if na == ne:\n return True\n # Remove extra whitespace\n na_clean = re.sub(r'\\s+', ' ', na)\n ne_clean = re.sub(r'\\s+', ' ', ne)\n if na_clean == ne_clean:\n return True\n return False\n\n # Pad row_values if shorter\n while len(row_values) < 15:\n row_values.append(None)\n\n issues = []\n # Column 0: APPT-20261124-003\n if not match_str(row_values[0], expected[0]):\n issues.append(f\"Col 1: expected '{expected[0]}', got '{row_values[0]}'\")\n\n # Column 1: date 11/24/2026\n if not match_date(row_values[1], expected[1]):\n issues.append(f\"Col 2: expected '{expected[1]}', got '{row_values[1]}'\")\n\n # Column 2: 12:00 CST\n if not match_str(row_values[2], expected[2]):\n issues.append(f\"Col 3: expected '{expected[2]}', got '{row_values[2]}'\")\n\n # Column 3: 13:30 CST\n if not match_str(row_values[3], expected[3]):\n issues.append(f\"Col 4: expected '{expected[3]}', got '{row_values[3]}'\")\n\n # Column 4: Dock 2\n if not match_str(row_values[4], expected[4]):\n issues.append(f\"Col 5: expected '{expected[4]}', got '{row_values[4]}'\")\n\n # Column 5: XPO Logistics\n if not match_str(row_values[5], expected[5]):\n issues.append(f\"Col 6: expected '{expected[5]}', got '{row_values[5]}'\")\n\n # Column 6: SUP-0147\n if not match_str(row_values[6], expected[6]):\n issues.append(f\"Col 7: expected '{expected[6]}', got '{row_values[6]}'\")\n\n # Column 7: Vantage Lubricants Inc.\n if not match_str(row_values[7], expected[7]):\n issues.append(f\"Col 8: expected '{expected[7]}', got '{row_values[7]}'\")\n\n Line truncated
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776721203166",
"sort_order": 12,
"rubric_text": "In `open_pos.xlsx`, row 5 must contain the following values in order: MER-PO-204871 | 001 | 10/31/2026 | 11/28/2026 | SUP-0147 | Vantage Lubricants Inc. | FL-08814-A1 | Gear oil, ISO VG 220, 20L drum | 24 | $38.50 | N | N | FL-08814-A0, FL-08814-A2, FL-08814-A3 | | | | | N | N | Open",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"open_pos.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'open_pos.xlsx' not found in workspace: {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'open_pos.xlsx': {e}\"}\n\n ws = wb.active\n\n row_5 = []\n for cell in ws[5]:\n row_5.append(cell.value)\n\n # Remove trailing None values\n while row_5 and row_5[-1] is None:\n row_5 = row_5[:-1]\n\n # Expected values (20 columns):\n # MER-PO-204871 | 001 | 10/31/2026 | 11/28/2026 | SUP-0147 | Vantage Lubricants Inc. | FL-08814-A1 | Gear oil, ISO VG 220, 20L drum | 24 | $38.50 | N | N | FL-08814-A0, FL-08814-A2, FL-08814-A3 | (empty) | (empty) | (empty) | (empty) | N | N | Open\n\n expected = [\n \"MER-PO-204871\",\n \"001\",\n \"10/31/2026\",\n \"11/28/2026\",\n \"SUP-0147\",\n \"Vantage Lubricants Inc.\",\n \"FL-08814-A1\",\n \"Gear oil, ISO VG 220, 20L drum\",\n \"24\",\n \"$38.50\",\n \"N\",\n \"N\",\n \"FL-08814-A0, FL-08814-A2, FL-08814-A3\",\n \"\",\n \"\",\n \"\",\n \"\",\n \"N\",\n \"N\",\n \"Open\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def values_match(actual_val, expected_val):\n a = normalize(actual_val)\n e = expected_val.strip()\n\n # Both empty\n if a == \"\" and e == \"\":\n return True\n if a == \"\" or e == \"\":\n # If expected is empty, actual should be empty\n return a == \"\" and e == \"\"\n\n # Case-insensitive comparison\n if a.lower() == e.lower():\n return True\n\n # Try numeric comparison with tolerance\n def extract_number(s):\n s_clean = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(s_clean)\n except ValueError:\n return None\n\n a_num = extract_number(a)\n e_num = extract_number(e)\n if a_num is not None and e_num is not None:\n if e_num == 0:\n return abs(a_num) < 0.01\n if abs(a_num - e_num) / max(abs(e_num), 0.001) <= 0.05:\n return True\n\n # For \"001\" vs \"1\" or \"1.0\"\n if e == \"001\":\n if a in [\"001\", \"1\", \"1.0\"]:\n return True\n\n # Date comparison: try multiple formats\n date_formats = [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\", \"%m/%d/%y\"]\n def parse_date(s):\n for fmt in date_formats:\n try:\n return datetime.strptime(s, fmt)\n except ValueError:\n continue\n return None\n\n a_date = parse_date(a)\n e_date = parse_date(e)\n if a_date and e_date and a_date == e_date:\n return True\n\n # For the alternates field, compare sets of items\n if \",\" in e:\n e_parts = set(x.strip().lower() for x in e.split(\",\"))\n a_parts = set(x.strip().lower() for x in a.split(\",\"))\n if e_parts == a_parts:\n return True\n\n return False\n\n # Ensure row has enough columns (pad with None if needed)\n while len(row_5) < len(expected):\n row_5.append(None)\n\n mismatches = []\n matches = 0\n details = []\n for i, (act, exp) in enumerate(zip(row_5[:len(expected)], expected)):\n if values_match(act, exp):\n matches += 1\n details.append(f\" Col {i+1}: OK (actual='{normalize(act)}', expected='{exp}')\")\n else:\n mismatches.append(i + 1)\n details.append(f\" Col {i+1}: MISMATCH (actual='{normalize(act)}', expected='{exp}')\")\n\n total = len(expected)\n score = matches / total if total > 0 else 0.0\n passed = len(mismatches) == 0\n\n detail_str = \"\\n\".join(details)\n if passed:\n feedback = f\"PASS: All {total} values in row 5 of open_pos.xlsx match expected values.\\n{detail_str}\"\n else:\n feedback = f\"FAIL: {len(mismatches)} of {total} values in row 5 do not match. Mismatched columns: {mismatches}\\n{detail_str}\"\n\n return {\"pass\": passed, \"score\": score, \"feedback\": feedback}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,7 @@
[Current date and time: 11/24/2026 14:21 CST]
[Current user: James Reed, Coordonator]
We’ve just had 3 transports arrive together, I want to make sure we can process, unload them and store before the end of the shift. Please take care of the standard one while I deal with the two hazard ones. Follow the procedures outline in Gear_Systems_Logistics_SOP. We’ve been waiting for the gear oil for a couple of weeks so I don’t want any unnecessary delays.
Use my name or contact info as needed, and if you need to fill in any free text fields that the SOP doesn’t already specify exact wording, write “Actioned per SOP protocol”. Also, if you are able to make the disposition decision, use exactly the current time (as of right now) for the corresponding timestamp.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py