Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,71 @@
{
"mailbox": {
"email": "analyst@prairiestar.com",
"name": "Houston Analyst"
},
"contacts": [
{
"email": "daniel.reyes@prairiestar.com",
"name": "Daniel Reyes"
},
{
"email": "emily.carter@prairiestar.com",
"name": "Emily Carter"
},
{
"email": "marcus.liu@prairiestar.com",
"name": "Marcus Liu"
},
{
"email": "hannah.brooks@prairiestar.com",
"name": "Hannah Brooks"
},
{
"email": "jacob.patel@prairiestar.com",
"name": "Jacob Patel"
},
{
"email": "carlos.mendoza@prairiestar.com",
"name": "Carlos Mendoza"
},
{
"email": "alicia.nguyen@prairiestar.com",
"name": "Alicia Nguyen"
},
{
"email": "sarah.kim@prairiestar.com",
"name": "Sarah Kim"
},
{
"email": "kevin.white@prairiestar.com",
"name": "Kevin White"
},
{
"email": "brandon.keller@prairiestar.com",
"name": "Brandon Keller"
},
{
"email": "luis.ramirez@prairiestar.com",
"name": "Luis Ramirez"
},
{
"email": "renee.dubois@prairiestar.com",
"name": "Renee Dubois"
},
{
"email": "wyatt.green@prairiestar.com",
"name": "Wyatt Green"
},
{
"email": "olivia.bennett@prairiestar.com",
"name": "Olivia Bennett"
},
{
"email": "dylan.harper@prairiestar.com",
"name": "Dylan Harper"
}
],
"folders": [],
"emails": [],
"next_email_id": 1
}
@@ -0,0 +1,354 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "analyst",
"deleted": false,
"real_name": "Inventory Operations Analyst",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Inventory Operations Analyst",
"display_name": "analyst",
"email": "analyst@prairiestar.com",
"title": "Inventory Operations Analyst"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "a.nguyen",
"deleted": false,
"real_name": "Alicia Nguyen",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Alicia Nguyen",
"display_name": "a.nguyen",
"email": "alicia.nguyen@prairiestar.com",
"title": "Inventory & Receiving Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "c.mendoza",
"deleted": false,
"real_name": "Carlos Mendoza",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Carlos Mendoza",
"display_name": "c.mendoza",
"email": "carlos.mendoza@prairiestar.com",
"title": "DC Manager"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "s.kim",
"deleted": false,
"real_name": "Sarah Kim",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Sarah Kim",
"display_name": "s.kim",
"email": "sarah.kim@prairiestar.com",
"title": "Warehouse Associate (Senior)"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "k.white",
"deleted": false,
"real_name": "Kevin White",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Kevin White",
"display_name": "k.white",
"email": "kevin.white@prairiestar.com",
"title": "Warehouse Associate"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "b.keller",
"deleted": false,
"real_name": "Brandon Keller",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Brandon Keller",
"display_name": "b.keller",
"email": "brandon.keller@prairiestar.com",
"title": "Dispatcher / Route Planner"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "j.patel",
"deleted": false,
"real_name": "Jacob Patel",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Jacob Patel",
"display_name": "j.patel",
"email": "jacob.patel@prairiestar.com",
"title": "Production Planner"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "d.reyes",
"deleted": false,
"real_name": "Daniel Reyes",
"tz": "America/Chicago",
"tz_label": "Central Time",
"tz_offset": -18000,
"profile": {
"real_name": "Daniel Reyes",
"display_name": "d.reyes",
"email": "daniel.reyes@prairiestar.com",
"title": "Director of Supply Chain"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "receiving-houston",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "receiving-houston",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Houston inbound receiving: arrivals, counts, temperatures, completions, flags",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Real-time receiving updates for Houston DC",
"creator": "U003",
"last_set": 1700000000
}
},
"C002": {
"id": "C002",
"name": "dc-escalations",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "dc-escalations",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "P1 and P2 priority events, escalations requiring Director attention",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Escalations for both Houston and Shreveport DCs",
"creator": "U003",
"last_set": 1700000000
}
},
"C003": {
"id": "C003",
"name": "cold-chain-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "cold-chain-alerts",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Temperature excursions, freezer zone alarms, cold chain breaches",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Cold chain alerts for both DCs",
"creator": "U003",
"last_set": 1700000000
}
},
"C004": {
"id": "C004",
"name": "waco-production-coord",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "waco-production-coord",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Variance issues, manifest questions, communication with Waco production",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Coordination between DCs and Waco production",
"creator": "U003",
"last_set": 1700000000
}
},
"C005": {
"id": "C005",
"name": "inventory-ops",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1700000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "inventory-ops",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1700000000,
"creator": "U003",
"shared_team_ids": ["T001"],
"is_member": true,
"topic": {
"value": "Cycle count results, expiry flags, inventory adjustments",
"creator": "U003",
"last_set": 1700000000
},
"purpose": {
"value": "Inventory operations across both DCs",
"creator": "U003",
"last_set": 1700000000
}
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "Waco transfer 2026-05-20-001 expected at 09:00 CT. Pallets: 24, Cases: 480. Dock crew standing by.",
"ts": "1747737600.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "@analyst Manifest 2026-05-20-001 receiving complete. Seal PS-88214 intact. Temps: -6.0, -4.5, -7.5 (front, middle, rear). Pallets: 24. Cases: 474 — short 6 on SC-003. Lot codes: A-20260519-01 ✓ matches manifest. No damage. Ready for put-away direction.",
"ts": "1747739100.000200",
"team": "T001"
}
]
},
"counters": {
"messageId": 1000000
}
}
@@ -0,0 +1,80 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R /F3 4 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/BaseFont /Symbol /Name /F3 /Subtype /Type1 /Type /Font
>>
endobj
5 0 obj
<<
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
6 0 obj
<<
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
>>
endobj
7 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260414162007+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260414162007+00'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
8 0 obj
<<
/Count 1 /Kids [ 5 0 R ] /Type /Pages
>>
endobj
9 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1292
>>
stream
Gatm;D0+\p&H9tYfT5[%FsL4>i?KYq9,]e8pu\G0S`r,d,eb0Y2.gasn%5kV"-.66,s\8/afBj^=M'VB&H'o7s*>(BRg4-n$C$+N!QoNph@iSr_7p(UcEnUr7LUZ>N>/hfAUoihb**"c(;2DpQLMbYn3NJ9RjQ$]07`$0*C%%E;cn<T(FJQl4J037rW:pen32DH1)Cc,MP&>tCeFu*RtQeQ$MChK&sXU`0NnH,65k=If-r8W1t]#'-r"+3YG4)"hk>keDJ#D6s,3:G3Dso$T@<`k7h>@A0$g9TIGo&V-@(t-<6[B2M1`Z@r&kO3#NXM8I?507[_PTFgDEs#cWc"/%p&+?dA,*H+6*WZkDg@Zg!k`OGk!JWY.W?$9b+%o#T251[rLGD/M8O4+'Dta%N;cY#b+U;+Via&%O%@TAr],s'uu'gG]\URmO-C/guYcmDkB,4TZ#4g9/^].X7kp/kTM1pdYOkWU(K^r(LTea9b=F5UGo^Wo<<?d_]le!^taDE!NFV],P_<=0YV)F99bt*6`kG>L:0J*aimI%h;`ht-,*LS$o`TRogpkJ(utn:]PP@^#O]CNr$XaULt%U>JFhPDNFUi/,j?XX<hUV-9*UQT4ueiYc#FT^*K46+\XEgrF%!;*j#G8,BBZRGUk/77"V8(00_'?qHi0%]*q=<,1cTp%h[cfm(B2u`DmA6c-<B*2C2Jm?KcO$7<@&D-Eag2I.<=pqF#GOcfMOQ.cdIOgARiB[@7T$"&b(KIfrLXK65j^![PAaJ.qd%33Ul^:BJ<SQB](,#!ZB't$]jZH3.66XcDM$\-=7N0[]<Fe7ttm^kNWISS!eX,osW?Z\1sLnKV@@J8EE+U)m.--=s(LG=f@Ni\\%q*+5bfFOFo=!LX37s_8>*<(N[TBq0kqWR&!$'82+-f^Lp<C_g64>R8e?4)E9$k\=0'71V.;diH?7G8bDS2qtlhbdhn>e\:4"s9js**`gY7_`2"W[6WXU9HT5q=OiK$HE8"DQ/l0)J>*R)(-o$n-=AQW\,W4BHFHKV#[7[MId8d3tO)cqLQ;"@3<^q_`WM2NmD-hOM]lKbhf5bU`j]]<!S#V;<]D6GU8f+mnR:.&Wou.4LGObqQI@5e-IMN.Wgb-!G+mTiWPW+p#7o]0-W-Ikerq/h<X-JF_fM@Y?8LJ979f0RDmL!_+#qAPd&!>al!Gcqn%+G4N('lu\%klRHBXpIq*36HuBEpi7hJ_qU!SG^b5L@h(Ib/"_=+LAF8J#KqB7@/@K\>8;o")NH41uEqDo&:T#R:;0lK81-~>endstream
endobj
xref
0 10
0000000000 65535 f
0000000061 00000 n
0000000112 00000 n
0000000219 00000 n
0000000331 00000 n
0000000408 00000 n
0000000601 00000 n
0000000669 00000 n
0000000949 00000 n
0000001008 00000 n
trailer
<<
/ID
[<97fefea0e165997abcb9996d0baf7562><97fefea0e165997abcb9996d0baf7562>]
% ReportLab generated PDF document -- digest (opensource)
/Info 7 0 R
/Root 6 0 R
/Size 10
>>
startxref
2391
%%EOF
@@ -0,0 +1,2 @@
You are the Inventory Operations Analyst at Prairie Star Creamery's Houston Distribution Center.
You have just received a receiving report from Alicia Nguyen, the Inventory & Receiving Coordinator, via the #receiving-houston Slack channel. Process this receipt in full accordance with the SOP. The inbound Waco transfer manifest for today's delivery is available (2026-05-20-001.pdf) and I also provided the receiving log (inbound_receiving_log.xlsx). If the SOP requires updating files other than the receiving log which I haven't attached, don't worry about them because I'll take care of them myself.
@@ -0,0 +1 @@
Today's date is, Wednesday, May, 20, 2026, 09:15 CT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/logistics_prairie_star_creamery_7b63c60d"
description = "SOP task from CSV row 7b63c60d"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,23 @@
[
{
"id": "149b0ac0-9db2-47d0-b674-c453f7c2d37d",
"sort_order": 0,
"rubric_text": "The inbound_receiving_log.xlsx file must contain a data row where: the Expected Pallets and Actual Pallets column both contain 24; the Actual Cases column contains 474; the three Temp Reading columns collectively contain −4.5, −6.0 (alternatively −6), and −7.5 in any order; the Variance (Cases) column contains −6; the Lot Codes Received column contains 'A-20260519-01'; the Disposition column contains both 'quarantine' (or 'quarantined') and 'H-Q'; and the Status column contains exactly 'P2 – Compound event.'",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n # Find the receiving log xlsx file\n xlsx_files = list(ws.rglob('*.xlsx'))\n log_file = None\n for f in xlsx_files:\n if 'receiv' in f.name.lower() or 'log' in f.name.lower():\n log_file = f\n break\n if log_file is None and xlsx_files:\n log_file = xlsx_files[0]\n\n if log_file is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No .xlsx file found in workspace.'}\n\n try:\n import openpyxl\n wb = openpyxl.load_workbook(log_file)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open {log_file.name}: {e}'}\n\n best_result = None\n best_score = -1\n\n for ws_sheet in wb.worksheets:\n rows = list(ws_sheet.iter_rows(values_only=True))\n if len(rows) < 2:\n continue\n\n # Find header row\n header_row_idx = None\n headers = []\n for i, row in enumerate(rows):\n non_none = [c for c in row if c is not None]\n if len(non_none) >= 3:\n header_row_idx = i\n headers = [str(c).strip().lower() if c is not None else '' for c in row]\n break\n\n if header_row_idx is None:\n continue\n\n def find_col(patterns):\n results = []\n for idx, h in enumerate(headers):\n for p in patterns:\n if p in h:\n results.append(idx)\n break\n return results\n\n expected_pallet_cols = find_col(['expected pallet'])\n actual_pallet_cols = find_col(['actual pallet'])\n pallet_cols = find_col(['pallet'])\n actual_case_cols = find_col(['actual case'])\n case_cols = find_col(['case'])\n temp_cols = find_col(['temp'])\n variance_cols = find_col(['variance'])\n lot_cols = find_col(['lot'])\n disposition_cols = find_col(['disposition'])\n status_cols = find_col(['status'])\n\n data_rows = rows[header_row_idx + 1:]\n for row in data_rows:\n row_str_list = [str(c).strip() if c is not None else '' for c in row]\n row_lower = ' '.join(row_str_list).lower()\n\n non_empty = [c for c in row if c is not None and str(c).strip() != '']\n if len(non_empty) < 3:\n continue\n\n def get_vals(col_indices):\n vals = []\n for ci in col_indices:\n if ci < len(row) and row[ci] is not None:\n vals.append(row[ci])\n return vals\n\n def num_eq(val, target, tol=0.5):\n try:\n return abs(float(val) - target) < tol\n except (ValueError, TypeError):\n s = str(val).replace('\\u2212', '-').replace('\\u2013', '-').replace('\\u2014', '-')\n m = re.search(r'-?\\d+\\.?\\d*', s)\n if m:\n try:\n return abs(float(m.group()) - target) < tol\n except:\n pass\n return False\n\n # Check Expected Pallets = 24\n expected_pallet_ok = False\n for v in get_vals(expected_pallet_cols):\n if num_eq(v, 24, 0.5):\n expected_pallet_ok = True\n if not expected_pallet_ok:\n for v in get_vals(pallet_cols):\n if num_eq(v, 24, 0.5):\n expected_pallet_ok = True\n break\n if not expected_pallet_ok:\n for v in row:\n if v is not None and num_eq(v, 24, 0.5):\n expected_pallet_ok = True\n break\n\n # Check Actual Pallets = 24\n actual_pallet_ok = False\n for v in get_vals(actual_pallet_cols):\n if num_eq(v, 24, 0.5):\n actual_pallet_ok = True\n if not actual_pallet_ok:\n for v in get_vals(pallet_cols):\n if num_eq(v, 24, 0.5):\n actual_pallet_ok = True\n break\n if not actual_pallet_ok:\n for v in row:\n if v is not None and num_eq(v, 24, 0.5):\n actual_pallet_ok = True\n break\n\n pallets_ok = expected_pallet_ok and actual_pallet_ok\n\n # Check Actual Cases = 474\n cases_ok = False\n for v in get_vals(actual_case_cols):\n if num_eq(v, 474, 0.5):\n cases_ok = True\n if not cases_ok:\n for v in get_vals(Line truncated
"criterion_type": "expected_output"
},
{
"id": "a20b335b-2dbd-4e0c-8f52-1500a16fd68f",
"sort_order": 1,
"rubric_text": "A message posted to the #dc-escalations Slack channel must begin with 'P2 escalation. Compound.' [exact string, 'compound event' not acceptable], contain manifest number '2026-05-20-001', contain the exact phrase 'Escalated to Director of Supply Chain.', and contain the exact phrase 'Incident report: awaiting Director decision.' The Detail and Action required fields may contain any text.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found at external_services_path'}\n\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n\n # Find the #dc-escalations channel\n channels = data.get('channels', {})\n escalation_channel_id = None\n for cid, cinfo in channels.items():\n cname = str(cinfo.get('name', '')).lower().strip()\n if cname == 'dc-escalations':\n escalation_channel_id = cid\n break\n\n if escalation_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find #dc-escalations channel in slack_data.json. Channels found: ' + str([c.get('name', '') for c in channels.values()])}\n\n # Get messages in that channel\n messages_by_channel = data.get('messages', {})\n channel_messages = messages_by_channel.get(escalation_channel_id, [])\n\n if not channel_messages:\n return {'pass': False, 'score': 0.0, 'feedback': f'No messages found in #dc-escalations channel (channel ID: {escalation_channel_id})'}\n\n # Score each message to find the best candidate\n best_msg = None\n best_score = -1\n\n for msg in channel_messages:\n text = str(msg.get('text', ''))\n text_lower = text.lower()\n score = 0\n if text_lower.lstrip().startswith('p2 escalation'):\n score += 5\n if '2026-05-20-001' in text:\n score += 5\n if 'compound' in text_lower:\n score += 3\n if 'director of supply chain' in text_lower:\n score += 3\n if 'awaiting director decision' in text_lower:\n score += 3\n if score > best_score:\n best_score = score\n best_msg = msg\n\n if best_msg is None or best_score <= 0:\n texts_preview = [str(m.get('text', ''))[:80] for m in channel_messages[:5]]\n return {'pass': False, 'score': 0.0, 'feedback': f'No plausible escalation message found in #dc-escalations among {len(channel_messages)} messages. Previews: {texts_preview}'}\n\n text = str(best_msg.get('text', ''))\n text_lower = text.lower()\n\n failures = []\n details = []\n\n # (1) Must begin with 'P2 escalation. Compound.' [exact string, 'compound event' not acceptable]\n # The rubric says exact string 'P2 escalation. Compound.' at the start.\n # We check case-insensitively for leniency, and allow minor whitespace variations,\n # but 'compound' must appear as a standalone word (not 'compound event' replacing it).\n # The key distinction: must say 'Compound' (with period/punctuation after), NOT 'Compound event'.\n start_pattern = re.compile(r'(?i)^\\s*p2\\s+escalation\\.\\s*compound\\s*\\.', re.DOTALL)\n if start_pattern.search(text):\n # Additionally verify it doesn't say 'compound event' right at the start\n bad_pattern = re.compile(r'(?i)^\\s*p2\\s+escalation\\.\\s*compound\\s+event', re.DOTALL)\n if bad_pattern.search(text):\n failures.append(f\"(1) Message begins with 'P2 escalation. Compound event...' but rubric requires exactly 'P2 escalation. Compound.' (not 'compound event'). Start: '{text[:80]}...'\")\n else:\n details.append(\"(1) Message begins with 'P2 escalation. Compound.' \\u2713\")\n else:\n failures.append(f\"(1) Message must begin with 'P2 escalation. Compound.' but starts with: '{text[:80]}...'\")\n\n # (2) Must contain manifest number '2026-05-20-001'\n if '2026-05-20-001' in text:\n details.append(\"(2) Manifest number '2026-05-20-001' found \\u2713\")\n else:\n failures.append(\"(2) Manifest number '2026-05-20-001' not found in message\")\n\n # (3) Must contain the exact phrase 'Escalated to Director of Supply Chain.'\n # Be loose: case-insensitive, allow with or without trailing period\n if re.search(r'(?i)escalated\\s+to\\s+director\\s+of\\s+supply\\s+chain\\.?', text):\n details.append(\"(3) 'Escalated to Director of Supply Chain.' found \\u2713\")\n else:\n failures.append(\"(3) Exact phrase 'Escalated to Director of Supply Chain.' not found in message\")\n\n # (4) Must contain the exact phrase 'Incident report: awaiting Director decision.'\n # Be loose: case-insensitive, allow with or without trailing period, flexible colon/space\n if re.search(r'(?i)incident\\s+report\\s*[:\\-]\\s*awaiting\\s+director\\s+decision\\.?', text):\n details.append(\"(4) 'Incident report: awaiting Director decision.' found \\u2713\")\n else:\n failures.append(\"Line truncated
"criterion_type": "expected_output"
},
{
"id": "98ee41bc-57a2-4c54-a5c1-57bf70a8aee9",
"sort_order": 2,
"rubric_text": "A sent email in mailbox.json must be addressed To: daniel.reyes@prairiestar.com only, CC: carlos.mendoza@prairiestar.com and jacob.patel@prairiestar.com with no additional recipients, have a subject containing 'P2', 'Compound Event', '2026-05-20-001', and '2026-05-20', and have a body containing all three temperature readings −4.5, −6.0 (alternatively −6), and −7.5, 'Tier 3', variance of −6 cases and 1.25%, 'Moderate' as the variance classification, 'quarantined to H-Q', and 'Director assessment required' [exact string].",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.json'}\n\n # Find the escalation email - look for one addressed to daniel.reyes\n matching_email = None\n for email in sent_emails:\n to_addr = (email.get('to_addr') or email.get('to') or '').lower()\n if 'daniel.reyes@prairiestar.com' in to_addr:\n matching_email = email\n break\n\n if matching_email is None:\n # Fallback: look for email with P2 in subject or relevant content\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or email.get('body') or '').lower()\n if 'p2' in subject or ('quarantine' in body and 'h-q' in body):\n matching_email = email\n break\n\n if matching_email is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No escalation email found in sent folder addressed to daniel.reyes@prairiestar.com'}\n\n failures = []\n\n # --- Check TO field ---\n to_addr = (matching_email.get('to_addr') or matching_email.get('to') or '').lower().strip()\n if 'daniel.reyes@prairiestar.com' not in to_addr:\n failures.append('To field does not contain daniel.reyes@prairiestar.com')\n else:\n to_emails = re.findall(r'[\\w.+-]+@[\\w.-]+', to_addr)\n to_emails_filtered = [e for e in to_emails if e != 'daniel.reyes@prairiestar.com']\n if to_emails_filtered:\n failures.append(f'To field contains additional recipients beyond daniel.reyes@prairiestar.com: {to_emails_filtered}')\n\n # --- Check CC field ---\n cc_addr = (matching_email.get('cc_addr') or matching_email.get('cc') or '').lower().strip()\n cc_emails = re.findall(r'[\\w.+-]+@[\\w.-]+', cc_addr)\n required_cc = {'carlos.mendoza@prairiestar.com', 'jacob.patel@prairiestar.com'}\n cc_set = set(cc_emails)\n missing_cc = required_cc - cc_set\n extra_cc = cc_set - required_cc\n if missing_cc:\n failures.append(f'CC field missing required recipients: {missing_cc}')\n if extra_cc:\n failures.append(f'CC field contains extra recipients: {extra_cc}')\n\n # --- Check Subject ---\n subject = matching_email.get('subject') or ''\n subject_lower = subject.lower()\n if 'p2' not in subject_lower:\n failures.append('Subject does not contain P2')\n if 'compound event' not in subject_lower:\n failures.append('Subject does not contain Compound Event')\n if '2026-05-20-001' not in subject:\n failures.append('Subject does not contain 2026-05-20-001')\n if '2026-05-20' not in subject:\n failures.append('Subject does not contain 2026-05-20')\n\n # --- Check Body ---\n body = matching_email.get('body_text') or matching_email.get('body') or ''\n # Normalize unicode minus signs and en-dashes to regular minus/hyphen\n normalized_body = body.replace('\\u2212', '-').replace('\\u2013', '-').replace('\\u2014', '-').replace('\\u00b0', ' ')\n normalized_lower = normalized_body.lower()\n\n # Temperature readings: -4.5, -6.0 (or -6), -7.5\n if not re.search(r'[-]\\s*4\\.5', normalized_lower):\n failures.append('Body does not contain temperature reading -4.5 (or \\u22124.5)')\n\n # -6.0 or -6 (but distinguish from -6 in other contexts like variance)\n has_minus6 = bool(re.search(r'[-]\\s*6\\.0', normalized_lower)) or bool(re.search(r'[-]\\s*6(?:\\b|\\s|f|$|,|\\)|\\*|\\xb0)', normalized_lower))\n if not has_minus6:\n failures.append('Body does not contain temperature reading -6.0 or -6 (or \\u22126.0/\\u22126)')\n\n if not re.search(r'[-]\\s*7\\.5', normalized_lower):\n failures.append('Body does not contain temperature reading -7.5 (or \\u22127.5)')\n\n # Tier 3\n if not re.search(r'tier[\\s\\-_]*3', normalized_lower):\n failures.append('Body does not contain Tier 3')\n\n # Variance: -6 cases and 1.25%\n has_variance_cases = bool(re.search(r'[-]\\s*6\\s*case', normalized_lower)) or \\\n bool(re.search(r'varianc.{0,80}[-]\\s*6', normalized_lower)) or \\\n bool(re.search(r'[-]\\s*6.{0,30}case', normalized_lower)) or \\\n bool(re.search(r'shortage.Line truncated
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,2 @@
It is 09:15 CT on Wednesday, 20 May 2026. You are the Inventory Operations Analyst at Prairie Star Creamery's Houston Distribution Center.
You have just received a receiving report from Alicia Nguyen, the Inventory & Receiving Coordinator, via the #receiving-houston Slack channel. Process this receipt in full accordance with the SOP. The inbound Waco transfer manifest for today's delivery is available (2026-05-20-001.pdf) and I also provided the receiving log (inbound_receiving_log.xlsx). If the SOP requires updating files other than the receiving log which I haven't attached, don't worry about them because I'll take care of them myself.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py