Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "analyst@prairiestar.com",
|
||||
"name": "Houston Analyst"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"name": "Daniel Reyes"
|
||||
},
|
||||
{
|
||||
"email": "emily.carter@prairiestar.com",
|
||||
"name": "Emily Carter"
|
||||
},
|
||||
{
|
||||
"email": "marcus.liu@prairiestar.com",
|
||||
"name": "Marcus Liu"
|
||||
},
|
||||
{
|
||||
"email": "hannah.brooks@prairiestar.com",
|
||||
"name": "Hannah Brooks"
|
||||
},
|
||||
{
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"name": "Jacob Patel"
|
||||
},
|
||||
{
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"name": "Carlos Mendoza"
|
||||
},
|
||||
{
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"name": "Alicia Nguyen"
|
||||
},
|
||||
{
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"name": "Sarah Kim"
|
||||
},
|
||||
{
|
||||
"email": "kevin.white@prairiestar.com",
|
||||
"name": "Kevin White"
|
||||
},
|
||||
{
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"name": "Brandon Keller"
|
||||
},
|
||||
{
|
||||
"email": "luis.ramirez@prairiestar.com",
|
||||
"name": "Luis Ramirez"
|
||||
},
|
||||
{
|
||||
"email": "renee.dubois@prairiestar.com",
|
||||
"name": "Renee Dubois"
|
||||
},
|
||||
{
|
||||
"email": "wyatt.green@prairiestar.com",
|
||||
"name": "Wyatt Green"
|
||||
},
|
||||
{
|
||||
"email": "olivia.bennett@prairiestar.com",
|
||||
"name": "Olivia Bennett"
|
||||
},
|
||||
{
|
||||
"email": "dylan.harper@prairiestar.com",
|
||||
"name": "Dylan Harper"
|
||||
}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [],
|
||||
"next_email_id": 1
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "analyst",
|
||||
"deleted": false,
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Inventory Operations Analyst",
|
||||
"display_name": "analyst",
|
||||
"email": "analyst@prairiestar.com",
|
||||
"title": "Inventory Operations Analyst"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "a.nguyen",
|
||||
"deleted": false,
|
||||
"real_name": "Alicia Nguyen",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Alicia Nguyen",
|
||||
"display_name": "a.nguyen",
|
||||
"email": "alicia.nguyen@prairiestar.com",
|
||||
"title": "Inventory & Receiving Coordinator"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "c.mendoza",
|
||||
"deleted": false,
|
||||
"real_name": "Carlos Mendoza",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Carlos Mendoza",
|
||||
"display_name": "c.mendoza",
|
||||
"email": "carlos.mendoza@prairiestar.com",
|
||||
"title": "DC Manager"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "s.kim",
|
||||
"deleted": false,
|
||||
"real_name": "Sarah Kim",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Sarah Kim",
|
||||
"display_name": "s.kim",
|
||||
"email": "sarah.kim@prairiestar.com",
|
||||
"title": "Warehouse Associate (Senior)"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "k.white",
|
||||
"deleted": false,
|
||||
"real_name": "Kevin White",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Kevin White",
|
||||
"display_name": "k.white",
|
||||
"email": "kevin.white@prairiestar.com",
|
||||
"title": "Warehouse Associate"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "b.keller",
|
||||
"deleted": false,
|
||||
"real_name": "Brandon Keller",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Brandon Keller",
|
||||
"display_name": "b.keller",
|
||||
"email": "brandon.keller@prairiestar.com",
|
||||
"title": "Dispatcher / Route Planner"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T001",
|
||||
"name": "j.patel",
|
||||
"deleted": false,
|
||||
"real_name": "Jacob Patel",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Jacob Patel",
|
||||
"display_name": "j.patel",
|
||||
"email": "jacob.patel@prairiestar.com",
|
||||
"title": "Production Planner"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008",
|
||||
"team_id": "T001",
|
||||
"name": "d.reyes",
|
||||
"deleted": false,
|
||||
"real_name": "Daniel Reyes",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Daniel Reyes",
|
||||
"display_name": "d.reyes",
|
||||
"email": "daniel.reyes@prairiestar.com",
|
||||
"title": "Director of Supply Chain"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "receiving-houston",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "receiving-houston",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Houston inbound receiving: arrivals, counts, temperatures, completions, flags",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Real-time receiving updates for Houston DC",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "dc-escalations",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "dc-escalations",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "P1 and P2 priority events, escalations requiring Director attention",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Escalations for both Houston and Shreveport DCs",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "cold-chain-alerts",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "cold-chain-alerts",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Temperature excursions, freezer zone alarms, cold chain breaches",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Cold chain alerts for both DCs",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "waco-production-coord",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "waco-production-coord",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Variance issues, manifest questions, communication with Waco production",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Coordination between DCs and Waco production",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
},
|
||||
"C005": {
|
||||
"id": "C005",
|
||||
"name": "inventory-ops",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1700000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "inventory-ops",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1700000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Cycle count results, expiry flags, inventory adjustments",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Inventory operations across both DCs",
|
||||
"creator": "U003",
|
||||
"last_set": 1700000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Waco transfer 2026-05-20-001 expected at 09:00 CT. Pallets: 24, Cases: 480. Dock crew standing by.",
|
||||
"ts": "1747737600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "@analyst Manifest 2026-05-20-001 receiving complete. Seal PS-88214 intact. Temps: -6.0, -4.5, -7.5 (front, middle, rear). Pallets: 24. Cases: 474 — short 6 on SC-003. Lot codes: A-20260519-01 ✓ matches manifest. No damage. Ready for put-away direction.",
|
||||
"ts": "1747739100.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
]
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R /F3 4 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/BaseFont /Symbol /Name /F3 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260414162007+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260414162007+00'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 5 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1292
|
||||
>>
|
||||
stream
|
||||
Gatm;D0+\p&H9tYfT5[%FsL4>i?KYq9,]e8pu\G0S`r,d,eb0Y2.gasn%5kV"-.66,s\8/afBj^=M'VB&H'o7s*>(BRg4-n$C$+N!QoNph@iSr_7p(UcEnUr7LUZ>N>/hfAUoihb**"c(;2DpQLMbYn3NJ9RjQ$]07`$0*C%%E;cn<T(FJQl4J037rW:pen32DH1)Cc,MP&>tCeFu*RtQeQ$MChK&sXU`0NnH,65k=If-r8W1t]#'-r"+3YG4)"hk>keDJ#D6s,3:G3Dso$T@<`k7h>@A0$g9TIGo&V-@(t-<6[B2M1`Z@r&kO3#NXM8I?507[_PTFgDEs#cWc"/%p&+?dA,*H+6*WZkDg@Zg!k`OGk!JWY.W?$9b+%o#T251[rLGD/M8O4+'Dta%N;cY#b+U;+Via&%O%@TAr],s'uu'gG]\URmO-C/guYcmDkB,4TZ#4g9/^].X7kp/kTM1pdYOkWU(K^r(LTea9b=F5UGo^Wo<<?d_]le!^taDE!NFV],P_<=0YV)F99bt*6`kG>L:0J*aimI%h;`ht-,*LS$o`TRogpkJ(utn:]PP@^#O]CNr$XaULt%U>JFhPDNFUi/,j?XX<hUV-9*UQT4ueiYc#FT^*K46+\XEgrF%!;*j#G8,BBZRGUk/77"V8(00_'?qHi0%]*q=<,1cTp%h[cfm(B2u`DmA6c-<B*2C2Jm?KcO$7<@&D-Eag2I.<=pqF#GOcfMOQ.cdIOgARiB[@7T$"&b(KIfrLXK65j^m.--=s(LG=f@Ni\\%q*+5bfFOFo=!LX37s_8>*<(N[TBq0kqWR&!$'82+-f^Lp<C_g64>R8e?4)E9$k\=0'71V.;diH?7G8bDS2qtlhbdhn>e\:4"s9js**`gY7_`2"W[6WXU9HT5q=OiK$HE8"DQ/l0)J>*R)(-o$n-=AQW\,W4BHFHKV#[7[MId8d3tO)cqLQ;"@3<^q_`WM2NmD-hOM]lKbhf5bU`j]]<!S#V;<]D6GU8f+mnR:.&Wou.4LGObqQI@5e-IMN.Wgb-!G+mTiWPW+p#7o]0-W-Ikerq/h<X-JF_fM@Y?8LJ979f0RDmL!_+#qAPd&!>al!Gcqn%+G4N('lu\%klRHBXpIq*36HuBEpi7hJ_qU!SG^b5L@h(Ib/"_=+LAF8J#KqB7@/@K\>8;o")NH41uEqDo&:T#R:;0lK81-~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 10
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000112 00000 n
|
||||
0000000219 00000 n
|
||||
0000000331 00000 n
|
||||
0000000408 00000 n
|
||||
0000000601 00000 n
|
||||
0000000669 00000 n
|
||||
0000000949 00000 n
|
||||
0000001008 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<97fefea0e165997abcb9996d0baf7562><97fefea0e165997abcb9996d0baf7562>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 7 0 R
|
||||
/Root 6 0 R
|
||||
/Size 10
|
||||
>>
|
||||
startxref
|
||||
2391
|
||||
%%EOF
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,2 @@
|
||||
You are the Inventory Operations Analyst at Prairie Star Creamery's Houston Distribution Center.
|
||||
You have just received a receiving report from Alicia Nguyen, the Inventory & Receiving Coordinator, via the #receiving-houston Slack channel. Process this receipt in full accordance with the SOP. The inbound Waco transfer manifest for today's delivery is available (2026-05-20-001.pdf) and I also provided the receiving log (inbound_receiving_log.xlsx). If the SOP requires updating files other than the receiving log which I haven't attached, don't worry about them because I'll take care of them myself.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, Wednesday, May, 20, 2026, 09:15 CT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/logistics_prairie_star_creamery_7b63c60d"
|
||||
description = "SOP task from CSV row 7b63c60d"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
It is 09:15 CT on Wednesday, 20 May 2026. You are the Inventory Operations Analyst at Prairie Star Creamery's Houston Distribution Center.
|
||||
You have just received a receiving report from Alicia Nguyen, the Inventory & Receiving Coordinator, via the #receiving-houston Slack channel. Process this receipt in full accordance with the SOP. The inbound Waco transfer manifest for today's delivery is available (2026-05-20-001.pdf) and I also provided the receiving log (inbound_receiving_log.xlsx). If the SOP requires updating files other than the receiving log which I haven't attached, don't worry about them because I'll take care of them myself.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user