Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "ioa@prairiestar.com",
|
||||
"name": "Inventory Operations Analyst"
|
||||
},
|
||||
"contacts": [
|
||||
{"email": "c.board@gcpack.com", "name": "Cara Board"},
|
||||
{"email": "b.wrap@swrapco.com", "name": "Bubb L. Wrapp"},
|
||||
{"email": "r.zin@hlbakery.com", "name": "Ray Zin"},
|
||||
{"email": "c.ontop@lstoppings.com", "name": "Cherry Ontop"},
|
||||
{"email": "d.mops@cleanproind.com", "name": "Dustin Hoover"},
|
||||
{"email": "c.ice@arcticexp.com", "name": "Cole D. Ice"},
|
||||
{"email": "t.downes@texpal.com", "name": "Ty Downes"},
|
||||
{"email": "daniel.reyes@prairiestar.com", "name": "Daniel Reyes"},
|
||||
{"email": "emily.carter@prairiestar.com", "name": "Emily Carter"},
|
||||
{"email": "marcus.liu@prairiestar.com", "name": "Marcus Liu"},
|
||||
{"email": "hannah.brooks@prairiestar.com", "name": "Hannah Brooks"},
|
||||
{"email": "jacob.patel@prairiestar.com", "name": "Jacob Patel"},
|
||||
{"email": "carlos.mendoza@prairiestar.com", "name": "Carlos Mendoza"},
|
||||
{"email": "alicia.nguyen@prairiestar.com", "name": "Alicia Nguyen"},
|
||||
{"email": "sarah.kim@prairiestar.com", "name": "Sarah Kim"},
|
||||
{"email": "kevin.white@prairiestar.com", "name": "Kevin White"},
|
||||
{"email": "brandon.keller@prairiestar.com", "name": "Brandon Keller"},
|
||||
{"email": "luis.ramirez@prairiestar.com", "name": "Luis Ramirez"},
|
||||
{"email": "renee.dubois@prairiestar.com", "name": "Renee Dubois"},
|
||||
{"email": "wyatt.green@prairiestar.com", "name": "Wyatt Green"},
|
||||
{"email": "olivia.bennett@prairiestar.com", "name": "Olivia Bennett"},
|
||||
{"email": "dylan.harper@prairiestar.com", "name": "Dylan Harper"}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Delivery Waiting!!!!!!!",
|
||||
"from_addr": "t.downes@texpal.com",
|
||||
"to_addr": "ioa@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T08:00:00-05:00",
|
||||
"message_id": "<msg001@texpal.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hey!\n\nSeriously need your help right now - one of my drivers has been sitting at your Houston gate for 20 minutes and there's NO security to let him in! \n\nIt's already 77 degrees out here and his cargo temp is reading 86F. We're losing time and I'm getting really worried about the integrity of everything in the back, but most importantly my driver's safety.\n\nCan you PLEASE get someone out there ASAP? I know it's early but this is critical.\n\nCall me when you get this!\n\nTy",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Pest Contamination",
|
||||
"from_addr": "sarah.kim@prairiestar.com",
|
||||
"to_addr": "ioa@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T08:15:00-05:00",
|
||||
"message_id": "<msg002@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nWe have a serious problem with the Texpal delivery that just came in this morning.\n\nWhile unloading the pallets, I discovered rodent droppings on at least 4-5 of them. The contamination looks recent and is definitely a health hazard.\n\nI've stopped the unloading process and quarantined the affected pallets. The rest of the shipment is on hold until we figure out next steps.\n\nWe need to document this immediately and decide whether to reject the entire shipment or just the contaminated pallets. I've taken photos for our records.\n\nSarah Kim\nWarehouse Associate (Senior)\nHouston Distribution Center",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "INBOX",
|
||||
"subject": "Upcoming Server Update",
|
||||
"from_addr": "hannah.brooks@prairiestar.com",
|
||||
"to_addr": "ioa@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T08:30:00-05:00",
|
||||
"message_id": "<msg003@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hey,\n\nJust wanted to give you a heads up about something I heard through the grapevine.\n\nApparently IT is doing some major server updates later this week and there's a chance some files on the shared drive might get wiped or corrupted during the migration. I know they're supposed to have backups, but... you know how that goes sometimes.\n\nAnyway, I'd strongly recommend saving a personal copy of the inventory_master.xlsx file, just to be safe. That thing would be a nightmare to rebuild from scratch if something goes wrong.\n\nThis isn't official or anything, just me looking out for you. Better safe than sorry!\n\nHannah\n\nHannah Brooks\nInventory Analyst\nPrairieStar",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Freezer Temp",
|
||||
"from_addr": "kevin.white@prairiestar.com",
|
||||
"to_addr": "ioa@prairiestar.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-01T08:15:00-05:00",
|
||||
"message_id": "<msg004@prairiestar.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi there,\n\nI'm really worried about something and I'm not sure what to do. I checked the freezer temperature in zone H-A this morning and it was reading -28 degrees. \n\nI know that seems really cold but I'm concerned it might be malfunctioning or something? I'm worried about all the stock in there getting damaged and if we miss orders because of this it would be a disaster. I've only been here a few months and I really don't want to mess anything up.\n\nShould I be concerned about this? I mean, the gauge seemed to be working fine and everything, but -28 just seems extreme to me. I double-checked and my thermometer was definitely reading accurately - I had it set to C when I took the measurement.\n\nI'm probably overthinking this. I just want to make sure we're not risking any product integrity or customer orders.?\n\nThanks so much,\n\nKevin White\nWarehouse Associate\nHouston Distribution Center",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 5
|
||||
}
|
||||
|
||||
+1253
File diff suppressed because it is too large
Load Diff
+1657
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Please read through unread emails and action each accordingly, as detailed to the SOP. I'd like you to go ahead with all SOP-prescribed actions right away, regardless SOP timelines like "within the next fifteen minutes" or within "4 hours".
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 1, 2026, 8:30am CT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/logistics_prairie_star_creamery_861b2650"
|
||||
description = "SOP task from CSV row 861b2650"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
[
|
||||
{
|
||||
"id": "3dc27ca8-d8b7-4cf5-9e1e-a047bd7ac982",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json`, the email with subject 'Delivery Waiting!!!!!!!' from t.downes@texpal.com must be forwarded to carlos.mendoza@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n # Look for a forwarded email about 'Delivery Waiting!!!!!!!' to carlos.mendoza\n for e in sent_emails:\n subject = e.get(\"subject\", \"\")\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n all_recipients = (to_addr + \" \" + cc_addr).lower()\n if \"delivery waiting\" in subject.lower() and \"carlos.mendoza@prairiestar.com\" in all_recipients:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarded email with subject '{subject}' to carlos.mendoza@prairiestar.com\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No forwarded email with subject containing 'Delivery Waiting' to carlos.mendoza@prairiestar.com found in Sent folder\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "25656f51-ffe3-4255-9b9a-f5efab012384",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `slack_data.json`, channel `#dm-c.mendoza` must contain a message with exact content 'OUT OF SCOPE: Delivery Waiting!!!!!!! -- please advise' (dash separator variations acceptable)",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\".\", \"-\").replace(\"_\", \"-\")\n if \"dm\" in name and \"mendoza\" in name:\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dm-c.mendoza channel in slack_data.json\"}\n channel_msgs = messages.get(target_channel_id, [])\n # Strict: anchored full-string match, no additional content allowed.\n # Rubric explicitly allows dash-separator variations (- / -- / – / —); everything else is literal.\n pattern = r'^out of scope:\\s*delivery waiting!{7}\\s*[-\\u2013\\u2014]+\\s*please advise$'\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if re.match(pattern, text, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #dm-c.mendoza: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No exact-match 'OUT OF SCOPE: Delivery Waiting!!!!!!! -- please advise' message found in #dm-c.mendoza. Messages: {[m.get('text','') for m in channel_msgs]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "efced52e-df02-424a-840a-6bd7ded35a82",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `slack_data.json`, channel `#dm-c.mendoza` must contain a message with exact content 'URGENT: Pest Contamination – TexPal shipment received 2026-04-01'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\".\", \"-\").replace(\"_\", \"-\")\n if \"dm\" in name and \"mendoza\" in name:\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dm-c.mendoza channel in slack_data.json\"}\n channel_msgs = messages.get(target_channel_id, [])\n # Strict: anchored full-string match, no additional content allowed.\n # Accepts \"TexPal\" or \"TexPal Logistics\"; accepts -, –, — as the separator.\n pattern = r'^urgent:\\s*pest\\s+contamination\\s*[-\\u2013\\u2014]\\s*texpal(\\s+logistics)?\\s+shipment\\s+received\\s+2026-04-01$'\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if re.match(pattern, text, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Pest Contamination URGENT message in #dm-c.mendoza: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No exact-match URGENT Pest Contamination message found in #dm-c.mendoza. Messages: {[m.get('text','') for m in channel_msgs]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f64ad1ee-0161-4f03-8e07-184943e71ed5",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, the email with subject 'Pest Contamination' from sarah.kim@prairiestar.com must be forwarded with both daniel.reyes@prairiestar.com and carlos.mendoza@prairiestar.com as recipients",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n subject = e.get(\"subject\", \"\")\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n all_recipients = (to_addr + \" \" + cc_addr).lower()\n if \"pest contamination\" in subject.lower():\n has_daniel = \"daniel.reyes@prairiestar.com\" in all_recipients\n has_carlos = \"carlos.mendoza@prairiestar.com\" in all_recipients\n if has_daniel and has_carlos:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarded Pest Contamination email with both daniel.reyes and carlos.mendoza as recipients\"}\n else:\n missing = []\n if not has_daniel: missing.append(\"daniel.reyes@prairiestar.com\")\n if not has_carlos: missing.append(\"carlos.mendoza@prairiestar.com\")\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found Pest Contamination forward but missing recipients: {missing}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No forwarded email with subject 'Pest Contamination' found in Sent folder\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "ef1969dc-8986-4758-a00f-d5c4796c8a83",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `slack_data.json`, channels `#cold-chain-alerts` and `#dc-escalations-houston` must each contain a message with content 'URGENT: issue with TexPal for shipment received on 2026-04-01' (or with 'TexPal Logistics' instead of 'TexPal'), with no additional content",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n def find_channel_id(target_name):\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\"_\", \"-\")\n if target_name.lower().replace(\"_\", \"-\") in name:\n return cid\n return None\n\n def msg_matches(text):\n # Accept TexPal or TexPal Logistics, case insensitive\n pattern = r'^urgent:\\s*issue with texpal(\\s+logistics)?\\s+for\\s+shipment\\s+received\\s+on\\s+2026-04-01$'\n return bool(re.match(pattern, text.strip(), re.IGNORECASE))\n\n results = {}\n for channel_name in [\"cold-chain-alerts\", \"dc-escalations-houston\"]:\n cid = find_channel_id(channel_name)\n if cid is None:\n results[channel_name] = f\"Channel #{channel_name} not found\"\n continue\n channel_msgs = messages.get(cid, [])\n found = False\n for msg in channel_msgs:\n text = msg.get(\"text\", \"\")\n if msg_matches(text):\n found = True\n break\n if not found:\n texts = [m.get(\"text\", \"\") for m in channel_msgs]\n results[channel_name] = f\"No matching URGENT TexPal message found. Messages: {texts}\"\n else:\n results[channel_name] = \"OK\"\n\n all_pass = all(v == \"OK\" for v in results.values())\n if all_pass:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found URGENT TexPal shipment messages in both #cold-chain-alerts and #dc-escalations-houston\"}\n else:\n feedback = \"; \".join(f\"{k}: {v}\" for k, v in results.items())\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a572ce7a-41ec-45ad-806c-60fe72c26842",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `inbound_receiving_log.xlsx`, cell H2 must be 'Class 4' and cell I2 must be 'Rejected – Class 4 Contamination'",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n files = list(Path(workspace_path).glob(\"*inbound_receiving_log*\"))\n if not files:\n files = list(Path(workspace_path).glob(\"*.xlsx\"))\n files = [f for f in files if \"inbound\" in f.name.lower() or \"receiving\" in f.name.lower()]\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No inbound_receiving_log.xlsx found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n h2 = ws[\"H2\"].value\n i2 = ws[\"I2\"].value\n h2_str = str(h2).strip() if h2 is not None else \"\"\n i2_str = str(i2).strip() if i2 is not None else \"\"\n h2_ok = h2_str.lower() == \"class 4\"\n # Accept dash variations: – — -\n i2_normalized = re.sub(r'[\\u2013\\u2014\\-]', '-', i2_str).lower()\n i2_ok = \"rejected\" in i2_normalized and \"class 4\" in i2_normalized and \"contamination\" in i2_normalized\n if h2_ok and i2_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"H2='{h2_str}', I2='{i2_str}' - both correct\"}\n feedback_parts = []\n if not h2_ok:\n feedback_parts.append(f\"H2 expected 'Class 4' but got '{h2_str}'\")\n if not i2_ok:\n feedback_parts.append(f\"I2 expected 'Rejected – Class 4 Contamination' but got '{i2_str}'\")\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(feedback_parts)}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4536b9b9-bea3-4876-a146-cf10b5529d90",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email to t.downes@texpal.com with daniel.reyes@prairiestar.com and marcus.liu@prairiestar.com CC'd, subject 'PRIORITY – Shipment Rejection – Manifest TP56734 – 2026-04-01', with body containing manifest TP56734, date 2026-04-01, reason Class 4 Contamination, and signed Prairie Star Creamery",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n subject = e.get(\"subject\", \"\") or \"\"\n body = e.get(\"body_text\", \"\") or \"\"\n if \"t.downes@texpal.com\" not in to_addr.lower():\n continue\n # Check subject - normalize dashes\n subj_norm = re.sub(r'[\\u2013\\u2014]', '-', subject)\n if not re.search(r'priority.*shipment rejection.*(manifest\\s+|po\\s*)?tp56734.*2026-04-01', subj_norm, re.IGNORECASE):\n continue\n # Check CC\n cc_lower = cc_addr.lower()\n has_daniel = \"daniel.reyes@prairiestar.com\" in cc_lower\n has_marcus = \"marcus.liu@prairiestar.com\" in cc_lower\n # Check body content\n body_lower = body.lower()\n has_manifest = \"tp56734\" in body_lower\n has_date = \"2026-04-01\" in body\n has_class4 = \"class 4\" in body_lower\n has_contamination = \"contamination\" in body_lower\n has_prairie = \"prairie star\" in body_lower\n checks = {\"daniel CC\": has_daniel, \"marcus CC\": has_marcus, \"manifest TP56734 in body\": has_manifest,\n \"date 2026-04-01 in body\": has_date, \"Class 4 in body\": has_class4,\n \"Contamination in body\": has_contamination, \"Prairie Star in body\": has_prairie}\n all_ok = all(checks.values())\n if all_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct rejection email to t.downes@texpal.com with subject '{subject}'\"}\n else:\n failed = [k for k, v in checks.items() if not v]\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found rejection email but missing: {failed}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email to t.downes@texpal.com with subject matching 'PRIORITY – Shipment Rejection – Manifest TP56734 – 2026-04-01' found\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e02229cd-4ddd-4fb6-8eab-17ba07579b31",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "File `IR-2026-04-01-Contamination.pdf` must exist in the workspace, must be the only file with 'IR' prefix, and must contain Incident Date: 2026-04-01, Incident Category: Contamination, Priority Level: P1",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find all files starting with IR (case insensitive)\n ir_files = [f for f in workspace.glob(\"*\") if f.is_file() and f.name.upper().startswith(\"IR\")]\n if len(ir_files) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No file with 'IR' prefix found in workspace\"}\n if len(ir_files) > 1:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Multiple IR-prefixed files found: {[f.name for f in ir_files]}. Expected exactly one.\"}\n ir_file = ir_files[0]\n if \"2026-04-01\" not in ir_file.name or \"contamination\" not in ir_file.name.lower():\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"IR file found as '{ir_file.name}' but expected name to contain '2026-04-01' and 'Contamination'\"}\n if ir_file.suffix.lower() != \".pdf\":\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"IR file '{ir_file.name}' is not a PDF\"}\n # Check PDF content\n try:\n import pdfplumber\n with pdfplumber.open(ir_file) as pdf:\n text = \"\"\n for page in pdf.pages:\n text += (page.extract_text() or \"\")\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Could not read PDF '{ir_file.name}': {ex}\"}\n text_lower = text.lower()\n checks = {\n \"Incident Date 2026-04-01\": \"2026-04-01\" in text,\n \"Incident Category Contamination\": \"contamination\" in text_lower,\n \"Priority Level P1\": bool(re.search(r'p1|priority.*1|1.*priority', text_lower))\n }\n all_ok = all(checks.values())\n if all_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"IR file '{ir_file.name}' exists with correct content: date 2026-04-01, Contamination, P1\"}\n failed = [k for k, v in checks.items() if not v]\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"IR file '{ir_file.name}' exists but missing content: {failed}. Extracted text snippet: {text[:500]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "7b25afa2-8e11-49bd-96d2-d9ea3dcdd1c9",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email to carlos.mendoza@prairiestar.com with subject 'IR-2026-04-01-Contamination.pdf for review' or 'IR-2026-04-01-Contamination for review', a blank body, and `IR-2026-04-01-Contamination.pdf` attached",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n subject = (e.get(\"subject\", \"\") or \"\")\n attachments = e.get(\"attachments\", []) or []\n if \"carlos.mendoza@prairiestar.com\" not in to_addr:\n continue\n # Check subject\n subj_ok = bool(re.search(r'IR-2026-04-01-Contamination(\\.pdf)?\\s+for\\s+review', subject, re.IGNORECASE))\n if not subj_ok:\n continue\n # Check attachment\n attach_ok = False\n for att in attachments:\n att_name = \"\"\n if isinstance(att, dict):\n att_name = att.get(\"filename\", \"\") or att.get(\"name\", \"\") or \"\"\n elif isinstance(att, str):\n att_name = att\n if \"IR-2026-04-01-Contamination\" in att_name:\n attach_ok = True\n break\n # Check body is blank/empty\n body = (e.get(\"body_text\", \"\") or \"\").strip()\n body_ok = len(body) == 0\n if subj_ok and attach_ok and body_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found email to carlos.mendoza with subject '{subject}', blank body, and IR attachment\"}\n issues = []\n if not attach_ok:\n issues.append(f\"Missing IR-2026-04-01-Contamination.pdf attachment (found: {attachments})\")\n if not body_ok:\n issues.append(f\"Body not blank: '{body}'\")\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found IR review email to carlos.mendoza but: {'; '.join(issues)}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email to carlos.mendoza@prairiestar.com with subject matching 'IR-2026-04-01-Contamination for review' found\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a4be86d2-af0c-4e30-881b-e6f259724fa8",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `mailbox.json`, there must be no reply to or forward of the email with subject 'Upcoming Server Update' from hannah.brooks@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find the original server update email's message_id\n original_msg_id = None\n for e in emails:\n subject = e.get(\"subject\", \"\") or \"\"\n from_addr = e.get(\"from_addr\", \"\") or \"\"\n if \"upcoming server update\" in subject.lower() and \"hannah.brooks@prairiestar.com\" in from_addr.lower():\n original_msg_id = e.get(\"message_id\", \"\")\n break\n # Check no sent email replies to or forwards this email\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n in_reply_to = (e.get(\"in_reply_to\", \"\") or \"\").lower()\n # Check if it's a reply by in_reply_to field matching original message_id\n if original_msg_id and in_reply_to and original_msg_id.lower() in in_reply_to:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a reply/forward to the 'Upcoming Server Update' email (via in_reply_to): '{e.get('subject', '')}'\"}\n # Check if subject looks like a reply or forward referencing server update\n if \"server update\" in subject:\n if \"re:\" in subject or \"fw:\" in subject or \"fwd:\" in subject:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found what appears to be a reply/forward to 'Upcoming Server Update' email: '{e.get('subject', '')}'\"}\n # Also catch cases where in_reply_to is set even without re:/fw: prefix\n if e.get(\"in_reply_to\"):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found what appears to be a reply to 'Upcoming Server Update' email (has in_reply_to set): '{e.get('subject', '')}'\"}\n # Also check if subject contains 'upcoming server update' with forward prefix\n if \"upcoming server update\" in subject:\n if \"fw:\" in subject or \"fwd:\" in subject or \"re:\" in subject:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a reply/forward referencing 'Upcoming Server Update': '{e.get('subject', '')}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No reply to or forward of 'Upcoming Server Update' email from hannah.brooks@prairiestar.com found in Sent folder\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "500679be-78dc-4566-9cda-7b53113a5eb2",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "There must be exactly one file in the workspace with any variation of the name `inventory_master.xlsx`",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find all files matching inventory_master pattern\n matches = list(workspace.glob(\"*inventory*master*\")) + list(workspace.glob(\"*inventory_master*\"))\n # Deduplicate\n matches = list(set(matches))\n # Filter to files only\n matches = [f for f in matches if f.is_file()]\n if len(matches) == 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Exactly one inventory_master file found: '{matches[0].name}'\"}\n elif len(matches) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No inventory_master file found in workspace\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Multiple inventory_master files found: {[f.name for f in matches]}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "32091992-2b49-4d22-a54a-885049f48f16",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `mailbox.json`, there must be no sent email to kevin.white@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n cc_addr = (e.get(\"cc_addr\", \"\") or \"\").lower()\n if \"kevin.white@prairiestar.com\" in to_addr or \"kevin.white@prairiestar.com\" in cc_addr:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent email to/cc kevin.white@prairiestar.com: subject='{e.get('subject', '')}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent email to kevin.white@prairiestar.com found\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "c023e4a4-31a4-4c1b-8c0c-079527ad3047",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `temperature_log.xlsx`, cell D2 must be '-28 C'",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n files = list(Path(workspace_path).glob(\"*temperature_log*\"))\n if not files:\n files = list(Path(workspace_path).glob(\"*temp*log*\")) + list(Path(workspace_path).glob(\"*log*temp*\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No temperature_log file found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n d2 = ws[\"D2\"].value\n d2_str = str(d2).strip() if d2 is not None else \"\"\n # Accept -28 C, -28C, -28°C etc.\n import re\n normalized = re.sub(r'\\s+', ' ', d2_str).strip()\n if re.match(r'^-28\\s*[°]?\\s*[Cc]$', normalized):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D2 = '{d2_str}' which matches -28 C\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"D2 expected '-28 C' but got '{d2_str}'\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "22121fad-e961-4671-886b-692a4cdc1688",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `mailbox.json`, there must be a total of exactly 4 emails in the Sent folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n count = len(sent_emails)\n if count == 4:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 4 sent emails in mailbox.json\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 4 sent emails but found {count}. Subjects: {[e.get('subject','') for e in sent_emails]}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "8bbdb09a-77eb-44dc-bcb6-34e0b6f0c2d2",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `slack_data.json`, there must be a total of exactly 11 messages across all channels",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n total = 0\n channel_counts = {}\n for channel_id, msgs in messages.items():\n count = len(msgs)\n total += count\n # Get channel name for feedback\n channels = data.get(\"channels\", {})\n ch_name = channels.get(channel_id, {}).get(\"name\", channel_id)\n channel_counts[ch_name] = count\n if total == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 11 messages total across channels: {channel_counts}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 11 total messages but found {total}. Per channel: {channel_counts}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Assume it's 8:30am CT on 1st April, 2026. Please read through unread emails and action each accordingly, as detailed to the SOP. I'd like you to go ahead with all SOP-prescribed actions right away, regardless SOP timelines like "within the next fifteen minutes" or within "4 hours".
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user