Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,7 @@
Hi, this is Jennifer Alexander and I could use some help scheduling some exit interviews (as per section 12.5 of our SOP, attached as Crestwood_University_HR_SOP.pdf). An intern was working on it earlier, I think, and I'm trying to make sure we haven't missed anyone.
I've given you access to some emails with some context, as well as the Jira tickets if you need them, as well as Nicole's Calendar.
All I need you to do is schedule the meetings into Nicole's calendar for everyone who (according to Jira and the SOP) needs to have one. The emails will help here. Nicole sent me some notes yesterday that you'll need to account for (she wrote the SOP, you'll note, so if there's a conflict anywhere go with her email), as well as those the workers gave in their replies. You'll also likely need to refer to employee_roster.xlsx. Anything I've given you is a valid source of info/context: make sure you don't miss any details.
I think something might have gotten lost in the shuffle with setting everything up, so don't schedule anything today or tomorrow, just so the workers have time to react. Thanks!
@@ -0,0 +1 @@
You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/hr_crestwood_university_d93daa01"
description = "SOP task from CSV row d93daa01"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,58 @@
[
{
"id": "7e23756f-b4ae-4663-a74d-790bd570af74",
"sort_order": 0,
"rubric_text": "In `calendar_data.json` (external services), there must be exactly 286 events in the events object.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found at external_services_path\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n events = data.get(\"events\", {})\n count = len(events)\n if count == 286:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"calendar_data.json contains exactly 286 events as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 286 events in calendar_data.json, but found {count}.\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "03437a49-686c-493e-8f2a-4af41397970e",
"sort_order": 1,
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – James Smith' (or close variant) starting at 3:30 PM and ending at 4:30 PM on February 12th, 2026.",
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*james\\s+smith', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - James Smith' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 12\n and start_dt.hour == 15 and start_dt.minute == 30\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 12\n and end_dt.hour == 16 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - James Smith' event on 2026-02-12 15:30-16:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - James Smith' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-12 15:30-16:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse start/end times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
"criterion_type": "expected_output"
},
{
"id": "96456488-26d3-46ed-adda-a47ce5e4da2c",
"sort_order": 2,
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – Teresa Romero' (or close variant) starting at 9:00 AM and ending at 9:30 AM on February 13th, 2026.",
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*teresa\\s+romero', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - Teresa Romero' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 13\n and start_dt.hour == 9 and start_dt.minute == 0\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 13\n and end_dt.hour == 9 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - Teresa Romero' event on 2026-02-13 09:00-09:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - Teresa Romero' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-13 09:00-09:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
"criterion_type": "expected_output"
},
{
"id": "b5f413ed-89f3-4cb6-af6a-32234779af2e",
"sort_order": 3,
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – Gabriel Tran' (or close variant) starting at 2:00 PM and ending at 2:30 PM on February 20th, 2026.",
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*gabriel\\s+tran', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - Gabriel Tran' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 20\n and start_dt.hour == 14 and start_dt.minute == 0\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 20\n and end_dt.hour == 14 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - Gabriel Tran' event on 2026-02-20 14:00-14:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - Gabriel Tran' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-20 14:00-14:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
"criterion_type": "expected_output"
},
{
"id": "8501d551-7380-4a22-b33a-59b96f322492",
"sort_order": 4,
"rubric_text": "In `employee_roster.xlsx` (workspace), rows with IDs E-1052, E-1064, and E-1158 in column A must have the correct values across columns A through O as specified.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n roster_files = [f for f in xlsx_files if \"roster\" in f.name.lower() or \"employee\" in f.name.lower()]\n if not roster_files:\n roster_files = xlsx_files\n if not roster_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n # Expected rows: key=ID, value=list of expected values for cols A-O (0-indexed 0-14)\n # Blanks are represented as None\n expected_rows = {\n \"E-1052\": [\"E-1052\", \"Teresa\", \"Romero\", \"t.romero@crestwood.edu\", \"Library Services\", \"Main\", \"B. Mendoza\", \"10/24/2013\", \"Non-Exempt\", \"Full-Time\", 1369, \"Active\", None, None, \"Remote work agreement on file — WFH Tuesdays and Thursdays.\"],\n \"E-1064\": [\"E-1064\", \"James\", \"Smith\", \"j.smith@crestwood.edu\", \"Student Affairs\", \"Main\", \"B. Fernandez\", \"04/29/2025\", \"Non-Exempt\", \"Full-Time\", 1908, \"Active\", None, None, None],\n \"E-1158\": [\"E-1158\", \"Gabriel\", \"Tran\", \"g.tran@crestwood.edu\", \"Student Affairs\", \"Main\", \"E. Baker\", \"01/13/2008\", \"Non-Exempt\", \"Full-Time\", 1961, \"Active\", None, None, \"Flex schedule approved Mon/Wed WFH.\"],\n }\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def date_match(actual, expected_str):\n # Accept various date formats - just check month/day/year numbers\n import re\n nums_actual = re.findall(r'\\d+', str(actual))\n nums_expected = re.findall(r'\\d+', expected_str)\n if len(nums_actual) >= 3 and len(nums_expected) >= 3:\n # expected is MM/DD/YYYY\n exp_month, exp_day, exp_year = nums_expected[0], nums_expected[1], nums_expected[2]\n act_flat = ''.join(nums_actual)\n return exp_month in act_flat and exp_day in act_flat and exp_year in act_flat\n return normalize(actual) == expected_str\n\n results = {}\n for xlsx_file in roster_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception as e:\n continue\n for ws in wb.worksheets:\n for row in ws.iter_rows():\n if not row:\n continue\n cell_a = normalize(row[0].value)\n if cell_a in expected_rows:\n exp = expected_rows[cell_a]\n row_vals = [cell.value for cell in row[:15]]\n mismatches = []\n for i, (actual_val, exp_val) in enumerate(zip(row_vals, exp)):\n col_letter = chr(ord('A') + i)\n if exp_val is None:\n # Blank expected\n if actual_val is not None and str(actual_val).strip() != \"\":\n mismatches.append(f\"Col {col_letter}: expected blank, got '{actual_val}'\")\n elif i == 7: # Date column\n if not date_match(actual_val, exp_val):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n elif i == 10: # Numeric (ext)\n try:\n if abs(float(str(actual_val).strip()) - float(str(exp_val))) > 1:\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n except:\n if normalize(actual_val) != str(exp_val):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n elif i == 14: # Notes - check key content loosely\n if exp_val and normalize(actual_val).lower() != normalize(exp_val).lower():\n # Allow partial/fuzzy match for notes\n if not (normalize(exp_val).lower()[:20] in normalize(actual_val).lower()):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n else:\n if normalize(actual_val).lower() != normalize(exp_val).lower():\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n results[cell_a] = mismatches\n\n missing = [eid for eid in expected_rows if eid not in results]\n if missing:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could noLine truncated
"criterion_type": "incorrect_behavior"
},
{
"id": "787cf3b0-251c-4627-a730-3d68eb3a234b",
"sort_order": 5,
"rubric_text": "In `jira_state.json` (external services), the issues 'Offboarding — Smith, James (E-1064)', 'Offboarding — Tran, Gabriel (E-1158)', and 'Offboarding — Romero, Teresa (E-1052)' must each have exactly 2 comments (the initial fixture count; no new comments added).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n if isinstance(data, list):\n issues = data\n elif isinstance(data, dict):\n raw = data.get(\"issues\", data)\n if isinstance(raw, dict):\n issues = list(raw.values())\n elif isinstance(raw, list):\n issues = raw\n else:\n issues = []\n else:\n issues = []\n\n def _summary(issue):\n if not isinstance(issue, dict):\n return \"\"\n fields = issue.get(\"fields\", {}) if isinstance(issue.get(\"fields\"), dict) else {}\n return fields.get(\"summary\") or issue.get(\"summary\") or issue.get(\"title\") or issue.get(\"name\") or \"\"\n\n def _comment_count(issue):\n if not isinstance(issue, dict):\n return 0\n fields = issue.get(\"fields\", {}) if isinstance(issue.get(\"fields\"), dict) else {}\n comment = fields.get(\"comment\")\n if isinstance(comment, dict):\n inner = comment.get(\"comments\", [])\n if isinstance(inner, list):\n return len(inner)\n if isinstance(comment, list):\n return len(comment)\n top = issue.get(\"comments\")\n if isinstance(top, list):\n return len(top)\n if isinstance(top, dict):\n inner = top.get(\"comments\", top)\n if isinstance(inner, list):\n return len(inner)\n return 0\n\n # Three issues as specified in the rubric\n targets = [\n (\"Offboarding \\u2014 Smith, James (E-1064)\", re.compile(r'smith.*james.*e-?1064', re.IGNORECASE), 2),\n (\"Offboarding \\u2014 Tran, Gabriel (E-1158)\", re.compile(r'tran.*gabriel.*e-?1158', re.IGNORECASE), 2),\n (\"Offboarding \\u2014 Romero, Teresa (E-1052)\", re.compile(r'romero.*teresa.*e-?1052', re.IGNORECASE), 2),\n ]\n\n found = {label: None for label, _, _ in targets}\n for issue in issues:\n summary = _summary(issue)\n if not summary:\n continue\n for label, pat, _ in targets:\n if found[label] is None and pat.search(summary):\n found[label] = issue\n\n missing = [label for label, iss in found.items() if iss is None]\n if missing:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find Jira issues: {missing}\"}\n\n failures = []\n for label, _, expected in targets:\n actual = _comment_count(found[label])\n if actual != expected:\n failures.append(f\"Issue '{label}' has {actual} comments (expected {expected})\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(failures)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All three issues ('Offboarding \\u2014 Smith, James (E-1064)', 'Offboarding \\u2014 Tran, Gabriel (E-1158)', and 'Offboarding \\u2014 Romero, Teresa (E-1052)') each have exactly 2 comments \\u2014 no new comments were added.\"}",
"criterion_type": "incorrect_behavior"
},
{
"id": "d034660f-c02f-4fc9-b048-904ebc881d05",
"sort_order": 6,
"rubric_text": "In `mailbox.json` (external services), there must be exactly 7 emails in the emails array.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n count = len(emails)\n if count == 7:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json contains exactly 7 emails as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 7 emails in mailbox.json, but found {count}.\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "576a38a2-aee1-4c3a-8ae7-ecdd26493cbc",
"sort_order": 7,
"rubric_text": "In `jira_state.json` (external services), there must be exactly 17 issues.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n if isinstance(data, list):\n issues = data\n elif isinstance(data, dict):\n if \"issues\" in data:\n issues = data[\"issues\"]\n else:\n issues = list(data.values())\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Unexpected jira_state.json format\"}\n\n count = len(issues)\n if count == 17:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"jira_state.json contains exactly 17 issues as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 17 issues in jira_state.json, but found {count}.\"}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,7 @@
Hi, this is Jennifer Alexander and I could use some help scheduling some exit interviews (as per section 12.5 of our SOP, attached as Crestwood_University_HR_SOP.pdf). An intern was working on it earlier, I think, and I'm trying to make sure we haven't missed anyone.
I've given you access to some emails with some context, as well as the Jira tickets if you need them, as well as Nicole's Calendar.
All I need you to do is schedule the meetings into Nicole's calendar for everyone who (according to Jira and the SOP) needs to have one. The emails will help here. Nicole sent me some notes yesterday that you'll need to account for (she wrote the SOP, you'll note, so if there's a conflict anywhere go with her email), as well as those the workers gave in their replies. You'll also likely need to refer to employee_roster.xlsx. Anything I've given you is a valid source of info/context: make sure you don't miss any details.
I think something might have gotten lost in the shuffle with setting everything up, so don't schedule anything today or tomorrow, just so the workers have time to react. Thanks!
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py