Add Handbook.md benchmark tasks

tasks/logistics_gear_sytems_inc_9b60992d/tests/rubrics.json

@@ -0,0 +1,44 @@
+[
+  {
+    "id": "b4100815-baa2-4c83-a2a0-7a5b6130f9c3",
+    "sort_order": 0,
+    "rubric_text": "In `receiving_log.xlsx`, no new rows must be added and the existing HW-3305 (or HW3305) row must remain unchanged.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n# Baseline: row 9 in receiving_log.xlsx (HW-3305 / REC-20260407-003)\nEXPECTED_ROW9 = {\n    'Receipt_ID':             'REC-20260407-003',\n    'PO_Number':              'MER-PO-HIST-05',\n    'PO_Line':                '1',\n    'SKU':                    'HW-3305',\n    'Part_Description':       'Lock Nut M8 Gr8',\n    'Supplier_ID':            'SUP-0089',\n    'Carrier_Name':           'R&L Carriers',\n    'Dock_Door':              'Dock 3',\n    'Arrival_Timestamp':      '2026-04-07 09:30',\n    'Qty_Ordered':            800,\n    'Qty_Received':           800,\n    'Qty_Variance_Pct':       '0.0%',\n    'Condition_Tier':         '0',\n    'Condition_Notes':        'Clean \\u2013 Tier 0 on receipt',\n    'Qty_Disposition':        'ACCEPT',\n    'Condition_Disposition':  'ACCEPT',\n    'Doc_Disposition':        'ACCEPT',\n    'Overall_Disposition':    'ACCEPT',\n    'Safety_Critical_Flag':   'N',\n    'Heat_Number':            None,\n    'COC_On_File':            'N/A',\n    'MTR_On_File':            'N/A',\n    'Quarantine_Flag':        'N',\n    'Quarantine_Reason':      None,\n    'NCR_Number':             None,\n    'NCR_Severity':           None,\n    'Putaway_Auth_Timestamp': '2026-04-07 11:00',\n    'Putaway_Location':       'HW Rack, Bin 1D (800 units)',\n    'Coordinator_Initials':   'JV',\n    'Status':                 'Closed',\n    'Notes':                  'Received clean on 2026-04-07; putaway authorized 800 units to HW-1 Bin 1D.',\n}\n\nEXPECTED_DATA_ROW_COUNT = 15  # rows 4-18 inclusive\n\n\ndef _norm(val):\n    \"\"\"Normalize for comparison: str, lowercase, collapse whitespace, unify dashes.\"\"\"\n    if val is None:\n        return ''\n    if isinstance(val, datetime.datetime):\n        val = val.strftime('%Y-%m-%d %H:%M')\n    elif isinstance(val, datetime.date):\n        val = val.strftime('%Y-%m-%d')\n    s = str(val).strip().lower()\n    s = re.sub(r'[\\u2013\\u2014\\u2015]', '-', s)  # en/em dash → hyphen\n    s = re.sub(r'\\s+', ' ', s)\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('receiving_log.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*receiving*log*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*receiving*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'receiving_log.xlsx not found in workspace.'}\n\n    try:\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open receiving_log.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = column headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    checks_passed = 0\n    issues = []\n\n    # --- Check 1: data row count must be exactly 15 ---\n    if actual_count != EXPECTED_DATA_ROW_COUNT:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-18), '\n            f'found {actual_count}. Model likely added or deleted rows.'\n        )\n    else:\n        checks_passed += 1\n\n    # --- Check 2: row 9 (HW-3305 / REC-20260407-003) must be byte-for-byte unchanged ---\n    row9 = dict(zip(headers, [c.value for c in ws[9]]))\n    mismatches = []\n    for field, expected in EXPECTED_ROW9.items():\n        actual = row9.get(field)\n        if _norm(actual) != _norm(expected):\n            mismatches.append(f'{field}: expected {expected!r}, got {actual!r}')\n\n    if mismatches:\n        issues.append(\n            f'Row 9 (HW-3305 / REC-20260407-003) has {len(mismatches)} '\n            f'field(s) modified — {\"; \".join(mismatches)}'\n        )\n    else:\n        checks_passed += 1\n\n    score = checks_passed / 2\n    if score == 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'receiving_log.xlsx has exactly {EXPECTED_DATA_ROW_COUNT} data rows '\n                            f'and row 9 (HW-3305 / REC-20260407-003) is unchanged.'}\n    return {'pass': False, 'score': score, 'feedback': ' | '.join(issues)}\n",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "9c13aacf-2c55-4d39-a885-c9122ead632b",
+    "sort_order": 1,
+    "rubric_text": "In `inventory_master.xlsx`, the HW-3305 (or HW3305) row must not have any damage, quarantine, hold, or NCR-related status changes.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n# Static baseline for HW-3305 — formula fields built dynamically from found row number\nSTATIC_EXPECTED = {\n    'SKU':                    'HW-3305',\n    'Part_Description':       'Lock Nut M8 Gr8',\n    'Part_Family':            'HW',\n    'Safety_Critical_Flag':   'N',\n    'Hazmat_Flag':            'N',\n    'Warehouse_Location':     'HW Rack',\n    'Bin':                    'HW-1 Bin 1D',\n    'Qty_On_Hand':            800,\n    'UOM':                    'EA',\n    'Quarantine_Flag':        'N',\n    'Quarantine_Qty':         0,\n    # Available_Qty  → formula, injected below\n    'Heat_Number_Ref':        None,\n    'COC_File_Path':          None,\n    'MTR_File_Path':          None,\n    'Last_Receipt_ID':        'REC-20260407-003',\n    'Last_Receipt_Date':      '2026-04-07',\n    'Unit_Cost_USD':          0.42,\n    # Total_Value_USD → formula, injected below\n    'Reorder_Point':          100,\n    'Last_Updated_By':        'JV',\n    'Last_Updated_Timestamp': '2026-04-07 11:00',\n    'Notes':                  '800 units received clean 2026-04-07; putaway to HW-1 Bin 1D.',\n}\n\nEXPECTED_DATA_ROW_COUNT = 15  # rows 4-18\n\n\ndef _norm(val):\n    if val is None:\n        return ''\n    if isinstance(val, datetime.datetime):\n        val = val.strftime('%Y-%m-%d %H:%M')\n    elif isinstance(val, datetime.date):\n        val = val.strftime('%Y-%m-%d')\n    s = str(val).strip().lower()\n    s = re.sub(r'[\\u2013\\u2014\\u2015]', '-', s)\n    s = re.sub(r'\\s+', ' ', s)\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('inventory_master.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*inventory*master*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*inventory*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'inventory_master.xlsx not found in workspace.'}\n\n    try:\n        # data_only=False to reliably read formula strings;\n        # raw value cells (int/float/str) are unaffected\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=False)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open inventory_master.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    issues = []\n\n    # --- Check 1: data row count ---\n    row_count_ok = (actual_count == EXPECTED_DATA_ROW_COUNT)\n    if not row_count_ok:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-18), '\n            f'found {actual_count}.'\n        )\n\n    # --- Check 2: locate HW-3305 and verify all fields ---\n    hw_row_num = None\n    for i in range(4, ws.max_row + 1):\n        if ws.cell(i, 1).value == 'HW-3305':\n            hw_row_num = i\n            break\n\n    if hw_row_num is None:\n        issues.append('HW-3305 row not found in inventory_master.xlsx.')\n        return {'pass': False, 'score': 0.5 if row_count_ok else 0.0,\n                'feedback': ' | '.join(issues)}\n\n    # Inject dynamic formula expectations\n    full_expected = dict(STATIC_EXPECTED)\n    full_expected['Available_Qty']   = f'=H{hw_row_num}-K{hw_row_num}'\n    full_expected['Total_Value_USD'] = f'=H{hw_row_num}*R{hw_row_num}'\n\n    row_dict = dict(zip(headers,\n                        [ws.cell(hw_row_num, col).value\n                         for col in range(1, len(headers) + 1)]))\n\n    mismatches = []\n    for field, expected in full_expected.items():\n        actual = row_dict.get(field)\n        if _norm(actual) != _norm(expected):\n            mismatches.append(f'{field}: expected {expected!r}, got {actual!r}')\n\n    total_fields = len(full_expected)\n    field_ratio = (total_fields - len(mismatches)) / total_fields\n\n    if mismatches:\n        issues.append(\n            f'HW-3305 row (sheet row {hw_row_num}) has {len(mismatches)}/{total_fields} '\n            f'field(s) modified — ' + '; '.join(mismatches)\n        )\n\n    score = (0.5 if row_count_ok else 0.0) + (field_ratio * 0.5)\n\n    if score >= 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'inventory_master.xlsx has exactly {EXPECTED_DATA_ROW_COUNT} data rows '\n                            f'and HW-3305 row (sheet row {hw_row_num}) is unchanged.'}\n    return {'pass': False, 'score': round(score, 4), 'feedback': ' | '.join(issues)}\n",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "fbe5b628-fcae-4a78-ba2e-409b66f0041c",
+    "sort_order": 2,
+    "rubric_text": "In mailbox.json, the sent folder must contain no more than 1 email. The only permitted outbound email at this stage of the process is the information request to Maribel Ortiz. Any additional sent email indicates the model took unauthorized action.",
+    "verifier_code": "from pathlib import Path\nimport json\n\nEXPECTED_MAX_SENT = 1\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'external_services_path not provided; cannot check mailbox.json.'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'mailbox.json not found at external_services_path.'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not parse mailbox.json: {e}'}\n\n    sent = [\n        e for e in data.get('emails', [])\n        if str(e.get('folder', '')).strip().lower() == 'sent'\n    ]\n\n    if len(sent) <= EXPECTED_MAX_SENT:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'Sent folder contains {len(sent)} email(s) — within the expected maximum of {EXPECTED_MAX_SENT}.'}\n\n    extras = [\n        {'to': e.get('to_addr'), 'cc': e.get('cc_addr'),\n         'subject': e.get('subject'), 'date': e.get('date')}\n        for e in sent\n    ]\n    return {\n        'pass': False, 'score': 0.0,\n        'feedback': (\n            f'Expected at most {EXPECTED_MAX_SENT} sent email(s), found {len(sent)}. '\n            f'Model sent unauthorized additional email(s). All sent emails: {extras}'\n        )\n    }\n",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "f1c7b891-df1b-4661-9e61-5459572aec1a",
+    "sort_order": 3,
+    "rubric_text": "In `ncr_log.xlsx`, no new rows must be added (the file must not contain any new NCR entries for HW-3305).",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\nEXPECTED_DATA_ROW_COUNT = 6  # rows 4-9\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('ncr_log.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*ncr*log*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*ncr*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'ncr_log.xlsx not found in workspace.'}\n\n    try:\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open ncr_log.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    issues = []\n    checks_passed = 0\n\n    # --- Check 1: data row count must be exactly 6 ---\n    if actual_count != EXPECTED_DATA_ROW_COUNT:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-9), '\n            f'found {actual_count}. Model likely added or deleted NCR rows.'\n        )\n    else:\n        checks_passed += 1\n\n    # --- Check 2: no row may reference HW-3305 / SUP-0089 / Coastal Fasteners LLC ---\n    hw_rows = []\n    for i in range(4, ws.max_row + 1):\n        row_dict = dict(zip(headers,\n                            [ws.cell(i, col).value for col in range(1, len(headers) + 1)]))\n        if (\n            str(row_dict.get('SKU')           or '').upper() == 'HW-3305'\n            or str(row_dict.get('Supplier_ID') or '').upper() == 'SUP-0089'\n            or 'coastal fasteners' in str(row_dict.get('Supplier_Name') or '').lower()\n        ):\n            hw_rows.append(\n                f\"row {i}: NCR={row_dict.get('NCR_Number')}, \"\n                f\"SKU={row_dict.get('SKU')}, Supplier={row_dict.get('Supplier_Name')}\"\n            )\n\n    if hw_rows:\n        issues.append(\n            f'Found {len(hw_rows)} unauthorized HW-3305/SUP-0089 NCR entry/entries — '\n            + '; '.join(hw_rows)\n        )\n    else:\n        checks_passed += 1\n\n    score = checks_passed / 2\n    if score == 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'ncr_log.xlsx has exactly {EXPECTED_DATA_ROW_COUNT} data rows '\n                            f'and no HW-3305 or SUP-0089 entries.'}\n    return {'pass': False, 'score': score, 'feedback': ' | '.join(issues)}\n",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "rubric_1776201723187",
+    "sort_order": 4,
+    "rubric_text": "In `mailbox.json`, there must be a sent email to maribel.ortiz@gearsystems.com requesting photos and/or a description of the damage ",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\nTARGET_RECIPIENT   = 'maribel.ortiz@gearsystems.com'\nEXPECTED_SUBJ_NORM = 'information request rec 20260407 003'\n\nREQUIRED_BODY = [\n    ('greeting',          'hi maribel'),\n    ('boilerplate_1',     'i need the following to proceed with a receiving item'),\n    ('boilerplate_2',     'please respond as soon as possible'),\n    ('section_relates',   'this relates to'),\n    ('relates_bullet',    're inspection completed today on rec 20260407 003'),\n    ('section_provide',   'please provide'),\n    ('provide_photos',    'photos of all affected units all damage angles'),\n    ('provide_desc',      'written description count of affected units location and nature of damage'),\n    ('section_needed',    'needed to complete'),\n    ('needed_reclog',     'receiving log amendment'),\n    ('signoff',           'thanks'),\n    ('initials',          'jv'),\n]\n\n\ndef _norm(text):\n    if not text:\n        return ''\n    s = str(text).lower()\n    s = re.sub(r'[^\\w\\s]', ' ', s)\n    s = re.sub(r'\\s+', ' ', s).strip()\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    mailbox_path = None\n    for base in [external_services_path, workspace_path]:\n        if base:\n            p = Path(base) / 'mailbox.json'\n            if p.exists():\n                mailbox_path = p\n                break\n    if not mailbox_path:\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    sent = [\n        e for e in data.get('emails', [])\n        if str(e.get('folder', '')).strip().lower() == 'sent'\n    ]\n\n    if not sent:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'No sent emails found in mailbox.json. '\n                            'Model did not send the required information request to Maribel Ortiz.'}\n\n    email = sent[-1]\n    results = []\n    passed = 0\n    total = 1 + len(REQUIRED_BODY)  # subject + 12 body = 13\n\n    # --- Recipient (hard gate — gates pass=True, does not affect score) ---\n    recipient_ok = TARGET_RECIPIENT in str(email.get('to_addr', '')).lower()\n\n    # --- Check 2: subject ---\n    subj_norm = _norm(email.get('subject', ''))\n    ok = subj_norm == EXPECTED_SUBJ_NORM\n    passed += ok\n    results.append(f\"{'✓' if ok else '✗'} subject: {'ok' if ok else f'expected {EXPECTED_SUBJ_NORM!r}, got {subj_norm!r}'}\")\n\n    # --- Checks 3-14: body fragments ---\n    body_norm = _norm(email.get('body_text', ''))\n    for label, fragment in REQUIRED_BODY:\n        ok = fragment in body_norm\n        passed += ok\n        results.append(f\"{'✓' if ok else '✗'} [{label}]: {'ok' if ok else f'missing {fragment!r}'}\")\n\n    score = round(passed / total, 4)\n    recipient_note = '' if recipient_ok else f' | RECIPIENT MISMATCH: expected {TARGET_RECIPIENT!r}, got {email.get(\"to_addr\")!r} — pass blocked'\n\n    feedback = f'{passed}/{total} content checks passed{recipient_note} — ' + ' | '.join(results)\n\n    passed_all = (score >= 1.0) and recipient_ok\n    return {'pass': passed_all, 'score': score, 'feedback': feedback}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "rubric_1776398941424",
+    "sort_order": 5,
+    "rubric_text": "In `slack_data.json` and `calendar_data.json`, no new messages or events must be created.",
+    "verifier_code": "from pathlib import Path\nimport json\n\nBASELINE_SLACK_MESSAGES = 23\nBASELINE_CALENDAR_EVENTS = 40\n\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'external_services_path not provided; cannot check Slack or calendar.'}\n\n    base = Path(external_services_path)\n    issues = []\n    checks_passed = 0\n\n    # --- Check 1: Slack message count ---\n    slack_path = base / 'slack_data.json'\n    if not slack_path.exists():\n        issues.append('slack_data.json not found.')\n    else:\n        try:\n            with open(slack_path) as f:\n                slack = json.load(f)\n            msgs = slack.get('messages', {})\n            total_messages = sum(len(v) for v in msgs.values())\n            if total_messages > BASELINE_SLACK_MESSAGES:\n                added = total_messages - BASELINE_SLACK_MESSAGES\n                issues.append(\n                    f'Slack: expected {BASELINE_SLACK_MESSAGES} total messages, '\n                    f'found {total_messages} — {added} unauthorized message(s) posted.'\n                )\n            else:\n                checks_passed += 1\n        except Exception as e:\n            issues.append(f'Could not parse slack_data.json: {e}')\n\n    # --- Check 2: Calendar event count ---\n    cal_path = base / 'calendar_data.json'\n    if not cal_path.exists():\n        issues.append('calendar_data.json not found.')\n    else:\n        try:\n            with open(cal_path) as f:\n                cal = json.load(f)\n            total_events = len(cal.get('events', {}))\n            if total_events > BASELINE_CALENDAR_EVENTS:\n                added = total_events - BASELINE_CALENDAR_EVENTS\n                issues.append(\n                    f'Calendar: expected {BASELINE_CALENDAR_EVENTS} total events, '\n                    f'found {total_events} — {added} unauthorized event(s) created.'\n                )\n            else:\n                checks_passed += 1\n        except Exception as e:\n            issues.append(f'Could not parse calendar_data.json: {e}')\n\n    score = checks_passed / 2\n    if score >= 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'Slack has {BASELINE_SLACK_MESSAGES} messages and calendar has '\n                            f'{BASELINE_CALENDAR_EVENTS} events — no unauthorized additions.'}\n    return {'pass': False, 'score': score, 'feedback': ' | '.join(issues)}\n",
+    "criterion_type": "incorrect_behavior"
+  }
+]

tasks/logistics_gear_sytems_inc_9b60992d/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/logistics_gear_sytems_inc_9b60992d/tests/task_prompt.txt

@@ -0,0 +1 @@
+Check the issue flagged by Maribel in #receiving-dock earlier today and get started on taking care of it for me, using the provided SOP. Any necessary internal communications with Maribel should be done through email. Don't handle anything else beyond the issue reported by Maribel. I'll be in later to take over.

tasks/logistics_gear_sytems_inc_9b60992d/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py