Add Handbook.md benchmark tasks

tasks/medical_pathfinder_billing_and_coding_a25684b9/tests/rubrics.json

@@ -0,0 +1,44 @@
+[
+  {
+    "id": "8b2fb62f-2aca-45b9-bda8-fa26cc4f08c9",
+    "sort_order": 0,
+    "rubric_text": "In file `PBC_Claim_Register.xlsx`, row 2 must have Claim_Status equal to 'On Hold'.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n        if not matches:\n            return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n        wb = openpyxl.load_workbook(matches[0])\n        ws = wb.active\n        # Find Claim_Status column\n        header_row = [cell.value for cell in ws[1]]\n        try:\n            claim_status_col = header_row.index('Claim_Status') + 1\n        except ValueError:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Claim_Status column not found. Headers: {header_row}'}\n        row2_status = ws.cell(row=2, column=claim_status_col).value\n        if row2_status is None:\n            return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 Claim_Status is empty/None, expected \"On Hold\"'}\n        if str(row2_status).strip() == 'On Hold':\n            return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 Claim_Status is \"On Hold\" as expected'}\n        else:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Row 2 Claim_Status is \"{row2_status}\", expected \"On Hold\"'}\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "e93679bd-96c4-43a4-bd26-29973129e54e",
+    "sort_order": 1,
+    "rubric_text": "In file `PBC_Claim_Register.xlsx`, row 3 must have Claim_Status equal to 'Written Off', Billed_Amount equal to 285, Paid_Amount equal to 285, and Adjustment_Amount equal to 0.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n        if not matches:\n            return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n        wb = openpyxl.load_workbook(matches[0])\n        ws = wb.active\n        header_row = [cell.value for cell in ws[1]]\n        def get_col(name):\n            try:\n                return header_row.index(name) + 1\n            except ValueError:\n                return None\n        claim_status_col = get_col('Claim_Status')\n        billed_col = get_col('Billed_Amount')\n        paid_col = get_col('Paid_Amount')\n        adj_col = get_col('Adjustment_Amount')\n        missing = [n for n, c in [('Claim_Status', claim_status_col), ('Billed_Amount', billed_col), ('Paid_Amount', paid_col), ('Adjustment_Amount', adj_col)] if c is None]\n        if missing:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n        row3_status = ws.cell(row=3, column=claim_status_col).value\n        row3_billed = ws.cell(row=3, column=billed_col).value\n        row3_paid = ws.cell(row=3, column=paid_col).value\n        row3_adj = ws.cell(row=3, column=adj_col).value\n        errors = []\n        if str(row3_status).strip() != 'Written Off':\n            errors.append(f'Claim_Status is \"{row3_status}\", expected \"Written Off\"')\n        try:\n            if abs(float(str(row3_billed).replace(',','')) - 285) > 2.85:\n                errors.append(f'Billed_Amount is \"{row3_billed}\", expected 285')\n        except:\n            errors.append(f'Billed_Amount \"{row3_billed}\" is not numeric')\n        try:\n            if abs(float(str(row3_paid).replace(',','')) - 285) > 2.85:\n                errors.append(f'Paid_Amount is \"{row3_paid}\", expected 285')\n        except:\n            errors.append(f'Paid_Amount \"{row3_paid}\" is not numeric')\n        try:\n            if abs(float(str(row3_adj).replace(',','')) - 0) > 2.85:\n                errors.append(f'Adjustment_Amount is \"{row3_adj}\", expected 0')\n        except:\n            errors.append(f'Adjustment_Amount \"{row3_adj}\" is not numeric')\n        if errors:\n            return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n        return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Claim_Status=\"Written Off\", Billed_Amount={row3_billed}, Paid_Amount={row3_paid}, Adjustment_Amount={row3_adj} as expected'}\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "9df589fb-fede-40cb-8361-20e290f6f684",
+    "sort_order": 2,
+    "rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 2 must have Recovery_Action equal to 'Route to PFR' and Recovery_Status equal to 'In Progress'.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n        if not matches:\n            return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n        wb = openpyxl.load_workbook(matches[0])\n        ws = wb.active\n        header_row = [cell.value for cell in ws[1]]\n        def get_col(name):\n            try:\n                return header_row.index(name) + 1\n            except ValueError:\n                return None\n        action_col = get_col('Recovery_Action')\n        status_col = get_col('Recovery_Status')\n        missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n        if missing:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n        row2_action = ws.cell(row=2, column=action_col).value\n        row2_status = ws.cell(row=2, column=status_col).value\n        errors = []\n        if str(row2_action).strip() != 'Route to PFR':\n            errors.append(f'Recovery_Action is \"{row2_action}\", expected \"Route to PFR\"')\n        if str(row2_status).strip() != 'In Progress':\n            errors.append(f'Recovery_Status is \"{row2_status}\", expected \"In Progress\"')\n        if errors:\n            return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 issues: ' + '; '.join(errors)}\n        return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 has Recovery_Action=\"Route to PFR\" and Recovery_Status=\"In Progress\" as expected'}\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "98830c8d-31c2-4120-8294-2c4bd73b5000",
+    "sort_order": 3,
+    "rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 3 must have Recovery_Action equal to 'Write Off' and Recovery_Status equal to 'Written Off'.",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n        if not matches:\n            return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n        wb = openpyxl.load_workbook(matches[0])\n        ws = wb.active\n        header_row = [cell.value for cell in ws[1]]\n        def get_col(name):\n            try:\n                return header_row.index(name) + 1\n            except ValueError:\n                return None\n        action_col = get_col('Recovery_Action')\n        status_col = get_col('Recovery_Status')\n        missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n        if missing:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n        row3_action = ws.cell(row=3, column=action_col).value\n        row3_status = ws.cell(row=3, column=status_col).value\n        errors = []\n        if str(row3_action).strip() != 'Write Off':\n            errors.append(f'Recovery_Action is \"{row3_action}\", expected \"Write Off\"')\n        if str(row3_status).strip() != 'Written Off':\n            errors.append(f'Recovery_Status is \"{row3_status}\", expected \"Written Off\"')\n        if errors:\n            return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n        return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Recovery_Action=\"Write Off\" and Recovery_Status=\"Written Off\" as expected'}\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "fd59b3bf-8c01-42a6-beae-7c1ba3b035ae",
+    "sort_order": 4,
+    "rubric_text": "In `slack_data.json`, the #special-authorization channel must contain exactly one message with text 'Alex Rivera: CLM-2026-0062 special action authorized (or authorised)'",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        # Check both external_services_path and workspace_path for slack_data.json\n        slack_path = None\n        candidates = []\n        if external_services_path is not None:\n            p = Path(external_services_path) / 'slack_data.json'\n            if p.exists():\n                candidates.append(p)\n        wp = Path(workspace_path) / 'slack_data.json'\n        if wp.exists():\n            candidates.append(wp)\n        if not candidates:\n            return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in workspace or external_services_path'}\n        slack_path = candidates[0]\n        with open(slack_path, 'r') as f:\n            data = json.load(f)\n\n        # Find the #special-authorization channel\n        channels = data.get('channels', {})\n        special_auth_channel_id = None\n        for ch_id, ch_info in channels.items():\n            ch_name = ch_info.get('name', '')\n            if 'special-authorization' in ch_name.lower() or 'special_authorization' in ch_name.lower() or 'special-authorisation' in ch_name.lower() or 'special_authorisation' in ch_name.lower():\n                special_auth_channel_id = ch_id\n                break\n\n        if special_auth_channel_id is None:\n            return {'pass': False, 'score': 0.0, 'feedback': f'#special-authorization channel not found. Channels: {[v.get(\"name\") for v in channels.values()]}'}\n\n        messages = data.get('messages', {})\n        channel_msgs = messages.get(special_auth_channel_id, [])\n\n        # Accept both \"authorized\" and \"authorised\" spellings\n        # Pattern: alex rivera: clm-2026-0062 special action authorized/authorised\n        # Be loose: allow extra whitespace, punctuation at end, case insensitive\n        target_pattern = re.compile(\n            r'alex\\s+rivera\\s*[:\\-]\\s*clm[\\-\\s]2026[\\-\\s]0062\\s+special\\s+action\\s+authori[sz]ed',\n            re.IGNORECASE\n        )\n\n        matching = []\n        for msg in channel_msgs:\n            msg_text = str(msg.get('text', '')).strip()\n            if target_pattern.search(msg_text):\n                matching.append(msg_text)\n\n        if len(matching) == 0:\n            all_texts = [m.get('text', '') for m in channel_msgs]\n            return {'pass': False, 'score': 0.0, 'feedback': f'No message matching \"Alex Rivera: CLM-2026-0062 special action authorized/authorised\" found in #special-authorization. Messages found: {all_texts}'}\n\n        if len(matching) > 1:\n            return {'pass': False, 'score': 0.5, 'feedback': f'Expected exactly 1 matching message but found {len(matching)}: {matching}'}\n\n        return {'pass': True, 'score': 1.0, 'feedback': f'Found exactly 1 message in #special-authorization matching the required text: \"{matching[0]}\"'}\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {str(e)}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "rubric_1775144961819",
+    "sort_order": 5,
+    "rubric_text": "There must be no sent emails in the file `mailbox.json`.",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    # Look for mailbox.json in external_services_path first, then workspace_path\n    mailbox_path = None\n\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n\n    if mailbox_path is None:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n\n    if mailbox_path is None:\n        # No mailbox.json found at all — no sent emails can exist\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json file found. No sent emails present.\"}\n\n    try:\n        with open(mailbox_path, 'r', encoding='utf-8') as f:\n            data = json.load(f)\n    except json.JSONDecodeError as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n    # Extract the list of emails from various possible structures\n    emails = []\n    if isinstance(data, dict):\n        if \"emails\" in data:\n            emails = data[\"emails\"]\n        elif \"messages\" in data:\n            emails = data[\"messages\"]\n        else:\n            for key, value in data.items():\n                if isinstance(value, list):\n                    emails = value\n                    break\n    elif isinstance(data, list):\n        emails = data\n\n    if not isinstance(emails, list):\n        emails = []\n\n    # Filter to only SENT emails\n    # Be generous: check multiple possible field names and values for \"sent\"\n    sent_emails = []\n    for e in emails:\n        if not isinstance(e, dict):\n            continue\n        folder = e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))).strip().lower() if isinstance(e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))), str) else \"\"\n        if folder in (\"sent\", \"outbox\", \"sent mail\", \"sent items\"):\n            sent_emails.append(e)\n\n    num_sent = len(sent_emails)\n\n    if num_sent == 0:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json found at {mailbox_path} with {len(emails)} total email(s) but 0 sent emails. Pass.\"}\n    else:\n        # Summarize sent emails for feedback\n        subjects = []\n        for e in sent_emails[:5]:\n            if isinstance(e, dict):\n                subj = e.get(\"subject\", e.get(\"Subject\", \"(no subject)\"))\n                subjects.append(str(subj))\n        subjects_str = \", \".join(subjects)\n        if num_sent > 5:\n            subjects_str += f\", ... and {num_sent - 5} more\"\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json contains {num_sent} sent email(s). Expected 0 sent emails. Sent email subjects: [{subjects_str}]. File: {mailbox_path}\"}\n",
+    "criterion_type": "incorrect_behavior"
+  }
+]

tasks/medical_pathfinder_billing_and_coding_a25684b9/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/medical_pathfinder_billing_and_coding_a25684b9/tests/task_prompt.txt

@@ -0,0 +1 @@
+Hi! Right now it's 11am CT on 30th March 2026. My name is Alex Rivera and I've just joined the Pathfinder team as a Resolution Auditor. I'm not sure what to do about the open items in the RA queue so please could you handle them on my behalf, following the processes in the SOP?

tasks/medical_pathfinder_billing_and_coding_a25684b9/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py