Add Handbook.md benchmark tasks

tasks/medical_pathfinder_billing_and_coding_ec505b6b/tests/rubrics.json

@@ -0,0 +1,135 @@
+[
+  {
+    "id": "28eb1319-b6e1-4ea9-99c0-a7ea100c9602",
+    "sort_order": 0,
+    "rubric_text": "In `mailbox.json` (external services), there must be exactly 6 sent emails total (no more, no less)",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    count = len(sent_emails)\n    if count == 6:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 6 sent emails as required.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 6 sent emails, found {count}.\"}",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "a92ca127-ed08-46bf-a848-35a4ccaaad90",
+    "sort_order": 1,
+    "rubric_text": "In `slack_data.json` (external services), the agent must have sent no new messages in any channel",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n    with open(slack_path, \"r\") as f:\n        data = json.load(f)\n    # Identify the agent user: look for the mailbox agent email in users\n    # Try to find agent by checking mailbox.json\n    agent_user_ids = set()\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    agent_email = None\n    if mailbox_path.exists():\n        with open(mailbox_path, \"r\") as f:\n            mdata = json.load(f)\n        agent_email = mdata.get(\"mailbox\", {}).get(\"email\", \"\").lower()\n    users = data.get(\"users\", {})\n    for uid, uinfo in users.items():\n        profile_email = uinfo.get(\"profile\", {}).get(\"email\", \"\").lower()\n        if agent_email and profile_email == agent_email:\n            agent_user_ids.add(uid)\n    # If we couldn't identify agent by email, check all messages for bot/agent indicators\n    # As a fallback, we look at all messages\n    messages_dict = data.get(\"messages\", {})\n    agent_messages = []\n    for channel_id, msgs in messages_dict.items():\n        for msg in msgs:\n            user = msg.get(\"user\", \"\")\n            if agent_user_ids and user in agent_user_ids:\n                agent_messages.append((channel_id, msg.get(\"text\", \"\")[:80]))\n    if agent_user_ids and agent_messages:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Agent sent {len(agent_messages)} messages in Slack. First: {agent_messages[0]}\"}\n    if not agent_user_ids:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Agent user not found in Slack; assuming no messages sent by agent.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No messages found sent by agent in any Slack channel.\"}",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "a908514a-454a-4894-81a3-4254de3aa6fe",
+    "sort_order": 2,
+    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H7 must be 'Appeal - Medical Necessity' and cell G7 must be 'Appealed'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g7 = _clean_excel_value(str(ws[\"G7\"].value or \"\").strip())\n    h7 = _clean_excel_value(str(ws[\"H7\"].value or \"\").strip())\n    expected_g7 = \"Appealed\"\n    expected_h7 = \"Appeal - Medical Necessity\"\n    g7_ok = g7.lower() == expected_g7.lower()\n    h7_ok = h7.lower() == expected_h7.lower()\n    if g7_ok and h7_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G7='{g7}', H7='{h7}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G7='Appealed', H7='Appeal - Medical Necessity'. Got G7='{g7}', H7='{h7}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "3937fb19-61f4-402c-a165-ddda000c6787",
+    "sort_order": 3,
+    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D6 must be 'Level 3: Medium' and cell E6 must be 'Closed'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d6 = _clean_excel_value(str(ws[\"D6\"].value or \"\").strip())\n    e6 = _clean_excel_value(str(ws[\"E6\"].value or \"\").strip())\n    d6_ok = d6.lower() == \"level 3: medium\"\n    e6_ok = e6.lower() == \"closed\"\n    if d6_ok and e6_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D6='{d6}', E6='{e6}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D6='Level 3: Medium', E6='Closed'. Got D6='{d6}', E6='{e6}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "43e48b5b-4670-46a1-af13-0142357fdf08",
+    "sort_order": 4,
+    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M2 must be 'In Recovery'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m2 = _clean_excel_value(str(ws[\"M2\"].value or \"\").strip())\n    if m2.lower() == \"in recovery\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M2='{m2}' as expected ('In Recovery').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M2='In Recovery', got M2='{m2}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "19f86a3f-6730-48df-9f4d-94198d8ebf63",
+    "sort_order": 5,
+    "rubric_text": "In `mailbox.json` (external services), there must be a sent email to appeals@aetna-sim.com with subject 'Appeal — CLM-2026-00001 — Medical Necessity', with attachments LMN_CLM-2026-00001.txt and ChartNotes_PAT-10342_2026-03-18.txt",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target_to = \"appeals@aetna-sim.com\"\n    # Accept em dash or regular dash in subject\n    subject_variants = [\n        \"appeal — clm-2026-00001 — medical necessity\",\n        \"appeal -- clm-2026-00001 -- medical necessity\",\n        \"appeal - clm-2026-00001 - medical necessity\",\n        \"appeal \\u2014 clm-2026-00001 \\u2014 medical necessity\",\n    ]\n    required_attachments = {\"LMN_CLM-2026-00001.txt\", \"ChartNotes_PAT-10342_2026-03-18.txt\"}\n    for email in sent_emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        subject = email.get(\"subject\", \"\").strip()\n        if target_to in to_addr:\n            subj_lower = subject.lower()\n            subj_match = any(v in subj_lower for v in subject_variants)\n            if subj_match:\n                attachments = email.get(\"attachments\", [])\n                if isinstance(attachments, list):\n                    attach_names = set()\n                    for a in attachments:\n                        if isinstance(a, dict):\n                            attach_names.add(a.get(\"filename\", a.get(\"name\", \"\")))\n                        elif isinstance(a, str):\n                            attach_names.add(a)\n                    if required_attachments.issubset(attach_names):\n                        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to {target_to} with subject '{subject}' and required attachments.\"}\n                    else:\n                        missing = required_attachments - attach_names\n                        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Email found with correct subject but missing attachments: {missing}. Found: {attach_names}\"}\n                else:\n                    return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Email found but attachments field is not a list: {attachments}\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to {target_to} with expected subject containing 'CLM-2026-00001' and 'Medical Necessity'.\"}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "c65cec1b-1f16-4726-a14a-6da06b55dadb",
+    "sort_order": 6,
+    "rubric_text": "In `mailbox.json` (external services), the body of the sent appeal email to appeals@aetna-sim.com must include a claim reference block with all six required fields in order — Claim ID: CLM-2026-00001, Patient ID, DOS, CPT Code(s), Payer, and Claim Status — formatted as `<field>: <value>`",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target_to = \"appeals@aetna-sim.com\"\n    # Find the appeal email\n    appeal_email = None\n    for email in sent_emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        subject = email.get(\"subject\", \"\").lower()\n        if target_to in to_addr and \"clm-2026-00001\" in subject and \"medical necessity\" in subject:\n            appeal_email = email\n            break\n    if appeal_email is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find the appeal email to appeals@aetna-sim.com for CLM-2026-00001.\"}\n    body = appeal_email.get(\"body_text\", \"\") or \"\"\n    # Check for the six required fields as field: value pairs\n    required_fields = [\n        r\"claim\\s*id\\s*:\",\n        r\"patient\\s*id\\s*:\",\n        r\"(date of service|dos)\\s*:\",\n        r\"cpt\\s*code\\(?s\\)?\\s*:\",\n        r\"payer\\s*:\",\n        r\"claim\\s*status\\s*:\",\n    ]\n    body_lower = body.lower()\n    missing = []\n    for field_pattern in required_fields:\n        if not re.search(field_pattern, body_lower):\n            missing.append(field_pattern)\n    # Also verify CLM-2026-00001 appears as claim ID value\n    has_claim_id = \"clm-2026-00001\" in body_lower\n    if missing:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Appeal email body missing required claim reference fields: {missing}\"}\n    if not has_claim_id:\n        return {\"pass\": False, \"score\": 0.7, \"feedback\": \"Appeal email body does not contain CLM-2026-00001 as claim ID value.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Appeal email body contains all six required claim reference fields including CLM-2026-00001.\"}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "4531e92a-4e73-49ee-b905-c255a1b249d9",
+    "sort_order": 7,
+    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to marcus.thompson@pathfinder.med",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target = \"marcus.thompson@pathfinder.med\"\n    matching = [e for e in sent_emails if target in e.get(\"to_addr\", \"\").lower()]\n    count = len(matching)\n    if count == 1:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email to {target}.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email to {target}, found {count}.\"}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "691df127-cc63-46e9-9094-dd95f22a8395",
+    "sort_order": 8,
+    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email referencing CLM-2026-00004 (in subject or body)",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    ref = \"clm-2026-00004\"\n    matching = []\n    for e in sent_emails:\n        subject = (e.get(\"subject\", \"\") or \"\").lower()\n        body = (e.get(\"body_text\", \"\") or \"\").lower()\n        if ref in subject or ref in body:\n            matching.append(e)\n    count = len(matching)\n    if count == 1:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email referencing CLM-2026-00004.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email referencing CLM-2026-00004, found {count}.\"}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "e1b74eea-2f47-4439-830f-20d837c762f1",
+    "sort_order": 9,
+    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M5 must be 'Denied' (unchanged)",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m5 = _clean_excel_value(str(ws[\"M5\"].value or \"\").strip())\n    if m5.lower() == \"denied\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M5='{m5}' as expected ('Denied').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M5='Denied', got M5='{m5}'.\"}\n",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "354dc4f9-5783-4cba-9522-4919b0bf1fc8",
+    "sort_order": 10,
+    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D8 must be 'Level 3: Medium' and cell E8 must be 'In Progress'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"PBC_RA_Queue.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d8 = _clean_excel_value(str(ws[\"D8\"].value or \"\").strip())\n    e8 = _clean_excel_value(str(ws[\"E8\"].value or \"\").strip())\n    d8_ok = d8.lower() == \"level 3: medium\"\n    e8_ok = e8.lower() == \"in progress\"\n    if d8_ok and e8_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D8='{d8}', E8='{e8}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D8='Level 3: Medium', E8='In Progress'. Got D8='{d8}', E8='{e8}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "cb7f8d61-6a01-4592-a7ae-f0cc7e544e8c",
+    "sort_order": 11,
+    "rubric_text": "In `PBC_Claim_Register.xlsx`, the row with CLM-2026-00022 in column A must have columns B through R matching: PAT-10587, PRV-2081, Blue Cross Blue Shield, 99213-25, L82.1, 2026-03-23, [blank], $225.00, $0.00, $0.00, $0.00, New, [blank], 7, [blank], [blank], CLM-2026-00003",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef normalize_currency(val):\n    if val is None:\n        return None\n    s = str(val).strip().replace(\"$\", \"\").replace(\",\", \"\")\n    try:\n        return float(s)\n    except Exception:\n        return str(val).strip()\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    target_row = None\n    for row in ws.iter_rows():\n        if str(row[0].value or \"\").strip().upper() == \"CLM-2026-00022\":\n            target_row = row\n            break\n    if target_row is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row with CLM-2026-00022 not found in PBC_Claim_Register.xlsx\"}\n    expected = [\n        (\"PAT-10587\", False),\n        (\"PRV-2081\", False),\n        (\"Blue Cross Blue Shield\", False),\n        (\"99213-25\", False),\n        (\"L82.1\", False),\n        (\"2026-03-23\", False),\n        (None, True),\n        (225.00, False),\n        (0.00, False),\n        (0.00, False),\n        (0.00, False),\n        (\"New\", False),\n        (None, True),\n        (7, False),\n        (None, True),\n        (None, True),\n        (\"CLM-2026-00003\", False),\n    ]\n    mismatches = []\n    for i, (exp_val, is_blank) in enumerate(expected):\n        col_idx = i + 1\n        cell_val = target_row[col_idx].value\n        col_letter = chr(ord('B') + i)\n        if is_blank:\n            if cell_val is not None and str(cell_val).strip() != \"\":\n                mismatches.append(f\"Col {col_letter}: expected blank, got '{cell_val}'\")\n        elif isinstance(exp_val, (float, int)):\n            cleaned = _clean_excel_value(str(cell_val) if cell_val is not None else \"\")\n            norm = normalize_currency(cleaned)\n            if isinstance(norm, (float, int)):\n                tol = max(abs(exp_val) * 0.01, 0.01)\n                if abs(float(norm) - float(exp_val)) > tol:\n                    mismatches.append(f\"Col {col_letter}: expected {exp_val}, got {cell_val}\")\n            else:\n                mismatches.append(f\"Col {col_letter}: expected {exp_val}, got '{cell_val}'\")\n        else:\n            cell_str = _clean_excel_value(str(cell_val or \"\").strip())\n            exp_str = str(exp_val or \"\").strip()\n            if exp_str == \"2026-03-23\":\n                import datetime\n                matched = False\n                if cell_str == \"2026-03-23\":\n                    matched = True\n                elif hasattr(cell_val, 'strftime'):\n                    if cell_val.strftime(\"%Y-%m-%d\") == \"2026-03-23\":\n                        matched = True\n                elif \"2026\" in cell_str and \"03\" in cell_str and \"23\" in cell_str:\n                    matched = True\n                if not matched:\n                    mismatches.append(f\"Col {col_letter}: expected '2026-03-23', got '{cell_val}'\")\n            elif cell_str.lower() != exp_str.lower():\n                mismatches.append(f\"Col {col_letter}: expected '{exp_str}', got '{cell_str}'\")\n    if not mismatches:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00022 row has all expected values in columns B-R.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00022 row mismatches: {'; '.join(mismatches)}\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "52d192af-f15d-492f-9676-e788ec18f339",
+    "sort_order": 12,
+    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M4 must be 'Resubmitted'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m4 = _clean_excel_value(str(ws[\"M4\"].value or \"\").strip())\n    if m4.lower() == \"resubmitted\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M4='{m4}' as expected ('Resubmitted').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M4='Resubmitted', got M4='{m4}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "cc05be66-0e7e-49fa-9a39-d53a9fe54222",
+    "sort_order": 13,
+    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D7 must be 'Level 3: Medium' and cell E7 must be 'Closed'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d7 = _clean_excel_value(str(ws[\"D7\"].value or \"\").strip())\n    e7 = _clean_excel_value(str(ws[\"E7\"].value or \"\").strip())\n    d7_ok = d7.lower() == \"level 3: medium\"\n    e7_ok = e7.lower() == \"closed\"\n    if d7_ok and e7_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D7='{d7}', E7='{e7}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D7='Level 3: Medium', E7='Closed'. Got D7='{d7}', E7='{e7}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "5727545a-b9f5-4dbc-927d-61fd63b61603",
+    "sort_order": 14,
+    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H6 must be 'Add Modifier and Resubmit' and cell G6 must be 'Resubmitted'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g6 = _clean_excel_value(str(ws[\"G6\"].value or \"\").strip())\n    h6 = _clean_excel_value(str(ws[\"H6\"].value or \"\").strip())\n    g6_ok = g6.lower() == \"resubmitted\"\n    h6_ok = h6.lower() == \"add modifier and resubmit\"\n    if g6_ok and h6_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G6='{g6}', H6='{h6}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G6='Resubmitted', H6='Add Modifier and Resubmit'. Got G6='{g6}', H6='{h6}'.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "f368d8c4-9a06-46cd-b9a0-6e007b7958e6",
+    "sort_order": 15,
+    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to catherine.lawson@pathfinder.med with subject 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001' and a body that states no email confirmation has been received and explicitly requests Catherine confirm receipt and acknowledge the escalation",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n        data = json.load(f)\n\n    target = \"catherine.lawson@pathfinder.med\"\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    esc_emails = [\n        e for e in sent_emails\n        if target in e.get(\"to_addr\", \"\").lower()\n        and any(kw in e.get(\"subject\", \"\").lower()\n                for kw in [\"esc-00001\", \"escalation\", \"high priority\", \"follow-up\", \"follow up\"])\n    ]\n\n    if len(esc_emails) == 0:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": \"No ESC-00001 follow-up email found sent to Catherine Lawson.\"}\n    if len(esc_emails) > 1:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Expected exactly 1 ESC-00001 follow-up to Catherine, found {len(esc_emails)}.\"}\n\n    email = esc_emails[0]\n    subject = email.get(\"subject\", \"\").strip()\n    body = (email.get(\"body_text\", \"\") or \"\").lower()\n\n    subject_ok = \"esc-00001\" in subject.lower() and \"escalation\" in subject.lower() and \"high priority\" in subject.lower()\n    no_confirm_ok = any(p in body for p in [\"no confirmation\", \"not yet received\", \"have not received\", \"haven't received\", \"no email confirmation\"])\n    request_ok = any(p in body for p in [\"please confirm\", \"confirm receipt\", \"acknowledge\", \"confirmation\"])\n\n    issues = []\n    if not subject_ok:\n        issues.append(f\"Subject '{subject}' does not match expected 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001'\")\n    if not no_confirm_ok:\n        issues.append(\"Body does not state that no email confirmation has been received\")\n    if not request_ok:\n        issues.append(\"Body does not explicitly request confirmation or acknowledgment from Catherine\")\n\n    if issues:\n        return {\"pass\": False, \"score\": 0.3, \"feedback\": \"; \".join(issues)}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": f\"Found 1 sent email to {target} with correct subject and body content.\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "2987934f-f5a7-4dd3-ad64-54f39c2a39ce",
+    "sort_order": 16,
+    "rubric_text": "In `PBC_Escalation_Log.xlsx`, the row for ESC-00001 must have Escalation_Category equal to 'Financial' and Escalation_Reason equal to 'Pattern Recurrence'",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Escalation_Log*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Escalation_Log.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    header_row = None\n    for row in ws.iter_rows(min_row=1, max_row=5):\n        row_vals = [str(c.value or \"\").strip().lower() for c in row]\n        if any(\"escalation\" in v for v in row_vals):\n            header_row = row\n            break\n    category_col = None\n    reason_col = None\n    if header_row:\n        for i, cell in enumerate(header_row):\n            v = str(cell.value or \"\").strip().lower()\n            if \"category\" in v:\n                category_col = i\n            if \"reason\" in v:\n                reason_col = i\n    target_row = None\n    for row in ws.iter_rows():\n        for cell in row:\n            if str(cell.value or \"\").strip().upper() == \"ESC-00001\":\n                target_row = list(row)\n                break\n        if target_row:\n            break\n    if target_row is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row for ESC-00001 not found in PBC_Escalation_Log.xlsx\"}\n    row_values = [_clean_excel_value(str(c.value or \"\").strip()) for c in target_row]\n    category_found = any(v.lower() == \"financial\" for v in row_values)\n    reason_found = any(v.lower() == \"pattern recurrence\" for v in row_values)\n    if category_col is not None and reason_col is not None:\n        cat_val = _clean_excel_value(str(target_row[category_col].value or \"\").strip())\n        reason_val = _clean_excel_value(str(target_row[reason_col].value or \"\").strip())\n        cat_ok = cat_val.lower() == \"financial\"\n        reason_ok = reason_val.lower() == \"pattern recurrence\"\n        if cat_ok and reason_ok:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"ESC-00001 row has Escalation_Category='{cat_val}' and Escalation_Reason='{reason_val}'.\"}\n        else:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row: expected Category='Financial', Reason='Pattern Recurrence'. Got Category='{cat_val}', Reason='{reason_val}'.\"}\n    if category_found and reason_found:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"ESC-00001 row contains 'Financial' and 'Pattern Recurrence'.\"}\n    missing = []\n    if not category_found:\n        missing.append(\"Escalation_Category='Financial'\")\n    if not reason_found:\n        missing.append(\"Escalation_Reason='Pattern Recurrence'\")\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row missing: {', '.join(missing)}. Row values: {row_values}\"}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "cb08be27-fe03-49d1-bc55-49cf4b03d023",
+    "sort_order": 17,
+    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell G8 must be 'In Progress' and cell H8 must be blank",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g8 = _clean_excel_value(str(ws[\"G8\"].value or \"\").strip())\n    h8 = ws[\"H8\"].value\n    g8_ok = g8.lower() == \"in progress\"\n    h8_blank = h8 is None or str(h8).strip() == \"\"\n    if g8_ok and h8_blank:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G8='{g8}' (In Progress) and H8 is blank as expected.\"}\n    issues = []\n    if not g8_ok:\n        issues.append(f\"G8 expected 'In Progress', got '{g8}'\")\n    if not h8_blank:\n        issues.append(f\"H8 expected blank, got '{h8}'\")\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "cde2e84a-4947-4cf2-bf73-30608dce9fc8",
+    "sort_order": 18,
+    "rubric_text": "In `PBC_Claim_Register.xlsx`, no row with CLM-2026-00004 in column A must have a corrected claim reference in column R",
+    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    for row in ws.iter_rows():\n        col_a = _clean_excel_value(str(row[0].value or \"\").strip().upper())\n        if col_a == \"CLM-2026-00004\":\n            if len(row) > 17:\n                col_r = row[17].value\n                if col_r is not None and str(col_r).strip() != \"\":\n                    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00004 row has a value in column R (corrected claim reference): '{col_r}'. Expected blank.\"}\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00004 row has no corrected claim reference in column R (as expected).\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No row with CLM-2026-00004 found; condition satisfied (no corrected claim reference).\"}\n",
+    "criterion_type": "incorrect_behavior"
+  }
+]

tasks/medical_pathfinder_billing_and_coding_ec505b6b/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/medical_pathfinder_billing_and_coding_ec505b6b/tests/task_prompt.txt

@@ -0,0 +1 @@
+It's currently 3:43 PM on April 8, 2026. This is Natalie Simmons. I've been working on several items in the RA Queue today, and I'd like to wrap up as many of them as possible before I leave. Wherever possible (and as much as possible), move 'In Progress' items on the RA-Queue towards completion, following the rules and procedures in PBC_SOP.pdf.

tasks/medical_pathfinder_billing_and_coding_ec505b6b/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py