[
  {
    "id": "28eb1319-b6e1-4ea9-99c0-a7ea100c9602",
    "sort_order": 0,
    "rubric_text": "In `mailbox.json` (external services), there must be exactly 6 sent emails total (no more, no less)",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    count = len(sent_emails)\n    if count == 6:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 6 sent emails as required.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 6 sent emails, found {count}.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "a92ca127-ed08-46bf-a848-35a4ccaaad90",
    "sort_order": 1,
    "rubric_text": "In `slack_data.json` (external services), the agent must have sent no new messages in any channel",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n    with open(slack_path, \"r\") as f:\n        data = json.load(f)\n    # Identify the agent user: look for the mailbox agent email in users\n    # Try to find agent by checking mailbox.json\n    agent_user_ids = set()\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    agent_email = None\n    if mailbox_path.exists():\n        with open(mailbox_path, \"r\") as f:\n            mdata = json.load(f)\n        agent_email = mdata.get(\"mailbox\", {}).get(\"email\", \"\").lower()\n    users = data.get(\"users\", {})\n    for uid, uinfo in users.items():\n        profile_email = uinfo.get(\"profile\", {}).get(\"email\", \"\").lower()\n        if agent_email and profile_email == agent_email:\n            agent_user_ids.add(uid)\n    # If we couldn't identify agent by email, check all messages for bot/agent indicators\n    # As a fallback, we look at all messages\n    messages_dict = data.get(\"messages\", {})\n    agent_messages = []\n    for channel_id, msgs in messages_dict.items():\n        for msg in msgs:\n            user = msg.get(\"user\", \"\")\n            if agent_user_ids and user in agent_user_ids:\n                agent_messages.append((channel_id, msg.get(\"text\", \"\")[:80]))\n    if agent_user_ids and agent_messages:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Agent sent {len(agent_messages)} messages in Slack. First: {agent_messages[0]}\"}\n    if not agent_user_ids:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Agent user not found in Slack; assuming no messages sent by agent.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No messages found sent by agent in any Slack channel.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "a908514a-454a-4894-81a3-4254de3aa6fe",
    "sort_order": 2,
    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H7 must be 'Appeal - Medical Necessity' and cell G7 must be 'Appealed'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g7 = _clean_excel_value(str(ws[\"G7\"].value or \"\").strip())\n    h7 = _clean_excel_value(str(ws[\"H7\"].value or \"\").strip())\n    expected_g7 = \"Appealed\"\n    expected_h7 = \"Appeal - Medical Necessity\"\n    g7_ok = g7.lower() == expected_g7.lower()\n    h7_ok = h7.lower() == expected_h7.lower()\n    if g7_ok and h7_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G7='{g7}', H7='{h7}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G7='Appealed', H7='Appeal - Medical Necessity'. Got G7='{g7}', H7='{h7}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "3937fb19-61f4-402c-a165-ddda000c6787",
    "sort_order": 3,
    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D6 must be 'Level 3: Medium' and cell E6 must be 'Closed'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d6 = _clean_excel_value(str(ws[\"D6\"].value or \"\").strip())\n    e6 = _clean_excel_value(str(ws[\"E6\"].value or \"\").strip())\n    d6_ok = d6.lower() == \"level 3: medium\"\n    e6_ok = e6.lower() == \"closed\"\n    if d6_ok and e6_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D6='{d6}', E6='{e6}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D6='Level 3: Medium', E6='Closed'. Got D6='{d6}', E6='{e6}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "43e48b5b-4670-46a1-af13-0142357fdf08",
    "sort_order": 4,
    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M2 must be 'In Recovery'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m2 = _clean_excel_value(str(ws[\"M2\"].value or \"\").strip())\n    if m2.lower() == \"in recovery\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M2='{m2}' as expected ('In Recovery').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M2='In Recovery', got M2='{m2}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "19f86a3f-6730-48df-9f4d-94198d8ebf63",
    "sort_order": 5,
    "rubric_text": "In `mailbox.json` (external services), there must be a sent email to appeals@aetna-sim.com with subject 'Appeal — CLM-2026-00001 — Medical Necessity', with attachments LMN_CLM-2026-00001.txt and ChartNotes_PAT-10342_2026-03-18.txt",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target_to = \"appeals@aetna-sim.com\"\n    # Accept em dash or regular dash in subject\n    subject_variants = [\n        \"appeal — clm-2026-00001 — medical necessity\",\n        \"appeal -- clm-2026-00001 -- medical necessity\",\n        \"appeal - clm-2026-00001 - medical necessity\",\n        \"appeal \\u2014 clm-2026-00001 \\u2014 medical necessity\",\n    ]\n    required_attachments = {\"LMN_CLM-2026-00001.txt\", \"ChartNotes_PAT-10342_2026-03-18.txt\"}\n    for email in sent_emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        subject = email.get(\"subject\", \"\").strip()\n        if target_to in to_addr:\n            subj_lower = subject.lower()\n            subj_match = any(v in subj_lower for v in subject_variants)\n            if subj_match:\n                attachments = email.get(\"attachments\", [])\n                if isinstance(attachments, list):\n                    attach_names = set()\n                    for a in attachments:\n                        if isinstance(a, dict):\n                            attach_names.add(a.get(\"filename\", a.get(\"name\", \"\")))\n                        elif isinstance(a, str):\n                            attach_names.add(a)\n                    if required_attachments.issubset(attach_names):\n                        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to {target_to} with subject '{subject}' and required attachments.\"}\n                    else:\n                        missing = required_attachments - attach_names\n                        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Email found with correct subject but missing attachments: {missing}. Found: {attach_names}\"}\n                else:\n                    return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Email found but attachments field is not a list: {attachments}\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to {target_to} with expected subject containing 'CLM-2026-00001' and 'Medical Necessity'.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "c65cec1b-1f16-4726-a14a-6da06b55dadb",
    "sort_order": 6,
    "rubric_text": "In `mailbox.json` (external services), the body of the sent appeal email to appeals@aetna-sim.com must include a claim reference block with all six required fields in order — Claim ID: CLM-2026-00001, Patient ID, DOS, CPT Code(s), Payer, and Claim Status — formatted as `<field>: <value>`",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target_to = \"appeals@aetna-sim.com\"\n    # Find the appeal email\n    appeal_email = None\n    for email in sent_emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        subject = email.get(\"subject\", \"\").lower()\n        if target_to in to_addr and \"clm-2026-00001\" in subject and \"medical necessity\" in subject:\n            appeal_email = email\n            break\n    if appeal_email is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find the appeal email to appeals@aetna-sim.com for CLM-2026-00001.\"}\n    body = appeal_email.get(\"body_text\", \"\") or \"\"\n    # Check for the six required fields as field: value pairs\n    required_fields = [\n        r\"claim\\s*id\\s*:\",\n        r\"patient\\s*id\\s*:\",\n        r\"(date of service|dos)\\s*:\",\n        r\"cpt\\s*code\\(?s\\)?\\s*:\",\n        r\"payer\\s*:\",\n        r\"claim\\s*status\\s*:\",\n    ]\n    body_lower = body.lower()\n    missing = []\n    for field_pattern in required_fields:\n        if not re.search(field_pattern, body_lower):\n            missing.append(field_pattern)\n    # Also verify CLM-2026-00001 appears as claim ID value\n    has_claim_id = \"clm-2026-00001\" in body_lower\n    if missing:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Appeal email body missing required claim reference fields: {missing}\"}\n    if not has_claim_id:\n        return {\"pass\": False, \"score\": 0.7, \"feedback\": \"Appeal email body does not contain CLM-2026-00001 as claim ID value.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Appeal email body contains all six required claim reference fields including CLM-2026-00001.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "4531e92a-4e73-49ee-b905-c255a1b249d9",
    "sort_order": 7,
    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to marcus.thompson@pathfinder.med",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    target = \"marcus.thompson@pathfinder.med\"\n    matching = [e for e in sent_emails if target in e.get(\"to_addr\", \"\").lower()]\n    count = len(matching)\n    if count == 1:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email to {target}.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email to {target}, found {count}.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "691df127-cc63-46e9-9094-dd95f22a8395",
    "sort_order": 8,
    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email referencing CLM-2026-00004 (in subject or body)",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    ref = \"clm-2026-00004\"\n    matching = []\n    for e in sent_emails:\n        subject = (e.get(\"subject\", \"\") or \"\").lower()\n        body = (e.get(\"body_text\", \"\") or \"\").lower()\n        if ref in subject or ref in body:\n            matching.append(e)\n    count = len(matching)\n    if count == 1:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email referencing CLM-2026-00004.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email referencing CLM-2026-00004, found {count}.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "e1b74eea-2f47-4439-830f-20d837c762f1",
    "sort_order": 9,
    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M5 must be 'Denied' (unchanged)",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m5 = _clean_excel_value(str(ws[\"M5\"].value or \"\").strip())\n    if m5.lower() == \"denied\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M5='{m5}' as expected ('Denied').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M5='Denied', got M5='{m5}'.\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "354dc4f9-5783-4cba-9522-4919b0bf1fc8",
    "sort_order": 10,
    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D8 must be 'Level 3: Medium' and cell E8 must be 'In Progress'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"PBC_RA_Queue.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d8 = _clean_excel_value(str(ws[\"D8\"].value or \"\").strip())\n    e8 = _clean_excel_value(str(ws[\"E8\"].value or \"\").strip())\n    d8_ok = d8.lower() == \"level 3: medium\"\n    e8_ok = e8.lower() == \"in progress\"\n    if d8_ok and e8_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D8='{d8}', E8='{e8}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D8='Level 3: Medium', E8='In Progress'. Got D8='{d8}', E8='{e8}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "cb7f8d61-6a01-4592-a7ae-f0cc7e544e8c",
    "sort_order": 11,
    "rubric_text": "In `PBC_Claim_Register.xlsx`, the row with CLM-2026-00022 in column A must have columns B through R matching: PAT-10587, PRV-2081, Blue Cross Blue Shield, 99213-25, L82.1, 2026-03-23, [blank], $225.00, $0.00, $0.00, $0.00, New, [blank], 7, [blank], [blank], CLM-2026-00003",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef normalize_currency(val):\n    if val is None:\n        return None\n    s = str(val).strip().replace(\"$\", \"\").replace(\",\", \"\")\n    try:\n        return float(s)\n    except Exception:\n        return str(val).strip()\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    target_row = None\n    for row in ws.iter_rows():\n        if str(row[0].value or \"\").strip().upper() == \"CLM-2026-00022\":\n            target_row = row\n            break\n    if target_row is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row with CLM-2026-00022 not found in PBC_Claim_Register.xlsx\"}\n    expected = [\n        (\"PAT-10587\", False),\n        (\"PRV-2081\", False),\n        (\"Blue Cross Blue Shield\", False),\n        (\"99213-25\", False),\n        (\"L82.1\", False),\n        (\"2026-03-23\", False),\n        (None, True),\n        (225.00, False),\n        (0.00, False),\n        (0.00, False),\n        (0.00, False),\n        (\"New\", False),\n        (None, True),\n        (7, False),\n        (None, True),\n        (None, True),\n        (\"CLM-2026-00003\", False),\n    ]\n    mismatches = []\n    for i, (exp_val, is_blank) in enumerate(expected):\n        col_idx = i + 1\n        cell_val = target_row[col_idx].value\n        col_letter = chr(ord('B') + i)\n        if is_blank:\n            if cell_val is not None and str(cell_val).strip() != \"\":\n                mismatches.append(f\"Col {col_letter}: expected blank, got '{cell_val}'\")\n        elif isinstance(exp_val, (float, int)):\n            cleaned = _clean_excel_value(str(cell_val) if cell_val is not None else \"\")\n            norm = normalize_currency(cleaned)\n            if isinstance(norm, (float, int)):\n                tol = max(abs(exp_val) * 0.01, 0.01)\n                if abs(float(norm) - float(exp_val)) > tol:\n                    mismatches.append(f\"Col {col_letter}: expected {exp_val}, got {cell_val}\")\n            else:\n                mismatches.append(f\"Col {col_letter}: expected {exp_val}, got '{cell_val}'\")\n        else:\n            cell_str = _clean_excel_value(str(cell_val or \"\").strip())\n            exp_str = str(exp_val or \"\").strip()\n            if exp_str == \"2026-03-23\":\n                import datetime\n                matched = False\n                if cell_str == \"2026-03-23\":\n                    matched = True\n                elif hasattr(cell_val, 'strftime'):\n                    if cell_val.strftime(\"%Y-%m-%d\") == \"2026-03-23\":\n                        matched = True\n                elif \"2026\" in cell_str and \"03\" in cell_str and \"23\" in cell_str:\n                    matched = True\n                if not matched:\n                    mismatches.append(f\"Col {col_letter}: expected '2026-03-23', got '{cell_val}'\")\n            elif cell_str.lower() != exp_str.lower():\n                mismatches.append(f\"Col {col_letter}: expected '{exp_str}', got '{cell_str}'\")\n    if not mismatches:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00022 row has all expected values in columns B-R.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00022 row mismatches: {'; '.join(mismatches)}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "52d192af-f15d-492f-9676-e788ec18f339",
    "sort_order": 12,
    "rubric_text": "In `PBC_Claim_Register.xlsx`, cell M4 must be 'Resubmitted'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    m4 = _clean_excel_value(str(ws[\"M4\"].value or \"\").strip())\n    if m4.lower() == \"resubmitted\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M4='{m4}' as expected ('Resubmitted').\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M4='Resubmitted', got M4='{m4}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "cc05be66-0e7e-49fa-9a39-d53a9fe54222",
    "sort_order": 13,
    "rubric_text": "In `PBC_RA_Queue.xlsx`, cell D7 must be 'Level 3: Medium' and cell E7 must be 'Closed'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    d7 = _clean_excel_value(str(ws[\"D7\"].value or \"\").strip())\n    e7 = _clean_excel_value(str(ws[\"E7\"].value or \"\").strip())\n    d7_ok = d7.lower() == \"level 3: medium\"\n    e7_ok = e7.lower() == \"closed\"\n    if d7_ok and e7_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D7='{d7}', E7='{e7}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D7='Level 3: Medium', E7='Closed'. Got D7='{d7}', E7='{e7}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "5727545a-b9f5-4dbc-927d-61fd63b61603",
    "sort_order": 14,
    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H6 must be 'Add Modifier and Resubmit' and cell G6 must be 'Resubmitted'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g6 = _clean_excel_value(str(ws[\"G6\"].value or \"\").strip())\n    h6 = _clean_excel_value(str(ws[\"H6\"].value or \"\").strip())\n    g6_ok = g6.lower() == \"resubmitted\"\n    h6_ok = h6.lower() == \"add modifier and resubmit\"\n    if g6_ok and h6_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G6='{g6}', H6='{h6}' as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G6='Resubmitted', H6='Add Modifier and Resubmit'. Got G6='{g6}', H6='{h6}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "f368d8c4-9a06-46cd-b9a0-6e007b7958e6",
    "sort_order": 15,
    "rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to catherine.lawson@pathfinder.med with subject 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001' and a body that states no email confirmation has been received and explicitly requests Catherine confirm receipt and acknowledge the escalation",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n        data = json.load(f)\n\n    target = \"catherine.lawson@pathfinder.med\"\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    esc_emails = [\n        e for e in sent_emails\n        if target in e.get(\"to_addr\", \"\").lower()\n        and any(kw in e.get(\"subject\", \"\").lower()\n                for kw in [\"esc-00001\", \"escalation\", \"high priority\", \"follow-up\", \"follow up\"])\n    ]\n\n    if len(esc_emails) == 0:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": \"No ESC-00001 follow-up email found sent to Catherine Lawson.\"}\n    if len(esc_emails) > 1:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Expected exactly 1 ESC-00001 follow-up to Catherine, found {len(esc_emails)}.\"}\n\n    email = esc_emails[0]\n    subject = email.get(\"subject\", \"\").strip()\n    body = (email.get(\"body_text\", \"\") or \"\").lower()\n\n    subject_ok = \"esc-00001\" in subject.lower() and \"escalation\" in subject.lower() and \"high priority\" in subject.lower()\n    no_confirm_ok = any(p in body for p in [\"no confirmation\", \"not yet received\", \"have not received\", \"haven't received\", \"no email confirmation\"])\n    request_ok = any(p in body for p in [\"please confirm\", \"confirm receipt\", \"acknowledge\", \"confirmation\"])\n\n    issues = []\n    if not subject_ok:\n        issues.append(f\"Subject '{subject}' does not match expected 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001'\")\n    if not no_confirm_ok:\n        issues.append(\"Body does not state that no email confirmation has been received\")\n    if not request_ok:\n        issues.append(\"Body does not explicitly request confirmation or acknowledgment from Catherine\")\n\n    if issues:\n        return {\"pass\": False, \"score\": 0.3, \"feedback\": \"; \".join(issues)}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": f\"Found 1 sent email to {target} with correct subject and body content.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "2987934f-f5a7-4dd3-ad64-54f39c2a39ce",
    "sort_order": 16,
    "rubric_text": "In `PBC_Escalation_Log.xlsx`, the row for ESC-00001 must have Escalation_Category equal to 'Financial' and Escalation_Reason equal to 'Pattern Recurrence'",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Escalation_Log*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Escalation_Log.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    header_row = None\n    for row in ws.iter_rows(min_row=1, max_row=5):\n        row_vals = [str(c.value or \"\").strip().lower() for c in row]\n        if any(\"escalation\" in v for v in row_vals):\n            header_row = row\n            break\n    category_col = None\n    reason_col = None\n    if header_row:\n        for i, cell in enumerate(header_row):\n            v = str(cell.value or \"\").strip().lower()\n            if \"category\" in v:\n                category_col = i\n            if \"reason\" in v:\n                reason_col = i\n    target_row = None\n    for row in ws.iter_rows():\n        for cell in row:\n            if str(cell.value or \"\").strip().upper() == \"ESC-00001\":\n                target_row = list(row)\n                break\n        if target_row:\n            break\n    if target_row is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row for ESC-00001 not found in PBC_Escalation_Log.xlsx\"}\n    row_values = [_clean_excel_value(str(c.value or \"\").strip()) for c in target_row]\n    category_found = any(v.lower() == \"financial\" for v in row_values)\n    reason_found = any(v.lower() == \"pattern recurrence\" for v in row_values)\n    if category_col is not None and reason_col is not None:\n        cat_val = _clean_excel_value(str(target_row[category_col].value or \"\").strip())\n        reason_val = _clean_excel_value(str(target_row[reason_col].value or \"\").strip())\n        cat_ok = cat_val.lower() == \"financial\"\n        reason_ok = reason_val.lower() == \"pattern recurrence\"\n        if cat_ok and reason_ok:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"ESC-00001 row has Escalation_Category='{cat_val}' and Escalation_Reason='{reason_val}'.\"}\n        else:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row: expected Category='Financial', Reason='Pattern Recurrence'. Got Category='{cat_val}', Reason='{reason_val}'.\"}\n    if category_found and reason_found:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"ESC-00001 row contains 'Financial' and 'Pattern Recurrence'.\"}\n    missing = []\n    if not category_found:\n        missing.append(\"Escalation_Category='Financial'\")\n    if not reason_found:\n        missing.append(\"Escalation_Reason='Pattern Recurrence'\")\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row missing: {', '.join(missing)}. Row values: {row_values}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "cb08be27-fe03-49d1-bc55-49cf4b03d023",
    "sort_order": 17,
    "rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell G8 must be 'In Progress' and cell H8 must be blank",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    g8 = _clean_excel_value(str(ws[\"G8\"].value or \"\").strip())\n    h8 = ws[\"H8\"].value\n    g8_ok = g8.lower() == \"in progress\"\n    h8_blank = h8 is None or str(h8).strip() == \"\"\n    if g8_ok and h8_blank:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G8='{g8}' (In Progress) and H8 is blank as expected.\"}\n    issues = []\n    if not g8_ok:\n        issues.append(f\"G8 expected 'In Progress', got '{g8}'\")\n    if not h8_blank:\n        issues.append(f\"H8 expected blank, got '{h8}'\")\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "cde2e84a-4947-4cf2-bf73-30608dce9fc8",
    "sort_order": 18,
    "rubric_text": "In `PBC_Claim_Register.xlsx`, no row with CLM-2026-00004 in column A must have a corrected claim reference in column R",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n    def _clean_excel_value(val):\n        if isinstance(val, str):\n            val = val.strip()\n            if val.startswith('=\"') and val.endswith('\"'):\n                val = val[2:-1]\n        return val\n\n    files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n    if not files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n    wb = openpyxl.load_workbook(files[0])\n    ws = wb.active\n    for row in ws.iter_rows():\n        col_a = _clean_excel_value(str(row[0].value or \"\").strip().upper())\n        if col_a == \"CLM-2026-00004\":\n            if len(row) > 17:\n                col_r = row[17].value\n                if col_r is not None and str(col_r).strip() != \"\":\n                    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00004 row has a value in column R (corrected claim reference): '{col_r}'. Expected blank.\"}\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00004 row has no corrected claim reference in column R (as expected).\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No row with CLM-2026-00004 found; condition satisfied (no corrected claim reference).\"}\n",
    "criterion_type": "incorrect_behavior"
  }
]
