DerekSurge
136 lines
36 KiB
JSON
136 lines
36 KiB
JSON
[
|
|
{
|
|
"id": "28eb1319-b6e1-4ea9-99c0-a7ea100c9602",
|
|
"sort_order": 0,
|
|
"rubric_text": "In `mailbox.json` (external services), there must be exactly 6 sent emails total (no more, no less)",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n count = len(sent_emails)\n if count == 6:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 6 sent emails as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 6 sent emails, found {count}.\"}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "a92ca127-ed08-46bf-a848-35a4ccaaad90",
|
|
"sort_order": 1,
|
|
"rubric_text": "In `slack_data.json` (external services), the agent must have sent no new messages in any channel",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n # Identify the agent user: look for the mailbox agent email in users\n # Try to find agent by checking mailbox.json\n agent_user_ids = set()\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n agent_email = None\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n mdata = json.load(f)\n agent_email = mdata.get(\"mailbox\", {}).get(\"email\", \"\").lower()\n users = data.get(\"users\", {})\n for uid, uinfo in users.items():\n profile_email = uinfo.get(\"profile\", {}).get(\"email\", \"\").lower()\n if agent_email and profile_email == agent_email:\n agent_user_ids.add(uid)\n # If we couldn't identify agent by email, check all messages for bot/agent indicators\n # As a fallback, we look at all messages\n messages_dict = data.get(\"messages\", {})\n agent_messages = []\n for channel_id, msgs in messages_dict.items():\n for msg in msgs:\n user = msg.get(\"user\", \"\")\n if agent_user_ids and user in agent_user_ids:\n agent_messages.append((channel_id, msg.get(\"text\", \"\")[:80]))\n if agent_user_ids and agent_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Agent sent {len(agent_messages)} messages in Slack. First: {agent_messages[0]}\"}\n if not agent_user_ids:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Agent user not found in Slack; assuming no messages sent by agent.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No messages found sent by agent in any Slack channel.\"}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "a908514a-454a-4894-81a3-4254de3aa6fe",
|
|
"sort_order": 2,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H7 must be 'Appeal - Medical Necessity' and cell G7 must be 'Appealed'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n g7 = _clean_excel_value(str(ws[\"G7\"].value or \"\").strip())\n h7 = _clean_excel_value(str(ws[\"H7\"].value or \"\").strip())\n expected_g7 = \"Appealed\"\n expected_h7 = \"Appeal - Medical Necessity\"\n g7_ok = g7.lower() == expected_g7.lower()\n h7_ok = h7.lower() == expected_h7.lower()\n if g7_ok and h7_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G7='{g7}', H7='{h7}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G7='Appealed', H7='Appeal - Medical Necessity'. Got G7='{g7}', H7='{h7}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "3937fb19-61f4-402c-a165-ddda000c6787",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, cell D6 must be 'Level 3: Medium' and cell E6 must be 'Closed'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n d6 = _clean_excel_value(str(ws[\"D6\"].value or \"\").strip())\n e6 = _clean_excel_value(str(ws[\"E6\"].value or \"\").strip())\n d6_ok = d6.lower() == \"level 3: medium\"\n e6_ok = e6.lower() == \"closed\"\n if d6_ok and e6_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D6='{d6}', E6='{e6}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D6='Level 3: Medium', E6='Closed'. Got D6='{d6}', E6='{e6}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "43e48b5b-4670-46a1-af13-0142357fdf08",
|
|
"sort_order": 4,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cell M2 must be 'In Recovery'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n m2 = _clean_excel_value(str(ws[\"M2\"].value or \"\").strip())\n if m2.lower() == \"in recovery\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M2='{m2}' as expected ('In Recovery').\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M2='In Recovery', got M2='{m2}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "19f86a3f-6730-48df-9f4d-94198d8ebf63",
|
|
"sort_order": 5,
|
|
"rubric_text": "In `mailbox.json` (external services), there must be a sent email to appeals@aetna-sim.com with subject 'Appeal — CLM-2026-00001 — Medical Necessity', with attachments LMN_CLM-2026-00001.txt and ChartNotes_PAT-10342_2026-03-18.txt",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n target_to = \"appeals@aetna-sim.com\"\n # Accept em dash or regular dash in subject\n subject_variants = [\n \"appeal — clm-2026-00001 — medical necessity\",\n \"appeal -- clm-2026-00001 -- medical necessity\",\n \"appeal - clm-2026-00001 - medical necessity\",\n \"appeal \\u2014 clm-2026-00001 \\u2014 medical necessity\",\n ]\n required_attachments = {\"LMN_CLM-2026-00001.txt\", \"ChartNotes_PAT-10342_2026-03-18.txt\"}\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n subject = email.get(\"subject\", \"\").strip()\n if target_to in to_addr:\n subj_lower = subject.lower()\n subj_match = any(v in subj_lower for v in subject_variants)\n if subj_match:\n attachments = email.get(\"attachments\", [])\n if isinstance(attachments, list):\n attach_names = set()\n for a in attachments:\n if isinstance(a, dict):\n attach_names.add(a.get(\"filename\", a.get(\"name\", \"\")))\n elif isinstance(a, str):\n attach_names.add(a)\n if required_attachments.issubset(attach_names):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to {target_to} with subject '{subject}' and required attachments.\"}\n else:\n missing = required_attachments - attach_names\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Email found with correct subject but missing attachments: {missing}. Found: {attach_names}\"}\n else:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Email found but attachments field is not a list: {attachments}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to {target_to} with expected subject containing 'CLM-2026-00001' and 'Medical Necessity'.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "c65cec1b-1f16-4726-a14a-6da06b55dadb",
|
|
"sort_order": 6,
|
|
"rubric_text": "In `mailbox.json` (external services), the body of the sent appeal email to appeals@aetna-sim.com must include a claim reference block with all six required fields in order — Claim ID: CLM-2026-00001, Patient ID, DOS, CPT Code(s), Payer, and Claim Status — formatted as `<field>: <value>`",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n target_to = \"appeals@aetna-sim.com\"\n # Find the appeal email\n appeal_email = None\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n subject = email.get(\"subject\", \"\").lower()\n if target_to in to_addr and \"clm-2026-00001\" in subject and \"medical necessity\" in subject:\n appeal_email = email\n break\n if appeal_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find the appeal email to appeals@aetna-sim.com for CLM-2026-00001.\"}\n body = appeal_email.get(\"body_text\", \"\") or \"\"\n # Check for the six required fields as field: value pairs\n required_fields = [\n r\"claim\\s*id\\s*:\",\n r\"patient\\s*id\\s*:\",\n r\"(date of service|dos)\\s*:\",\n r\"cpt\\s*code\\(?s\\)?\\s*:\",\n r\"payer\\s*:\",\n r\"claim\\s*status\\s*:\",\n ]\n body_lower = body.lower()\n missing = []\n for field_pattern in required_fields:\n if not re.search(field_pattern, body_lower):\n missing.append(field_pattern)\n # Also verify CLM-2026-00001 appears as claim ID value\n has_claim_id = \"clm-2026-00001\" in body_lower\n if missing:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Appeal email body missing required claim reference fields: {missing}\"}\n if not has_claim_id:\n return {\"pass\": False, \"score\": 0.7, \"feedback\": \"Appeal email body does not contain CLM-2026-00001 as claim ID value.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Appeal email body contains all six required claim reference fields including CLM-2026-00001.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "4531e92a-4e73-49ee-b905-c255a1b249d9",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to marcus.thompson@pathfinder.med",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n target = \"marcus.thompson@pathfinder.med\"\n matching = [e for e in sent_emails if target in e.get(\"to_addr\", \"\").lower()]\n count = len(matching)\n if count == 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email to {target}.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email to {target}, found {count}.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "691df127-cc63-46e9-9094-dd95f22a8395",
|
|
"sort_order": 8,
|
|
"rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email referencing CLM-2026-00004 (in subject or body)",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n ref = \"clm-2026-00004\"\n matching = []\n for e in sent_emails:\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n body = (e.get(\"body_text\", \"\") or \"\").lower()\n if ref in subject or ref in body:\n matching.append(e)\n count = len(matching)\n if count == 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 sent email referencing CLM-2026-00004.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 sent email referencing CLM-2026-00004, found {count}.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "e1b74eea-2f47-4439-830f-20d837c762f1",
|
|
"sort_order": 9,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cell M5 must be 'Denied' (unchanged)",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n m5 = _clean_excel_value(str(ws[\"M5\"].value or \"\").strip())\n if m5.lower() == \"denied\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M5='{m5}' as expected ('Denied').\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M5='Denied', got M5='{m5}'.\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "354dc4f9-5783-4cba-9522-4919b0bf1fc8",
|
|
"sort_order": 10,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, cell D8 must be 'Level 3: Medium' and cell E8 must be 'In Progress'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"PBC_RA_Queue.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n d8 = _clean_excel_value(str(ws[\"D8\"].value or \"\").strip())\n e8 = _clean_excel_value(str(ws[\"E8\"].value or \"\").strip())\n d8_ok = d8.lower() == \"level 3: medium\"\n e8_ok = e8.lower() == \"in progress\"\n if d8_ok and e8_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D8='{d8}', E8='{e8}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D8='Level 3: Medium', E8='In Progress'. Got D8='{d8}', E8='{e8}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "cb7f8d61-6a01-4592-a7ae-f0cc7e544e8c",
|
|
"sort_order": 11,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, the row with CLM-2026-00022 in column A must have columns B through R matching: PAT-10587, PRV-2081, Blue Cross Blue Shield, 99213-25, L82.1, 2026-03-23, [blank], $225.00, $0.00, $0.00, $0.00, New, [blank], 7, [blank], [blank], CLM-2026-00003",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef normalize_currency(val):\n if val is None:\n return None\n s = str(val).strip().replace(\"$\", \"\").replace(\",\", \"\")\n try:\n return float(s)\n except Exception:\n return str(val).strip()\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n target_row = None\n for row in ws.iter_rows():\n if str(row[0].value or \"\").strip().upper() == \"CLM-2026-00022\":\n target_row = row\n break\n if target_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row with CLM-2026-00022 not found in PBC_Claim_Register.xlsx\"}\n expected = [\n (\"PAT-10587\", False),\n (\"PRV-2081\", False),\n (\"Blue Cross Blue Shield\", False),\n (\"99213-25\", False),\n (\"L82.1\", False),\n (\"2026-03-23\", False),\n (None, True),\n (225.00, False),\n (0.00, False),\n (0.00, False),\n (0.00, False),\n (\"New\", False),\n (None, True),\n (7, False),\n (None, True),\n (None, True),\n (\"CLM-2026-00003\", False),\n ]\n mismatches = []\n for i, (exp_val, is_blank) in enumerate(expected):\n col_idx = i + 1\n cell_val = target_row[col_idx].value\n col_letter = chr(ord('B') + i)\n if is_blank:\n if cell_val is not None and str(cell_val).strip() != \"\":\n mismatches.append(f\"Col {col_letter}: expected blank, got '{cell_val}'\")\n elif isinstance(exp_val, (float, int)):\n cleaned = _clean_excel_value(str(cell_val) if cell_val is not None else \"\")\n norm = normalize_currency(cleaned)\n if isinstance(norm, (float, int)):\n tol = max(abs(exp_val) * 0.01, 0.01)\n if abs(float(norm) - float(exp_val)) > tol:\n mismatches.append(f\"Col {col_letter}: expected {exp_val}, got {cell_val}\")\n else:\n mismatches.append(f\"Col {col_letter}: expected {exp_val}, got '{cell_val}'\")\n else:\n cell_str = _clean_excel_value(str(cell_val or \"\").strip())\n exp_str = str(exp_val or \"\").strip()\n if exp_str == \"2026-03-23\":\n import datetime\n matched = False\n if cell_str == \"2026-03-23\":\n matched = True\n elif hasattr(cell_val, 'strftime'):\n if cell_val.strftime(\"%Y-%m-%d\") == \"2026-03-23\":\n matched = True\n elif \"2026\" in cell_str and \"03\" in cell_str and \"23\" in cell_str:\n matched = True\n if not matched:\n mismatches.append(f\"Col {col_letter}: expected '2026-03-23', got '{cell_val}'\")\n elif cell_str.lower() != exp_str.lower():\n mismatches.append(f\"Col {col_letter}: expected '{exp_str}', got '{cell_str}'\")\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00022 row has all expected values in columns B-R.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00022 row mismatches: {'; '.join(mismatches)}\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "52d192af-f15d-492f-9676-e788ec18f339",
|
|
"sort_order": 12,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cell M4 must be 'Resubmitted'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n m4 = _clean_excel_value(str(ws[\"M4\"].value or \"\").strip())\n if m4.lower() == \"resubmitted\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"M4='{m4}' as expected ('Resubmitted').\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected M4='Resubmitted', got M4='{m4}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "cc05be66-0e7e-49fa-9a39-d53a9fe54222",
|
|
"sort_order": 13,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, cell D7 must be 'Level 3: Medium' and cell E7 must be 'Closed'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*RA_Queue*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n d7 = _clean_excel_value(str(ws[\"D7\"].value or \"\").strip())\n e7 = _clean_excel_value(str(ws[\"E7\"].value or \"\").strip())\n d7_ok = d7.lower() == \"level 3: medium\"\n e7_ok = e7.lower() == \"closed\"\n if d7_ok and e7_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D7='{d7}', E7='{e7}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected D7='Level 3: Medium', E7='Closed'. Got D7='{d7}', E7='{e7}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "5727545a-b9f5-4dbc-927d-61fd63b61603",
|
|
"sort_order": 14,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell H6 must be 'Add Modifier and Resubmit' and cell G6 must be 'Resubmitted'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n g6 = _clean_excel_value(str(ws[\"G6\"].value or \"\").strip())\n h6 = _clean_excel_value(str(ws[\"H6\"].value or \"\").strip())\n g6_ok = g6.lower() == \"resubmitted\"\n h6_ok = h6.lower() == \"add modifier and resubmit\"\n if g6_ok and h6_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G6='{g6}', H6='{h6}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected G6='Resubmitted', H6='Add Modifier and Resubmit'. Got G6='{g6}', H6='{h6}'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "f368d8c4-9a06-46cd-b9a0-6e007b7958e6",
|
|
"sort_order": 15,
|
|
"rubric_text": "In `mailbox.json` (external services), there must be exactly one sent email to catherine.lawson@pathfinder.med with subject 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001' and a body that states no email confirmation has been received and explicitly requests Catherine confirm receipt and acknowledge the escalation",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n\n target = \"catherine.lawson@pathfinder.med\"\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n esc_emails = [\n e for e in sent_emails\n if target in e.get(\"to_addr\", \"\").lower()\n and any(kw in e.get(\"subject\", \"\").lower()\n for kw in [\"esc-00001\", \"escalation\", \"high priority\", \"follow-up\", \"follow up\"])\n ]\n\n if len(esc_emails) == 0:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No ESC-00001 follow-up email found sent to Catherine Lawson.\"}\n if len(esc_emails) > 1:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Expected exactly 1 ESC-00001 follow-up to Catherine, found {len(esc_emails)}.\"}\n\n email = esc_emails[0]\n subject = email.get(\"subject\", \"\").strip()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n\n subject_ok = \"esc-00001\" in subject.lower() and \"escalation\" in subject.lower() and \"high priority\" in subject.lower()\n no_confirm_ok = any(p in body for p in [\"no confirmation\", \"not yet received\", \"have not received\", \"haven't received\", \"no email confirmation\"])\n request_ok = any(p in body for p in [\"please confirm\", \"confirm receipt\", \"acknowledge\", \"confirmation\"])\n\n issues = []\n if not subject_ok:\n issues.append(f\"Subject '{subject}' does not match expected 'HIGH PRIORITY - Escalation Follow-Up - ESC-00001'\")\n if not no_confirm_ok:\n issues.append(\"Body does not state that no email confirmation has been received\")\n if not request_ok:\n issues.append(\"Body does not explicitly request confirmation or acknowledgment from Catherine\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": \"; \".join(issues)}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found 1 sent email to {target} with correct subject and body content.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "2987934f-f5a7-4dd3-ad64-54f39c2a39ce",
|
|
"sort_order": 16,
|
|
"rubric_text": "In `PBC_Escalation_Log.xlsx`, the row for ESC-00001 must have Escalation_Category equal to 'Financial' and Escalation_Reason equal to 'Pattern Recurrence'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Escalation_Log*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Escalation_Log.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n header_row = None\n for row in ws.iter_rows(min_row=1, max_row=5):\n row_vals = [str(c.value or \"\").strip().lower() for c in row]\n if any(\"escalation\" in v for v in row_vals):\n header_row = row\n break\n category_col = None\n reason_col = None\n if header_row:\n for i, cell in enumerate(header_row):\n v = str(cell.value or \"\").strip().lower()\n if \"category\" in v:\n category_col = i\n if \"reason\" in v:\n reason_col = i\n target_row = None\n for row in ws.iter_rows():\n for cell in row:\n if str(cell.value or \"\").strip().upper() == \"ESC-00001\":\n target_row = list(row)\n break\n if target_row:\n break\n if target_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row for ESC-00001 not found in PBC_Escalation_Log.xlsx\"}\n row_values = [_clean_excel_value(str(c.value or \"\").strip()) for c in target_row]\n category_found = any(v.lower() == \"financial\" for v in row_values)\n reason_found = any(v.lower() == \"pattern recurrence\" for v in row_values)\n if category_col is not None and reason_col is not None:\n cat_val = _clean_excel_value(str(target_row[category_col].value or \"\").strip())\n reason_val = _clean_excel_value(str(target_row[reason_col].value or \"\").strip())\n cat_ok = cat_val.lower() == \"financial\"\n reason_ok = reason_val.lower() == \"pattern recurrence\"\n if cat_ok and reason_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"ESC-00001 row has Escalation_Category='{cat_val}' and Escalation_Reason='{reason_val}'.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row: expected Category='Financial', Reason='Pattern Recurrence'. Got Category='{cat_val}', Reason='{reason_val}'.\"}\n if category_found and reason_found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"ESC-00001 row contains 'Financial' and 'Pattern Recurrence'.\"}\n missing = []\n if not category_found:\n missing.append(\"Escalation_Category='Financial'\")\n if not reason_found:\n missing.append(\"Escalation_Reason='Pattern Recurrence'\")\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"ESC-00001 row missing: {', '.join(missing)}. Row values: {row_values}\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "cb08be27-fe03-49d1-bc55-49cf4b03d023",
|
|
"sort_order": 17,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, cell G8 must be 'In Progress' and cell H8 must be blank",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Denial_Worklist*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Denial_Worklist.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n g8 = _clean_excel_value(str(ws[\"G8\"].value or \"\").strip())\n h8 = ws[\"H8\"].value\n g8_ok = g8.lower() == \"in progress\"\n h8_blank = h8 is None or str(h8).strip() == \"\"\n if g8_ok and h8_blank:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G8='{g8}' (In Progress) and H8 is blank as expected.\"}\n issues = []\n if not g8_ok:\n issues.append(f\"G8 expected 'In Progress', got '{g8}'\")\n if not h8_blank:\n issues.append(f\"H8 expected blank, got '{h8}'\")\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "cde2e84a-4947-4cf2-bf73-30608dce9fc8",
|
|
"sort_order": 18,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, no row with CLM-2026-00004 in column A must have a corrected claim reference in column R",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n\n def _clean_excel_value(val):\n if isinstance(val, str):\n val = val.strip()\n if val.startswith('=\"') and val.endswith('\"'):\n val = val[2:-1]\n return val\n\n files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n for row in ws.iter_rows():\n col_a = _clean_excel_value(str(row[0].value or \"\").strip().upper())\n if col_a == \"CLM-2026-00004\":\n if len(row) > 17:\n col_r = row[17].value\n if col_r is not None and str(col_r).strip() != \"\":\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"CLM-2026-00004 row has a value in column R (corrected claim reference): '{col_r}'. Expected blank.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"CLM-2026-00004 row has no corrected claim reference in column R (as expected).\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No row with CLM-2026-00004 found; condition satisfied (no corrected claim reference).\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|