178 lines
90 KiB
JSON
178 lines
90 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "07fafb80-83f3-401a-bef5-05533f53057d",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, rows in cells A3:F11 must contain all 9 claim entries with denial_category='Contractual/Payer Error', CO-45 denial code, recovery_action='Appeal - Contractual Rate Dispute', date='02/17/2026', and recovery_status='Escalated' for claim IDs: HIC-19887899, HIC-19887900, HIC-19887901, HIC-25124778, HIC-25124779, HIC-25124780, HIC-26877757, HIC-26877758, HIC-26877795",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n\n xlsx_files = list(workspace.glob('**/PBC_Denial_Worklist*'))\n if not xlsx_files:\n xlsx_files = list(workspace.glob('**/*Denial_Worklist*'))\n if not xlsx_files:\n xlsx_files = [f for f in workspace.glob('**/*.xlsx')\n if 'denial' in f.name.lower() or 'worklist' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No PBC_Denial_Worklist.xlsx found in workspace'}\n\n required_ids = {\n 'HIC-19887899', 'HIC-19887900', 'HIC-19887901',\n 'HIC-25124778', 'HIC-25124779', 'HIC-25124780',\n 'HIC-26877757', 'HIC-26877758', 'HIC-26877795'\n }\n\n best_result = {'pass': False, 'score': 0.0, 'feedback': 'No matching claim IDs found in denial worklist'}\n\n for xlsx_file in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception:\n continue\n\n for sheet in wb.worksheets:\n # Detect column positions from the header row so layout changes don't break checks\n col_map = {}\n for cell in sheet[1]:\n if cell.value:\n key = str(cell.value).strip().lower()\n col_map[key] = cell.column - 1 # 0-based index into row_vals\n\n # Fall back to known fixed positions if headers are absent or renamed\n cat_col = col_map.get('category', col_map.get('denial_category', 0))\n id_col = col_map.get('claim_id', 1)\n code_col = col_map.get('co_code', col_map.get('denial_code', 2))\n action_col = col_map.get('recovery_action', col_map.get('action', 3))\n date_col = col_map.get('date_received', col_map.get('date', 4))\n status_col = col_map.get('recovery_status', col_map.get('status', 5))\n\n found_ids = set()\n issues = []\n\n for row in sheet.iter_rows(min_row=2, max_row=100, values_only=False):\n raw_vals = [cell.value for cell in row]\n row_vals = []\n for v in raw_vals:\n if v is None:\n row_vals.append('')\n elif isinstance(v, datetime):\n row_vals.append(v.strftime('%m/%d/%Y'))\n else:\n row_vals.append(str(v).strip())\n\n # Identify which target claim ID appears in this row\n row_id = None\n for rid in required_ids:\n rid_lower = rid.lower()\n rid_nodash = rid_lower.replace('-', '')\n for v in row_vals:\n v_lower = v.lower()\n if rid_lower in v_lower or rid_nodash in v_lower.replace('-', ''):\n row_id = rid\n break\n if row_id:\n break\n if row_id is None:\n continue\n found_ids.add(row_id)\n\n # --- Category: check the Category column value ONLY ---\n # Intentionally NOT searching the combined row string — 'contractual'\n # appearing in the action field ('Appeal - Contractual Rate Dispute')\n # must not satisfy the category check.\n cat_val = row_vals[cat_col] if cat_col < len(row_vals) else ''\n cat_ok = (\n 'contractual/payer error' in cat_val.lower() or\n ('contractual' in cat_val.lower() and 'payer' in cat_val.lower()) or\n 'payer error' in cat_val.lower()\n )\n\n # --- CO code: check the CO_Code column value
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "de776c20-2323-48f3-b2f3-2536938b7ac7",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), exactly 1 email in draft to no one (\"to_addr\": empty or NULL)",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services path first, then workspace\n paths_to_check = []\n if external_services_path:\n paths_to_check.append(Path(external_services_path) / 'mailbox.json')\n paths_to_check.append(Path(workspace_path) / 'mailbox.json')\n\n for mailbox_path in paths_to_check:\n if not mailbox_path.exists():\n continue\n try:\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get('emails', [])\n\n # The rubric says: exactly 1 email in draft to no one (\"to_addr\": empty or NULL)\n # This means we need a draft email where to_addr is empty or null\n draft_emails_no_to = []\n for email in emails:\n folder = str(email.get('folder', '')).lower().strip()\n to_addr = email.get('to_addr', None)\n\n is_draft = 'draft' in folder\n # to_addr should be empty or NULL\n to_empty = (\n to_addr is None\n or str(to_addr).strip() == ''\n or str(to_addr).strip().lower() == 'null'\n or str(to_addr).strip().lower() == 'none'\n or to_addr == []\n )\n\n if is_draft and to_empty:\n draft_emails_no_to.append(email)\n\n if len(draft_emails_no_to) == 1:\n target = draft_emails_no_to[0]\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 1 draft email with empty/null to_addr. Email ID: {target.get(\"email_id\")}, from: {target.get(\"from_addr\")}, subject: {target.get(\"subject\")}'\n }\n elif len(draft_emails_no_to) == 0:\n # Also check for drafts more loosely - maybe to_addr is present but it's a new draft\n all_drafts = [e for e in emails if 'draft' in str(e.get('folder', '')).lower()]\n if all_drafts:\n details = []\n for d in all_drafts:\n details.append(f'Email ID {d.get(\"email_id\")}: from={d.get(\"from_addr\")}, to={d.get(\"to_addr\")}')\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'Found {len(all_drafts)} draft email(s) but none have empty/null to_addr. Drafts found: ' + '; '.join(details)\n }\n else:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': 'No draft emails found in mailbox.json'\n }\n else:\n details = []\n for d in draft_emails_no_to:\n details.append(f'Email ID {d.get(\"email_id\")}: from={d.get(\"from_addr\")}, to={d.get(\"to_addr\")}')\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'Expected exactly 1 draft email with empty/null to_addr, found {len(draft_emails_no_to)}: ' + '; '.join(details)\n }\n except Exception as e:\n continue\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found or unreadable in any checked path'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d35a5f4e-e42c-4687-bf4f-8324873d2149",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), the draft email has \"body_text\":\"Dear [Payer Billing/Appeals Contact Name],\\n\\nI am writing on behalf of Pathfinder Billing and Coding to formally dispute the payment amounts received for CPT 11241 under Payer ID A4789P. The following claims were paid below the contracted rate with no adjustment reason code provided:\\n\\nHIC-19887899 | 2025-12-01 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-19887900 | 2025-12-01 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-19887901 | 2025-12-03 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-25124778 | 2025-12-10 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-25124779 | 2025-12-12 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-25124780 | 2025-12-15 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-26877757 | 2025-12-29 | Contracted: $80 | Paid: $70 | Variance: $10\\nHIC-26877758 | 2026-01-02 | Contracted: $75 | Paid: $65 | Variance: $10\\nHIC-26877795 | 2026-01-05 | Contracted: $75 | Paid: $65 | Variance: $10\\n\\nTotal outstanding variance: $90\\n\\nWe request review and remittance of the outstanding balance within [X] Business Days. Please confirm receipt of this dispute and provide a reference number for tracking.\\n\\nJoe Malfroy | Pathfinder Billing and Coding | rcm@pathfinder.com\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # The rubric specifies a very specific body_text in a draft email in mailbox.json (external services).\n # We need to find a draft email whose body_text matches the expected content.\n \n # Expected body text from the rubric\n expected_body = (\n \"Dear [Payer Billing/Appeals Contact Name],\\n\\n\"\n \"I am writing on behalf of Pathfinder Billing and Coding to formally dispute the payment amounts received for CPT 11241 under Payer ID A4789P. \"\n \"The following claims were paid below the contracted rate with no adjustment reason code provided:\\n\\n\"\n \"HIC-19887899 | 2025-12-01 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-19887900 | 2025-12-01 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-19887901 | 2025-12-03 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-25124778 | 2025-12-10 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-25124779 | 2025-12-12 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-25124780 | 2025-12-15 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-26877757 | 2025-12-29 | Contracted: $80 | Paid: $70 | Variance: $10\\n\"\n \"HIC-26877758 | 2026-01-02 | Contracted: $75 | Paid: $65 | Variance: $10\\n\"\n \"HIC-26877795 | 2026-01-05 | Contracted: $75 | Paid: $65 | Variance: $10\\n\\n\"\n \"Total outstanding variance: $90\\n\\n\"\n \"We request review and remittance of the outstanding balance within [X] Business Days. \"\n \"Please confirm receipt of this dispute and provide a reference number for tracking.\\n\\n\"\n \"Joe Malfroy | Pathfinder Billing and Coding | rcm@pathfinder.com\"\n )\n\n # Key fragments to check for partial credit\n key_fragments = [\n (\"Payer Billing/Appeals Contact Name\", \"Salutation placeholder\"),\n (\"Pathfinder Billing and Coding\", \"Company name\"),\n (\"CPT 11241\", \"CPT code\"),\n (\"Payer ID A4789P\", \"Payer ID\"),\n (\"HIC-19887899\", \"Claim HIC-19887899\"),\n (\"HIC-19887900\", \"Claim HIC-19887900\"),\n (\"HIC-19887901\", \"Claim HIC-19887901\"),\n (\"HIC-25124778\", \"Claim HIC-25124778\"),\n (\"HIC-25124779\", \"Claim HIC-25124779\"),\n (\"HIC-25124780\", \"Claim HIC-25124780\"),\n (\"HIC-26877757\", \"Claim HIC-26877757\"),\n (\"HIC-26877758\", \"Claim HIC-26877758\"),\n (\"HIC-26877795\", \"Claim HIC-26877795\"),\n (\"2025-12-01\", \"DOS 2025-12-01\"),\n (\"2025-12-03\", \"DOS 2025-12-03\"),\n (\"2025-12-10\", \"DOS 2025-12-10\"),\n (\"2025-12-12\", \"DOS 2025-12-12\"),\n (\"2025-12-15\", \"DOS 2025-12-15\"),\n (\"2025-12-29\", \"DOS 2025-12-29\"),\n (\"2026-01-02\", \"DOS 2026-01-02\"),\n (\"2026-01-05\", \"DOS 2026-01-05\"),\n (\"Total outstanding variance: $90\", \"Total variance $90\"),\n (\"Joe Malfroy\", \"Joe Malfroy signature\"),\n (\"rcm@pathfinder.com\", \"Email address\"),\n ]\n\n # Try to find mailbox.json\n paths_to_check = []\n if external_services_path:\n paths_to_check.append(Path(external_services_path) / 'mailbox.json')\n paths_to_check.append(Path(workspace_path) / 'mailbox.json')\n\n mailbox_data = None\n mailbox_found_path = None\n for mailbox_path in paths_to_check:\n if mailbox_path.exists():\n try:\n with open(mailbox_path) as f:\n mailbox_data = json.load(f)\n mailbox_found_path = str(mailbox_path)\n break\n except Exception as e:\n continue\n\n if mailbox_data is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found or unreadable in workspace or external services paths.'}\n\n emails = mailbox_data.get('emails', [])\n if not emails:\n return {'pass': Fa
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "bbecdc58-6aa5-4de9-9e56-4d70926c42dc",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In file `PBC_Escalation_Log.xlsx`, row 3 must contain Claim_ID with 'HIC-19887899, HIC-19887900, HIC-19887901, HIC-25124778, HIC-25124779, HIC-25124780, HIC-26877757, HIC-26877758, HIC-26877795', Payer_ID with 'A4789P', Category 'Operational', Esc_Reason 'Pattern Recurrence', and Resolution_Tkg must be 'RCM Manager on 02/17/2026'",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find escalation log files with increasingly broad patterns\n xlsx_files = list(Path(workspace_path).glob('*Escalation_Log*'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*escalation*log*'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'escalation' in f.name.lower() or 'esc' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx file found in workspace'}\n\n required_claim_ids = [\n 'HIC-19887899', 'HIC-19887900', 'HIC-19887901',\n 'HIC-25124778', 'HIC-25124779', 'HIC-25124780',\n 'HIC-26877757', 'HIC-26877758', 'HIC-26877795'\n ]\n\n def check_date_present(text):\n \"\"\"Check for 02/17/2026 or similar date formats.\"\"\"\n date_patterns = [\n r'02[/\\-.]17[/\\-.]2026',\n r'2[/\\-.]17[/\\-.]2026'\n ]\n for pat in date_patterns:\n if re.search(pat, text, re.IGNORECASE):\n return True\n return False\n\n def check_row(row_vals):\n \"\"\"Check a row for all required fields. Returns (pass, details_dict).\"\"\"\n row_str = ' '.join(row_vals)\n row_str_lower = row_str.lower()\n\n # Check Claim_IDs - at least some of the required claim IDs should be present\n found_claims = [cid for cid in required_claim_ids if cid.lower() in row_str_lower]\n has_claim_ids = len(found_claims) >= 5 # at least majority present (be lenient)\n\n has_payer_id = 'a4789p' in row_str_lower\n has_operational = 'operational' in row_str_lower\n has_pattern_recurrence = 'pattern recurrence' in row_str_lower or ('pattern' in row_str_lower and 'recur' in row_str_lower)\n\n # Resolution_Tkg: 'RCM Manager on 02/17/2026'\n # Accept variations: \"RCM Manager\", date present\n has_manager_ref = 'rcm manager' in row_str_lower\n has_date = check_date_present(row_str)\n # Also accept notified/escalated language as part of resolution\n has_notified_or_escalated = (\n 'notified' in row_str_lower or 'notify' in row_str_lower or 'notification' in row_str_lower or\n 'escalated' in row_str_lower or 'escalate' in row_str_lower or 'escalation' in row_str_lower or\n ' on ' in row_str_lower # \"RCM Manager on 02/17/2026\"\n )\n has_resolution = has_manager_ref and has_date\n\n details = {\n 'Claim_IDs': has_claim_ids,\n 'A4789P': has_payer_id,\n 'Operational': has_operational,\n 'Pattern Recurrence': has_pattern_recurrence,\n 'RCM Manager in Resolution': has_manager_ref,\n 'date in Resolution': has_date,\n }\n total = len(details)\n matched = sum(1 for v in details.values() if v)\n all_pass = all(details.values())\n return all_pass, details, matched / total, found_claims\n\n best_score = 0.0\n best_feedback = ''\n\n for xlsx_file in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception as e:\n continue\n for sheet in wb.worksheets:\n # First check row 3 specifically (as rubric states)\n if sheet.max_row >= 3:\n row3 = sheet[3]\n row_vals = [str(cell.value).strip() if cell.value is not None else '' for cell in row3]\n all_pass, details, score, found_claims = check_row(row_vals)\n if all_pass:\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 in {xlsx_file.name} contains all required escalation fields: Claim IDs ({len(found_claims)} found), A4789P, Operational, Pattern Recurrence, and Resolution with RCM Manager on 02/17/2026. Values:
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "431da804-2bbf-45fb-bc08-0b94279781f4",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In file `PBC_Escalation_Log.xlsx`, cell A3 must be either blank/null OR contain any one of the following 9 claim IDs: HIC-19887899, HIC-19887900, HIC-19887901, HIC-25124778, HIC-25124779, HIC-25124780, HIC-26877757, HIC-26877758, HIC-26877795",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Target only the Escalation Log - do not fall back to all PBC files\n workspace = Path(workspace_path)\n xlsx_files = list(workspace.glob('*Escalation_Log*.xlsx'))\n if not xlsx_files:\n xlsx_files = list(workspace.glob('*escalation*log*.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in workspace.glob('*.xlsx') if 'escalation' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Escalation_Log.xlsx not found in workspace'}\n\n required_ids = [\n 'HIC-19887899', 'HIC-19887900', 'HIC-19887901',\n 'HIC-25124778', 'HIC-25124779', 'HIC-25124780',\n 'HIC-26877757', 'HIC-26877758', 'HIC-26877795'\n ]\n\n # Collect results across all matching files; do NOT return early on failure\n best_result = {'pass': False, 'score': 0.0, 'feedback': 'No matching escalation log found'}\n\n for xlsx_file in xlsx_files:\n try:\n wb = openpyxl.load_workbook(str(xlsx_file), data_only=True)\n except Exception as e:\n continue\n\n for sheet in wb.worksheets:\n cell_a3 = sheet['A3']\n val = cell_a3.value\n\n # Blank/null is acceptable\n if val is None or str(val).strip() == '':\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'A3 in {xlsx_file.name} is blank/null (acceptable per rubric)'\n }\n\n val_str = str(val).strip().upper()\n\n # Check if A3 contains ANY ONE of the 9 claim IDs\n for rid in required_ids:\n if rid.upper() in val_str:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'A3 in {xlsx_file.name} contains claim ID {rid}. Value: {str(val)[:200]}'\n }\n\n # Flexible match: without hyphen\n for rid in required_ids:\n rid_nohyphen = rid.replace('-', '')\n if rid_nohyphen in val_str.replace('-', '').replace(' ', ''):\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'A3 in {xlsx_file.name} contains claim ID {rid} (flexible match). Value: {str(val)[:200]}'\n }\n\n # Also scan other cells in row 3 in case the agent put the ID in a different column\n max_col = sheet.max_column if sheet.max_column else 20\n for col in range(1, max_col + 1):\n cell_val = sheet.cell(row=3, column=col).value\n if cell_val is not None:\n cell_str = str(cell_val).strip().upper()\n for rid in required_ids:\n if rid.upper() in cell_str:\n return {\n 'pass': True,\n 'score': 0.8,\n 'feedback': f'Claim ID {rid} found in row 3 column {col} of {xlsx_file.name} (not A3, but close). Value: {cell_str[:200]}'\n }\n\n # This file's A3 does not match — record as best failure but keep checking other files\n best_result = {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'A3 in {xlsx_file.name} is not blank/null and does not contain any of the 9 required claim IDs. A3 value: {str(val)[:300]}'\n }\n # NOTE: do NOT return here — continue to next file\n\n return best_result",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e2c9d058-5586-457e-afde-9b5d50ed6b4d",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), there must be an email in the Sent folder to 'joem@pathfinder.com' ",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n paths_to_check = []\n if external_services_path:\n paths_to_check.append(Path(external_services_path) / 'mailbox.json')\n paths_to_check.append(Path(workspace_path) / 'mailbox.json')\n\n mailbox_found = False\n all_emails_info = []\n\n for mailbox_path in paths_to_check:\n if not mailbox_path.exists():\n continue\n mailbox_found = True\n try:\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get('emails', [])\n for email in emails:\n folder = (email.get('folder', '') or '').lower()\n to_addr = (email.get('to_addr', '') or email.get('to', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n body_text = (email.get('body_text', '') or email.get('body', '') or '').lower()\n\n is_sent = 'sent' in folder\n is_to_joe = 'joem@pathfinder.com' in to_addr\n\n if is_sent and is_to_joe:\n all_emails_info.append({\n 'to': to_addr,\n 'subject': subject[:80],\n 'folder': folder\n })\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found sent email to joem@pathfinder.com. Subject: {subject[:80]}'\n }\n\n except Exception as e:\n continue\n\n if not mailbox_found:\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found or unreadable in either workspace or external services path.'}\n\n detail = ''\n if all_emails_info:\n detail = f' Found {len(all_emails_info)} email(s) but none were in the Sent folder addressed to joem@pathfinder.com. Email snippets: {json.dumps(all_emails_info[:3])}'\n else:\n detail = ' No sent emails to joem@pathfinder.com were found at all.'\n\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'No email to joem@pathfinder.com found in the Sent folder.{detail}'\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774861529578",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, rows in cells A3:E11 must contain all 9 claim entries with the following properties: trigger=\"CO-45\", trigger_date=\"02/17/2026\", staff_assigned=\"Joe Malfroy\", status=\"Open\" for claim IDs: HIC-19887899, HIC-19887900, HIC-19887901, HIC-25124778, HIC-25124779, HIC-25124780, HIC-26877757, HIC-26877758, HIC-26877795.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime as dt\n\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file flexibly\n target_file = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n if not target_file.exists():\n candidates = list(Path(workspace_path).rglob(\"PBC_RA_Queue.xlsx\"))\n if not candidates:\n # Also try case-insensitive\n candidates = [p for p in Path(workspace_path).rglob(\"*.xlsx\") if \"pbc_ra_queue\" in p.name.lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n target_file = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n expected_claim_ids = {\n \"HIC-19887899\", \"HIC-19887900\", \"HIC-19887901\",\n \"HIC-25124778\", \"HIC-25124779\", \"HIC-25124780\",\n \"HIC-26877757\", \"HIC-26877758\", \"HIC-26877795\"\n }\n\n # Scan the entire sheet to find claim entries, being very flexible about location\n found_claims = {}\n detail_msgs = []\n\n # Scan broadly\n max_row = max(ws.max_row or 1, 50)\n max_col = max(ws.max_column or 1, 20)\n\n for row_num in range(1, max_row + 1):\n row_values = []\n for col in range(1, max_col + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_values.append(cell_val)\n\n # Try to find a claim ID in this row\n claim_id_found = None\n for col_idx, val in enumerate(row_values):\n if val is not None:\n val_str = str(val).strip().upper()\n for cid in expected_claim_ids:\n if val_str == cid.upper():\n claim_id_found = cid\n break\n # Also try without hyphen\n if val_str.replace(\"-\", \"\").replace(\" \", \"\") == cid.replace(\"-\", \"\"):\n claim_id_found = cid\n break\n if claim_id_found:\n break\n\n if claim_id_found is None:\n continue\n\n if claim_id_found in found_claims:\n continue # Already found this claim\n\n # Check all cells in the row for trigger, trigger_date, staff_assigned, status\n trigger_ok = False\n trigger_date_ok = False\n staff_ok = False\n status_ok = False\n\n for col_idx, val in enumerate(row_values):\n if val is None:\n continue\n val_str = str(val).strip()\n\n # Check trigger - accept CO-45, CO 45, CO45, co-45, etc.\n if re.match(r'^CO[\\s\\-]?45$', val_str, re.IGNORECASE):\n trigger_ok = True\n\n # Check trigger_date - accept various date formats for 02/17/2026\n if _is_matching_date(val, 2026, 2, 17):\n trigger_date_ok = True\n\n # Check staff_assigned - Joe Malfroy\n normalized = val_str.lower().replace(\" \", \"\").replace(\",\", \"\").replace(\".\", \"\")\n if \"joemalfroy\" in normalized or \"malfroyj\" in normalized:\n staff_ok = True\n if \"malfroy\" in val_str.lower() and \"joe\" in val_str.lower():\n staff_ok = True\n\n # Check status - Open\n if val_str.lower().strip() == \"open\":\n status_ok = True\n\n found_claims[claim_id_found] = {\n \"row\": row_num,\n \"trigger_ok\": trigger_ok,\n \"trigger_date_ok\": trigger_date_ok,\n \"staff_ok\": staff_ok,\n \"status_ok\": status_ok,\n \"row_values\": row_values[:max_col]\n }\n\n # Evaluate results\n total_pass = 0\n total_partial = 0\n\n for cid in sorted(
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016401116",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "`PBC_Denial_Worklist.xlsx` has exactly 11 rows (1 header row and 10 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n \n if not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Denial_Worklist.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(target_file, read_only=True)\n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n all_rows = list(ws.iter_rows(values_only=True))\n total_rows = len(all_rows)\n \n # Also count non-empty rows (rows with at least one non-None value)\n non_empty_rows = [row for row in all_rows if any(cell is not None for cell in row)]\n non_empty_count = len(non_empty_rows)\n \n wb.close()\n \n # Check if total rows == 11 (1 header + 10 data)\n if total_rows == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_Denial_Worklist.xlsx has exactly {total_rows} rows (1 header + 10 data rows). PASS.\"}\n \n # Be lenient: also check non-empty rows in case there are trailing empty rows\n if non_empty_count == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_Denial_Worklist.xlsx has {non_empty_count} non-empty rows (1 header + 10 data rows) out of {total_rows} total rows. PASS.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PBC_Denial_Worklist.xlsx has {total_rows} total rows and {non_empty_count} non-empty rows. Expected exactly 11 rows (1 header + 10 data rows).\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Denial_Worklist.xlsx: {str(e)}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016407757",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "The file `PBC_RA_Queue.xlsx` must exist with the following headers in row 1: Claim_ID | Trigger | Trigger_Date (MM/DD/YYYY) | Staff_Assigned | Status. Row 2 (the existing data row) must contain: HOH-7199789 | CO-16 | 12/23/2025 | Russell Commy | Closed.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n filepath = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n if not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in workspace at {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(filepath))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n \n # Check headers in row 1\n expected_headers = [\"Claim_ID\", \"Trigger\", \"Trigger_Date\", \"Staff_Assigned\", \"Status\"]\n # Also accept Trigger_Date (MM/DD/YYYY) as header\n actual_headers = []\n for col in range(1, 6):\n cell_val = ws.cell(row=1, column=col).value\n if cell_val is not None:\n actual_headers.append(str(cell_val).strip())\n else:\n actual_headers.append(\"\")\n \n header_pass = True\n header_feedback = []\n for i, expected in enumerate(expected_headers):\n if i < len(actual_headers):\n actual = actual_headers[i]\n # Normalize for comparison: remove parenthetical notes, strip whitespace, compare case-insensitively\n norm_expected = re.sub(r'\\s*\\(.*?\\)', '', expected).strip().lower().replace('_', '').replace(' ', '')\n norm_actual = re.sub(r'\\s*\\(.*?\\)', '', actual).strip().lower().replace('_', '').replace(' ', '')\n if norm_expected != norm_actual:\n header_pass = False\n header_feedback.append(f\"Column {i+1} header: expected '{expected}', got '{actual}'\")\n else:\n header_pass = False\n header_feedback.append(f\"Column {i+1} header missing, expected '{expected}'\")\n \n if not header_pass:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Header mismatch: \" + \"; \".join(header_feedback)}\n \n # Check row 2 data\n expected_row2 = {\n 1: \"HOH-7199789\",\n 2: \"CO-16\",\n 3: \"12/23/2025\",\n 4: \"Russell Commy\",\n 5: \"Closed\"\n }\n \n row2_pass = True\n row2_feedback = []\n \n for col, expected_val in expected_row2.items():\n cell = ws.cell(row=2, column=col)\n actual_val = cell.value\n \n if actual_val is None:\n row2_pass = False\n row2_feedback.append(f\"Row 2, Col {col}: expected '{expected_val}', got empty/None\")\n continue\n \n # Handle date column specially\n if col == 3:\n # Could be a datetime object or a string\n import datetime\n if isinstance(actual_val, datetime.datetime) or isinstance(actual_val, datetime.date):\n if isinstance(actual_val, datetime.datetime):\n actual_str = actual_val.strftime(\"%m/%d/%Y\")\n else:\n actual_str = actual_val.strftime(\"%m/%d/%Y\")\n if actual_str == expected_val:\n continue\n else:\n row2_pass = False\n row2_feedback.append(f\"Row 2, Col {col}: expected '{expected_val}', got date '{actual_str}'\")\n continue\n else:\n actual_str = str(actual_val).strip()\n if actual_str == expected_val:\n continue\n else:\n row2_pass = False\n row2_feedback.append(f\"Row 2, Col {col}: expected '{expected_val}', got '{actual_str}'\")\n continue\n \n actual_str = str(actual_val).strip()\n if actual_str.lower() == str(expected_val).strip().lower():\n continue\n else:\n row2_pass = False\n row2_feedback.append(f\"Row 2, Col {col}: expected '{expected_val}', got '{actual_str}'\")\n \n if not row2_pass:\n return {\"pass\"
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016420194",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "`PBC_RA_Queue.xlsx` has exactly 11 rows (1 header row and 10 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_RA_Queue.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n total_rows = 0\n for row in ws.iter_rows(min_row=1):\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n wb.close()\n \n if total_rows == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has exactly 11 non-empty rows (1 header + 10 data rows) as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {total_rows} non-empty rows, but expected exactly 11 (1 header + 10 data rows).\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_RA_Queue.xlsx: {str(e)}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016448496",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "The file `PBC_Denial_Worklist.xlsx` must exist with the headers: Category | Claim_ID | CO_Code | Recovery_Action | Date_Logged (MM/DD/YYYY) | Recovery_Status. Row 2 (the existing data row) must contain: Technical/Administrative | HOH-7199789 | CO-16 | Correct and Resubmit | 12/23/2025 | Resubmitted.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {target_file}\"}\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # Check headers in row 1\n expected_headers = [\"Category\", \"Claim_ID\", \"CO_Code\", \"Recovery_Action\", \"Date_Logged\", \"Recovery_Status\"]\n actual_headers = []\n for col in range(1, 7):\n val = ws.cell(row=1, column=col).value\n actual_headers.append(str(val).strip() if val is not None else \"\")\n\n def normalize_header(h):\n # Remove parenthetical, strip, lower, collapse whitespace/underscores\n h = re.sub(r'\\(.*?\\)', '', h)\n h = h.strip().lower().replace('_', '').replace(' ', '').replace('-', '')\n return h\n\n headers_ok = True\n header_feedback = []\n for i, (exp, act) in enumerate(zip(expected_headers, actual_headers)):\n if normalize_header(exp) != normalize_header(act):\n headers_ok = False\n header_feedback.append(f\"Column {i+1}: expected '{exp}', got '{act}'\")\n\n if not headers_ok:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Header mismatch: {'; '.join(header_feedback)}. Actual headers: {actual_headers}\"}\n\n # Check row 2 values\n expected_row = {\n 1: \"Technical/Administrative\",\n 2: \"HOH-7199789\",\n 3: \"CO-16\",\n 4: \"Correct and Resubmit\",\n 5: \"12/23/2025\",\n 6: \"Resubmitted\"\n }\n\n row2_values = {}\n for col in range(1, 7):\n val = ws.cell(row=2, column=col).value\n row2_values[col] = val\n\n def normalize(s):\n if s is None:\n return \"\"\n s = str(s).strip().lower()\n # collapse multiple spaces\n s = re.sub(r'\\s+', ' ', s)\n return s\n\n def check_date(actual, expected_str):\n \"\"\"Check if actual value matches expected date string 12/23/2025\"\"\"\n if actual is None:\n return False\n # If it's a datetime object\n import datetime\n if isinstance(actual, (datetime.datetime, datetime.date)):\n # Check month=12, day=23, year=2025\n if actual.month == 12 and actual.day == 23 and actual.year == 2025:\n return True\n # String comparison\n s = str(actual).strip()\n # Try various date formats\n if '12/23/2025' in s or '12-23-2025' in s or '2025-12-23' in s:\n return True\n # Check for datetime string like 2025-12-23 00:00:00\n if '2025-12-23' in s:\n return True\n return False\n\n mismatches = []\n for col, expected_val in expected_row.items():\n actual_val = row2_values[col]\n if col == 5:\n # Date column\n if not check_date(actual_val, expected_val):\n mismatches.append(f\"Column {col} (Date_Logged): expected '{expected_val}', got '{actual_val}'\")\n else:\n if normalize(actual_val) != normalize(expected_val):\n # Try looser matching\n act_n = normalize(actual_val).replace('/', '').replace('-', '').replace('_', '')\n exp_n = normalize(expected_val).replace('/', '').replace('-', '').replace('_', '')\n if act_n != exp_n:\n mismatches.append(f\"Column {col}: expected '{expected_val}', got '{actual_val}'\")\n\n if mismatches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 2 mismatches: {'; '.join(mismatches)}. Full row 2: {row2_values}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File PBC_Denial_Worklist.xlsx found with correct headers and row
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016520300",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "`PBC_Escalation_Log.xlsx` has exactly 3 rows (1 header row and 2 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Escalation_Log.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n ws = wb.active\n \n # Count non-empty rows (rows that have at least one non-None cell)\n total_rows = 0\n for row in ws.iter_rows():\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n if total_rows == 3:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_Escalation_Log.xlsx has exactly 3 non-empty rows (1 header + 2 data rows) as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PBC_Escalation_Log.xlsx has {total_rows} non-empty rows, but expected exactly 3 (1 header + 2 data rows).\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Escalation_Log.xlsx: {str(e)}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016522106",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "The file `PBC_Escalation_Log.xlsx` must exist with the following headers in row 1: `Claim_ID, if any, comma-separated list if multiple` | `Payer_ID, if any` | `Category` | `Esc_Reason` | `Resolution_Tkg ([Role] on [Date of escalation, MM/DD/YYYY])`. Row 2 (the existing data row) must contain: [empty] | A748AG | Critical | Payer System Outage | RCM Manager on 12/10/2025.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n import openpyxl\n\n target_file = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if not target_file.exists():\n # Search recursively\n candidates = list(Path(workspace_path).rglob(\"PBC_Escalation_Log.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Escalation_Log.xlsx not found in workspace.\"}\n target_file = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Escalation_Log.xlsx: {e}\"}\n\n ws = wb.active\n\n # ---- Check headers in row 1 ----\n expected_headers = [\n \"Claim_ID, if any, comma-separated list if multiple\",\n \"Payer_ID, if any\",\n \"Category\",\n \"Esc_Reason\",\n \"Resolution_Tkg ([Role] on [Date of escalation, MM/DD/YYYY])\"\n ]\n\n def normalize(s):\n if s is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(s).strip()).lower()\n\n header_issues = []\n for idx, expected in enumerate(expected_headers, start=1):\n actual = ws.cell(row=1, column=idx).value\n if normalize(actual) != normalize(expected):\n # Be lenient: check if it contains key substrings\n actual_n = normalize(actual)\n expected_n = normalize(expected)\n # Check if at least the main keyword is present\n if expected_n not in actual_n and actual_n not in expected_n:\n # Try even looser: check main keyword\n key_parts = {\n 1: [\"claim_id\"],\n 2: [\"payer_id\"],\n 3: [\"category\"],\n 4: [\"esc_reason\"],\n 5: [\"resolution_tkg\"]\n }\n found_key = any(kp in actual_n for kp in key_parts.get(idx, []))\n if not found_key:\n header_issues.append(f\"Column {idx} header: expected '{expected}', got '{actual}'\")\n\n # ---- Check row 2 data ----\n row2_issues = []\n\n # Column A (Claim_ID) should be empty\n cell_a2 = ws.cell(row=2, column=1).value\n if cell_a2 is not None and str(cell_a2).strip() != \"\":\n row2_issues.append(f\"Cell A2 should be empty but got '{cell_a2}'\")\n\n # Column B (Payer_ID) should be A748AG\n cell_b2 = ws.cell(row=2, column=2).value\n if normalize(cell_b2) != normalize(\"A748AG\"):\n row2_issues.append(f\"Cell B2: expected 'A748AG', got '{cell_b2}'\")\n\n # Column C (Category) should be Critical\n cell_c2 = ws.cell(row=2, column=3).value\n if normalize(cell_c2) != normalize(\"Critical\"):\n row2_issues.append(f\"Cell C2: expected 'Critical', got '{cell_c2}'\")\n\n # Column D (Esc_Reason) should be Payer System Outage\n cell_d2 = ws.cell(row=2, column=4).value\n if normalize(cell_d2) != normalize(\"Payer System Outage\"):\n # Looser check\n if \"payer system outage\" not in normalize(cell_d2):\n row2_issues.append(f\"Cell D2: expected 'Payer System Outage', got '{cell_d2}'\")\n\n # Column E (Resolution_Tkg) should be \"RCM Manager on 12/10/2025\"\n cell_e2 = ws.cell(row=2, column=5).value\n cell_e2_n = normalize(cell_e2)\n expected_e2 = \"rcm manager on 12/10/2025\"\n if expected_e2 not in cell_e2_n:\n # Try alternate date formats\n alt_dates = [\"12/10/2025\", \"12-10-2025\"]\n found_match = False\n for ad in alt_dates:\n if \"rcm manager\" in cell_e2_n and ad in cell_e2_n:\n found_match = True\n break\n if not found_match:\n row2_issues.append(f\"Cell E2: expected 'RCM Manager on 12/10/2025', got '{cell_e2}'\")\n\n all_issues = header_issues + row2_issues\n if all_issues:\n feedback = \"Issues found in PBC_Escalation_Log
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016528253",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "`PBC_Claim_Register.xlsx` has exactly 28 rows (1 header row and 27 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Claim_Register.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)\n ws = wb.active\n \n # Count all rows that have at least one non-None cell\n total_rows = 0\n for row in ws.iter_rows():\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n wb.close()\n \n if total_rows == 28:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_Claim_Register.xlsx has exactly 28 non-empty rows (1 header + 27 data rows) as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PBC_Claim_Register.xlsx has {total_rows} non-empty rows, but expected exactly 28 (1 header + 27 data rows).\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Claim_Register.xlsx: {str(e)}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016548731",
|
||
|
|
"sort_order": 14,
|
||
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cells I14 through I19 must all contain the value `Reconciled`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n # Try the active sheet first, then iterate sheets if needed\n ws = wb.active\n \n target_cells = [f\"I{row}\" for row in range(14, 20)] # I14 through I19\n results = {}\n all_pass = True\n \n for cell_ref in target_cells:\n val = ws[cell_ref].value\n if val is None:\n results[cell_ref] = None\n all_pass = False\n else:\n val_str = str(val).strip().lower()\n if val_str == \"reconciled\":\n results[cell_ref] = str(val).strip()\n else:\n results[cell_ref] = str(val).strip()\n all_pass = False\n \n # If not all pass on active sheet, try all sheets\n if not all_pass:\n for sheet_name in wb.sheetnames:\n ws_try = wb[sheet_name]\n results_try = {}\n all_pass_try = True\n for cell_ref in target_cells:\n val = ws_try[cell_ref].value\n if val is None:\n results_try[cell_ref] = None\n all_pass_try = False\n else:\n val_str = str(val).strip().lower()\n if val_str == \"reconciled\":\n results_try[cell_ref] = str(val).strip()\n else:\n results_try[cell_ref] = str(val).strip()\n all_pass_try = False\n if all_pass_try:\n all_pass = True\n results = results_try\n break\n \n wb.close()\n \n if all_pass:\n feedback = \"PASS: All cells I14:I19 contain 'Reconciled'. Values found: \" + \", \".join(\n [f\"{k}={v}\" for k, v in results.items()]\n )\n return {\"pass\": True, \"score\": 1.0, \"feedback\": feedback}\n else:\n failing = {k: v for k, v in results.items() if v is None or str(v).strip().lower() != \"reconciled\"}\n passing = {k: v for k, v in results.items() if v is not None and str(v).strip().lower() == \"reconciled\"}\n score = len(passing) / len(target_cells)\n feedback = (\n f\"FAIL: Not all cells I14:I19 contain 'Reconciled'. \"\n f\"Failing cells: {failing}. \"\n f\"Passing cells: {passing}.\"\n )\n return {\"pass\": False, \"score\": score, \"feedback\": feedback}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016553638",
|
||
|
|
"sort_order": 15,
|
||
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cells I20, I21, and I22 must each contain the value `Held - Contractual Variance`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n \n if not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {target_file}\"}\n \n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n # Try the active sheet first, then iterate sheets if needed\n expected = \"held - contractual variance\"\n cells_to_check = [\"I20\", \"I21\", \"I22\"]\n \n results = {}\n found_sheet = None\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n all_match = True\n sheet_results = {}\n for cell_ref in cells_to_check:\n val = ws[cell_ref].value\n val_str = str(val).strip().lower() if val is not None else \"\"\n sheet_results[cell_ref] = val\n if val_str != expected:\n all_match = False\n if all_match:\n found_sheet = sheet_name\n results = sheet_results\n break\n # Keep results from last checked sheet for error reporting\n results = sheet_results\n \n if found_sheet:\n details = \", \".join([f\"{c}='{results[c]}'\" for c in cells_to_check])\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All three cells contain 'Held - Contractual Variance' in sheet '{found_sheet}'. Values: {details}\"\n }\n \n # Partial credit: count how many match across all sheets\n best_count = 0\n best_sheet = None\n best_results = {}\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n count = 0\n sheet_results = {}\n for cell_ref in cells_to_check:\n val = ws[cell_ref].value\n val_str = str(val).strip().lower() if val is not None else \"\"\n sheet_results[cell_ref] = val\n if val_str == expected:\n count += 1\n if count > best_count:\n best_count = count\n best_sheet = sheet_name\n best_results = sheet_results\n \n details = \", \".join([f\"{c}='{best_results.get(c, 'N/A')}'\" for c in cells_to_check])\n score = best_count / 3.0\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Expected all of I20, I21, I22 to be 'Held - Contractual Variance'. Best match found in sheet '{best_sheet}' with {best_count}/3 matching. Values: {details}\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016558874",
|
||
|
|
"sort_order": 16,
|
||
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, cells K14 through M22 must all be 0.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n # Try glob for any xlsx file with similar name\n candidates = list(Path(workspace_path).glob(\"*Claim*Register*.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n # Use the active sheet or first sheet\n ws = wb.active\n \n non_zero_cells = []\n total_cells = 0\n zero_cells = 0\n \n # Rubric says K14 through M22: Columns K=11, L=12, M=13; Rows 14-22\n for row in range(14, 23): # 14 through 22 inclusive\n for col_letter, col_idx in [('K', 11), ('L', 12), ('M', 13)]:\n total_cells += 1\n cell = ws.cell(row=row, column=col_idx)\n val = cell.value\n \n # Check if the value is effectively 0\n is_zero = False\n if val is None or val == '' or val == 0:\n is_zero = True\n elif isinstance(val, (int, float)):\n if abs(val) < 0.001: # tolerance for floating point\n is_zero = True\n elif isinstance(val, str):\n stripped = val.strip().replace('$', '').replace(',', '').replace(' ', '')\n if stripped == '' or stripped == '0' or stripped == '0.00' or stripped == '0.0':\n is_zero = True\n else:\n try:\n numeric = float(stripped)\n if abs(numeric) < 0.001:\n is_zero = True\n except ValueError:\n pass\n \n if is_zero:\n zero_cells += 1\n else:\n non_zero_cells.append(f\"{col_letter}{row}={val}\")\n \n wb.close()\n \n if not non_zero_cells:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All {total_cells} cells in K14:M22 are 0 as required.\"\n }\n else:\n score = zero_cells / total_cells if total_cells > 0 else 0.0\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"{len(non_zero_cells)} cell(s) in K14:M22 are not 0: {', '.join(non_zero_cells)}\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016573480",
|
||
|
|
"sort_order": 17,
|
||
|
|
"rubric_text": "Cells I27 and I28 in `PBC_Claim_Register.xlsx` must be set to `Unapplied`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n \n if not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {target_file}\"}\n \n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Try to find the values in any sheet\n results = {}\n cells_to_check = {\"I27\": None, \"I28\": None}\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for cell_ref in cells_to_check:\n val = ws[cell_ref].value\n if val is not None and cells_to_check[cell_ref] is None:\n cells_to_check[cell_ref] = (sheet_name, val)\n \n # If not found with non-None in any sheet, check the first/active sheet specifically\n ws = wb.active if wb.active else wb[wb.sheetnames[0]]\n for cell_ref in cells_to_check:\n if cells_to_check[cell_ref] is None:\n val = ws[cell_ref].value\n cells_to_check[cell_ref] = (ws.title, val)\n \n feedback_parts = []\n pass_count = 0\n \n for cell_ref, info in cells_to_check.items():\n if info is None:\n feedback_parts.append(f\"{cell_ref}: No value found in any sheet.\")\n continue\n sheet_name, val = info\n val_str = str(val).strip().lower() if val is not None else \"\"\n if val_str == \"unapplied\":\n pass_count += 1\n feedback_parts.append(f\"{cell_ref} in sheet '{sheet_name}' = '{val}' — PASS\")\n else:\n feedback_parts.append(f\"{cell_ref} in sheet '{sheet_name}' = '{val}' (expected 'Unapplied') — FAIL\")\n \n all_pass = pass_count == 2\n score = pass_count / 2.0\n \n return {\n \"pass\": all_pass,\n \"score\": score,\n \"feedback\": \"; \".join(feedback_parts)\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016631059",
|
||
|
|
"sort_order": 18,
|
||
|
|
"rubric_text": "`mailbox.json` must contain exactly 1 email in inbox, 2 emails in sent, and 1 email in drafts.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks email state, so we need external_services_path\n # But also check workspace_path for mailbox.json as a fallback\n mailbox_path = None\n \n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n inbox_count = 0\n sent_count = 0\n drafts_count = 0\n \n for email in emails:\n folder = email.get(\"folder\", \"\").strip().upper()\n if folder == \"INBOX\":\n inbox_count += 1\n elif folder in (\"SENT\", \"SEND\"):\n sent_count += 1\n elif folder in (\"DRAFTS\", \"DRAFT\"):\n drafts_count += 1\n \n details = f\"Found: inbox={inbox_count}, sent={sent_count}, drafts={drafts_count}. Expected: inbox=1, sent=2, drafts=1.\"\n \n passed = (inbox_count == 1 and sent_count == 2 and drafts_count == 1)\n \n if passed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS. {details}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL. {details}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016669182",
|
||
|
|
"sort_order": 19,
|
||
|
|
"rubric_text": "In `mailbox.json`, there must be no email with `to_addr` set to `ursula.james@hull.com`, and no email with \"Ursula James\" in `body_text`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the absence of certain emails.\n # mailbox.json could be in external_services_path or workspace_path.\n \n mailbox_path = None\n \n # Check external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # No mailbox.json found at all - this means no emails exist,\n # so the condition is trivially satisfied (no offending emails).\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so no offending emails exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n issues = []\n \n for i, email in enumerate(emails):\n to_addr = email.get(\"to_addr\", \"\")\n if isinstance(to_addr, str) and to_addr.strip().lower() == \"ursula.james@hull.com\":\n issues.append(f\"Email index {i} (subject: {email.get('subject', 'N/A')}, folder: {email.get('folder', 'N/A')}) has to_addr='ursula.james@hull.com'.\")\n \n body_text = email.get(\"body_text\", \"\")\n if isinstance(body_text, str) and \"ursula james\" in body_text.lower():\n issues.append(f\"Email index {i} (subject: {email.get('subject', 'N/A')}, folder: {email.get('folder', 'N/A')}) contains 'Ursula James' in body_text.\")\n \n if issues:\n feedback = \"FAIL - Found offending emails:\\n\" + \"\\n\".join(issues)\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS - Checked {len(emails)} emails in mailbox.json. No email has to_addr='ursula.james@hull.com' and no email contains 'Ursula James' in body_text.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016710395",
|
||
|
|
"sort_order": 20,
|
||
|
|
"rubric_text": "In `mailbox.json`, the draft email has the subject: \"Contractual Rate Dispute Hull Insurance / A4789P / CPT 11241\".",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for a draft email with the specified subject.\n # mailbox.json could be in the workspace or in external_services_path.\n \n mailbox_path = None\n candidates = []\n \n # Check external_services_path first\n if external_services_path:\n candidates.append(Path(external_services_path) / \"mailbox.json\")\n \n # Also check workspace_path\n candidates.append(Path(workspace_path) / \"mailbox.json\")\n \n for candidate in candidates:\n if candidate.exists():\n mailbox_path = candidate\n break\n \n if mailbox_path is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Could not find mailbox.json in either external_services_path or workspace_path.\"\n }\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Error reading mailbox.json: {e}\"\n }\n \n emails = mailbox_data.get(\"emails\", [])\n if not emails:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No emails found in mailbox.json.\"\n }\n \n expected_subject = \"Contractual Rate Dispute Hull Insurance / A4789P / CPT 11241\"\n expected_subject_lower = expected_subject.lower().strip()\n \n # Also create a more flexible pattern: just check key components are present\n key_components = [\"contractual rate dispute\", \"hull insurance\", \"a4789p\", \"cpt 11241\"]\n \n draft_emails = []\n matching_drafts = []\n all_subjects = []\n \n for email in emails:\n subject = email.get(\"subject\", \"\").strip()\n folder = email.get(\"folder\", \"\").strip()\n all_subjects.append(f\"[folder={folder}] {subject}\")\n \n # Check if it's a draft\n is_draft = folder.lower() in [\"drafts\", \"draft\"]\n \n if is_draft:\n draft_emails.append(email)\n \n # Check subject match - be generous\n subject_lower = subject.lower().strip()\n \n # Exact match (case-insensitive)\n exact_match = (subject_lower == expected_subject_lower)\n \n # Flexible match: normalize slashes and whitespace\n def normalize(s):\n s = re.sub(r'\\s*/\\s*', ' / ', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.strip().lower()\n \n normalized_match = (normalize(subject) == normalize(expected_subject))\n \n # Component match: all key components present\n component_match = all(comp in subject_lower for comp in key_components)\n \n if exact_match or normalized_match or component_match:\n if is_draft:\n matching_drafts.append(email)\n # Be lenient: even if not explicitly in \"Drafts\" folder, if the subject matches\n # and the task says \"do not send\", it might be in another folder\n elif not is_draft:\n # Still track it but note it's not in Drafts\n matching_drafts.append(email) # Be generous\n \n if matching_drafts:\n found_email = matching_drafts[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found email with matching subject: \\\"{found_email.get('subject', '')}\\\" in folder \\\"{found_email.get('folder', '')}\\\"\"\n }\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No email found with subject matching 'Contractual Rate Dispute Hull Insurance / A4789P / CPT 11241'. Found {len(draft_emails)} draft(s). All email subjects: {all_subjects}\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016711173",
|
||
|
|
"sort_order": 21,
|
||
|
|
"rubric_text": "`calendar_data.json` has exactly 10 events.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for calendar_data.json\n calendar_file = None\n if external_services_path:\n candidate = Path(external_services_path) / \"calendar_data.json\"\n if candidate.exists():\n calendar_file = candidate\n\n # Also check workspace_path as fallback\n if calendar_file is None:\n candidate = Path(workspace_path) / \"calendar_data.json\"\n if candidate.exists():\n calendar_file = candidate\n\n if calendar_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found in external_services_path or workspace_path.\"}\n\n try:\n with open(calendar_file, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse calendar_data.json: {e}\"}\n\n # Count events\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n num_events = len(events)\n elif isinstance(events, list):\n num_events = len(events)\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unexpected 'events' type: {type(events)}. Could not count events.\"}\n\n if num_events == 10:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"calendar_data.json has exactly 10 events as expected. Found {num_events} events.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 10 events in calendar_data.json, but found {num_events}.\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016717601",
|
||
|
|
"sort_order": 22,
|
||
|
|
"rubric_text": "`slack_data.json` has exactly 2 messages.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n if slack_path is None:\n slack_path = Path(workspace_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'slack_data.json' not found in workspace or external services path.\"}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except json.JSONDecodeError as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json as JSON: {e}\"}\n\n messages_dict = data.get(\"messages\", {})\n count = sum(len(v) for v in messages_dict.values() if isinstance(v, list))\n\n if count == 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"slack_data.json contains exactly 2 messages as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json contains {count} message(s), but expected exactly 2.\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016761864",
|
||
|
|
"sort_order": 23,
|
||
|
|
"rubric_text": "In `mailbox.json`, there is an email to `joem@pathfinder.com` with `body_text` containing:\n\n\"Joe,\\n\\nI am escalating a pattern recurrence for Hull Insurance (A4789P) affecting 9 claims for CPT 11241. All instances share the same discrepancy:\\n\\nHIC-19887899 | Contracted: $80 | Paid: $70\\nHIC-19887900 | Contracted: $80 | Paid: $70\\nHIC-19887901 | Contracted: $80 | Paid: $70\\nHIC-25124778 | Contracted: $80 | Paid: $70\\nHIC-25124779 | Contracted: $80 | Paid: $70\\nHIC-25124780 | Contracted: $80 | Paid: $70\\nHIC-26877757 | Contracted: $80 | Paid: $70\\nHIC-26877758 | Contracted: $75 | Paid: $65\\nHIC-26877795 | Contracted: $75 | Paid: $65\\n\\nTotal variance: $90.\\n\\nPathfinder Billing and Coding | rcm@pathfinder.com\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the email\n # First try external_services_path, then workspace_path\n mailbox_path = None\n if external_services_path:\n p = Path(external_services_path) / \"mailbox.json\"\n if p.exists():\n mailbox_path = p\n if not mailbox_path:\n p = Path(workspace_path) / \"mailbox.json\"\n if p.exists():\n mailbox_path = p\n \n if not mailbox_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in either external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Find email to joem@pathfinder.com\n target_emails = []\n for email in emails:\n to_addr = email.get(\"to_addr\", \"\").lower().strip()\n if \"joem@pathfinder.com\" in to_addr:\n target_emails.append(email)\n \n if not target_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email found addressed to joem@pathfinder.com.\"}\n \n key_fragments = [\n \"Joe,\",\n \"pattern recurrence\",\n \"Hull Insurance (A4789P)\",\n \"9 claims\",\n \"CPT 11241\",\n \"All instances share the same discrepancy\",\n \"HIC-19887899 | Contracted: $80 | Paid: $70\",\n \"HIC-19887900 | Contracted: $80 | Paid: $70\",\n \"HIC-19887901 | Contracted: $80 | Paid: $70\",\n \"HIC-25124778 | Contracted: $80 | Paid: $70\",\n \"HIC-25124779 | Contracted: $80 | Paid: $70\",\n \"HIC-25124780 | Contracted: $80 | Paid: $70\",\n \"HIC-26877757 | Contracted: $80 | Paid: $70\",\n \"HIC-26877758 | Contracted: $75 | Paid: $65\",\n \"HIC-26877795 | Contracted: $75 | Paid: $65\",\n \"Total variance: $90\",\n \"Pathfinder Billing and Coding\",\n \"rcm@pathfinder.com\",\n ]\n\n best_score = 0.0\n best_feedback = \"\"\n\n for email in target_emails:\n body = email.get(\"body_text\", \"\") or \"\"\n found = [f for f in key_fragments if f.lower() in body.lower()]\n missing = [f for f in key_fragments if f.lower() not in body.lower()]\n score = len(found) / len(key_fragments)\n if score > best_score:\n best_score = score\n if missing:\n best_feedback = f\"Found {len(found)}/{len(key_fragments)} fragments. Missing: {missing}\"\n else:\n best_feedback = \"All checks passed. Email to joem@pathfinder.com contains all required content.\"\n\n passed = best_score == 1.0\n return {\"pass\": passed, \"score\": best_score, \"feedback\": best_feedback}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775016771886",
|
||
|
|
"sort_order": 24,
|
||
|
|
"rubric_text": "In `mailbox.json`, there is an email to `joem@pathfinder.com` with `\"subject\": \"Pattern Recurrence Hull Insurance / A4789P / CPT 11241\"`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # First, check external_services_path for mailbox.json\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path for mailbox.json\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n expected_subject = \"Pattern Recurrence Hull Insurance / A4789P / CPT 11241\"\n expected_to = \"joem@pathfinder.com\"\n \n for email in emails:\n to_addr = (email.get(\"to_addr\", \"\") or \"\").strip().lower()\n subject = (email.get(\"subject\", \"\") or \"\").strip()\n \n # Check to_addr - be flexible, it could be in a comma-separated list\n to_match = expected_to.lower() in to_addr\n \n # Check subject - be loose: normalize whitespace and compare case-insensitively\n def normalize(s):\n import re\n return re.sub(r'\\s+', ' ', s).strip().lower()\n \n subject_match = normalize(expected_subject) == normalize(subject)\n \n # Also try a more lenient check: just see if the key components are present\n if not subject_match:\n subj_lower = normalize(subject)\n subject_match = (\n \"pattern recurrence\" in subj_lower and\n \"hull insurance\" in subj_lower and\n \"a4789p\" in subj_lower and\n \"cpt 11241\" in subj_lower\n )\n \n if to_match and subject_match:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found matching email to '{email.get('to_addr')}' with subject '{email.get('subject')}' (folder: {email.get('folder', 'unknown')}).\"\n }\n \n # Build diagnostic info\n found_subjects = [(e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\"), e.get(\"folder\", \"\")) for e in emails]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n f\"No email found to '{expected_to}' with subject '{expected_subject}'. \"\n f\"Emails found: {found_subjects}\"\n )\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|