129 lines
91 KiB
JSON
129 lines
91 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "292496ce-fecf-4a85-a8ac-244d4df88aae",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In file `claim.xlsx`, the row where `DOS` = `02/14/2026` must have `Claim_Status` = `Denied`, `ERA_Received_Date` = `04/08/2026` or 8 April 2026, `Payment_Posted_Date` blank/null, `Benefit_Tier_Billed` = `Medical`, `Paid_Amount` = `0` and `Claim_Frequency_Code` = `1`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find claim.xlsx - be flexible with naming\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n # Try glob for any xlsx with 'claim' in the name\n candidates = list(Path(workspace_path).glob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n # Also try recursively\n candidates = list(Path(workspace_path).rglob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n headers_upper = [h.upper().replace(' ', '_') for h in headers]\n\n # Find columns flexibly\n dos_col = None\n status_col = None\n era_received_col = None\n payment_posted_col = None\n tier_col = None\n freq_col = None\n paid_amount_col = None\n\n for i, h in enumerate(headers_upper):\n if dos_col is None and 'DOS' in h:\n dos_col = i\n if status_col is None and ('CLAIM_STATUS' in h or ('CLAIM' in h and 'STATUS' in h) or h == 'STATUS' or h == 'CLAIM_STATUS'):\n status_col = i\n if era_received_col is None and ('ERA' in h and 'RECEIVED' in h):\n era_received_col = i\n if payment_posted_col is None and ('PAYMENT_POSTED' in h or ('PAYMENT' in h and 'POSTED' in h)):\n payment_posted_col = i\n if tier_col is None and ('BENEFIT_TIER' in h or ('TIER' in h and 'BILLED' in h) or ('BENEFIT' in h and 'TIER' not in h)):\n tier_col = i\n if tier_col is None and 'BENEFIT' in h:\n tier_col = i\n if freq_col is None and ('FREQUENCY' in h or 'FREQ' in h):\n freq_col = i\n if paid_amount_col is None and ('PAID_AMOUNT' in h or ('PAID' in h and 'AMOUNT' in h) or h == 'PAID_AMOUNT'):\n paid_amount_col = i\n\n if dos_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'DOS column not found. Headers: {headers}'}\n\n # Find the target row with DOS = 02/14/2026\n target_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n dos_val = row[dos_col]\n if dos_val is None:\n continue\n dos_str = str(dos_val).strip()\n match = False\n if '02/14/2026' in dos_str or '2/14/2026' in dos_str or '2026-02-14' in dos_str:\n match = True\n if isinstance(dos_val, datetime.datetime):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n elif isinstance(dos_val, datetime.date):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n if match:\n target_row = row\n break\n\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=02/14/2026 found in claim.xlsx'}\n\n issues = []\n details = []\n\n # Check Claim_Status = Denied\n if status_col is not None:\n status_val = str(target_row[status_col]).strip().lower() if target_row[status_col] is not None else ''\n if 'denied' in status_val or 'deny' in status_val or 'denial' in status_val:\n details.append(f'Claim_Status={target_row[status_col]}')\n else:\n issues.append(f'Claim_Status expected Denied, got: {target_row[status_col]}')\n else:\n issues.append(f'Claim_Status column not found. Headers: {headers}')\n\n # Check ERA_Received_Date = 04/08/2026 or 8 April 2026\n if era_received_col is not None:\n era_val = target_row[era_received_col]\n era_match = False\n if era_val is not None:\n if isinstance(
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "826aeb1f-eea5-4a46-b9d1-3bb715ba3013",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `claim.xlsx`, total row count must be exactly 5 (1 header + 4 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n \n row_count = sum(1 for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row))\n \n if row_count == 5:\n return {'pass': True, 'score': 1.0, 'feedback': f'claim.xlsx has exactly 5 rows (1 header + 4 data rows)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'claim.xlsx has {row_count} non-empty rows, expected 5 (1 header + 4 data rows)'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "f7996998-b356-45fc-a368-c4ad88757797",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "A file named `appeal_L1_02142026.pdf` must exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n all_files = [f for f in ws.rglob('*') if f.is_file()]\n\n # --- Pass 1: exact stem match, PDF only ---\n target_stem = 'appeal_l1_02142026'\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n if f.stem.lower() == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found appeal_L1_02142026.pdf at {f.relative_to(ws)}'\n }\n\n # --- Pass 2: normalised separators (hyphens → underscores), PDF only ---\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n normalised = f.stem.lower().replace('-', '_').replace(' ', '_')\n if normalised == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found matching PDF with alternate separators: '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Pass 3: component match, PDF only ---\n date_variants = ['02142026', '02_14_2026', '02-14-2026']\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n n = f.name.lower()\n if 'appeal' in n and 'l1' in n and any(d in n for d in date_variants):\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found appeal letter PDF (component match): '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Diagnosis: did the model produce the file in a non-PDF format? ---\n non_pdf_matches = [\n f for f in all_files\n if f.suffix.lower() != '.pdf'\n and 'appeal' in f.name.lower()\n and 'l1' in f.name.lower()\n and any(d in f.name.lower() for d in date_variants)\n ]\n if non_pdf_matches:\n names = [f.name for f in non_pdf_matches]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'Appeal letter found but not in PDF format: {names}. '\n f'The criterion requires a .pdf file. '\n f'Rename or re-export the file as appeal_L1_02142026.pdf.'\n )\n }\n\n all_names = [f.name for f in all_files]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No appeal_L1_02142026.pdf found in workspace. '\n f'Expected a .pdf file whose name contains \"appeal\", \"l1\", '\n f'and date \"02142026\". '\n f'Files present: {all_names}'\n )\n }",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "f7446601-8f7d-4712-9097-7620699d893c",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), exactly 1 email must exist in the `Sent` folder with `to_addr` = `appeals@silverpath-claims.com`, with subject containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `SPA-00291847`, `DOS:`, `02/14/2026`, `Claim:`, `CLM-REF-20260218-9931`, `CARC:`, `CO-97`, and attachments containing exactly `appeal_L1_02142026.pdf`, `claim_02142026_Holloway_SilverPath.pdf`, `pa_approval_SilverPath_Holloway.pdf`, `lmn_DrOsei_Holloway.pdf`, `era_02142026_SilverPath.pdf`, and filename containing `visit_note`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find sent emails to the appeals address (case-insensitive)\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@silverpath-claims.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent email to appeals@silverpath-claims.com found'}\n\n if len(target_emails) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 sent email to appeals@silverpath-claims.com, found {len(target_emails)}'}\n\n email = target_emails[0]\n\n # Check subject for required substrings\n subject = str(email.get('subject', ''))\n required_substrings = [\n 'APPEAL L1',\n 'CareIG',\n 'Holloway',\n 'MemberID:',\n 'SPA-00291847',\n 'DOS:',\n '02/14/2026',\n 'Claim:',\n 'CLM-REF-20260218-9931',\n 'CARC:',\n 'CO-97'\n ]\n\n # Case-sensitive IDs that must match exactly\n exact_match_required = {'SPA-00291847', 'CLM-REF-20260218-9931', 'CO-97', '02/14/2026'}\n\n missing = []\n for s in required_substrings:\n if s in subject:\n continue\n # For non-exact fields, try case-insensitive\n if s not in exact_match_required and s.upper() in subject.upper():\n continue\n missing.append(s)\n\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Email subject missing required substrings: {missing}. Subject was: {subject}'}\n\n # Check attachments\n attachments = email.get('attachments', [])\n attach_names = []\n for a in attachments:\n if isinstance(a, str):\n attach_names.append(a)\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', str(a)))\n attach_names.append(name)\n else:\n attach_names.append(str(a))\n\n attach_names_lower = [n.lower() for n in attach_names]\n\n # Required exact attachment filenames\n required_exact_attachments = [\n 'appeal_L1_02142026.pdf',\n 'claim_02142026_Holloway_SilverPath.pdf',\n 'pa_approval_SilverPath_Holloway.pdf',\n 'lmn_DrOsei_Holloway.pdf',\n 'era_02142026_SilverPath.pdf'\n ]\n\n missing_attachments = []\n for req in required_exact_attachments:\n # Check case-insensitive\n found = any(req.lower() in n for n in attach_names_lower)\n if not found:\n missing_attachments.append(req)\n\n if missing_attachments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required attachments: {missing_attachments}. Found attachments: {attach_names}'}\n\n # Check for an attachment whose filename contains 'visit_note' (case-insensitive)\n found_visit_note = any('visit_note' in n.lower() for n in attach_names)\n if not found_visit_note:\n return {'pass': False, 'score': 0.0, 'feedback': f'No attachment with filename containing \"visit_note\" found. Found attachments: {attach_names}'}\n\n # Total expected: 5 exact + 1 visit_note = 6\n # Be lenient: as long as all required ones are present, pass even if there are extras\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Sent email to appeals@silverpath-claims.com found with correct subject substrings and all required attachments. Subject: {subject}. Attachments: {attach_names
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "74e31c33-9667-455a-8fd5-dd989a7231eb",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), no email in any folder must have `to_addr` containing `billing@silverpath-claims.com` with a date of `2026-04-01` or later.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n \n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n cutoff = datetime(2026, 4, 1)\n \n violations = []\n for e in emails:\n to_addr = str(e.get('to_addr', ''))\n if 'billing@silverpath-claims.com' in to_addr.lower():\n date_str = str(e.get('date', ''))\n try:\n email_date = datetime.fromisoformat(date_str[:10])\n if email_date >= cutoff:\n violations.append(f\"Subject: {e.get('subject', 'N/A')}, Date: {date_str}\")\n except Exception:\n pass\n \n if violations:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found emails to billing@silverpath-claims.com on/after 2026-04-01: {violations}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No emails sent to billing@silverpath-claims.com on or after 2026-04-01 (no corrected claim submitted to payer)'}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "bce26050-60a1-4108-8bf3-836752efd837",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "File `appeals_log.xlsx` must contain exactly 4 rows (1 header + 3 data rows), with exactly 1 row having `DOS` = `02/14/2026`, and that row must have `CARC_Code` = `CO-97`, `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, `Appeal_Status` = `Submitted`, `Payer` = `SilverPath Medicare Advantage`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) != 4:\n return {'pass': False, 'score': 0.0, 'feedback': f'appeals_log.xlsx has {len(rows)} rows, expected 4 (1 header + 3 data rows)'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n carc_col = col_idx('CARC')\n level_col = col_idx('LEVEL')\n subdate_col = col_idx('SUBMITTED_DATE') or col_idx('SUBMIT')\n status_col = col_idx('STATUS')\n payer_col = col_idx('PAYER')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '02/14/2026' in dos_val or '2026-02-14' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 row with DOS=02/14/2026 in appeals_log.xlsx, found {len(target_rows)}'}\n \n row = target_rows[0]\n issues = []\n \n if carc_col is not None:\n carc_val = str(row[carc_col]).strip() if row[carc_col] is not None else ''\n if 'CO-97' not in carc_val.upper() and 'CO97' not in carc_val.upper():\n issues.append(f'CARC_Code expected CO-97, got: {carc_val}')\n else:\n issues.append('CARC_Code column not found')\n \n if level_col is not None:\n level_val = str(row[level_col]).strip().upper() if row[level_col] is not None else ''\n if 'L1' not in level_val:\n issues.append(f'Appeal_Level expected L1, got: {row[level_col]}')\n else:\n issues.append('Appeal_Level column not found')\n \n if subdate_col is not None:\n date_val = str(row[subdate_col]).strip() if row[subdate_col] is not None else ''\n if '04/08/2026' not in date_val and '2026-04-08' not in date_val:\n issues.append(f'Appeal_Submitted_Date expected 04/08/2026, got: {date_val}')\n else:\n issues.append('Appeal_Submitted_Date column not found')\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if payer_col is not None:\n payer_val = str(row[payer_col]).strip() if row[payer_col] is not None else ''\n if 'SILVERPATH' not in payer_val.upper():\n issues.append(f'Payer expected SilverPath Medicare Advantage, got: {payer_val}')\n else:\n issues.append('Payer column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=02/14/2026 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx has 4 rows, DOS=02/14/2026 row has CARC_Code=CO-97, Appeal_Level=L1, Appeal_Submitted_Date=04/08/2026, Appeal_Status=Submitted, Payer=SilverPath Medicare Advantage'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "3e721c8d-53cd-47f9-a9a0-19515432c546",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `appeals_log.xlsx`, the row where `DOS` = `12/09/2025` must have `Appeal_Status` = `Submitted` and `Followup_Log` that is non-blank and contains a date matching `04/08/2026` or `8 April 2026`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx has no data rows'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n status_col = col_idx('STATUS')\n followup_col = col_idx('FOLLOWUP') or col_idx('FOLLOW_UP') or col_idx('FOLLOW')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '12/09/2025' in dos_val or '2025-12-09' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=12/09/2025 found in appeals_log.xlsx'}\n \n row = target_rows[0]\n issues = []\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if followup_col is not None:\n followup_val = str(row[followup_col]).strip() if row[followup_col] is not None else ''\n if followup_val in ('', 'None', 'nan', 'NaN', 'NULL'):\n issues.append('Followup_Log is blank')\n else:\n has_date = '04/08/2026' in followup_val or '2026-04-08' in followup_val or '8 april 2026' in followup_val.lower() or 'april 8, 2026' in followup_val.lower()\n if not has_date:\n issues.append(f'Followup_Log does not contain date 04/08/2026 or 8 April 2026. Value: {followup_val}')\n else:\n issues.append('Followup_Log column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=12/09/2025 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx DOS=12/09/2025 row has Appeal_Status=Submitted and Followup_Log containing 04/08/2026'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "fc648229-7c0f-4abc-9ba6-d8d05942a39a",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), exactly 3 emails must exist in the `Sent` folder with `to_addr` = `appeals@bluecrest-health.com`. The 2 of the emails must have subjects containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `BCR-00184523`, `DOS:`, `12/09/2025`, `Claim:`, `CLM-REF-20251215-6612`, `CARC:`, `PI-204`, email attachment name = `appeal_L1_12092025.pdf`, email body containing `BlueCrest Commercial PPO — Provider Appeals Department`, `Member ID: BCR-00184523`, `DOS: 12/09/2025`, `Claim Reference: CLM-REF-20251215-6612`, `Billed: $5,840.00`",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find all sent emails to appeals@bluecrest-health.com\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@bluecrest-health.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails to appeals@bluecrest-health.com, found {len(target_emails)}'}\n\n # Required substrings in the subject (case-insensitive, whitespace-flexible)\n required_subject_substrings = [\n 'APPEAL L1', 'CareIG', 'Holloway', 'MemberID:',\n 'BCR-00184523', 'DOS:', '12/09/2025', 'Claim:',\n 'CLM-REF-20251215-6612', 'CARC:', 'PI-204'\n ]\n\n # Required body substrings\n required_body_substrings = [\n 'BlueCrest Commercial PPO',\n 'Provider Appeals Department',\n 'Member ID: BCR-00184523',\n 'DOS: 12/09/2025',\n 'Claim Reference: CLM-REF-20251215-6612',\n 'Billed: $5,840.00'\n ]\n\n required_attachment = 'appeal_L1_12092025.pdf'\n\n def normalize(text):\n \"\"\"Normalize whitespace for flexible matching.\"\"\"\n return re.sub(r'\\s+', ' ', str(text)).strip()\n\n def check_subject(subject):\n subj_norm = normalize(subject).lower()\n # Also try without spaces around colons\n subj_no_space = re.sub(r'\\s*:\\s*', ':', subj_norm)\n missing = []\n for s in required_subject_substrings:\n s_lower = s.lower()\n s_no_space = re.sub(r'\\s*:\\s*', ':', s_lower)\n # Try multiple matching strategies\n if s_lower in subj_norm:\n continue\n if s_no_space in subj_no_space:\n continue\n # Try without space: \"MemberID:\" might appear as \"Member ID:\" or \"MemberID:\"\n if s == 'MemberID:':\n if 'memberid:' in subj_norm or 'member id:' in subj_norm or 'memberid :' in subj_norm:\n continue\n missing.append(s)\n return missing\n\n def check_body(body):\n body_norm = normalize(body)\n missing = []\n for s in required_body_substrings:\n # Case-insensitive check with normalized whitespace\n s_norm = normalize(s)\n if s_norm.lower() in body_norm.lower():\n continue\n # Try with em-dash variants\n s_variants = [\n s_norm,\n s_norm.replace('\\u2014', '-'),\n s_norm.replace('\\u2014', '--'),\n s_norm.replace('-', '\\u2014'),\n ]\n found = False\n for variant in s_variants:\n if variant.lower() in body_norm.lower():\n found = True\n break\n if not found:\n missing.append(s)\n return missing\n\n def get_attachment_names(attachments):\n names = []\n for a in (attachments or []):\n if isinstance(a, str):\n names.append(a.strip())\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', a.get('file_name', '')))\n names.append(str(name).strip())\n else:\n names.append(str(a).strip())\n return names\n\n def check_attachment(email_obj):\n att_names = get_attachment_names(email_obj.get('attachments', [])
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "52595972-13bc-49f5-b35b-4674e117b8ff",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "In `slack_data.json` (external services), channel `billing-appeals` must contain a message with text containing all of: `APPEAL FILED L1`, `Holloway_DOB_03141961`, `02/14/2026`, `SilverPath`, `CO-97`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check Slack'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n \n with open(slack_path, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n messages = data.get('messages', {})\n \n # Find #billing-appeals channel (be flexible with naming)\n billing_appeals_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if ch_name == 'billingappeals':\n billing_appeals_id = ch_id\n break\n \n if billing_appeals_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Channel #billing-appeals not found in slack_data.json. Available channels: {[ch.get(\"name\") for ch in channels.values()]}'}\n \n channel_messages = messages.get(billing_appeals_id, [])\n \n # Required content pieces to find in a message\n # Be loose: case-insensitive where appropriate, flexible date formats, flexible separators\n required_checks = {\n 'APPEAL FILED L1': lambda t: 'appeal' in t.lower() and ('filed' in t.lower() or 'file' in t.lower()) and ('l1' in t.lower() or 'level 1' in t.lower() or 'level1' in t.lower()),\n 'Holloway_DOB_03141961': lambda t: 'holloway' in t.lower() and ('03141961' in t.replace('/', '').replace('-', '').replace(' ', '') or '03/14/1961' in t or '03-14-1961' in t or '3/14/1961' in t),\n '02/14/2026': lambda t: any(d in t for d in ['02/14/2026', '2/14/2026', '02-14-2026', '2-14-2026', '2026-02-14', '02142026']),\n 'SilverPath': lambda t: 'silverpath' in t.lower().replace(' ', '').replace('-', '').replace('_', ''),\n 'CO-97': lambda t: 'co-97' in t.lower() or 'co 97' in t.lower() or 'co97' in t.lower(),\n }\n \n best_match_count = 0\n best_match_text = ''\n matched_keys_best = []\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n \n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n \n if len(matched_keys) == len(required_checks):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found matching message in #billing-appeals containing all required elements: {text[:300]}'}\n \n # Also check all messages across all channels in case channel name matching failed\n for ch_id, ch_msgs in messages.items():\n if ch_id == billing_appeals_id:\n continue\n for msg in ch_msgs:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n # Note: don't pass here since it's wrong channel, but track for feedback\n \n missing = [k for k in required_checks if k not in matched_keys_best]\n return {\n 'pass': False,\n 'score': best_match_count / len(required_checks) * 0.5, # partial credit\n 'feedback': f'No message in #billing-appeals found containing all required elements. '\n f'Best match had {best_match_count}/{len(required_checks)} elements. '\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "29097e98-d53d-4af6-ac8a-8df8f3ec442e",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "In `audit_log.xlsx`, total row count must be exactly 37 (1 header + 36 data rows).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find audit_log.xlsx - be flexible with location\n audit_path = Path(workspace_path) / 'audit_log.xlsx'\n if not audit_path.exists():\n # Try glob search\n candidates = list(Path(workspace_path).rglob('audit_log.xlsx'))\n if candidates:\n audit_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n\n try:\n wb = openpyxl.load_workbook(audit_path)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = wb.active\n\n # Collect all non-empty rows\n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n\n total_row_count = len(rows)\n\n # Rubric says exactly 37 rows (1 header + 36 data rows)\n if total_row_count != 37:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'audit_log.xlsx has {total_row_count} non-empty rows, expected exactly 37 (1 header + 36 data rows).'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'audit_log.xlsx has exactly {total_row_count} non-empty rows (1 header + 36 data rows), as required.'\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775206985975",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "File `DENIAL_CO97_02142026.xlsx` must exist and contain exactly 2 rows (1 header + 1 data row) with the following values in the data row: `CASE_ID` = `Holloway_DOB_03141961`, `CARC_Code` = `CO-97`, `CARC_Description` = `Benefit not assigned`, `DOS` = `02/14/2026`, `Payer` = `SilverPath Medicare Advantage`, `Denied_Code` containing `J1560`, `99600`, `99603`, `J7030`, `J0171`, `A4221`, and `Denied_Amount` = `6140.00` (numerical or string). It must also have `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, and `Appeal_Status` = `Submitted`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with flexible matching\n target_file = None\n exact_path = Path(workspace_path) / \"DENIAL_CO97_02142026.xlsx\"\n if exact_path.exists():\n target_file = exact_path\n else:\n # Try glob for similar filenames\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*02142026*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*denial*co*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*denial*co*97*.xlsx\"))\n if not candidates:\n # Even more lenient: any xlsx with denial and 97\n candidates = list(Path(workspace_path).glob(\"**/*enial*97*.xlsx\"))\n if not candidates:\n # Try any xlsx with denial in name\n candidates = list(Path(workspace_path).glob(\"**/*enial*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ENIAL*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n # List all xlsx files to help debug\n all_xlsx = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File DENIAL_CO97_02142026.xlsx not found in workspace. Found xlsx files: {[str(f.name) for f in all_xlsx]}\"}\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n ws = wb.active\n # If active sheet is empty, try other sheets\n if ws.max_row < 2:\n for sheet_name in wb.sheetnames:\n candidate = wb[sheet_name]\n if candidate.max_row >= 2:\n ws = candidate\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n rows = list(ws.iter_rows(values_only=False))\n if len(rows) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 rows (1 header + 1 data), but found {len(rows)} rows.\"}\n\n # Be lenient: ignore completely empty rows beyond row 2\n non_empty = [r for r in rows if any(c.value is not None for c in r)]\n if len(non_empty) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n if len(non_empty) > 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n\n # Build header map from first non-empty row\n header_row = non_empty[0]\n headers = {}\n for idx, cell in enumerate(header_row):\n if cell.value is not None:\n headers[str(cell.value).strip()] = idx\n\n # Helper to find a column by flexible name matching\n def find_col(possible_names):\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if h_lower == pn_lower:\n return idx\n # Also try substring matching\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if pn_lower in h_lower or h_lo
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775268135625",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "File `era_02142026_SilverPath.pdf` must exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"era_02142026_SilverPath.pdf\"\n if target_file.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' exists at {target_file}.\"}\n else:\n # Also do a case-insensitive search as a generous fallback\n found = [f for f in Path(workspace_path).rglob(\"*\") if f.name.lower() == \"era_02142026_silverpath.pdf\"]\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File found (case-insensitive match): {found[0]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' was NOT found in the workspace at '{workspace_path}'. Files present: {[f.name for f in Path(workspace_path).rglob('*') if f.is_file()]}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775268234332",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "File `statement_02142026.pdf` must not exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"statement_02142026.pdf\"\n if target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'statement_02142026.pdf' still exists in the workspace at '{target_file}'. It should have been removed.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'statement_02142026.pdf' does not exist in the workspace, as expected.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775268384702",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "In `mailbox.json`, an email with the following details must not exist: `to_addr` = `m.holloway@emailhost.com`, subject containing `CareIG Statement of Account` and `02/14/2026` (or the equivalent date in any format).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the absence of a specific email.\n # The mailbox.json could be in the workspace or in external_services_path.\n \n mailbox_path = None\n \n # Check external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json found at all, the email trivially doesn't exist -> PASS\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n # Define date patterns that all represent February 14, 2026\n date_patterns = [\n r'02/14/2026',\n r'02-14-2026',\n r'2/14/2026',\n r'2-14-2026',\n r'14/02/2026',\n r'14-02-2026',\n r'2026-02-14',\n r'2026/02/14',\n r'Feb(?:ruary)?\\s*14,?\\s*2026',\n r'14\\s*Feb(?:ruary)?,?\\s*2026',\n ]\n \n def subject_contains_date(subject):\n for pat in date_patterns:\n if re.search(pat, subject, re.IGNORECASE):\n return True\n return False\n \n violating_emails = []\n \n for email in emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n subject = email.get(\"subject\", \"\")\n \n # Check to_addr matches\n if to_addr != \"m.holloway@emailhost.com\":\n continue\n \n # Check subject contains \"CareIG Statement of Account\" (case-insensitive)\n if \"careig statement of account\" not in subject.lower():\n continue\n \n # Check subject contains the date 02/14/2026 in any format\n if subject_contains_date(subject):\n violating_emails.append(email)\n \n if len(violating_emails) > 0:\n details = []\n for e in violating_emails:\n details.append(f\" - email_id={e.get('email_id','?')}, to={e.get('to_addr','?')}, subject='{e.get('subject','?')}', folder={e.get('folder','?')}\")\n detail_str = \"\\n\".join(details)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) that should NOT exist (to=m.holloway@emailhost.com, subject containing 'CareIG Statement of Account' and date 02/14/2026):\\n{detail_str}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No email found in mailbox.json with to_addr='m.holloway@emailhost.com' and subject containing both 'CareIG Statement of Account' and the date 02/14/2026.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775268649908",
|
||
|
|
"sort_order": 14,
|
||
|
|
"rubric_text": "The file `appeal_L1_02142026.pdf` must contain the following:\n- Date: `04/08/2026`\n- Patient name: `Margaret` `Holloway`\n- Member ID: `SPA-00291847`\n- Date of Service (DOS): `02/14/2026`\n- Claim #: `CLM-REF-20260218-9931`\n- CARC: `CO-97`\n- Billed amount: `$6,140.00`\n- The letter body must contain: `#SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026` and `D83.9`.\n- Contracted rate: '$4,800.00'",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n pdf_file = None\n\n # Search for the file - try exact name and case-insensitive\n target_name = \"appeal_L1_02142026.pdf\"\n candidates = list(workspace.rglob(\"*.pdf\"))\n for c in candidates:\n if c.name.lower() == target_name.lower():\n pdf_file = c\n break\n\n # Also try broader matching in case of slight name variations\n if pdf_file is None:\n for c in candidates:\n if \"appeal\" in c.name.lower() and \"02142026\" in c.name:\n pdf_file = c\n break\n\n if pdf_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{target_name}' not found in workspace. PDF files found: {[str(c.relative_to(workspace)) for c in candidates]}\"}\n\n # Extract text from PDF\n text = \"\"\n try:\n import pdfplumber\n with pdfplumber.open(str(pdf_file)) as pdf:\n for page in pdf.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e1:\n try:\n import PyPDF2\n with open(str(pdf_file), 'rb') as f:\n reader = PyPDF2.PdfReader(f)\n for page in reader.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read PDF. pdfplumber error: {e1}; PyPDF2 error: {e2}\"}\n\n if not text.strip():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PDF file exists but no text could be extracted.\"}\n\n # Normalize text for searching\n text_lower = text.lower()\n # Also create a version with collapsed whitespace for matching longer strings\n text_collapsed = re.sub(r'\\s+', ' ', text).strip()\n text_collapsed_lower = text_collapsed.lower()\n\n checks = []\n total = 0\n passed = 0\n\n # Helper function for flexible matching\n def check_present(label, patterns, required=True):\n nonlocal total, passed\n total += 1\n for pattern in patterns:\n if pattern.lower() in text_collapsed_lower:\n passed += 1\n checks.append(f\"PASS: {label} - found '{pattern}'\")\n return True\n checks.append(f\"FAIL: {label} - none of {patterns} found in PDF text\")\n return False\n\n # 1. Date = 04/08/2026\n check_present(\"Date 04/08/2026\", [\n \"04/08/2026\", \"04-08-2026\", \"april 8, 2026\", \"april 08, 2026\",\n \"4/8/2026\", \"8 april 2026\", \"08 april 2026\"\n ])\n\n # 2. Margaret\n check_present(\"Patient first name 'Margaret'\", [\"margaret\"])\n\n # 3. Holloway\n check_present(\"Patient last name 'Holloway'\", [\"holloway\"])\n\n # 4. Member ID SPA-00291847\n check_present(\"Member ID SPA-00291847\", [\n \"spa-00291847\", \"spa00291847\", \"spa 00291847\"\n ])\n\n # 5. DOS 02/14/2026\n check_present(\"DOS 02/14/2026\", [\n \"02/14/2026\", \"02-14-2026\", \"2/14/2026\", \"february 14, 2026\",\n \"february 14 2026\", \"14 february 2026\"\n ])\n\n # 6. Claim # CLM-REF-20260218-9931\n check_present(\"Claim # CLM-REF-20260218-9931\", [\n \"clm-ref-20260218-9931\", \"clm ref 20260218 9931\"\n ])\n\n # 7. CARC CO-97\n check_present(\"CARC CO-97\", [\n \"co-97\", \"co 97\", \"co97\"\n ])\n\n # 8. Billed $6,140.00\n check_present(\"Billed $6,140.00\", [\n \"$6,140.00\", \"$6140.00\", \"6,140.00\", \"6140.00\", \"$ 6,140.00\", \"$6,140\"\n ])\n\n # 9. Letter body: #SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026\n # Check for the key components since exact whitespace/formatting may vary\n s
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775269539963",
|
||
|
|
"sort_order": 15,
|
||
|
|
"rubric_text": "In `audit_log.xlsx`, rows 36 and 37 should have: `CASE_ID` = `Holloway_DOB_03141961`, the date in `Datetime` = `04/08/2026`, `Action` = `Denial Received — CO-97` (case-insensitive) for row 36, and `Action` = `Appeal Submitted` (case-insensitive) for row 37.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"audit_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening audit_log.xlsx: {e}\"}\n\n # Find column indices by header row (row 1)\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip().upper()] = col_idx\n\n # Try to locate relevant columns\n case_id_col = None\n datetime_col = None\n action_col = None\n\n for key, idx in headers.items():\n if 'CASE' in key and 'ID' in key:\n case_id_col = idx\n if key == 'DATETIME': # FIX: was ('DATE' in key or 'TIME' in key or key == 'DATETIME')\n datetime_col = idx # 'DATE' in key matches FILE_UPDATED and FIELD_UPDATED,\n if key == 'ACTION': # overwriting datetime_col with the wrong column index.\n action_col = idx # Exact match eliminates the substring collision.\n\n if case_id_col is None or datetime_col is None or action_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find required columns. Headers found: {headers}\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_case_id(val):\n normed = normalize(val).lower()\n return normed == \"holloway_dob_03141961\"\n\n def check_date(val):\n if val is None:\n return False\n import datetime as dt\n if isinstance(val, (dt.datetime, dt.date)):\n d = val.date() if isinstance(val, dt.datetime) else val\n return d.month == 4 and d.day == 8 and d.year == 2026\n s = normalize(val)\n patterns = [\n r'0?4[/\\-]0?8[/\\-]2026',\n r'2026[/\\-]0?4[/\\-]0?8',\n r'April\\s+8,?\\s+2026',\n r'8\\s+April\\s+2026',\n r'04082026',\n ]\n for p in patterns:\n if re.search(p, s, re.IGNORECASE):\n return True\n return False\n\n def check_action_denial(val):\n normed = normalize(val).lower()\n normed_dashes = re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\-]+', '-', normed)\n if 'denial received' in normed_dashes and ('co-97' in normed_dashes or 'co 97' in normed_dashes or 'co97' in normed_dashes):\n return True\n return False\n\n def check_action_appeal(val):\n normed = normalize(val).lower()\n return 'appeal submitted' in normed\n\n # Check row 36\n row36_case = ws.cell(row=36, column=case_id_col).value\n row36_date = ws.cell(row=36, column=datetime_col).value\n row36_action = ws.cell(row=36, column=action_col).value\n\n # Check row 37\n row37_case = ws.cell(row=37, column=case_id_col).value\n row37_date = ws.cell(row=37, column=datetime_col).value\n row37_action = ws.cell(row=37, column=action_col).value\n\n issues = []\n passes = []\n\n if check_case_id(row36_case):\n passes.append(f\"Row 36 CASE_ID: '{row36_case}' matches\")\n else:\n issues.append(f\"Row 36 CASE_ID: expected 'Holloway_DOB_03141961', got '{row36_case}'\")\n\n if check_date(row36_date):\n passes.append(f\"Row 36 Datetime: '{row36_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 36 Datetime: expected date 04/08/2026, got '{row36_date}'\")\n\n if check_action_denial(row36_action):\n passes.append(f\"Row 36 Action: '{row36_action}' matches 'Denial Received — CO-97'\")\n else:\n issues.append(f\"Row
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775271368334",
|
||
|
|
"sort_order": 16,
|
||
|
|
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must contain 2 messages with text containing all of: `APPEAL`, `L1`, `Holloway_DOB_03141961`, `12/09/2025`, `BlueCrest Commercial PPO`, `PI-204` and with exactly one message containing `FOLLOW-UP`",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_path}\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n if not isinstance(slack_data, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json top-level is not a dict.\"}\n\n # Collect all messages from billing-appeals channel using multiple possible structures\n all_channel_messages = []\n\n channels_map = slack_data.get(\"channels\", {})\n messages_map = slack_data.get(\"messages\", {})\n\n # Standard format: channels dict keyed by channel ID, messages dict keyed by channel ID\n if isinstance(channels_map, dict) and isinstance(messages_map, dict):\n for ch_id, ch_info in channels_map.items():\n ch_name = \"\"\n if isinstance(ch_info, dict):\n ch_name = ch_info.get(\"name\", \"\") or ch_info.get(\"channel_name\", \"\") or \"\"\n elif isinstance(ch_info, str):\n ch_name = ch_info\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n msgs = messages_map.get(ch_id, [])\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n # Also check if messages_map is keyed by channel name directly\n if not all_channel_messages:\n for key, msgs in messages_map.items():\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n\n # Fallback: channels is a dict with channel names as keys containing messages\n if not all_channel_messages and isinstance(channels_map, dict):\n for ch_name, ch_data in channels_map.items():\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n all_channel_messages.extend(ch_data[\"messages\"])\n elif isinstance(ch_data, list):\n all_channel_messages.extend(ch_data)\n break\n\n # Fallback: channels is a list of channel objects\n if not all_channel_messages and isinstance(channels_map, list):\n for ch in channels_map:\n if isinstance(ch, dict):\n ch_name = ch.get(\"name\", \"\") or ch.get(\"channel_name\", \"\")\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n all_channel_messages.extend(ch.get(\"messages\", []))\n break\n\n # Fallback: top-level keys are channel names\n if not all_channel_messages:\n for key in slack_data:\n if key in (\"channels\", \"messages\", \"users\", \"counters\"):\n continue\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n val = slack_data[key]\n if isinstance(val, dict) and \"messages\" in val:\n all_channel_messages.extend(val[\"messages\"])\n elif isinstance(val, list):\n all_channel_messages.extend(val)\n break\n\n if not all_channel_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775274130585",
|
||
|
|
"sort_order": 17,
|
||
|
|
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must have a message containing: `[DENIAL]`, `Holloway_DOB_03141961`, and `CO-97`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_file = Path(external_services_path) / \"slack_data.json\"\n if not slack_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_file}\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n # According to the documented Slack data format:\n # \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } }\n # \"messages\": { \"C001\": [ { \"text\": \"...\", ... } ] }\n\n # Step 1: Find the channel ID for billing-appeals\n channels = slack_data.get(\"channels\", {})\n messages_by_channel = slack_data.get(\"messages\", {})\n\n billing_appeals_channel_id = None\n\n if isinstance(channels, dict):\n for cid, cinfo in channels.items():\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\":\n billing_appeals_channel_id = cid\n break\n # Also check loosely\n if \"billing\" in cname and \"appeal\" in cname:\n billing_appeals_channel_id = cid\n break\n elif isinstance(channels, list):\n for cinfo in channels:\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\" or (\"billing\" in cname and \"appeal\" in cname):\n billing_appeals_channel_id = cinfo.get(\"id\", cname)\n break\n\n # Collect message texts from the identified channel\n message_texts = []\n\n if billing_appeals_channel_id and isinstance(messages_by_channel, dict):\n channel_msgs = messages_by_channel.get(billing_appeals_channel_id, [])\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for key in [\"text\", \"message\", \"content\", \"body\"]:\n if key in msg and isinstance(msg[key], str):\n message_texts.append(msg[key])\n elif isinstance(msg, str):\n message_texts.append(msg)\n\n # Also try direct key lookup by name in messages dict (fallback)\n if not message_texts and isinstance(messages_by_channel, dict):\n for key in messages_by_channel:\n key_lower = key.lower().replace(\"#\", \"\").strip()\n if key_lower == \"billing-appeals\" or (\"billing\" in key_lower and \"appeal\" in key_lower):\n channel_msgs = messages_by_channel[key]\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for mkey in [\"text\", \"message\", \"content\", \"body\"]:\n if mkey in msg and isinstance(msg[mkey], str):\n message_texts.append(msg[mkey])\n elif isinstance(msg, str):\n message_texts.append(msg)\n break\n\n # Fallback: recursively search the entire structure for billing-appeals messages\n if not message_texts:\n def find_channel_messages(data):\n results = []\n if isinstance(data, dict):\n for key, value in data.items():\n key_lower = key.lower().replace
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|