[ { "id": "292496ce-fecf-4a85-a8ac-244d4df88aae", "sort_order": 0, "rubric_text": "In file `claim.xlsx`, the row where `DOS` = `02/14/2026` must have `Claim_Status` = `Denied`, `ERA_Received_Date` = `04/08/2026` or 8 April 2026, `Payment_Posted_Date` blank/null, `Benefit_Tier_Billed` = `Medical`, `Paid_Amount` = `0` and `Claim_Frequency_Code` = `1`.", "verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find claim.xlsx - be flexible with naming\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n # Try glob for any xlsx with 'claim' in the name\n candidates = list(Path(workspace_path).glob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n # Also try recursively\n candidates = list(Path(workspace_path).rglob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n headers_upper = [h.upper().replace(' ', '_') for h in headers]\n\n # Find columns flexibly\n dos_col = None\n status_col = None\n era_received_col = None\n payment_posted_col = None\n tier_col = None\n freq_col = None\n paid_amount_col = None\n\n for i, h in enumerate(headers_upper):\n if dos_col is None and 'DOS' in h:\n dos_col = i\n if status_col is None and ('CLAIM_STATUS' in h or ('CLAIM' in h and 'STATUS' in h) or h == 'STATUS' or h == 'CLAIM_STATUS'):\n status_col = i\n if era_received_col is None and ('ERA' in h and 'RECEIVED' in h):\n era_received_col = i\n if payment_posted_col is None and ('PAYMENT_POSTED' in h or ('PAYMENT' in h and 'POSTED' in h)):\n payment_posted_col = i\n if tier_col is None and ('BENEFIT_TIER' in h or ('TIER' in h and 'BILLED' in h) or ('BENEFIT' in h and 'TIER' not in h)):\n tier_col = i\n if tier_col is None and 'BENEFIT' in h:\n tier_col = i\n if freq_col is None and ('FREQUENCY' in h or 'FREQ' in h):\n freq_col = i\n if paid_amount_col is None and ('PAID_AMOUNT' in h or ('PAID' in h and 'AMOUNT' in h) or h == 'PAID_AMOUNT'):\n paid_amount_col = i\n\n if dos_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'DOS column not found. Headers: {headers}'}\n\n # Find the target row with DOS = 02/14/2026\n target_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n dos_val = row[dos_col]\n if dos_val is None:\n continue\n dos_str = str(dos_val).strip()\n match = False\n if '02/14/2026' in dos_str or '2/14/2026' in dos_str or '2026-02-14' in dos_str:\n match = True\n if isinstance(dos_val, datetime.datetime):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n elif isinstance(dos_val, datetime.date):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n if match:\n target_row = row\n break\n\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=02/14/2026 found in claim.xlsx'}\n\n issues = []\n details = []\n\n # Check Claim_Status = Denied\n if status_col is not None:\n status_val = str(target_row[status_col]).strip().lower() if target_row[status_col] is not None else ''\n if 'denied' in status_val or 'deny' in status_val or 'denial' in status_val:\n details.append(f'Claim_Status={target_row[status_col]}')\n else:\n issues.append(f'Claim_Status expected Denied, got: {target_row[status_col]}')\n else:\n issues.append(f'Claim_Status column not found. Headers: {headers}')\n\n # Check ERA_Received_Date = 04/08/2026 or 8 April 2026\n if era_received_col is not None:\n era_val = target_row[era_received_col]\n era_match = False\n if era_val is not None:\n if isinstance(era_val, (datetime.datetime, datetime.date)):\n if era_val.year == 2026 and era_val.month == 4 and era_val.day == 8:\n era_match = True\n else:\n era_str = str(era_val).strip()\n if '04/08/2026' in era_str or '4/8/2026' in era_str or '2026-04-08' in era_str or '8 April 2026' in era_str or 'April 8' in era_str or '08-Apr' in era_str or '8-Apr' in era_str or '04-08-2026' in era_str:\n era_match = True\n if era_match:\n details.append(f'ERA_Received_Date={era_val}')\n else:\n issues.append(f'ERA_Received_Date expected 04/08/2026 (8 April 2026), got: {era_val}')\n else:\n issues.append(f'ERA_Received_Date column not found. Headers: {headers}')\n\n # Check Payment_Posted_Date blank/null\n if payment_posted_col is not None:\n payment_val = target_row[payment_posted_col]\n if payment_val is None or str(payment_val).strip() in ('', 'None', 'nan', 'NaN', 'NULL', 'null', 'N/A', 'n/a'):\n details.append('Payment_Posted_Date=blank')\n else:\n issues.append(f'Payment_Posted_Date expected blank/null, got: {payment_val}')\n else:\n details.append('Payment_Posted_Date column not found (treated as blank)')\n\n # Check Benefit_Tier_Billed = Medical\n if tier_col is not None:\n tier_val = str(target_row[tier_col]).strip().lower() if target_row[tier_col] is not None else ''\n if 'medical' in tier_val:\n details.append(f'Benefit_Tier_Billed={target_row[tier_col]}')\n else:\n issues.append(f'Benefit_Tier_Billed expected Medical, got: {target_row[tier_col]}')\n else:\n issues.append(f'Benefit_Tier_Billed column not found. Headers: {headers}')\n\n # Check Paid_Amount = 0\n if paid_amount_col is not None:\n paid_val = target_row[paid_amount_col]\n paid_str = str(paid_val).strip().lower() if paid_val is not None else ''\n try:\n if paid_str in ('', 'none', 'nan', 'null', 'n/a'):\n numeric_val = 0.0\n else:\n numeric_val = float(paid_str.replace('$', '').replace(',', ''))\n if abs(numeric_val) < 0.01:\n details.append(f'Paid_Amount={paid_val}')\n else:\n issues.append(f'Paid_Amount expected 0, got: {paid_val}')\n except (ValueError, TypeError):\n if paid_str in ('', 'none', 'nan', 'null', 'n/a', '0', '0.0', '0.00'):\n details.append(f'Paid_Amount={paid_val} (treated as 0)')\n else:\n issues.append(f'Paid_Amount expected 0, got: {paid_val}')\n else:\n issues.append(f'Paid_Amount column not found. Headers: {headers}')\n\n # Check Claim_Frequency_Code = 1\n if freq_col is not None:\n freq_val = str(target_row[freq_col]).strip() if target_row[freq_col] is not None else ''\n if freq_val in ('1', '1.0', '01'):\n details.append(f'Claim_Frequency_Code={freq_val}')\n else:\n issues.append(f'Claim_Frequency_Code expected 1, got: {freq_val}')\n else:\n issues.append(f'Claim_Frequency_Code column not found. Headers: {headers}')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues found in claim.xlsx row DOS=02/14/2026: ' + '; '.join(issues)}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'claim.xlsx row DOS=02/14/2026 verified: ' + '; '.join(details)}\n", "criterion_type": "expected_output" }, { "id": "826aeb1f-eea5-4a46-b9d1-3bb715ba3013", "sort_order": 1, "rubric_text": "In file `claim.xlsx`, total row count must be exactly 5 (1 header + 4 data rows).", "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n \n row_count = sum(1 for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row))\n \n if row_count == 5:\n return {'pass': True, 'score': 1.0, 'feedback': f'claim.xlsx has exactly 5 rows (1 header + 4 data rows)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'claim.xlsx has {row_count} non-empty rows, expected 5 (1 header + 4 data rows)'}\n", "criterion_type": "expected_output" }, { "id": "f7996998-b356-45fc-a368-c4ad88757797", "sort_order": 2, "rubric_text": "A file named `appeal_L1_02142026.pdf` must exist in the workspace.", "verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n all_files = [f for f in ws.rglob('*') if f.is_file()]\n\n # --- Pass 1: exact stem match, PDF only ---\n target_stem = 'appeal_l1_02142026'\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n if f.stem.lower() == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found appeal_L1_02142026.pdf at {f.relative_to(ws)}'\n }\n\n # --- Pass 2: normalised separators (hyphens → underscores), PDF only ---\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n normalised = f.stem.lower().replace('-', '_').replace(' ', '_')\n if normalised == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found matching PDF with alternate separators: '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Pass 3: component match, PDF only ---\n date_variants = ['02142026', '02_14_2026', '02-14-2026']\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n n = f.name.lower()\n if 'appeal' in n and 'l1' in n and any(d in n for d in date_variants):\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found appeal letter PDF (component match): '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Diagnosis: did the model produce the file in a non-PDF format? ---\n non_pdf_matches = [\n f for f in all_files\n if f.suffix.lower() != '.pdf'\n and 'appeal' in f.name.lower()\n and 'l1' in f.name.lower()\n and any(d in f.name.lower() for d in date_variants)\n ]\n if non_pdf_matches:\n names = [f.name for f in non_pdf_matches]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'Appeal letter found but not in PDF format: {names}. '\n f'The criterion requires a .pdf file. '\n f'Rename or re-export the file as appeal_L1_02142026.pdf.'\n )\n }\n\n all_names = [f.name for f in all_files]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No appeal_L1_02142026.pdf found in workspace. '\n f'Expected a .pdf file whose name contains \"appeal\", \"l1\", '\n f'and date \"02142026\". '\n f'Files present: {all_names}'\n )\n }", "criterion_type": "expected_output" }, { "id": "f7446601-8f7d-4712-9097-7620699d893c", "sort_order": 3, "rubric_text": "In `mailbox.json` (external services), exactly 1 email must exist in the `Sent` folder with `to_addr` = `appeals@silverpath-claims.com`, with subject containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `SPA-00291847`, `DOS:`, `02/14/2026`, `Claim:`, `CLM-REF-20260218-9931`, `CARC:`, `CO-97`, and attachments containing exactly `appeal_L1_02142026.pdf`, `claim_02142026_Holloway_SilverPath.pdf`, `pa_approval_SilverPath_Holloway.pdf`, `lmn_DrOsei_Holloway.pdf`, `era_02142026_SilverPath.pdf`, and filename containing `visit_note`.", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find sent emails to the appeals address (case-insensitive)\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@silverpath-claims.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent email to appeals@silverpath-claims.com found'}\n\n if len(target_emails) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 sent email to appeals@silverpath-claims.com, found {len(target_emails)}'}\n\n email = target_emails[0]\n\n # Check subject for required substrings\n subject = str(email.get('subject', ''))\n required_substrings = [\n 'APPEAL L1',\n 'CareIG',\n 'Holloway',\n 'MemberID:',\n 'SPA-00291847',\n 'DOS:',\n '02/14/2026',\n 'Claim:',\n 'CLM-REF-20260218-9931',\n 'CARC:',\n 'CO-97'\n ]\n\n # Case-sensitive IDs that must match exactly\n exact_match_required = {'SPA-00291847', 'CLM-REF-20260218-9931', 'CO-97', '02/14/2026'}\n\n missing = []\n for s in required_substrings:\n if s in subject:\n continue\n # For non-exact fields, try case-insensitive\n if s not in exact_match_required and s.upper() in subject.upper():\n continue\n missing.append(s)\n\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Email subject missing required substrings: {missing}. Subject was: {subject}'}\n\n # Check attachments\n attachments = email.get('attachments', [])\n attach_names = []\n for a in attachments:\n if isinstance(a, str):\n attach_names.append(a)\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', str(a)))\n attach_names.append(name)\n else:\n attach_names.append(str(a))\n\n attach_names_lower = [n.lower() for n in attach_names]\n\n # Required exact attachment filenames\n required_exact_attachments = [\n 'appeal_L1_02142026.pdf',\n 'claim_02142026_Holloway_SilverPath.pdf',\n 'pa_approval_SilverPath_Holloway.pdf',\n 'lmn_DrOsei_Holloway.pdf',\n 'era_02142026_SilverPath.pdf'\n ]\n\n missing_attachments = []\n for req in required_exact_attachments:\n # Check case-insensitive\n found = any(req.lower() in n for n in attach_names_lower)\n if not found:\n missing_attachments.append(req)\n\n if missing_attachments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required attachments: {missing_attachments}. Found attachments: {attach_names}'}\n\n # Check for an attachment whose filename contains 'visit_note' (case-insensitive)\n found_visit_note = any('visit_note' in n.lower() for n in attach_names)\n if not found_visit_note:\n return {'pass': False, 'score': 0.0, 'feedback': f'No attachment with filename containing \"visit_note\" found. Found attachments: {attach_names}'}\n\n # Total expected: 5 exact + 1 visit_note = 6\n # Be lenient: as long as all required ones are present, pass even if there are extras\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Sent email to appeals@silverpath-claims.com found with correct subject substrings and all required attachments. Subject: {subject}. Attachments: {attach_names}'\n }\n", "criterion_type": "expected_output" }, { "id": "74e31c33-9667-455a-8fd5-dd989a7231eb", "sort_order": 4, "rubric_text": "In `mailbox.json` (external services), no email in any folder must have `to_addr` containing `billing@silverpath-claims.com` with a date of `2026-04-01` or later.", "verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n \n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n cutoff = datetime(2026, 4, 1)\n \n violations = []\n for e in emails:\n to_addr = str(e.get('to_addr', ''))\n if 'billing@silverpath-claims.com' in to_addr.lower():\n date_str = str(e.get('date', ''))\n try:\n email_date = datetime.fromisoformat(date_str[:10])\n if email_date >= cutoff:\n violations.append(f\"Subject: {e.get('subject', 'N/A')}, Date: {date_str}\")\n except Exception:\n pass\n \n if violations:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found emails to billing@silverpath-claims.com on/after 2026-04-01: {violations}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No emails sent to billing@silverpath-claims.com on or after 2026-04-01 (no corrected claim submitted to payer)'}\n", "criterion_type": "incorrect_behavior" }, { "id": "bce26050-60a1-4108-8bf3-836752efd837", "sort_order": 5, "rubric_text": "File `appeals_log.xlsx` must contain exactly 4 rows (1 header + 3 data rows), with exactly 1 row having `DOS` = `02/14/2026`, and that row must have `CARC_Code` = `CO-97`, `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, `Appeal_Status` = `Submitted`, `Payer` = `SilverPath Medicare Advantage`.", "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) != 4:\n return {'pass': False, 'score': 0.0, 'feedback': f'appeals_log.xlsx has {len(rows)} rows, expected 4 (1 header + 3 data rows)'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n carc_col = col_idx('CARC')\n level_col = col_idx('LEVEL')\n subdate_col = col_idx('SUBMITTED_DATE') or col_idx('SUBMIT')\n status_col = col_idx('STATUS')\n payer_col = col_idx('PAYER')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '02/14/2026' in dos_val or '2026-02-14' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 row with DOS=02/14/2026 in appeals_log.xlsx, found {len(target_rows)}'}\n \n row = target_rows[0]\n issues = []\n \n if carc_col is not None:\n carc_val = str(row[carc_col]).strip() if row[carc_col] is not None else ''\n if 'CO-97' not in carc_val.upper() and 'CO97' not in carc_val.upper():\n issues.append(f'CARC_Code expected CO-97, got: {carc_val}')\n else:\n issues.append('CARC_Code column not found')\n \n if level_col is not None:\n level_val = str(row[level_col]).strip().upper() if row[level_col] is not None else ''\n if 'L1' not in level_val:\n issues.append(f'Appeal_Level expected L1, got: {row[level_col]}')\n else:\n issues.append('Appeal_Level column not found')\n \n if subdate_col is not None:\n date_val = str(row[subdate_col]).strip() if row[subdate_col] is not None else ''\n if '04/08/2026' not in date_val and '2026-04-08' not in date_val:\n issues.append(f'Appeal_Submitted_Date expected 04/08/2026, got: {date_val}')\n else:\n issues.append('Appeal_Submitted_Date column not found')\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if payer_col is not None:\n payer_val = str(row[payer_col]).strip() if row[payer_col] is not None else ''\n if 'SILVERPATH' not in payer_val.upper():\n issues.append(f'Payer expected SilverPath Medicare Advantage, got: {payer_val}')\n else:\n issues.append('Payer column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=02/14/2026 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx has 4 rows, DOS=02/14/2026 row has CARC_Code=CO-97, Appeal_Level=L1, Appeal_Submitted_Date=04/08/2026, Appeal_Status=Submitted, Payer=SilverPath Medicare Advantage'}\n", "criterion_type": "expected_output" }, { "id": "3e721c8d-53cd-47f9-a9a0-19515432c546", "sort_order": 6, "rubric_text": "In `appeals_log.xlsx`, the row where `DOS` = `12/09/2025` must have `Appeal_Status` = `Submitted` and `Followup_Log` that is non-blank and contains a date matching `04/08/2026` or `8 April 2026`.", "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx has no data rows'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n status_col = col_idx('STATUS')\n followup_col = col_idx('FOLLOWUP') or col_idx('FOLLOW_UP') or col_idx('FOLLOW')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '12/09/2025' in dos_val or '2025-12-09' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=12/09/2025 found in appeals_log.xlsx'}\n \n row = target_rows[0]\n issues = []\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if followup_col is not None:\n followup_val = str(row[followup_col]).strip() if row[followup_col] is not None else ''\n if followup_val in ('', 'None', 'nan', 'NaN', 'NULL'):\n issues.append('Followup_Log is blank')\n else:\n has_date = '04/08/2026' in followup_val or '2026-04-08' in followup_val or '8 april 2026' in followup_val.lower() or 'april 8, 2026' in followup_val.lower()\n if not has_date:\n issues.append(f'Followup_Log does not contain date 04/08/2026 or 8 April 2026. Value: {followup_val}')\n else:\n issues.append('Followup_Log column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=12/09/2025 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx DOS=12/09/2025 row has Appeal_Status=Submitted and Followup_Log containing 04/08/2026'}\n", "criterion_type": "expected_output" }, { "id": "fc648229-7c0f-4abc-9ba6-d8d05942a39a", "sort_order": 7, "rubric_text": "In `mailbox.json` (external services), exactly 3 emails must exist in the `Sent` folder with `to_addr` = `appeals@bluecrest-health.com`. The 2 of the emails must have subjects containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `BCR-00184523`, `DOS:`, `12/09/2025`, `Claim:`, `CLM-REF-20251215-6612`, `CARC:`, `PI-204`, email attachment name = `appeal_L1_12092025.pdf`, email body containing `BlueCrest Commercial PPO — Provider Appeals Department`, `Member ID: BCR-00184523`, `DOS: 12/09/2025`, `Claim Reference: CLM-REF-20251215-6612`, `Billed: $5,840.00`", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find all sent emails to appeals@bluecrest-health.com\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@bluecrest-health.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails to appeals@bluecrest-health.com, found {len(target_emails)}'}\n\n # Required substrings in the subject (case-insensitive, whitespace-flexible)\n required_subject_substrings = [\n 'APPEAL L1', 'CareIG', 'Holloway', 'MemberID:',\n 'BCR-00184523', 'DOS:', '12/09/2025', 'Claim:',\n 'CLM-REF-20251215-6612', 'CARC:', 'PI-204'\n ]\n\n # Required body substrings\n required_body_substrings = [\n 'BlueCrest Commercial PPO',\n 'Provider Appeals Department',\n 'Member ID: BCR-00184523',\n 'DOS: 12/09/2025',\n 'Claim Reference: CLM-REF-20251215-6612',\n 'Billed: $5,840.00'\n ]\n\n required_attachment = 'appeal_L1_12092025.pdf'\n\n def normalize(text):\n \"\"\"Normalize whitespace for flexible matching.\"\"\"\n return re.sub(r'\\s+', ' ', str(text)).strip()\n\n def check_subject(subject):\n subj_norm = normalize(subject).lower()\n # Also try without spaces around colons\n subj_no_space = re.sub(r'\\s*:\\s*', ':', subj_norm)\n missing = []\n for s in required_subject_substrings:\n s_lower = s.lower()\n s_no_space = re.sub(r'\\s*:\\s*', ':', s_lower)\n # Try multiple matching strategies\n if s_lower in subj_norm:\n continue\n if s_no_space in subj_no_space:\n continue\n # Try without space: \"MemberID:\" might appear as \"Member ID:\" or \"MemberID:\"\n if s == 'MemberID:':\n if 'memberid:' in subj_norm or 'member id:' in subj_norm or 'memberid :' in subj_norm:\n continue\n missing.append(s)\n return missing\n\n def check_body(body):\n body_norm = normalize(body)\n missing = []\n for s in required_body_substrings:\n # Case-insensitive check with normalized whitespace\n s_norm = normalize(s)\n if s_norm.lower() in body_norm.lower():\n continue\n # Try with em-dash variants\n s_variants = [\n s_norm,\n s_norm.replace('\\u2014', '-'),\n s_norm.replace('\\u2014', '--'),\n s_norm.replace('-', '\\u2014'),\n ]\n found = False\n for variant in s_variants:\n if variant.lower() in body_norm.lower():\n found = True\n break\n if not found:\n missing.append(s)\n return missing\n\n def get_attachment_names(attachments):\n names = []\n for a in (attachments or []):\n if isinstance(a, str):\n names.append(a.strip())\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', a.get('file_name', '')))\n names.append(str(name).strip())\n else:\n names.append(str(a).strip())\n return names\n\n def check_attachment(email_obj):\n att_names = get_attachment_names(email_obj.get('attachments', []))\n for name in att_names:\n if name.lower() == required_attachment.lower():\n return True\n return False\n\n # Find emails that match all criteria\n fully_matching = []\n partial_details = []\n\n for email in target_emails:\n subject = str(email.get('subject', ''))\n body = str(email.get('body_text', email.get('body', '')))\n\n subj_missing = check_subject(subject)\n body_missing = check_body(body)\n has_attachment = check_attachment(email)\n\n if not subj_missing and not body_missing and has_attachment:\n fully_matching.append(email)\n else:\n partial_details.append({\n 'subject': subject[:100],\n 'subj_missing': subj_missing,\n 'body_missing': body_missing,\n 'has_attachment': has_attachment\n })\n\n if len(fully_matching) >= 2:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 3 sent emails to appeals@bluecrest-health.com. {len(fully_matching)} of them have correct subjects containing all required substrings, correct body content, and the required attachment ({required_attachment}).'\n }\n\n # If we didn't find 2 fully matching, try a more lenient approach:\n # Check subject and body separately, being more lenient on subject formatting\n lenient_matching = []\n for email in target_emails:\n subject = str(email.get('subject', ''))\n body = str(email.get('body_text', email.get('body', '')))\n subj_norm = normalize(subject).lower()\n\n # Lenient subject check: just check key identifiers are present\n key_subject_items = ['appeal', 'l1', 'careig', 'holloway', 'bcr-00184523',\n '12/09/2025', 'clm-ref-20251215-6612', 'pi-204']\n subj_key_missing = [k for k in key_subject_items if k not in subj_norm]\n\n body_missing = check_body(body)\n has_attachment = check_attachment(email)\n\n if not subj_key_missing and not body_missing and has_attachment:\n lenient_matching.append(email)\n\n if len(lenient_matching) >= 2:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 3 sent emails to appeals@bluecrest-health.com. {len(lenient_matching)} of them have correct subjects (lenient match), correct body content, and the required attachment ({required_attachment}).'\n }\n\n # Build detailed failure feedback\n feedback_parts = [f'Expected at least 2 fully matching emails out of 3 sent to appeals@bluecrest-health.com.']\n feedback_parts.append(f'Fully matching (strict): {len(fully_matching)}, Lenient matching: {len(lenient_matching)}')\n for i, detail in enumerate(partial_details):\n feedback_parts.append(\n f'Email {i+1} (subject: \"{detail[\"subject\"]}...\"): '\n f'subject missing={detail[\"subj_missing\"]}, '\n f'body missing={detail[\"body_missing\"]}, '\n f'has_attachment={detail[\"has_attachment\"]}'\n )\n for i, email in enumerate(fully_matching):\n feedback_parts.append(f'Matching email {i+1} subject: \"{str(email.get(\"subject\", \"\"))[:100]}\"')\n\n score = 0.0\n if len(fully_matching) == 1 or len(lenient_matching) == 1:\n score = 0.5\n\n return {\n 'pass': False,\n 'score': score,\n 'feedback': ' | '.join(feedback_parts)\n }\n", "criterion_type": "expected_output" }, { "id": "52595972-13bc-49f5-b35b-4674e117b8ff", "sort_order": 8, "rubric_text": "In `slack_data.json` (external services), channel `billing-appeals` must contain a message with text containing all of: `APPEAL FILED L1`, `Holloway_DOB_03141961`, `02/14/2026`, `SilverPath`, `CO-97`.", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check Slack'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n \n with open(slack_path, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n messages = data.get('messages', {})\n \n # Find #billing-appeals channel (be flexible with naming)\n billing_appeals_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if ch_name == 'billingappeals':\n billing_appeals_id = ch_id\n break\n \n if billing_appeals_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Channel #billing-appeals not found in slack_data.json. Available channels: {[ch.get(\"name\") for ch in channels.values()]}'}\n \n channel_messages = messages.get(billing_appeals_id, [])\n \n # Required content pieces to find in a message\n # Be loose: case-insensitive where appropriate, flexible date formats, flexible separators\n required_checks = {\n 'APPEAL FILED L1': lambda t: 'appeal' in t.lower() and ('filed' in t.lower() or 'file' in t.lower()) and ('l1' in t.lower() or 'level 1' in t.lower() or 'level1' in t.lower()),\n 'Holloway_DOB_03141961': lambda t: 'holloway' in t.lower() and ('03141961' in t.replace('/', '').replace('-', '').replace(' ', '') or '03/14/1961' in t or '03-14-1961' in t or '3/14/1961' in t),\n '02/14/2026': lambda t: any(d in t for d in ['02/14/2026', '2/14/2026', '02-14-2026', '2-14-2026', '2026-02-14', '02142026']),\n 'SilverPath': lambda t: 'silverpath' in t.lower().replace(' ', '').replace('-', '').replace('_', ''),\n 'CO-97': lambda t: 'co-97' in t.lower() or 'co 97' in t.lower() or 'co97' in t.lower(),\n }\n \n best_match_count = 0\n best_match_text = ''\n matched_keys_best = []\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n \n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n \n if len(matched_keys) == len(required_checks):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found matching message in #billing-appeals containing all required elements: {text[:300]}'}\n \n # Also check all messages across all channels in case channel name matching failed\n for ch_id, ch_msgs in messages.items():\n if ch_id == billing_appeals_id:\n continue\n for msg in ch_msgs:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n # Note: don't pass here since it's wrong channel, but track for feedback\n \n missing = [k for k in required_checks if k not in matched_keys_best]\n return {\n 'pass': False,\n 'score': best_match_count / len(required_checks) * 0.5, # partial credit\n 'feedback': f'No message in #billing-appeals found containing all required elements. '\n f'Best match had {best_match_count}/{len(required_checks)} elements. '\n f'Missing: {missing}. '\n f'Found {len(channel_messages)} messages in #billing-appeals. '\n f'Best matching text: \"{best_match_text[:300]}\"'\n }\n", "criterion_type": "expected_output" }, { "id": "29097e98-d53d-4af6-ac8a-8df8f3ec442e", "sort_order": 9, "rubric_text": "In `audit_log.xlsx`, total row count must be exactly 37 (1 header + 36 data rows).", "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find audit_log.xlsx - be flexible with location\n audit_path = Path(workspace_path) / 'audit_log.xlsx'\n if not audit_path.exists():\n # Try glob search\n candidates = list(Path(workspace_path).rglob('audit_log.xlsx'))\n if candidates:\n audit_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n\n try:\n wb = openpyxl.load_workbook(audit_path)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = wb.active\n\n # Collect all non-empty rows\n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n\n total_row_count = len(rows)\n\n # Rubric says exactly 37 rows (1 header + 36 data rows)\n if total_row_count != 37:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'audit_log.xlsx has {total_row_count} non-empty rows, expected exactly 37 (1 header + 36 data rows).'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'audit_log.xlsx has exactly {total_row_count} non-empty rows (1 header + 36 data rows), as required.'\n }\n", "criterion_type": "expected_output" }, { "id": "rubric_1775206985975", "sort_order": 10, "rubric_text": "File `DENIAL_CO97_02142026.xlsx` must exist and contain exactly 2 rows (1 header + 1 data row) with the following values in the data row: `CASE_ID` = `Holloway_DOB_03141961`, `CARC_Code` = `CO-97`, `CARC_Description` = `Benefit not assigned`, `DOS` = `02/14/2026`, `Payer` = `SilverPath Medicare Advantage`, `Denied_Code` containing `J1560`, `99600`, `99603`, `J7030`, `J0171`, `A4221`, and `Denied_Amount` = `6140.00` (numerical or string). It must also have `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, and `Appeal_Status` = `Submitted`.", "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with flexible matching\n target_file = None\n exact_path = Path(workspace_path) / \"DENIAL_CO97_02142026.xlsx\"\n if exact_path.exists():\n target_file = exact_path\n else:\n # Try glob for similar filenames\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*02142026*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*denial*co*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*denial*co*97*.xlsx\"))\n if not candidates:\n # Even more lenient: any xlsx with denial and 97\n candidates = list(Path(workspace_path).glob(\"**/*enial*97*.xlsx\"))\n if not candidates:\n # Try any xlsx with denial in name\n candidates = list(Path(workspace_path).glob(\"**/*enial*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ENIAL*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n # List all xlsx files to help debug\n all_xlsx = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File DENIAL_CO97_02142026.xlsx not found in workspace. Found xlsx files: {[str(f.name) for f in all_xlsx]}\"}\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n ws = wb.active\n # If active sheet is empty, try other sheets\n if ws.max_row < 2:\n for sheet_name in wb.sheetnames:\n candidate = wb[sheet_name]\n if candidate.max_row >= 2:\n ws = candidate\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n rows = list(ws.iter_rows(values_only=False))\n if len(rows) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 rows (1 header + 1 data), but found {len(rows)} rows.\"}\n\n # Be lenient: ignore completely empty rows beyond row 2\n non_empty = [r for r in rows if any(c.value is not None for c in r)]\n if len(non_empty) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n if len(non_empty) > 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n\n # Build header map from first non-empty row\n header_row = non_empty[0]\n headers = {}\n for idx, cell in enumerate(header_row):\n if cell.value is not None:\n headers[str(cell.value).strip()] = idx\n\n # Helper to find a column by flexible name matching\n def find_col(possible_names):\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if h_lower == pn_lower:\n return idx\n # Also try substring matching\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if pn_lower in h_lower or h_lower in pn_lower:\n return idx\n return None\n\n data_row = non_empty[1]\n\n def cell_val(row, col_idx):\n if col_idx is None:\n return None\n if col_idx < len(row):\n return row[col_idx].value\n return None\n\n def normalize_str(v):\n if v is None:\n return \"\"\n return str(v).strip()\n\n def normalize_date(v):\n \"\"\"Try to normalize a date value to MM/DD/YYYY for comparison.\"\"\"\n if v is None:\n return \"\"\n if isinstance(v, (datetime.datetime, datetime.date)):\n return v.strftime(\"%m/%d/%Y\")\n s = str(v).strip()\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\", \"%d/%m/%Y\"]:\n try:\n dt = datetime.datetime.strptime(s, fmt)\n return dt.strftime(\"%m/%d/%Y\")\n except ValueError:\n continue\n return s\n\n def nums_close(a, b, tol=0.02):\n try:\n fa = float(re.sub(r'[^\\d.\\-]', '', str(a)))\n fb = float(re.sub(r'[^\\d.\\-]', '', str(b)))\n if fb == 0:\n return fa == 0\n return abs(fa - fb) / abs(fb) <= tol\n except:\n return False\n\n issues = []\n found_details = []\n\n # Check CASE_ID = Holloway_DOB_03141961\n col = find_col([\"CASE_ID\", \"CaseID\", \"Case ID\", \"Case_ID\", \"CaseId\"])\n if col is None:\n issues.append(f\"Column 'CASE_ID' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n expected = \"Holloway_DOB_03141961\"\n # Be lenient: compare lowered and stripped, allow flexible separators\n val_cmp = val.lower().replace(\" \", \"_\").replace(\"-\", \"_\")\n exp_cmp = expected.lower().replace(\" \", \"_\").replace(\"-\", \"_\")\n if val_cmp != exp_cmp:\n issues.append(f\"CASE_ID: expected '{expected}', got '{val}'.\")\n else:\n found_details.append(f\"CASE_ID='{val}'\")\n\n # Check CARC_Code = CO-97\n col = find_col([\"CARC_Code\", \"CARCCode\", \"CARC Code\", \"CARC\", \"CARC_code\"])\n if col is None:\n issues.append(f\"Column 'CARC_Code' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n # Accept CO-97, CO97, co-97, co97\n val_cmp = val.upper().replace(\" \", \"\").replace(\"-\", \"\")\n if val_cmp != \"CO97\":\n issues.append(f\"CARC_Code: expected 'CO-97', got '{val}'.\")\n else:\n found_details.append(f\"CARC_Code='{val}'\")\n\n # Check CARC_Description = \"Benefit not assigned\" - be lenient, accept variations\n col = find_col([\"CARC_Description\", \"CARCDescription\", \"CARC Description\", \"CARC_Desc\", \"Description\"])\n if col is None:\n issues.append(f\"Column 'CARC_Description' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n val_lower = val.lower()\n # The rubric says \"Benefit not assigned\" but the original code checked for bundling-related words.\n # Let's accept either the exact phrase or close variants, and also accept the CO-97 standard description\n # which is \"The benefit for this service is included in the payment/allowance for another service/procedure\n # that has already been adjudicated.\"\n # Be lenient: accept if it contains \"benefit\" and either \"not assigned\" or \"included\" or \"another\"\n has_benefit = \"benefit\" in val_lower\n has_not_assigned = \"not assigned\" in val_lower or \"not been assigned\" in val_lower\n has_included = \"includ\" in val_lower\n has_another = \"another\" in val_lower\n if has_benefit and (has_not_assigned or (has_included and has_another)):\n found_details.append(f\"CARC_Description='{val}'\")\n elif val_lower == \"\" or not has_benefit:\n issues.append(f\"CARC_Description: expected 'Benefit not assigned' or similar CO-97 description, got '{val}'.\")\n else:\n # Has benefit but no other match - still be lenient if it seems reasonable\n found_details.append(f\"CARC_Description='{val}' (accepted with leniency)\")\n\n # Check DOS = 02/14/2026\n col = find_col([\"DOS\", \"Date of Service\", \"DateOfService\", \"Date_of_Service\", \"Service_Date\", \"ServiceDate\"])\n if col is None:\n issues.append(f\"Column 'DOS' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_date(cell_val(data_row, col))\n expected = \"02/14/2026\"\n if val != expected:\n issues.append(f\"DOS: expected '{expected}', got '{val}'.\")\n else:\n found_details.append(f\"DOS='{val}'\")\n\n # Check Payer = SilverPath Medicare Advantage\n col = find_col([\"Payer\", \"PayerName\", \"Payer Name\", \"Payer_Name\", \"Insurance\", \"Insurance_Name\"])\n if col is None:\n issues.append(f\"Column 'Payer' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n expected = \"SilverPath Medicare Advantage\"\n # Lenient: compare ignoring case and extra whitespace\n val_cmp = re.sub(r'\\s+', ' ', val.lower().strip())\n exp_cmp = re.sub(r'\\s+', ' ', expected.lower().strip())\n if val_cmp != exp_cmp:\n # Also accept if it contains the key parts\n if \"silverpath\" in val_cmp and \"medicare\" in val_cmp:\n found_details.append(f\"Payer='{val}' (accepted with leniency)\")\n else:\n issues.append(f\"Payer: expected '{expected}', got '{val}'.\")\n else:\n found_details.append(f\"Payer='{val}'\")\n\n # Check Denied_Code containing J1560, 99600, 99603, J7030, J0171, A4221\n col = find_col([\"Denied_Code\", \"DeniedCode\", \"Denied Code\", \"Denied_Codes\", \"DeniedCodes\", \"Denied Codes\", \"Denial_Code\", \"DenialCode\", \"Denial_Codes\", \"Codes\", \"Code\", \"CPT_Codes\", \"CPT\", \"HCPCS\"])\n if col is None:\n issues.append(f\"Column 'Denied_Code' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n val_upper = val.upper()\n required_codes = [\"J1560\", \"99600\", \"99603\", \"J7030\", \"J0171\", \"A4221\"]\n missing_codes = [c for c in required_codes if c not in val_upper]\n if missing_codes:\n issues.append(f\"Denied_Code: expected to contain {required_codes}, but missing {missing_codes}. Got '{val}'.\")\n else:\n found_details.append(f\"Denied_Code='{val}'\")\n\n # Check Denied_Amount = 6140.00\n col = find_col([\"Denied_Amount\", \"DeniedAmount\", \"Denied Amount\", \"Denied_Amt\", \"DeniedAmt\", \"Amount\", \"Denial_Amount\", \"Total_Amount\", \"TotalAmount\"])\n if col is None:\n issues.append(f\"Column 'Denied_Amount' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = cell_val(data_row, col)\n if not nums_close(val, 6140.00):\n issues.append(f\"Denied_Amount: expected 6140.00, got '{val}'.\")\n else:\n found_details.append(f\"Denied_Amount='{val}'\")\n\n # Check Appeal_Level = L1\n col = find_col([\"Appeal_Level\", \"AppealLevel\", \"Appeal Level\", \"Level\"])\n if col is None:\n issues.append(f\"Column 'Appeal_Level' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n val_upper = val.upper().replace(\" \", \"\").replace(\"-\", \"\").replace(\"_\", \"\")\n if val_upper not in [\"L1\", \"LEVEL1\", \"1\", \"LVL1\"]:\n issues.append(f\"Appeal_Level: expected 'L1', got '{val}'.\")\n else:\n found_details.append(f\"Appeal_Level='{val}'\")\n\n # Check Appeal_Submitted_Date = 04/08/2026\n col = find_col([\"Appeal_Submitted_Date\", \"AppealSubmittedDate\", \"Appeal Submitted Date\", \"Appeal_Submit_Date\", \"Appeal_Date\", \"AppealDate\", \"Submitted_Date\", \"SubmittedDate\"])\n if col is None:\n issues.append(f\"Column 'Appeal_Submitted_Date' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_date(cell_val(data_row, col))\n expected = \"04/08/2026\"\n if val != expected:\n issues.append(f\"Appeal_Submitted_Date: expected '{expected}', got '{val}'.\")\n else:\n found_details.append(f\"Appeal_Submitted_Date='{val}'\")\n\n # Check Appeal_Status = Submitted\n col = find_col([\"Appeal_Status\", \"AppealStatus\", \"Appeal Status\", \"Status\"])\n if col is None:\n issues.append(f\"Column 'Appeal_Status' not found in header. Available headers: {list(headers.keys())}\")\n else:\n val = normalize_str(cell_val(data_row, col))\n if val.lower() != \"submitted\":\n issues.append(f\"Appeal_Status: expected 'Submitted', got '{val}'.\")\n else:\n found_details.append(f\"Appeal_Status='{val}'\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File found: {target_file.name}. Issues found: \" + \"; \".join(issues)}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"All checks passed for {target_file.name}. Found: \" + \"; \".join(found_details)}\n", "criterion_type": "expected_output" }, { "id": "rubric_1775268135625", "sort_order": 11, "rubric_text": "File `era_02142026_SilverPath.pdf` must exist in the workspace.", "verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"era_02142026_SilverPath.pdf\"\n if target_file.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' exists at {target_file}.\"}\n else:\n # Also do a case-insensitive search as a generous fallback\n found = [f for f in Path(workspace_path).rglob(\"*\") if f.name.lower() == \"era_02142026_silverpath.pdf\"]\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File found (case-insensitive match): {found[0]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' was NOT found in the workspace at '{workspace_path}'. Files present: {[f.name for f in Path(workspace_path).rglob('*') if f.is_file()]}\"}\n", "criterion_type": "expected_output" }, { "id": "rubric_1775268234332", "sort_order": 12, "rubric_text": "File `statement_02142026.pdf` must not exist in the workspace.", "verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"statement_02142026.pdf\"\n if target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'statement_02142026.pdf' still exists in the workspace at '{target_file}'. It should have been removed.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'statement_02142026.pdf' does not exist in the workspace, as expected.\"}\n", "criterion_type": "incorrect_behavior" }, { "id": "rubric_1775268384702", "sort_order": 13, "rubric_text": "In `mailbox.json`, an email with the following details must not exist: `to_addr` = `m.holloway@emailhost.com`, subject containing `CareIG Statement of Account` and `02/14/2026` (or the equivalent date in any format).", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the absence of a specific email.\n # The mailbox.json could be in the workspace or in external_services_path.\n \n mailbox_path = None\n \n # Check external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json found at all, the email trivially doesn't exist -> PASS\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n # Define date patterns that all represent February 14, 2026\n date_patterns = [\n r'02/14/2026',\n r'02-14-2026',\n r'2/14/2026',\n r'2-14-2026',\n r'14/02/2026',\n r'14-02-2026',\n r'2026-02-14',\n r'2026/02/14',\n r'Feb(?:ruary)?\\s*14,?\\s*2026',\n r'14\\s*Feb(?:ruary)?,?\\s*2026',\n ]\n \n def subject_contains_date(subject):\n for pat in date_patterns:\n if re.search(pat, subject, re.IGNORECASE):\n return True\n return False\n \n violating_emails = []\n \n for email in emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n subject = email.get(\"subject\", \"\")\n \n # Check to_addr matches\n if to_addr != \"m.holloway@emailhost.com\":\n continue\n \n # Check subject contains \"CareIG Statement of Account\" (case-insensitive)\n if \"careig statement of account\" not in subject.lower():\n continue\n \n # Check subject contains the date 02/14/2026 in any format\n if subject_contains_date(subject):\n violating_emails.append(email)\n \n if len(violating_emails) > 0:\n details = []\n for e in violating_emails:\n details.append(f\" - email_id={e.get('email_id','?')}, to={e.get('to_addr','?')}, subject='{e.get('subject','?')}', folder={e.get('folder','?')}\")\n detail_str = \"\\n\".join(details)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) that should NOT exist (to=m.holloway@emailhost.com, subject containing 'CareIG Statement of Account' and date 02/14/2026):\\n{detail_str}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No email found in mailbox.json with to_addr='m.holloway@emailhost.com' and subject containing both 'CareIG Statement of Account' and the date 02/14/2026.\"\n }\n", "criterion_type": "incorrect_behavior" }, { "id": "rubric_1775268649908", "sort_order": 14, "rubric_text": "The file `appeal_L1_02142026.pdf` must contain the following:\n- Date: `04/08/2026`\n- Patient name: `Margaret` `Holloway`\n- Member ID: `SPA-00291847`\n- Date of Service (DOS): `02/14/2026`\n- Claim #: `CLM-REF-20260218-9931`\n- CARC: `CO-97`\n- Billed amount: `$6,140.00`\n- The letter body must contain: `#SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026` and `D83.9`.\n- Contracted rate: '$4,800.00'", "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n pdf_file = None\n\n # Search for the file - try exact name and case-insensitive\n target_name = \"appeal_L1_02142026.pdf\"\n candidates = list(workspace.rglob(\"*.pdf\"))\n for c in candidates:\n if c.name.lower() == target_name.lower():\n pdf_file = c\n break\n\n # Also try broader matching in case of slight name variations\n if pdf_file is None:\n for c in candidates:\n if \"appeal\" in c.name.lower() and \"02142026\" in c.name:\n pdf_file = c\n break\n\n if pdf_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{target_name}' not found in workspace. PDF files found: {[str(c.relative_to(workspace)) for c in candidates]}\"}\n\n # Extract text from PDF\n text = \"\"\n try:\n import pdfplumber\n with pdfplumber.open(str(pdf_file)) as pdf:\n for page in pdf.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e1:\n try:\n import PyPDF2\n with open(str(pdf_file), 'rb') as f:\n reader = PyPDF2.PdfReader(f)\n for page in reader.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read PDF. pdfplumber error: {e1}; PyPDF2 error: {e2}\"}\n\n if not text.strip():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PDF file exists but no text could be extracted.\"}\n\n # Normalize text for searching\n text_lower = text.lower()\n # Also create a version with collapsed whitespace for matching longer strings\n text_collapsed = re.sub(r'\\s+', ' ', text).strip()\n text_collapsed_lower = text_collapsed.lower()\n\n checks = []\n total = 0\n passed = 0\n\n # Helper function for flexible matching\n def check_present(label, patterns, required=True):\n nonlocal total, passed\n total += 1\n for pattern in patterns:\n if pattern.lower() in text_collapsed_lower:\n passed += 1\n checks.append(f\"PASS: {label} - found '{pattern}'\")\n return True\n checks.append(f\"FAIL: {label} - none of {patterns} found in PDF text\")\n return False\n\n # 1. Date = 04/08/2026\n check_present(\"Date 04/08/2026\", [\n \"04/08/2026\", \"04-08-2026\", \"april 8, 2026\", \"april 08, 2026\",\n \"4/8/2026\", \"8 april 2026\", \"08 april 2026\"\n ])\n\n # 2. Margaret\n check_present(\"Patient first name 'Margaret'\", [\"margaret\"])\n\n # 3. Holloway\n check_present(\"Patient last name 'Holloway'\", [\"holloway\"])\n\n # 4. Member ID SPA-00291847\n check_present(\"Member ID SPA-00291847\", [\n \"spa-00291847\", \"spa00291847\", \"spa 00291847\"\n ])\n\n # 5. DOS 02/14/2026\n check_present(\"DOS 02/14/2026\", [\n \"02/14/2026\", \"02-14-2026\", \"2/14/2026\", \"february 14, 2026\",\n \"february 14 2026\", \"14 february 2026\"\n ])\n\n # 6. Claim # CLM-REF-20260218-9931\n check_present(\"Claim # CLM-REF-20260218-9931\", [\n \"clm-ref-20260218-9931\", \"clm ref 20260218 9931\"\n ])\n\n # 7. CARC CO-97\n check_present(\"CARC CO-97\", [\n \"co-97\", \"co 97\", \"co97\"\n ])\n\n # 8. Billed $6,140.00\n check_present(\"Billed $6,140.00\", [\n \"$6,140.00\", \"$6140.00\", \"6,140.00\", \"6140.00\", \"$ 6,140.00\", \"$6,140\"\n ])\n\n # 9. Letter body: #SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026\n # Check for the key components since exact whitespace/formatting may vary\n spa_pa_found = False\n spa_patterns = [\"spa-pa-2026-3302\"]\n for p in spa_patterns:\n if p in text_collapsed_lower:\n spa_pa_found = True\n break\n\n octagam_found = False\n octagam_patterns = [\"octagam 10% 40g\", \"octagam 10% 40 g\", \"octagam10% 40g\", \"octagam 10%, 40g\", \"octagam 10% 40g\"]\n for p in octagam_patterns:\n if p in text_collapsed_lower:\n octagam_found = True\n break\n # Also try regex for more flexible matching\n if not octagam_found:\n if re.search(r'octagam\\s*10\\s*%\\s*,?\\s*40\\s*g', text_collapsed_lower):\n octagam_found = True\n\n dates_found = False\n start_date_found = any(d in text_collapsed for d in [\"01/28/2026\", \"1/28/2026\", \"01-28-2026\"]) or \\\n any(d in text_collapsed_lower for d in [\"january 28, 2026\", \"january 28 2026\"])\n end_date_found = any(d in text_collapsed for d in [\"07/28/2026\", \"7/28/2026\", \"07-28-2026\"]) or \\\n any(d in text_collapsed_lower for d in [\"july 28, 2026\", \"july 28 2026\"])\n if start_date_found and end_date_found:\n dates_found = True\n\n approved_phrase_found = \"approved\" in text_collapsed_lower and spa_pa_found and octagam_found and dates_found\n\n total += 1\n if approved_phrase_found:\n passed += 1\n checks.append(\"PASS: Letter body contains approval phrase with SPA-PA-2026-3302, Octagam 10% 40g, and effective dates 01/28/2026 through 07/28/2026\")\n else:\n details = []\n if not spa_pa_found:\n details.append(\"SPA-PA-2026-3302 not found\")\n if not octagam_found:\n details.append(\"Octagam 10% 40g not found\")\n if not dates_found:\n if not start_date_found:\n details.append(\"Start date 01/28/2026 not found\")\n if not end_date_found:\n details.append(\"End date 07/28/2026 not found\")\n if \"approved\" not in text_collapsed_lower:\n details.append(\"'approved' not found\")\n checks.append(f\"FAIL: Approval phrase incomplete - {'; '.join(details)}\")\n\n # 10. D83.9\n check_present(\"Diagnosis code D83.9\", [\"d83.9\"])\n\n # 11. Contracted rate $4,800.00\n check_present(\"Contracted rate $4,800.00\", [\n \"$4,800.00\", \"$4800.00\", \"4,800.00\", \"4800.00\", \"$ 4,800.00\", \"$4,800\"\n ])\n\n score = passed / total if total > 0 else 0.0\n all_passed = (passed == total)\n\n feedback = f\"Checked {total} criteria, {passed} passed, {total - passed} failed.\\n\"\n feedback += \"\\n\".join(checks)\n\n # Show a snippet of the extracted text on failure for debugging\n if not all_passed:\n snippet = text_collapsed[:2000]\n feedback += f\"\\n\\n--- Extracted PDF text (first 2000 chars) ---\\n{snippet}\"\n\n return {\"pass\": all_passed, \"score\": score, \"feedback\": feedback}\n", "criterion_type": "expected_output" }, { "id": "rubric_1775269539963", "sort_order": 15, "rubric_text": "In `audit_log.xlsx`, rows 36 and 37 should have: `CASE_ID` = `Holloway_DOB_03141961`, the date in `Datetime` = `04/08/2026`, `Action` = `Denial Received — CO-97` (case-insensitive) for row 36, and `Action` = `Appeal Submitted` (case-insensitive) for row 37.", "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"audit_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening audit_log.xlsx: {e}\"}\n\n # Find column indices by header row (row 1)\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip().upper()] = col_idx\n\n # Try to locate relevant columns\n case_id_col = None\n datetime_col = None\n action_col = None\n\n for key, idx in headers.items():\n if 'CASE' in key and 'ID' in key:\n case_id_col = idx\n if key == 'DATETIME': # FIX: was ('DATE' in key or 'TIME' in key or key == 'DATETIME')\n datetime_col = idx # 'DATE' in key matches FILE_UPDATED and FIELD_UPDATED,\n if key == 'ACTION': # overwriting datetime_col with the wrong column index.\n action_col = idx # Exact match eliminates the substring collision.\n\n if case_id_col is None or datetime_col is None or action_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find required columns. Headers found: {headers}\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_case_id(val):\n normed = normalize(val).lower()\n return normed == \"holloway_dob_03141961\"\n\n def check_date(val):\n if val is None:\n return False\n import datetime as dt\n if isinstance(val, (dt.datetime, dt.date)):\n d = val.date() if isinstance(val, dt.datetime) else val\n return d.month == 4 and d.day == 8 and d.year == 2026\n s = normalize(val)\n patterns = [\n r'0?4[/\\-]0?8[/\\-]2026',\n r'2026[/\\-]0?4[/\\-]0?8',\n r'April\\s+8,?\\s+2026',\n r'8\\s+April\\s+2026',\n r'04082026',\n ]\n for p in patterns:\n if re.search(p, s, re.IGNORECASE):\n return True\n return False\n\n def check_action_denial(val):\n normed = normalize(val).lower()\n normed_dashes = re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\-]+', '-', normed)\n if 'denial received' in normed_dashes and ('co-97' in normed_dashes or 'co 97' in normed_dashes or 'co97' in normed_dashes):\n return True\n return False\n\n def check_action_appeal(val):\n normed = normalize(val).lower()\n return 'appeal submitted' in normed\n\n # Check row 36\n row36_case = ws.cell(row=36, column=case_id_col).value\n row36_date = ws.cell(row=36, column=datetime_col).value\n row36_action = ws.cell(row=36, column=action_col).value\n\n # Check row 37\n row37_case = ws.cell(row=37, column=case_id_col).value\n row37_date = ws.cell(row=37, column=datetime_col).value\n row37_action = ws.cell(row=37, column=action_col).value\n\n issues = []\n passes = []\n\n if check_case_id(row36_case):\n passes.append(f\"Row 36 CASE_ID: '{row36_case}' matches\")\n else:\n issues.append(f\"Row 36 CASE_ID: expected 'Holloway_DOB_03141961', got '{row36_case}'\")\n\n if check_date(row36_date):\n passes.append(f\"Row 36 Datetime: '{row36_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 36 Datetime: expected date 04/08/2026, got '{row36_date}'\")\n\n if check_action_denial(row36_action):\n passes.append(f\"Row 36 Action: '{row36_action}' matches 'Denial Received — CO-97'\")\n else:\n issues.append(f\"Row 36 Action: expected 'Denial Received — CO-97', got '{row36_action}'\")\n\n if check_case_id(row37_case):\n passes.append(f\"Row 37 CASE_ID: '{row37_case}' matches\")\n else:\n issues.append(f\"Row 37 CASE_ID: expected 'Holloway_DOB_03141961', got '{row37_case}'\")\n\n if check_date(row37_date):\n passes.append(f\"Row 37 Datetime: '{row37_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 37 Datetime: expected date 04/08/2026, got '{row37_date}'\")\n\n if check_action_appeal(row37_action):\n passes.append(f\"Row 37 Action: '{row37_action}' matches 'Appeal Submitted'\")\n else:\n issues.append(f\"Row 37 Action: expected 'Appeal Submitted', got '{row37_action}'\")\n\n total_checks = 6\n passed_checks = total_checks - len(issues)\n score = passed_checks / total_checks\n\n if len(issues) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All checks passed. \" + \"; \".join(passes)}\n else:\n return {\"pass\": False, \"score\": score, \"feedback\": \"Issues found: \" + \"; \".join(issues) + \". Passed: \" + \"; \".join(passes)}", "criterion_type": "expected_output" }, { "id": "rubric_1775271368334", "sort_order": 16, "rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must contain 2 messages with text containing all of: `APPEAL`, `L1`, `Holloway_DOB_03141961`, `12/09/2025`, `BlueCrest Commercial PPO`, `PI-204` and with exactly one message containing `FOLLOW-UP`", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_path}\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n if not isinstance(slack_data, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json top-level is not a dict.\"}\n\n # Collect all messages from billing-appeals channel using multiple possible structures\n all_channel_messages = []\n\n channels_map = slack_data.get(\"channels\", {})\n messages_map = slack_data.get(\"messages\", {})\n\n # Standard format: channels dict keyed by channel ID, messages dict keyed by channel ID\n if isinstance(channels_map, dict) and isinstance(messages_map, dict):\n for ch_id, ch_info in channels_map.items():\n ch_name = \"\"\n if isinstance(ch_info, dict):\n ch_name = ch_info.get(\"name\", \"\") or ch_info.get(\"channel_name\", \"\") or \"\"\n elif isinstance(ch_info, str):\n ch_name = ch_info\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n msgs = messages_map.get(ch_id, [])\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n # Also check if messages_map is keyed by channel name directly\n if not all_channel_messages:\n for key, msgs in messages_map.items():\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n\n # Fallback: channels is a dict with channel names as keys containing messages\n if not all_channel_messages and isinstance(channels_map, dict):\n for ch_name, ch_data in channels_map.items():\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n all_channel_messages.extend(ch_data[\"messages\"])\n elif isinstance(ch_data, list):\n all_channel_messages.extend(ch_data)\n break\n\n # Fallback: channels is a list of channel objects\n if not all_channel_messages and isinstance(channels_map, list):\n for ch in channels_map:\n if isinstance(ch, dict):\n ch_name = ch.get(\"name\", \"\") or ch.get(\"channel_name\", \"\")\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n all_channel_messages.extend(ch.get(\"messages\", []))\n break\n\n # Fallback: top-level keys are channel names\n if not all_channel_messages:\n for key in slack_data:\n if key in (\"channels\", \"messages\", \"users\", \"counters\"):\n continue\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n val = slack_data[key]\n if isinstance(val, dict) and \"messages\" in val:\n all_channel_messages.extend(val[\"messages\"])\n elif isinstance(val, list):\n all_channel_messages.extend(val)\n break\n\n if not all_channel_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find any messages in #billing-appeals channel in slack_data.json. Top-level keys: {list(slack_data.keys())}\"\n }\n\n def get_text(msg):\n if isinstance(msg, dict):\n return msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"content\", \"\") or msg.get(\"body\", \"\") or \"\"\n elif isinstance(msg, str):\n return msg\n return \"\"\n\n def term_in_text(term, text):\n \"\"\"Check if a required term appears in text, with flexible matching.\"\"\"\n text_upper = text.upper()\n term_upper = term.upper()\n if term_upper in text_upper:\n return True\n # Flexible matching for specific terms\n if term == \"Holloway_DOB_03141961\":\n pattern = re.compile(r\"holloway[\\s_\\-]*dob[\\s_\\-]*0?3[\\s_\\-/]*14[\\s_\\-/]*1961\", re.IGNORECASE)\n if pattern.search(text):\n return True\n # Also accept Holloway ... 03/14/1961 style\n if re.search(r\"holloway\", text, re.IGNORECASE) and re.search(r\"03[/\\-.]14[/\\-.]1961\", text):\n return True\n return False\n elif term == \"12/09/2025\":\n if re.search(r\"12[/\\-.]09[/\\-.]2025\", text):\n return True\n return False\n elif term == \"BlueCrest Commercial PPO\":\n # Accept variations in spacing/casing\n pattern = re.compile(r\"bluecrest\\s+commercial\\s+ppo\", re.IGNORECASE)\n if pattern.search(text):\n return True\n return False\n elif term == \"APPEAL\":\n if re.search(r\"appeal\", text, re.IGNORECASE):\n return True\n return False\n elif term == \"L1\":\n if re.search(r\"\\bL1\\b\", text, re.IGNORECASE):\n return True\n # Also accept \"Level 1\" or \"Level-1\"\n if re.search(r\"level[\\s\\-]?1\", text, re.IGNORECASE):\n return True\n return False\n elif term == \"PI-204\":\n if re.search(r\"PI[\\-\\s]?204\", text, re.IGNORECASE):\n return True\n return False\n return False\n\n required_terms = [\"APPEAL\", \"L1\", \"Holloway_DOB_03141961\", \"12/09/2025\", \"BlueCrest Commercial PPO\", \"PI-204\"]\n\n matching_messages = []\n follow_up_count = 0\n\n for msg in all_channel_messages:\n text = get_text(msg)\n if not text:\n continue\n\n all_found = True\n for term in required_terms:\n if not term_in_text(term, text):\n all_found = False\n break\n if all_found:\n matching_messages.append(text[:500])\n # Check for FOLLOW-UP in this message\n if re.search(r\"follow[\\s\\-_]?up\", text, re.IGNORECASE):\n follow_up_count += 1\n\n feedback_parts = []\n score = 0.0\n passed = False\n\n # Check: need exactly 2 matching messages\n if len(matching_messages) >= 2:\n score += 0.6\n feedback_parts.append(f\"Found {len(matching_messages)} message(s) in #billing-appeals containing all required terms.\")\n else:\n feedback_parts.append(f\"Expected 2 messages in #billing-appeals containing all of {required_terms}. Found {len(matching_messages)} matching message(s). Total messages in channel: {len(all_channel_messages)}.\")\n # Provide diagnostic info\n sample_texts = []\n for msg in all_channel_messages[:5]:\n t = get_text(msg)\n sample_texts.append(t[:150])\n feedback_parts.append(f\"Sample messages: {sample_texts}\")\n\n # Check: exactly one message should contain FOLLOW-UP\n if follow_up_count == 1:\n score += 0.4\n feedback_parts.append(\"Exactly 1 matching message contains FOLLOW-UP as required.\")\n else:\n feedback_parts.append(f\"Expected exactly 1 matching message to contain FOLLOW-UP, but found {follow_up_count}.\")\n\n if len(matching_messages) >= 2 and follow_up_count == 1:\n passed = True\n score = 1.0\n\n return {\n \"pass\": passed,\n \"score\": score,\n \"feedback\": \" \".join(feedback_parts)\n }\n", "criterion_type": "expected_output" }, { "id": "rubric_1775274130585", "sort_order": 17, "rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must have a message containing: `[DENIAL]`, `Holloway_DOB_03141961`, and `CO-97`.", "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_file = Path(external_services_path) / \"slack_data.json\"\n if not slack_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_file}\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n # According to the documented Slack data format:\n # \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } }\n # \"messages\": { \"C001\": [ { \"text\": \"...\", ... } ] }\n\n # Step 1: Find the channel ID for billing-appeals\n channels = slack_data.get(\"channels\", {})\n messages_by_channel = slack_data.get(\"messages\", {})\n\n billing_appeals_channel_id = None\n\n if isinstance(channels, dict):\n for cid, cinfo in channels.items():\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\":\n billing_appeals_channel_id = cid\n break\n # Also check loosely\n if \"billing\" in cname and \"appeal\" in cname:\n billing_appeals_channel_id = cid\n break\n elif isinstance(channels, list):\n for cinfo in channels:\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\" or (\"billing\" in cname and \"appeal\" in cname):\n billing_appeals_channel_id = cinfo.get(\"id\", cname)\n break\n\n # Collect message texts from the identified channel\n message_texts = []\n\n if billing_appeals_channel_id and isinstance(messages_by_channel, dict):\n channel_msgs = messages_by_channel.get(billing_appeals_channel_id, [])\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for key in [\"text\", \"message\", \"content\", \"body\"]:\n if key in msg and isinstance(msg[key], str):\n message_texts.append(msg[key])\n elif isinstance(msg, str):\n message_texts.append(msg)\n\n # Also try direct key lookup by name in messages dict (fallback)\n if not message_texts and isinstance(messages_by_channel, dict):\n for key in messages_by_channel:\n key_lower = key.lower().replace(\"#\", \"\").strip()\n if key_lower == \"billing-appeals\" or (\"billing\" in key_lower and \"appeal\" in key_lower):\n channel_msgs = messages_by_channel[key]\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for mkey in [\"text\", \"message\", \"content\", \"body\"]:\n if mkey in msg and isinstance(msg[mkey], str):\n message_texts.append(msg[mkey])\n elif isinstance(msg, str):\n message_texts.append(msg)\n break\n\n # Fallback: recursively search the entire structure for billing-appeals messages\n if not message_texts:\n def find_channel_messages(data):\n results = []\n if isinstance(data, dict):\n for key, value in data.items():\n key_lower = key.lower().replace('#', '').replace(' ', '-')\n if 'billing' in key_lower and 'appeal' in key_lower:\n results.extend(_extract_texts(value))\n else:\n results.extend(find_channel_messages(value))\n elif isinstance(data, list):\n for item in data:\n if isinstance(item, dict):\n name = (item.get('name', '') or item.get('channel_name', '') or '').lower().replace('#', '').replace(' ', '-')\n if 'billing' in name and 'appeal' in name:\n results.extend(_extract_texts(item.get('messages', item)))\n else:\n results.extend(find_channel_messages(item))\n return results\n\n def _extract_texts(obj):\n texts = []\n if isinstance(obj, list):\n for item in obj:\n if isinstance(item, dict):\n for k in ['text', 'message', 'content', 'body']:\n if k in item and isinstance(item[k], str):\n texts.append(item[k])\n elif isinstance(item, str):\n texts.append(item)\n elif isinstance(obj, dict):\n for k, v in obj.items():\n if isinstance(v, str):\n texts.append(v)\n elif isinstance(v, list):\n texts.extend(_extract_texts(v))\n return texts\n\n message_texts = find_channel_messages(slack_data)\n\n if not message_texts:\n available_channels = []\n if isinstance(channels, dict):\n for cid, cinfo in channels.items():\n if isinstance(cinfo, dict):\n available_channels.append(cinfo.get('name', cid))\n else:\n available_channels.append(cid)\n msg_keys = list(messages_by_channel.keys())[:20] if isinstance(messages_by_channel, dict) else []\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find messages in #billing-appeals channel. Available channels: {available_channels}. Message keys: {msg_keys}\"}\n\n # Now search messages for the three required strings\n required = ['[DENIAL]', 'Holloway_DOB_03141961', 'CO-97']\n\n for msg_text in message_texts:\n msg_upper = msg_text.upper()\n found_all = True\n missing = []\n for req in required:\n req_upper = req.upper()\n if req_upper in msg_upper:\n continue\n # Looser matching: for [DENIAL], also accept DENIAL without brackets\n elif req == '[DENIAL]' and 'DENIAL' in msg_upper:\n continue\n # For Holloway DOB, accept various separators\n elif req == 'Holloway_DOB_03141961':\n # Accept variations like Holloway DOB 03141961, Holloway-DOB-03141961, etc.\n pattern = re.compile(r'HOLLOWAY[_\\s\\-]*DOB[_\\s\\-]*03[\\-/]?14[\\-/]?1961', re.IGNORECASE)\n if pattern.search(msg_text):\n continue\n else:\n found_all = False\n missing.append(req)\n # For CO-97, accept CO 97 or CO97\n elif req == 'CO-97':\n pattern = re.compile(r'CO[\\-\\s]?97', re.IGNORECASE)\n if pattern.search(msg_text):\n continue\n else:\n found_all = False\n missing.append(req)\n else:\n found_all = False\n missing.append(req)\n if found_all:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found a message in #billing-appeals containing all required strings: [DENIAL], Holloway_DOB_03141961, CO-97. Message: {msg_text[:300]}\"}\n\n # If no single message has all three, report what was found\n found_anywhere = {}\n for req in required:\n for msg_text in message_texts:\n msg_upper = msg_text.upper()\n req_upper = req.upper()\n if req_upper in msg_upper:\n found_anywhere[req] = True\n break\n elif req == '[DENIAL]' and 'DENIAL' in msg_upper:\n found_anywhere[req] = True\n break\n elif req == 'Holloway_DOB_03141961':\n pattern = re.compile(r'HOLLOWAY[_\\s\\-]*DOB[_\\s\\-]*03[\\-/]?14[\\-/]?1961', re.IGNORECASE)\n if pattern.search(msg_text):\n found_anywhere[req] = True\n break\n elif req == 'CO-97':\n pattern = re.compile(r'CO[\\-\\s]?97', re.IGNORECASE)\n if pattern.search(msg_text):\n found_anywhere[req] = True\n break\n if req not in found_anywhere:\n found_anywhere[req] = False\n\n found_list = [r for r in required if found_anywhere.get(r)]\n missing_list = [r for r in required if not found_anywhere.get(r)]\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No single message in #billing-appeals contains all three required strings. Found across all messages: {found_list}. Missing: {missing_list}. Total messages checked: {len(message_texts)}. Sample messages: {[t[:200] for t in message_texts[:5]]}\"\n }\n", "criterion_type": "expected_output" } ]