87 lines
66 KiB
JSON
87 lines
66 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "556a749d-f292-4fb9-abe2-d92e0eb17b29",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Header worksheet must contain exactly 2 rows (header + 1 data row) with the correct headers in A1:AF1 and the correct claim data in A2:AF2, including CASE_ID=CIG_2026_0341, DOS=2026-10-05, Patient_Last_Name=Thornton, Patient_First_Name=Margaret, Patient_DOB=1961-06-15, Member_ID=SHC-884729103, Group_Number=GRP-20145, Payer_Name=Sunflower Health Commercial PPO, Payer_Type=Commercial PPO, PA_Number=PA-2026-SHC-41092, Physician_NPI=1528074836, CareIG_NPI=1234567893, CareIG_Tax_ID=47-3821056, CareIG_Taxonomy_Code=3336C0003X, Diagnosis_ICD10=D83.9, Drug_Source=CareIG, Claim_Frequency_Code=1, Total_Billed=6154.80, Submission_Date=2026-10-08, Claim_Status=Submitted, and Partial_Infusion_Note containing administered dose info.",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n \"\"\"Lowercase, strip all punctuation, hyphens, em-dashes, apostrophes, whitespace.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019\\u201c\\u201d'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Header' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Header\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Header']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data[idx] if idx < len(data) else None\n\n # ── Headers A1:AF1 ─────────────────────────────────────────────────────────\n expected_headers = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Pa
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "cd924150-af76-44ea-8a22-8bafaf26cb9a",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Lines worksheet must contain exactly 8 rows (header + 7 data rows) with correct headers in A1:I1 and correct line data for all 7 data lines in any order including HCPCS codes J1459 (71 units, $5,573.50), 99600 (1 unit, $185.00), 99603 (2 units, $290.00), J7030 (2 units, $13.50), J0171 (1 unit, $12.30), A4221 (1 unit, $42.00), A4222 (1 unit, $38.50).",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passed_list = [f'{lbl}' for lbl, p, det in results if p]\n feedback = (f'All {total} checks passed: {\", \".join(passed_list)}'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Lines' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Lines\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Lines']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n\n # ── Headers A1:I1 ──────────────────────────────────────────────────────────\n expected_headers = ['CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier']\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data rows (matched in any order by HCPCS code) ─────────────────────────\n # Expected: (hcpcs, units, line_total)\n expected_lines = [\n ('J1459', 71, 5573.50),\n ('99600', 1, 185.00),\n ('99603', 2, 290.00),\n ('J7030', 2, 13.50),\n ('J0171', 1, 12.30),\n ('A4221', 1, 42.00),\n ('A4222', 1, 38.50),\n ]\n\n # Build a lookup from HCPCS code (normalized) -> list of data rows\n data_rows = rows[1:] if len(rows) > 1 else []\n hcpcs_col = 3 # Column index for HCPCS_Code\n units_col = 5 # Column index for Units\n total_col = 7 # Column index for Line_Total\
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "4e4620e4-5676-488b-9792-26fe4528035f",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In file `audit_log.xlsx`, the Audit Log worksheet must contain exactly 8 rows (header + 7 data rows), with the 8th data row (row 8) recording the claim submission: Date_Time must be within 24 hours after 2026-10-08 9:30 AM, CASE_ID=CIG_2026_0341, Action=Claim Submitted, File_Updated=claim_CIG_2026_0341.xlsx, Field_Updated=Claim_Status, New_Value=Submitted, Staff_User=Julio Martinez, Result=SUCCESS.",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d %H:%M:%S')\n return str(value).strip()\n\n\ndef _parse_datetime(cell_value):\n \"\"\"Try to parse a cell value into a datetime object.\"\"\"\n if cell_value is None:\n return None\n if isinstance(cell_value, datetime.datetime):\n return cell_value\n if isinstance(cell_value, datetime.date):\n return datetime.datetime(cell_value.year, cell_value.month, cell_value.day)\n s = str(cell_value).strip()\n # Try many common datetime formats\n formats = [\n '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d %I:%M %p', '%Y-%m-%d %I:%M:%S %p',\n '%m/%d/%Y %H:%M:%S', '%m/%d/%Y %H:%M', '%m/%d/%Y %I:%M %p', '%m/%d/%Y %I:%M:%S %p',\n '%d/%m/%Y %H:%M:%S', '%d/%m/%Y %H:%M', '%d/%m/%Y %I:%M %p',\n '%Y/%m/%d %H:%M:%S', '%Y/%m/%d %H:%M',\n '%m-%d-%Y %H:%M:%S', '%m-%d-%Y %H:%M',\n '%Y-%m-%dT%H:%M:%S', '%Y-%m-%dT%H:%M',\n '%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y',\n ]\n for fmt in formats:\n try:\n return datetime.datetime.strptime(s, fmt)\n except ValueError:\n continue\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n # Locate worksheet — prefer exact name, fall back to first sheet with audit/log\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n if len(rows) < 8:\n # Can't check row 8 details if not enough rows; still report partial\n return score_and_return()\n\n header_row = rows[0]\n # The 8th row overall = row index 7 = 7th data row\n data_row8 = rows[7]\n\n # ── Headers (informational, lenient) ───────â”
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "59c8eabd-b7f0-415d-a697-6d8c7e64adff",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), a sent email must exist with Subject containing 'CLAIM SUBMISSION', CareIG NPI 1234567893, patient Thornton, MemberID SHC-884729103, DOS 2026-10-05, sent From billing@careig.com To claims@sunflowerhealthks.com, with attachment claim_CIG_2026_0341.xlsx, and body containing patient details (Thornton, Margaret; DOB 1961-06-15; Member ID SHC-884729103; Group GRP-20145; DOS 2026-10-05; PA-2026-SHC-41092; NPI 1234567893; Tax ID 47-3821056; Total Billed $6,154.80) and signed by Julio Martinez.",
|
||
|
|
"verifier_code": "import re\nimport json\nimport datetime\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails\n if e.get('folder', '').lower() in ('sent', 'sent items')]\n\n # Find the claim email — any sent email whose subject contains 'claim submission'\n claim_email = next(\n (e for e in sent_emails\n if 'claimsubmission' in _normalize(e.get('subject', ''))),\n None\n )\n\n if claim_email is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No claim submission email found in sent folder. '\n f'Sent subjects: {[e.get(\"subject\",\"\") for e in sent_emails]}'}\n\n # ── Subject ────────────────────────────────────────────────────────────────\n expected_subject = ('CLAIM SUBMISSION CareIG NPI: 1234567893 | Thornton | '\n 'MemberID: SHC-884729103 | DOS: 2026-10-05')\n actual_subject = claim_email.get('subject', '')\n chk('subject',\n _normalize(actual_subject) == _normalize(expected_subject),\n f'expected \"{expected_subject}\", got \"{actual_subject}\"')\n\n # ── From / To ──────────────────────────────────────────────────────────────\n chk('from_addr',\n _normalize(claim_email.get('from_addr', '')) == _normalize('billing@careig.com'),\n f'expected billing@careig.com, got \"{claim_email.get(\"from_addr\",\"\")}\"')\n chk('to_addr',\n _normalize(claim_email.get('to_addr', '')) == _normalize('claims@sunflowerhealthks.com'),\n f'expected claims@sunflowerhealthks.com, got \"{claim_email.get(\"to_addr\",\"\")}\"')\n\n # ── Attachment ─────────────────────────────────────────────────────────────\n attachments = claim_email.get('attachments', [])\n att_names = []\n if isinstance(attachments, list):\n att_names = [a.get('filename', '') if isinstance(a, dict) else str(a)\n for a in attachments]\n has_att = any(_normalize('claim_CIG_2026_0341.xlsx') in _normalize(n)\n for n in att_names)\n chk('attachment',\n has_att,\n f
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "8051d798-11b8-4f3c-b809-b76770f40632",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `slack_data.json` (external services), channel C005 must contain a message with text 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80' (not a BILLING HOLD message).",
|
||
|
|
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n \"\"\"Normalize a string for loose comparison: lowercase, strip whitespace and punctuation.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n # Remove whitespace, hyphens, dashes, quotes, punctuation, dollar signs\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|$]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n c005_messages = data.get('messages', {}).get('C005', [])\n\n # The rubric says the text should be:\n # 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80'\n # We accept with or without a dollar sign before the amount.\n # We'll check for key fragments rather than exact match for leniency.\n\n # Key fragments that must all be present in a single message\n required_fragments = [\n 'claimsubmitted',\n 'cig_2026_0341',\n 'dos',\n '20261005',\n 'sunflowerhealthcommercialppo',\n '615480', # 6,154.80 normalized\n ]\n\n # Also accept alternate claim id normalizations\n alt_claim_ids = ['cig20260341']\n\n matched_msg = None\n for m in c005_messages:\n norm_text = _normalize(m.get('text', ''))\n # Check all required fragments\n has_claim_id = 'cig_2026_0341' in norm_text or 'cig20260341' in norm_text\n all_others = all(frag in norm_text for frag in required_fragments if frag not in ('cig_2026_0341',))\n if has_claim_id and all_others:\n matched_msg = m\n break\n\n # Also do a looser check: look for the key parts with the pipe-separated format\n if matched_msg is None:\n for m in c005_messages:\n text = m.get('text', '')\n text_upper = text.upper()\n if ('CLAIM SUBMITTED' in text_upper and\n 'CIG_2026_0341' in text_upper and\n '2026-10-05' in text and\n 'SUNFLOWER' in text_upper and\n ('6,154.80' in text or '6154.80' in text)):\n matched_msg = m\n break\n\n # Check: No BILLING HOLD message present in C005\n billing_hold_present = any(\n 'billing hold' in m.get('text', '').lower() or\n 'billinghold' in _normalize(m.get('text', ''))\n for m in c005_messages\n )\n chk('c005_no_billing_hold',\n not billing_hold_present,\n 'A BILLING HOLD message was found in C005' if billing_hold_present else 'No BILLING HOLD message found in C005 (good)')\n\n # Check: Expected CLAIM SUBMITTED message exists\n msg_texts =
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e9993229-323e-401c-8d4a-ee8ce5e1554a",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In file `claim_blank.xlsx`, the template file must remain unchanged with Claim Header worksheet containing exactly 1 row (header only) with headers A1:AF1, and Claim Lines worksheet containing exactly 1 row (header only) with headers A1:I1.",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_blank.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_blank.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_blank.xlsx: {e}'}\n\n # ── Claim Header sheet ─────────────────────────────────────────────────────\n expected_header_cols = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Payer_Member_ID',\n 'COB_Other_Payer_Paid', 'COB_Other_Payer_EOB_Date', 'AR_Followup_Log',\n ]\n\n chk('claim_header_sheet_exists', 'Claim Header' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_h = wb['Claim Header'] if 'Claim Header' in wb.sheetnames else None\n if ws_h is not None:\n rows_h = [r for r in ws_h.iter_rows(values_only=True) if any(c is not None for c in r)]\n chk('claim_header_row_count', len(rows_h) == 1,\n f'expected 1 row (header only), got {len(rows_h)}')\n hrow = rows_h[0] if rows_h else ()\n for i, exp in enumerate(expected_header_cols):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'claim_header_col_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Claim Lines sheet ──────────────────────────────────────────────────────\n expected_lines_cols = [\n 'CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier',\n ]\n\n chk('claim_lines_sheet_exists', 'Claim Lines' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_l = wb['Claim Lines'] if 'Claim Lines' in wb.sheetnames else None\n if ws_l is not None:\n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "af850a02-2c83-4f4b-b248-d9026ae57faa",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `calendar_data.json` (external services), there must be exactly 1 event and it must have id 'CIG_2026_0341'.",
|
||
|
|
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check calendar_data.json'}\n\n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'calendar_data.json not found at {cal_path}'}\n\n try:\n with open(cal_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse calendar_data.json: {e}'}\n\n events = data.get('events', {})\n event_ids = list(events.keys())\n\n # ── Exactly 1 event ────────────────────────────────────────────────────────\n chk('event_count', len(events) == 1,\n f'expected 1 event, got {len(events)}: {event_ids}')\n\n # ── Event id = CIG_2026_0341 ───────────────────────────────────────────────\n # Check both the dict key and the id field inside the event object\n expected_id = 'CIG_2026_0341'\n found_id = any(\n _normalize(k) == _normalize(expected_id) or\n _normalize(str(v.get('id', ''))) == _normalize(expected_id)\n for k, v in events.items()\n )\n chk('event_id',\n found_id,\n f'expected event id \"{expected_id}\", got event ids: {event_ids}')\n\n return score_and_return()\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "2256d643-3276-468d-a6d8-8ef0b3207ca5",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In `mailbox.json` (external services), the inbox must contain exactly 4 emails, no emails in Drafts, no emails in Trash, and no more than 1 email in Sent.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse mailbox.json: {e}'}\n \n emails = data.get('emails', [])\n \n inbox_emails = [e for e in emails if e.get('folder', '').upper() in ['INBOX']]\n draft_emails = [e for e in emails if e.get('folder', '').lower() in ['drafts', 'draft']]\n trash_emails = [e for e in emails if e.get('folder', '').lower() in ['trash', 'deleted']]\n sent_emails = [e for e in emails if e.get('folder', '').lower() in ['sent', 'sent items']]\n \n errors = []\n \n if len(inbox_emails) != 4:\n errors.append(f'Inbox has {len(inbox_emails)} emails, expected exactly 4')\n \n if len(draft_emails) != 0:\n errors.append(f'Drafts has {len(draft_emails)} emails, expected 0')\n \n if len(trash_emails) != 0:\n errors.append(f'Trash has {len(trash_emails)} emails, expected 0')\n \n if len(sent_emails) > 1:\n errors.append(f'Sent has {len(sent_emails)} emails, expected no more than 1')\n \n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json folder counts incorrect: ' + '; '.join(errors)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'mailbox.json folder counts verified: inbox={len(inbox_emails)}, drafts={len(draft_emails)}, trash={len(trash_emails)}, sent={len(sent_emails)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "47d9923f-1c23-4706-a08d-1a467e9ca9ec",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "In `slack_data.json` (external services), channel message counts must be: C001 no more than 2 messages, C002 no messages, C003 no messages, C004 no more than 3 messages, C005 no more than 1 message, C006 no messages, C007 no messages, C008 no messages.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n\n # (channel_id, max_allowed)\n channel_limits = [\n ('C001', 2),\n ('C002', 0),\n ('C003', 0),\n ('C004', 3),\n ('C005', 1),\n ('C006', 0),\n ('C007', 0),\n ('C008', 0),\n ]\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n for channel_id, max_allowed in channel_limits:\n count = len(messages.get(channel_id, []))\n passed = count <= max_allowed\n detail = f'has {count} messages, max allowed is {max_allowed}' if not passed else f'{count} messages'\n chk(f'channel_{channel_id}', passed, detail)\n\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4)\n failures = [f'{lbl} ({det})' for lbl, p, det in results if not p]\n feedback = (f'All {total} channel count checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Violations: ' + ' | '.join(failures))\n\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774579377910",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "The file `claim_CIG_2026_0341.xlsx` exists in the workspace and has exactly 2 worksheets named \"Claim Header\" and \"Claim Lines\".",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"claim_CIG_2026_0341.xlsx\"\n \n # Check file existence\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' does not exist in the workspace. Files found: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, read_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'claim_CIG_2026_0341.xlsx': {e}\"}\n \n sheet_names = wb.sheetnames\n wb.close()\n \n # Check exactly 2 worksheets\n if len(sheet_names) != 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 worksheets but found {len(sheet_names)}: {sheet_names}\"}\n \n # Check worksheet names (case-insensitive and whitespace-tolerant)\n expected_names = {\"claim header\", \"claim lines\"}\n actual_names = {name.strip().lower() for name in sheet_names}\n \n if actual_names != expected_names:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected worksheets named 'Claim Header' and 'Claim Lines', but found: {sheet_names}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' exists with exactly 2 worksheets: {sheet_names}\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774579558077",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "In `audit_log.xlsx`, the \"Audit Log\" worksheet must preserve all 6 pre-existing data rows (rows 2–7) unchanged and exactly:\n- Row 2: 2026-09-02, Referral Received, intake.xlsx, All required fields, Populated from referral, Lisa Choi, SUCCESS\n- Row 3: 2026-09-02, BVS Assigned, intake.xlsx, BVS_Assigned, Rachel Kim, Lisa Choi, SUCCESS\n- Row 4: 2026-09-08, BV Complete, benefits.xlsx, BV_Complete, YES, Rachel Kim, SUCCESS\n- Row 5: 2026-09-12, PA Submitted, auth.xlsx, PA_Status, Pending, Rachel Kim, SUCCESS\n- Row 6: 2026-09-20, PA Approved — Clinical Handoff, auth.xlsx, PA_Status, Approved, Rachel Kim, SUCCESS\n- Row 7: 2026-10-05, Visit Note Signed, visit_note.xlsx, Nurse_Signature, Patricia Nolan RN, Patricia Nolan, SUCCESS\n\nAll rows must have CASE_ID = CIG_2026_0341 and Result = SUCCESS.",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef _flexible_match(actual_str, expected_str):\n \"\"\"More lenient matching: normalize both sides and compare.\n Also tries checking if one contains the other after normalization.\"\"\"\n na = _normalize(actual_str)\n ne = _normalize(expected_str)\n if na == ne:\n return True\n # Also try: if one contains the other (for slight variations)\n if na and ne and (na in ne or ne in na):\n return True\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # Need at least 7 rows to check existing data (1 header + 6 data)\n if len(rows) < 7:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'audit_log.xlsx has only {len(rows)} non-empty rows; expected at least 7 (header + 6 data rows)'}\n\n # ── Row 1: headers ─────────────────────────────────────────────────────────\n expected_headers = ['Date_Time', 'CASE_ID', 'Action', 'File_Updated',\n 'Field_Updated', 'New_Value', 'Staff_User', 'Result']\n hrow = rows[0]\n for i, exp in enumerate(expected_headers):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'header_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Rows 2–7: existing data ──────────────────────────────────────â”
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774579634882",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "In file `visit_note_CIG_2026_0341.xlsx`, the worksheet must be unchanged with exactly 2 rows (header + 1 data row). Headers A1:Z1 must match the 26 expected column names.",
|
||
|
|
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n\n def _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n def _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n def _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n def _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n def _time_matches(cell_value, hour, minute):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.time, datetime.datetime)):\n return cell_value.hour == hour and cell_value.minute == minute\n s = str(cell_value).strip()\n for fmt in ('%I:%M %p', '%H:%M:%S', '%H:%M', '%I:%M%p'):\n try:\n t = datetime.datetime.strptime(s, fmt)\n return t.hour == hour and t.minute == minute\n except ValueError:\n continue\n return False\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'visit_note_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'visit_note_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open visit_note_CIG_2026_0341.xlsx: {e}'}\n\n ws = wb[wb.sheetnames[0]]\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data_row = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data_row[idx] if idx < len(data_row) else None\n\n # ── Headers A1:Z1 ──────────────────────────────────────────────────────────\n expected_head
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|