DerekSurge
87 lines
66 KiB
JSON
87 lines
66 KiB
JSON
[
|
||
{
|
||
"id": "556a749d-f292-4fb9-abe2-d92e0eb17b29",
|
||
"sort_order": 0,
|
||
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Header worksheet must contain exactly 2 rows (header + 1 data row) with the correct headers in A1:AF1 and the correct claim data in A2:AF2, including CASE_ID=CIG_2026_0341, DOS=2026-10-05, Patient_Last_Name=Thornton, Patient_First_Name=Margaret, Patient_DOB=1961-06-15, Member_ID=SHC-884729103, Group_Number=GRP-20145, Payer_Name=Sunflower Health Commercial PPO, Payer_Type=Commercial PPO, PA_Number=PA-2026-SHC-41092, Physician_NPI=1528074836, CareIG_NPI=1234567893, CareIG_Tax_ID=47-3821056, CareIG_Taxonomy_Code=3336C0003X, Diagnosis_ICD10=D83.9, Drug_Source=CareIG, Claim_Frequency_Code=1, Total_Billed=6154.80, Submission_Date=2026-10-08, Claim_Status=Submitted, and Partial_Infusion_Note containing administered dose info.",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n \"\"\"Lowercase, strip all punctuation, hyphens, em-dashes, apostrophes, whitespace.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019\\u201c\\u201d'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Header' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Header\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Header']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data[idx] if idx < len(data) else None\n\n # ── Headers A1:AF1 ─────────────────────────────────────────────────────────\n expected_headers = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Payer_Member_ID',\n 'COB_Other_Payer_Paid', 'COB_Other_Payer_EOB_Date', 'AR_Followup_Log',\n ]\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data row A2:AF2 — string fields ───────────────────────────────────────\n str_fields = [\n (0, 'CIG_2026_0341', 'CASE_ID'),\n (2, 'Thornton', 'Patient_Last_Name'),\n (3, 'Margaret', 'Patient_First_Name'),\n (5, 'SHC-884729103', 'Member_ID'),\n (6, 'GRP-20145', 'Group_Number'),\n (7, 'Sunflower Health Commercial PPO', 'Payer_Name'),\n (8, 'Commercial PPO', 'Payer_Type'),\n (9, 'PA-2026-SHC-41092', 'PA_Number'),\n (10, '1528074836', 'Physician_NPI'),\n (11, '1234567893', 'CareIG_NPI'),\n (12, '47-3821056', 'CareIG_Tax_ID'),\n (13, '3336C0003X', 'CareIG_Taxonomy_Code'),\n (14, 'D83.9', 'Diagnosis_ICD10'),\n (15, 'CareIG', 'Drug_Source'),\n (21, 'Submitted', 'Claim_Status'),\n ]\n for idx, exp, name in str_fields:\n actual = _cell_str(d(idx))\n chk(f'data_{name}',\n _normalize(actual) == _normalize(exp),\n f'expected \"{exp}\", got \"{actual}\"')\n\n # ── Dates ─────────────────────────────────────────────────────────────────\n chk('data_DOS',\n _date_matches(d(1), 2026, 10, 5),\n f'expected 2026-10-05, got \"{_cell_str(d(1))}\"')\n chk('data_Patient_DOB',\n _date_matches(d(4), 1961, 6, 15),\n f'expected 1961-06-15, got \"{_cell_str(d(4))}\"')\n chk('data_Submission_Date',\n _date_matches(d(20), 2026, 10, 8),\n f'expected 2026-10-08, got \"{_cell_str(d(20))}\"')\n\n # ── Claim_Frequency_Code = 1 ───────────────────────────────────────────────\n try:\n chk('data_Claim_Frequency_Code',\n int(float(_cell_str(d(18)))) == 1,\n f'expected 1, got \"{_cell_str(d(18))}\"')\n except (ValueError, TypeError):\n chk('data_Claim_Frequency_Code', False,\n f'could not parse as integer: \"{_cell_str(d(18))}\"')\n\n # ── Total_Billed = 6154.80 (exact to the cent) ────────────────────────────\n try:\n total_num = float(re.sub(r'[,$\\s]', '', _cell_str(d(19))))\n chk('data_Total_Billed',\n abs(total_num - 6154.80) < 0.005,\n f'expected 6154.80, got {total_num}')\n except (ValueError, TypeError):\n chk('data_Total_Billed', False,\n f'could not parse as number: \"{_cell_str(d(19))}\"')\n\n # ── Partial_Infusion_Note — exact normalized match ─────────────────────────\n exp_note = 'Ordered 40g, administered 35.8g. Billed on administered dose.'\n actual_note = _cell_str(d(22))\n chk('data_Partial_Infusion_Note',\n _normalize(actual_note) == _normalize(exp_note),\n f'expected \"{exp_note}\", got \"{actual_note}\"')\n\n # ── Billing_Model and Drug_Modifier must be empty ─────────────────────────\n for col_idx, col_name in [(16, 'Billing_Model'), (17, 'Drug_Modifier')]:\n val = _cell_str(d(col_idx))\n chk(f'data_{col_name}_empty',\n val == '',\n f'expected empty, got \"{val}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "cd924150-af76-44ea-8a22-8bafaf26cb9a",
|
||
"sort_order": 1,
|
||
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Lines worksheet must contain exactly 8 rows (header + 7 data rows) with correct headers in A1:I1 and correct line data for all 7 data lines in any order including HCPCS codes J1459 (71 units, $5,573.50), 99600 (1 unit, $185.00), 99603 (2 units, $290.00), J7030 (2 units, $13.50), J0171 (1 unit, $12.30), A4221 (1 unit, $42.00), A4222 (1 unit, $38.50).",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passed_list = [f'{lbl}' for lbl, p, det in results if p]\n feedback = (f'All {total} checks passed: {\", \".join(passed_list)}'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Lines' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Lines\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Lines']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n\n # ── Headers A1:I1 ──────────────────────────────────────────────────────────\n expected_headers = ['CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier']\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data rows (matched in any order by HCPCS code) ─────────────────────────\n # Expected: (hcpcs, units, line_total)\n expected_lines = [\n ('J1459', 71, 5573.50),\n ('99600', 1, 185.00),\n ('99603', 2, 290.00),\n ('J7030', 2, 13.50),\n ('J0171', 1, 12.30),\n ('A4221', 1, 42.00),\n ('A4222', 1, 38.50),\n ]\n\n # Build a lookup from HCPCS code (normalized) -> list of data rows\n data_rows = rows[1:] if len(rows) > 1 else []\n hcpcs_col = 3 # Column index for HCPCS_Code\n units_col = 5 # Column index for Units\n total_col = 7 # Column index for Line_Total\n\n hcpcs_to_rows = {}\n for dr in data_rows:\n hcpcs_val = _normalize(_cell_str(dr[hcpcs_col] if hcpcs_col < len(dr) else None))\n if hcpcs_val not in hcpcs_to_rows:\n hcpcs_to_rows[hcpcs_val] = []\n hcpcs_to_rows[hcpcs_val].append(dr)\n\n for (hcpcs, exp_units, exp_total) in expected_lines:\n prefix = f'line_{hcpcs}'\n norm_hcpcs = _normalize(hcpcs)\n\n # Find matching row\n matched_rows = hcpcs_to_rows.get(norm_hcpcs, [])\n found = len(matched_rows) > 0\n chk(f'{prefix}_found', found,\n f'HCPCS code \"{hcpcs}\" not found among data rows. Available: {list(hcpcs_to_rows.keys())}')\n\n if not found:\n # Still add checks as failed so score reflects missing data\n chk(f'{prefix}_Units', False, f'Row for {hcpcs} not found, cannot check units')\n chk(f'{prefix}_Line_Total', False, f'Row for {hcpcs} not found, cannot check line total')\n continue\n\n # Use first matching row\n row = matched_rows[0]\n\n def cell(col):\n return row[col] if col < len(row) else None\n\n # Units (accept within ±1% or exact integer match)\n u_val = _to_float(cell(units_col))\n units_ok = False\n if u_val is not None:\n units_ok = int(round(u_val)) == exp_units\n chk(f'{prefix}_Units',\n units_ok,\n f'expected {exp_units} units, got \"{_cell_str(cell(units_col))}\"')\n\n # Line_Total (within 1% tolerance or $0.01)\n t_val = _to_float(cell(total_col))\n total_ok = False\n if t_val is not None:\n tolerance = max(abs(exp_total) * 0.01, 0.01)\n total_ok = abs(t_val - exp_total) <= tolerance\n chk(f'{prefix}_Line_Total',\n total_ok,\n f'expected ${exp_total:.2f}, got \"{_cell_str(cell(total_col))}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "4e4620e4-5676-488b-9792-26fe4528035f",
|
||
"sort_order": 2,
|
||
"rubric_text": "In file `audit_log.xlsx`, the Audit Log worksheet must contain exactly 8 rows (header + 7 data rows), with the 8th data row (row 8) recording the claim submission: Date_Time must be within 24 hours after 2026-10-08 9:30 AM, CASE_ID=CIG_2026_0341, Action=Claim Submitted, File_Updated=claim_CIG_2026_0341.xlsx, Field_Updated=Claim_Status, New_Value=Submitted, Staff_User=Julio Martinez, Result=SUCCESS.",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d %H:%M:%S')\n return str(value).strip()\n\n\ndef _parse_datetime(cell_value):\n \"\"\"Try to parse a cell value into a datetime object.\"\"\"\n if cell_value is None:\n return None\n if isinstance(cell_value, datetime.datetime):\n return cell_value\n if isinstance(cell_value, datetime.date):\n return datetime.datetime(cell_value.year, cell_value.month, cell_value.day)\n s = str(cell_value).strip()\n # Try many common datetime formats\n formats = [\n '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d %I:%M %p', '%Y-%m-%d %I:%M:%S %p',\n '%m/%d/%Y %H:%M:%S', '%m/%d/%Y %H:%M', '%m/%d/%Y %I:%M %p', '%m/%d/%Y %I:%M:%S %p',\n '%d/%m/%Y %H:%M:%S', '%d/%m/%Y %H:%M', '%d/%m/%Y %I:%M %p',\n '%Y/%m/%d %H:%M:%S', '%Y/%m/%d %H:%M',\n '%m-%d-%Y %H:%M:%S', '%m-%d-%Y %H:%M',\n '%Y-%m-%dT%H:%M:%S', '%Y-%m-%dT%H:%M',\n '%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y',\n ]\n for fmt in formats:\n try:\n return datetime.datetime.strptime(s, fmt)\n except ValueError:\n continue\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n # Locate worksheet — prefer exact name, fall back to first sheet with audit/log\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n if len(rows) < 8:\n # Can't check row 8 details if not enough rows; still report partial\n return score_and_return()\n\n header_row = rows[0]\n # The 8th row overall = row index 7 = 7th data row\n data_row8 = rows[7]\n\n # ── Headers (informational, lenient) ───────────────────────────────────────\n expected_headers = ['Date_Time', 'CASE_ID', 'Action', 'File_Updated',\n 'Field_Updated', 'New_Value', 'Staff_User', 'Result']\n headers_ok = True\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n if _normalize(actual_h) != _normalize(exp_h):\n headers_ok = False\n chk('headers', headers_ok, f'Header row: {[_cell_str(header_row[i]) if i < len(header_row) else \"\" for i in range(len(expected_headers))]}')\n\n # ── Row 8 field checks ─────────────────────────────────────────────────────\n def c(idx):\n return data_row8[idx] if idx < len(data_row8) else None\n\n # Date_Time: must be within 24 hours AFTER 2026-10-08 9:30 AM\n target_dt = datetime.datetime(2026, 10, 8, 9, 30)\n dt_val = c(0)\n parsed_dt = _parse_datetime(dt_val)\n if parsed_dt is not None:\n diff_seconds = (parsed_dt - target_dt).total_seconds()\n # Within 24 hours after: 0 <= diff <= 86400, but be lenient (allow a small negative window for rounding)\n dt_ok = -300 <= diff_seconds <= 86400 # allow 5 min before for leniency\n chk('row8_Date_Time', dt_ok,\n f'expected within 24h after 2026-10-08 09:30, got \"{_cell_str(dt_val)}\" (diff={diff_seconds:.0f}s)')\n else:\n # Fallback: check if the string contains the date at minimum\n dt_str = str(dt_val) if dt_val else ''\n date_present = any(p in dt_str for p in ['2026-10-08', '10/08/2026', '10/8/2026', '2026/10/08'])\n chk('row8_Date_Time', date_present,\n f'could not parse datetime, raw value=\"{_cell_str(dt_val)}\", date_present={date_present}')\n\n # Exact normalized matches for remaining fields\n str_checks = [\n (1, 'CIG_2026_0341', 'CASE_ID'),\n (2, 'Claim Submitted', 'Action'),\n (3, 'claim_CIG_2026_0341.xlsx', 'File_Updated'),\n (4, 'Claim_Status', 'Field_Updated'),\n (5, 'Submitted', 'New_Value'),\n (6, 'Julio Martinez', 'Staff_User'),\n (7, 'SUCCESS', 'Result'),\n ]\n for idx, exp, name in str_checks:\n actual = _cell_str(c(idx))\n chk(f'row8_{name}',\n _normalize(actual) == _normalize(exp),\n f'expected \"{exp}\", got \"{actual}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "59c8eabd-b7f0-415d-a697-6d8c7e64adff",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `mailbox.json` (external services), a sent email must exist with Subject containing 'CLAIM SUBMISSION', CareIG NPI 1234567893, patient Thornton, MemberID SHC-884729103, DOS 2026-10-05, sent From billing@careig.com To claims@sunflowerhealthks.com, with attachment claim_CIG_2026_0341.xlsx, and body containing patient details (Thornton, Margaret; DOB 1961-06-15; Member ID SHC-884729103; Group GRP-20145; DOS 2026-10-05; PA-2026-SHC-41092; NPI 1234567893; Tax ID 47-3821056; Total Billed $6,154.80) and signed by Julio Martinez.",
|
||
"verifier_code": "import re\nimport json\nimport datetime\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails\n if e.get('folder', '').lower() in ('sent', 'sent items')]\n\n # Find the claim email — any sent email whose subject contains 'claim submission'\n claim_email = next(\n (e for e in sent_emails\n if 'claimsubmission' in _normalize(e.get('subject', ''))),\n None\n )\n\n if claim_email is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No claim submission email found in sent folder. '\n f'Sent subjects: {[e.get(\"subject\",\"\") for e in sent_emails]}'}\n\n # ── Subject ────────────────────────────────────────────────────────────────\n expected_subject = ('CLAIM SUBMISSION CareIG NPI: 1234567893 | Thornton | '\n 'MemberID: SHC-884729103 | DOS: 2026-10-05')\n actual_subject = claim_email.get('subject', '')\n chk('subject',\n _normalize(actual_subject) == _normalize(expected_subject),\n f'expected \"{expected_subject}\", got \"{actual_subject}\"')\n\n # ── From / To ──────────────────────────────────────────────────────────────\n chk('from_addr',\n _normalize(claim_email.get('from_addr', '')) == _normalize('billing@careig.com'),\n f'expected billing@careig.com, got \"{claim_email.get(\"from_addr\",\"\")}\"')\n chk('to_addr',\n _normalize(claim_email.get('to_addr', '')) == _normalize('claims@sunflowerhealthks.com'),\n f'expected claims@sunflowerhealthks.com, got \"{claim_email.get(\"to_addr\",\"\")}\"')\n\n # ── Attachment ─────────────────────────────────────────────────────────────\n attachments = claim_email.get('attachments', [])\n att_names = []\n if isinstance(attachments, list):\n att_names = [a.get('filename', '') if isinstance(a, dict) else str(a)\n for a in attachments]\n has_att = any(_normalize('claim_CIG_2026_0341.xlsx') in _normalize(n)\n for n in att_names)\n chk('attachment',\n has_att,\n f'expected claim_CIG_2026_0341.xlsx, got {att_names}')\n\n # ── Body — full template line checks ──────────────────────────────────────\n body = claim_email.get('body_text', '') or ''\n nbody = _normalize(body)\n\n template_lines = [\n 'To Whom It May Concern',\n 'Please find attached the claim for the following patient and date of service',\n 'Patient: Thornton, Margaret',\n 'DOB: 1961-06-15',\n 'Member ID: SHC-884729103',\n 'Group: GRP-20145',\n 'Date of Service: 2026-10-05',\n 'PA Number: PA-2026-SHC-41092',\n 'Rendering Provider: CareIG Specialty Pharmacy',\n 'NPI: 1234567893',\n 'Tax ID: 47-3821056',\n 'Taxonomy: 3336C0003X',\n 'Referring Physician: Dr. Elena Vasquez, NPI: 1528074836',\n 'Diagnosis: D83.9',\n 'Total Billed: $6,154.80',\n 'Attached: claim_CIG_2026_0341.xlsx',\n 'For questions, contact billing@careig.com or (816) 555-0200.',\n 'Julio Martinez',\n 'CareIG Specialty Pharmacy, Billing',\n 'billing@careig.com',\n '(816) 555-0200',\n ]\n\n for line in template_lines:\n chk(f'body_line_{_normalize(line)[:30]}',\n _normalize(line) in nbody,\n f'body missing line: \"{line}\"')\n\n\n return score_and_return()\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "8051d798-11b8-4f3c-b809-b76770f40632",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `slack_data.json` (external services), channel C005 must contain a message with text 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80' (not a BILLING HOLD message).",
|
||
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n \"\"\"Normalize a string for loose comparison: lowercase, strip whitespace and punctuation.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n # Remove whitespace, hyphens, dashes, quotes, punctuation, dollar signs\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|$]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n c005_messages = data.get('messages', {}).get('C005', [])\n\n # The rubric says the text should be:\n # 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80'\n # We accept with or without a dollar sign before the amount.\n # We'll check for key fragments rather than exact match for leniency.\n\n # Key fragments that must all be present in a single message\n required_fragments = [\n 'claimsubmitted',\n 'cig_2026_0341',\n 'dos',\n '20261005',\n 'sunflowerhealthcommercialppo',\n '615480', # 6,154.80 normalized\n ]\n\n # Also accept alternate claim id normalizations\n alt_claim_ids = ['cig20260341']\n\n matched_msg = None\n for m in c005_messages:\n norm_text = _normalize(m.get('text', ''))\n # Check all required fragments\n has_claim_id = 'cig_2026_0341' in norm_text or 'cig20260341' in norm_text\n all_others = all(frag in norm_text for frag in required_fragments if frag not in ('cig_2026_0341',))\n if has_claim_id and all_others:\n matched_msg = m\n break\n\n # Also do a looser check: look for the key parts with the pipe-separated format\n if matched_msg is None:\n for m in c005_messages:\n text = m.get('text', '')\n text_upper = text.upper()\n if ('CLAIM SUBMITTED' in text_upper and\n 'CIG_2026_0341' in text_upper and\n '2026-10-05' in text and\n 'SUNFLOWER' in text_upper and\n ('6,154.80' in text or '6154.80' in text)):\n matched_msg = m\n break\n\n # Check: No BILLING HOLD message present in C005\n billing_hold_present = any(\n 'billing hold' in m.get('text', '').lower() or\n 'billinghold' in _normalize(m.get('text', ''))\n for m in c005_messages\n )\n chk('c005_no_billing_hold',\n not billing_hold_present,\n 'A BILLING HOLD message was found in C005' if billing_hold_present else 'No BILLING HOLD message found in C005 (good)')\n\n # Check: Expected CLAIM SUBMITTED message exists\n msg_texts = [m.get('text', '') for m in c005_messages]\n chk('c005_claim_submitted_message',\n matched_msg is not None,\n f'Found matching CLAIM SUBMITTED message: {matched_msg.get(\"text\", \"\")!r}' if matched_msg\n else f'Expected CLAIM SUBMITTED message with CIG_2026_0341, DOS 2026-10-05, Sunflower Health Commercial PPO, Billed 6,154.80 not found in C005. Messages found: {msg_texts}')\n\n return score_and_return()\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e9993229-323e-401c-8d4a-ee8ce5e1554a",
|
||
"sort_order": 5,
|
||
"rubric_text": "In file `claim_blank.xlsx`, the template file must remain unchanged with Claim Header worksheet containing exactly 1 row (header only) with headers A1:AF1, and Claim Lines worksheet containing exactly 1 row (header only) with headers A1:I1.",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_blank.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_blank.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_blank.xlsx: {e}'}\n\n # ── Claim Header sheet ─────────────────────────────────────────────────────\n expected_header_cols = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Payer_Member_ID',\n 'COB_Other_Payer_Paid', 'COB_Other_Payer_EOB_Date', 'AR_Followup_Log',\n ]\n\n chk('claim_header_sheet_exists', 'Claim Header' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_h = wb['Claim Header'] if 'Claim Header' in wb.sheetnames else None\n if ws_h is not None:\n rows_h = [r for r in ws_h.iter_rows(values_only=True) if any(c is not None for c in r)]\n chk('claim_header_row_count', len(rows_h) == 1,\n f'expected 1 row (header only), got {len(rows_h)}')\n hrow = rows_h[0] if rows_h else ()\n for i, exp in enumerate(expected_header_cols):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'claim_header_col_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Claim Lines sheet ──────────────────────────────────────────────────────\n expected_lines_cols = [\n 'CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier',\n ]\n\n chk('claim_lines_sheet_exists', 'Claim Lines' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_l = wb['Claim Lines'] if 'Claim Lines' in wb.sheetnames else None\n if ws_l is not None:\n rows_l = [r for r in ws_l.iter_rows(values_only=True) if any(c is not None for c in r)]\n chk('claim_lines_row_count', len(rows_l) == 1,\n f'expected 1 row (header only), got {len(rows_l)}')\n lrow = rows_l[0] if rows_l else ()\n for i, exp in enumerate(expected_lines_cols):\n actual = _cell_str(lrow[i]) if i < len(lrow) else ''\n chk(f'claim_lines_col_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "af850a02-2c83-4f4b-b248-d9026ae57faa",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `calendar_data.json` (external services), there must be exactly 1 event and it must have id 'CIG_2026_0341'.",
|
||
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check calendar_data.json'}\n\n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'calendar_data.json not found at {cal_path}'}\n\n try:\n with open(cal_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse calendar_data.json: {e}'}\n\n events = data.get('events', {})\n event_ids = list(events.keys())\n\n # ── Exactly 1 event ────────────────────────────────────────────────────────\n chk('event_count', len(events) == 1,\n f'expected 1 event, got {len(events)}: {event_ids}')\n\n # ── Event id = CIG_2026_0341 ───────────────────────────────────────────────\n # Check both the dict key and the id field inside the event object\n expected_id = 'CIG_2026_0341'\n found_id = any(\n _normalize(k) == _normalize(expected_id) or\n _normalize(str(v.get('id', ''))) == _normalize(expected_id)\n for k, v in events.items()\n )\n chk('event_id',\n found_id,\n f'expected event id \"{expected_id}\", got event ids: {event_ids}')\n\n return score_and_return()\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "2256d643-3276-468d-a6d8-8ef0b3207ca5",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `mailbox.json` (external services), the inbox must contain exactly 4 emails, no emails in Drafts, no emails in Trash, and no more than 1 email in Sent.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse mailbox.json: {e}'}\n \n emails = data.get('emails', [])\n \n inbox_emails = [e for e in emails if e.get('folder', '').upper() in ['INBOX']]\n draft_emails = [e for e in emails if e.get('folder', '').lower() in ['drafts', 'draft']]\n trash_emails = [e for e in emails if e.get('folder', '').lower() in ['trash', 'deleted']]\n sent_emails = [e for e in emails if e.get('folder', '').lower() in ['sent', 'sent items']]\n \n errors = []\n \n if len(inbox_emails) != 4:\n errors.append(f'Inbox has {len(inbox_emails)} emails, expected exactly 4')\n \n if len(draft_emails) != 0:\n errors.append(f'Drafts has {len(draft_emails)} emails, expected 0')\n \n if len(trash_emails) != 0:\n errors.append(f'Trash has {len(trash_emails)} emails, expected 0')\n \n if len(sent_emails) > 1:\n errors.append(f'Sent has {len(sent_emails)} emails, expected no more than 1')\n \n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json folder counts incorrect: ' + '; '.join(errors)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'mailbox.json folder counts verified: inbox={len(inbox_emails)}, drafts={len(draft_emails)}, trash={len(trash_emails)}, sent={len(sent_emails)}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "47d9923f-1c23-4706-a08d-1a467e9ca9ec",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `slack_data.json` (external services), channel message counts must be: C001 no more than 2 messages, C002 no messages, C003 no messages, C004 no more than 3 messages, C005 no more than 1 message, C006 no messages, C007 no messages, C008 no messages.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n\n # (channel_id, max_allowed)\n channel_limits = [\n ('C001', 2),\n ('C002', 0),\n ('C003', 0),\n ('C004', 3),\n ('C005', 1),\n ('C006', 0),\n ('C007', 0),\n ('C008', 0),\n ]\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n for channel_id, max_allowed in channel_limits:\n count = len(messages.get(channel_id, []))\n passed = count <= max_allowed\n detail = f'has {count} messages, max allowed is {max_allowed}' if not passed else f'{count} messages'\n chk(f'channel_{channel_id}', passed, detail)\n\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4)\n failures = [f'{lbl} ({det})' for lbl, p, det in results if not p]\n feedback = (f'All {total} channel count checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Violations: ' + ' | '.join(failures))\n\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1774579377910",
|
||
"sort_order": 9,
|
||
"rubric_text": "The file `claim_CIG_2026_0341.xlsx` exists in the workspace and has exactly 2 worksheets named \"Claim Header\" and \"Claim Lines\".",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"claim_CIG_2026_0341.xlsx\"\n \n # Check file existence\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' does not exist in the workspace. Files found: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, read_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'claim_CIG_2026_0341.xlsx': {e}\"}\n \n sheet_names = wb.sheetnames\n wb.close()\n \n # Check exactly 2 worksheets\n if len(sheet_names) != 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 worksheets but found {len(sheet_names)}: {sheet_names}\"}\n \n # Check worksheet names (case-insensitive and whitespace-tolerant)\n expected_names = {\"claim header\", \"claim lines\"}\n actual_names = {name.strip().lower() for name in sheet_names}\n \n if actual_names != expected_names:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected worksheets named 'Claim Header' and 'Claim Lines', but found: {sheet_names}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' exists with exactly 2 worksheets: {sheet_names}\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1774579558077",
|
||
"sort_order": 10,
|
||
"rubric_text": "In `audit_log.xlsx`, the \"Audit Log\" worksheet must preserve all 6 pre-existing data rows (rows 2–7) unchanged and exactly:\n- Row 2: 2026-09-02, Referral Received, intake.xlsx, All required fields, Populated from referral, Lisa Choi, SUCCESS\n- Row 3: 2026-09-02, BVS Assigned, intake.xlsx, BVS_Assigned, Rachel Kim, Lisa Choi, SUCCESS\n- Row 4: 2026-09-08, BV Complete, benefits.xlsx, BV_Complete, YES, Rachel Kim, SUCCESS\n- Row 5: 2026-09-12, PA Submitted, auth.xlsx, PA_Status, Pending, Rachel Kim, SUCCESS\n- Row 6: 2026-09-20, PA Approved — Clinical Handoff, auth.xlsx, PA_Status, Approved, Rachel Kim, SUCCESS\n- Row 7: 2026-10-05, Visit Note Signed, visit_note.xlsx, Nurse_Signature, Patricia Nolan RN, Patricia Nolan, SUCCESS\n\nAll rows must have CASE_ID = CIG_2026_0341 and Result = SUCCESS.",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef _flexible_match(actual_str, expected_str):\n \"\"\"More lenient matching: normalize both sides and compare.\n Also tries checking if one contains the other after normalization.\"\"\"\n na = _normalize(actual_str)\n ne = _normalize(expected_str)\n if na == ne:\n return True\n # Also try: if one contains the other (for slight variations)\n if na and ne and (na in ne or ne in na):\n return True\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # Need at least 7 rows to check existing data (1 header + 6 data)\n if len(rows) < 7:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'audit_log.xlsx has only {len(rows)} non-empty rows; expected at least 7 (header + 6 data rows)'}\n\n # ── Row 1: headers ─────────────────────────────────────────────────────────\n expected_headers = ['Date_Time', 'CASE_ID', 'Action', 'File_Updated',\n 'Field_Updated', 'New_Value', 'Staff_User', 'Result']\n hrow = rows[0]\n for i, exp in enumerate(expected_headers):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'header_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Rows 2–7: existing data ────────────────────────────────────────────────\n # The rubric says:\n # Row 2: 2026-09-02, Referral Received, intake.xlsx, All required fields, Populated from referral, Lisa Choi, SUCCESS\n # Row 3: 2026-09-02, BVS Assigned, intake.xlsx, BVS_Assigned, Rachel Kim, Lisa Choi, SUCCESS\n # Row 4: 2026-09-08, BV Complete, benefits.xlsx, BV_Complete, YES, Rachel Kim, SUCCESS\n # Row 5: 2026-09-12, PA Submitted, auth.xlsx, PA_Status, Pending, Rachel Kim, SUCCESS\n # Row 6: 2026-09-20, PA Approved — Clinical Handoff, auth.xlsx, PA_Status, Approved, Rachel Kim, SUCCESS\n # Row 7: 2026-10-05, Visit Note Signed, visit_note.xlsx, Nurse_Signature, Patricia Nolan RN, Patricia Nolan, SUCCESS\n # All rows have CASE_ID = CIG_2026_0341 and Result = SUCCESS\n #\n # The rubric uses short filenames like \"intake.xlsx\" but actual files may include case ID suffix.\n # We match leniently: the actual value must contain the rubric's base name (normalized).\n\n # (date_ymd, CASE_ID, Action, File_Updated, Field_Updated, New_Value, Staff_User, Result)\n expected_rows = [\n ((2026, 9, 2), 'CIG_2026_0341', 'Referral Received', 'intake.xlsx', 'All required fields', 'Populated from referral', 'Lisa Choi', 'SUCCESS'),\n ((2026, 9, 2), 'CIG_2026_0341', 'BVS Assigned', 'intake.xlsx', 'BVS_Assigned', 'Rachel Kim', 'Lisa Choi', 'SUCCESS'),\n ((2026, 9, 8), 'CIG_2026_0341', 'BV Complete', 'benefits.xlsx', 'BV_Complete', 'YES', 'Rachel Kim', 'SUCCESS'),\n ((2026, 9, 12), 'CIG_2026_0341', 'PA Submitted', 'auth.xlsx', 'PA_Status', 'Pending', 'Rachel Kim', 'SUCCESS'),\n ((2026, 9, 20), 'CIG_2026_0341', 'PA Approved \\u2014 Clinical Handoff', 'auth.xlsx', 'PA_Status', 'Approved', 'Rachel Kim', 'SUCCESS'),\n ((2026, 10, 5), 'CIG_2026_0341', 'Visit Note Signed', 'visit_note.xlsx', 'Nurse_Signature', 'Patricia Nolan RN', 'Patricia Nolan', 'SUCCESS'),\n ]\n\n for i, (date_ymd, case_id, action, file_upd, field_upd, new_val, staff, result) in enumerate(expected_rows):\n row_num = i + 2 # human-readable row number\n row = rows[i + 1] # skip header\n\n def c(idx):\n return row[idx] if idx < len(row) else None\n\n prefix = f'row{row_num}'\n\n chk(f'{prefix}_Date_Time',\n _date_matches(c(0), *date_ymd),\n f'expected date {date_ymd}, got \"{_cell_str(c(0))}\"')\n\n # CASE_ID check\n chk(f'{prefix}_CASE_ID',\n _flexible_match(_cell_str(c(1)), case_id),\n f'expected \"{case_id}\", got \"{_cell_str(c(1))}\"')\n\n # Action check\n chk(f'{prefix}_Action',\n _flexible_match(_cell_str(c(2)), action),\n f'expected \"{action}\", got \"{_cell_str(c(2))}\"')\n\n # File_Updated: be lenient - the rubric says e.g. \"intake.xlsx\" but actual might be\n # \"intake_CIG_2026_0341.xlsx\" or \"intake.xlsx\". Accept if the base name is contained.\n actual_file = _cell_str(c(3))\n file_base = file_upd.replace('.xlsx', '').lower()\n file_match = (file_base in actual_file.lower()) or _flexible_match(actual_file, file_upd)\n chk(f'{prefix}_File_Updated',\n file_match,\n f'expected contains \"{file_upd}\", got \"{actual_file}\"')\n\n # Field_Updated\n chk(f'{prefix}_Field_Updated',\n _flexible_match(_cell_str(c(4)), field_upd),\n f'expected \"{field_upd}\", got \"{_cell_str(c(4))}\"')\n\n # New_Value\n chk(f'{prefix}_New_Value',\n _flexible_match(_cell_str(c(5)), new_val),\n f'expected \"{new_val}\", got \"{_cell_str(c(5))}\"')\n\n # Staff_User\n chk(f'{prefix}_Staff_User',\n _flexible_match(_cell_str(c(6)), staff),\n f'expected \"{staff}\", got \"{_cell_str(c(6))}\"')\n\n # Result\n chk(f'{prefix}_Result',\n _flexible_match(_cell_str(c(7)), result),\n f'expected \"{result}\", got \"{_cell_str(c(7))}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1774579634882",
|
||
"sort_order": 11,
|
||
"rubric_text": "In file `visit_note_CIG_2026_0341.xlsx`, the worksheet must be unchanged with exactly 2 rows (header + 1 data row). Headers A1:Z1 must match the 26 expected column names.",
|
||
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n\n def _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n def _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n def _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n def _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n def _time_matches(cell_value, hour, minute):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.time, datetime.datetime)):\n return cell_value.hour == hour and cell_value.minute == minute\n s = str(cell_value).strip()\n for fmt in ('%I:%M %p', '%H:%M:%S', '%H:%M', '%I:%M%p'):\n try:\n t = datetime.datetime.strptime(s, fmt)\n return t.hour == hour and t.minute == minute\n except ValueError:\n continue\n return False\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'visit_note_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'visit_note_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open visit_note_CIG_2026_0341.xlsx: {e}'}\n\n ws = wb[wb.sheetnames[0]]\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data_row = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data_row[idx] if idx < len(data_row) else None\n\n # ── Headers A1:Z1 ──────────────────────────────────────────────────────────\n expected_headers = [\n 'CASE_ID', 'Visit_Date', 'Visit_Start_Time', 'Visit_End_Time',\n 'Visit_Duration_Minutes', 'Drug_Name_Trade', 'Drug_Name_Generic',\n 'Drug_NDC', 'Drug_Lot_Number', 'Drug_Expiration_Date',\n 'Dose_Ordered_Grams', 'Dose_Administered_Grams', 'Route',\n 'Infusion_Start_Rate_mL_hr', 'Infusion_Rate_Adjustments',\n 'Premeds_Administered', 'NS_500mL_Bags_Used', 'NS_250mL_Bags_Used',\n 'Pump_Used', 'Vital_Signs_Pre', 'Vital_Signs_Mid', 'Vital_Signs_Post',\n 'Patient_Response', 'Education_Provided', 'Nurse_Signature',\n 'Dextrose_500mL_Bags_Used',\n ]\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data row — string fields ───────────────────────────────────────────────\n str_fields = [\n (0, 'CIG_2026_0341', 'CASE_ID'),\n (5, 'Privigen', 'Drug_Name_Trade'),\n (6, 'Immune Globulin Intravenous (Human)', 'Drug_Name_Generic'),\n (7, '44206-0455-10', 'Drug_NDC'),\n (8, 'LG26K0489', 'Drug_Lot_Number'),\n (12, 'IV', 'Route'),\n (14, '9:45 AM increased to 60 mL/hr; 10:15 AM increased to 120 mL/hr', 'Infusion_Rate_Adjustments'),\n (15, 'Diphenhydramine 50mg IV at 9:20 AM for infusion reaction prophylaxis', 'Premeds_Administered'),\n (18, 'YES', 'Pump_Used'),\n (19, 'BP 132/78, HR 74, Temp 98.2°F, RR 16', 'Vital_Signs_Pre'),\n (20, 'BP 128/76, HR 78, Temp 98.4°F, RR 16', 'Vital_Signs_Mid'),\n (21, 'BP 126/74, HR 72, Temp 98.1°F, RR 14', 'Vital_Signs_Post'),\n (22, 'Patient tolerated infusion without adverse events.', 'Patient_Response'),\n (23, 'Reviewed signs/symptoms of adverse reactions, when to contact provider.', 'Education_Provided'),\n (24, 'Patricia Nolan, RN', 'Nurse_Signature'),\n (25, 'N/A', 'Dextrose_500mL_Bags_Used'),\n ]\n for idx, exp, name in str_fields:\n actual = _cell_str(d(idx))\n chk(f'data_{name}',\n _normalize(actual) == _normalize(exp),\n f'expected \"{exp}\", got \"{actual}\"')\n\n # ── Dates ─────────────────────────────────────────────────────────────────\n chk('data_Visit_Date',\n _date_matches(d(1), 2026, 10, 5),\n f'expected 2026-10-05, got \"{_cell_str(d(1))}\"')\n chk('data_Drug_Expiration_Date',\n _date_matches(d(9), 2027, 6, 15),\n f'expected 2027-06-15, got \"{_cell_str(d(9))}\"')\n\n # ── Time fields ───────────────────────────────────────────────────────────\n chk('data_Visit_Start_Time',\n _time_matches(d(2), 9, 15),\n f'expected 9:15 AM, got \"{_cell_str(d(2))}\"')\n chk('data_Visit_End_Time',\n _time_matches(d(3), 12, 50),\n f'expected 12:50 PM, got \"{_cell_str(d(3))}\"')\n\n # ── Numeric fields ────────────────────────────────────────────────────────\n numeric_fields = [\n (4, 215, 0.5, 'Visit_Duration_Minutes'),\n (10, 40, 0.05, 'Dose_Ordered_Grams'),\n (11, 35.8, 0.05, 'Dose_Administered_Grams'),\n (13, 30, 0.05, 'Infusion_Start_Rate_mL_hr'),\n (16, 2, 0.05, 'NS_500mL_Bags_Used'),\n (17, 0, 0.05, 'NS_250mL_Bags_Used'),\n ]\n for idx, exp_val, tol, name in numeric_fields:\n val = _to_float(d(idx))\n chk(f'data_{name}',\n val is not None and abs(val - exp_val) <= tol,\n f'expected {exp_val}, got \"{_cell_str(d(idx))}\"')\n\n return score_and_return()\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|