DerekSurge
59 lines
32 KiB
JSON
59 lines
32 KiB
JSON
[
|
||
{
|
||
"id": "e82385e0-38ac-490b-8f93-5166cac146c9",
|
||
"sort_order": 0,
|
||
"rubric_text": "File `benefits.xlsx` must exist in the workspace and contain a header row with all 24 required columns in any order: 'Home_Infusion_Covered', 'Benefit_Type', 'PA_Required', 'PA_Phone', 'PA_Submission_Email', 'Specialty_Pharmacy_Required', 'Specialty_Pharmacy_Name', 'CareIG_Network_Status', 'Deductible_Individual', 'Deductible_Met_YTD', 'OOP_Max_Individual', 'OOP_Max_Met_YTD', 'Coinsurance_Pct', 'Copay_Amount', 'Plan_Year_Type', 'Payer_Type', 'Nursing_Code_Pref', 'Coverage_Effective_Date', 'Coverage_Term_Date', 'BV_Call_Date', 'BV_Rep_Name', 'BV_Call_Ref_Number', 'BV_Complete', 'Financial_Counseling_Log'",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'benefits.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'benefits.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n expected_headers = [\n 'Home_Infusion_Covered', 'Benefit_Type', 'PA_Required', 'PA_Phone',\n 'PA_Submission_Email', 'Specialty_Pharmacy_Required', 'Specialty_Pharmacy_Name',\n 'CareIG_Network_Status', 'Deductible_Individual', 'Deductible_Met_YTD',\n 'OOP_Max_Individual', 'OOP_Max_Met_YTD', 'Coinsurance_Pct', 'Copay_Amount',\n 'Plan_Year_Type', 'Payer_Type', 'Nursing_Code_Pref', 'Coverage_Effective_Date',\n 'Coverage_Term_Date', 'BV_Call_Date', 'BV_Rep_Name', 'BV_Call_Ref_Number',\n 'BV_Complete', 'Financial_Counseling_Log'\n ]\n actual_headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n actual_headers_clean = [h for h in actual_headers if h]\n actual_lower = set(h.lower() for h in actual_headers_clean)\n missing = []\n found = []\n for exp in expected_headers:\n if exp.lower() in actual_lower:\n found.append(exp)\n else:\n missing.append(exp)\n if missing:\n return {'pass': False, 'score': len(found) / len(expected_headers), 'feedback': f'benefits.xlsx is missing {len(missing)} required header(s): {missing}. Found headers: {actual_headers_clean}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'benefits.xlsx has all 24 required headers. Found headers: {actual_headers_clean}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading benefits.xlsx: {e}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e41ee32e-8d97-45f1-bd51-40c3ce243abd",
|
||
"sort_order": 1,
|
||
"rubric_text": "In file `benefits.xlsx`, the first data row must contain exactly two rows total (header + 1 data row) and the data row must contain: 'YES' for Home_Infusion_Covered, 'Medical' for Benefit_Type, 'YES' for PA_Required, '(800) 555-7221' for PA_Phone, 'pa@aetna.com' for PA_Submission_Email, 'NO' for Specialty_Pharmacy_Required, 'In-Network' for CareIG_Network_Status, '$1,500' for Deductible_Individual, '$750' for Deductible_Met_YTD, '$5,000' for OOP_Max_Individual, '$750' for OOP_Max_Met_YTD, '20%' for Coinsurance_Pct, '$0' for Copay_Amount, 'Calendar' for Plan_Year_Type, 'Commercial PPO' for Payer_Type, '99600' for Nursing_Code_Pref, '01/01/2026' for Coverage_Effective_Date, '12/31/2026' for Coverage_Term_Date, '04/02/2026' for BV_Call_Date, 'Marcus' or 'Marcus T.' for BV_Rep_Name, 'AET-BV-20260402-884471' for BV_Call_Ref_Number, 'YES' for BV_Complete, '04/03/2026 11:22' and 'Est OOP: $1,639' in Financial_Counseling_Log",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'benefits.xlsx'\n if not wb_path.exists():\n candidates = list(Path(workspace_path).glob('*benefits*.xlsx'))\n if candidates:\n wb_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'benefits.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected at least 2 rows (header + data) in benefits.xlsx, found {len(rows)}'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) < 1:\n return {'pass': False, 'score': 0.0, 'feedback': 'No data rows found in benefits.xlsx'}\n if len(data_rows) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in benefits.xlsx, found {len(data_rows)}'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n data = data_rows[0]\n\n def get_val(col_name):\n col_lower = col_name.lower().replace('_', '').replace(' ', '')\n for i, h in enumerate(headers):\n h_norm = h.lower().replace('_', '').replace(' ', '')\n if h_norm == col_lower:\n if i < len(data):\n return str(data[i]).strip() if data[i] is not None else ''\n return None\n\n def currency_eq(val, expected):\n def clean(s):\n return re.sub(r'[\\s,$%]', '', str(s).strip())\n return clean(val) == clean(expected)\n\n def loose_match(val, expected):\n v = val.strip().lower().replace(' ', '').replace(',', '')\n e = expected.strip().lower().replace(' ', '').replace(',', '')\n if v == e:\n return True\n if currency_eq(val, expected):\n return True\n if '%' in expected:\n num = expected.replace('%', '').strip()\n val_clean = val.replace('%', '').strip()\n try:\n if float(val_clean) == float(num) or float(val_clean) == float(num) / 100:\n return True\n except:\n pass\n return False\n\n def normalize_date(val, expected):\n # Strip time component if present\n val = re.split(r'\\s+', val.strip())[0]\n val_parts = re.split(r'[/\\-]', val.strip())\n if len(val_parts) == 3 and len(val_parts[0]) == 4:\n val = f'{val_parts[1]}/{val_parts[2]}/{val_parts[0]}'\n return val.lower().replace(' ', '') == expected.lower().replace(' ', '')\n\n errors = []\n checks = [\n ('Home_Infusion_Covered', 'YES'),\n ('Benefit_Type', 'Medical'),\n ('PA_Required', 'YES'),\n ('PA_Phone', '(800) 555-7221'),\n ('PA_Submission_Email', 'pa@aetna.com'),\n ('Specialty_Pharmacy_Required', 'NO'),\n ('CareIG_Network_Status', 'In-Network'),\n ('Deductible_Individual', '$1,500'),\n ('Deductible_Met_YTD', '$750'),\n ('OOP_Max_Individual', '$5,000'),\n ('OOP_Max_Met_YTD', '$750'),\n ('Coinsurance_Pct', '20%'),\n ('Copay_Amount', '$0'),\n ('Plan_Year_Type', 'Calendar'),\n ('Payer_Type', 'Commercial PPO'),\n ('Nursing_Code_Pref', '99600'),\n ('Coverage_Effective_Date', '01/01/2026'),\n ('Coverage_Term_Date', '12/31/2026'),\n ('BV_Call_Date', '04/02/2026'),\n ('BV_Call_Ref_Number', 'AET-BV-20260402-884471'),\n ('BV_Complete', 'YES'),\n ]\n for col, expected in checks:\n val = get_val(col)\n if val is None:\n errors.append(f'Column {col} not found in benefits.xlsx (headers found: {\", \".join(headers[:10])}...)')\n continue\n if not loose_match(val, expected):\n if col == 'PA_Phone':\n if re.sub(r'\\D', '', val) == re.sub(r'\\D', '', expected):\n continue\n if 'Date' in col or 'date' in col:\n if normalize_date(val, expected):\n continue\n errors.append(f'Column {col}: expected \"{expected}\", got \"{val}\"')\n\n rep = get_val('BV_Rep_Name')\n if rep is None:\n errors.append('Column BV_Rep_Name not found in benefits.xlsx')\n elif 'marcus' not in rep.lower():\n errors.append(f'BV_Rep_Name expected to contain \"Marcus\", got \"{rep}\"')\n\n log = get_val('Financial_Counseling_Log')\n if log is None:\n errors.append('Column Financial_Counseling_Log not found in benefits.xlsx')\n else:\n if '04/03/2026' not in log and '2026-04-03' not in log and '4/3/2026' not in log:\n errors.append(f'Financial_Counseling_Log missing date 04/03/2026, got: \"{log}\"')\n if '11:22' not in log:\n errors.append(f'Financial_Counseling_Log missing time 11:22, got: \"{log}\"')\n if '1639' not in log.replace(' ', '').replace(',', ''):\n errors.append(f'Financial_Counseling_Log missing Est OOP $1,639, got: \"{log}\"')\n\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'benefits.xlsx verification failed: ' + '; '.join(errors)}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'benefits.xlsx data row contains all required values including Home_Infusion_Covered=YES, Benefit_Type=Medical, PA_Required=YES, BV_Rep_Name contains Marcus, Financial_Counseling_Log contains 04/03/2026 11:22 and Est OOP $1,639'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading benefits.xlsx: {e}'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "910ad5ca-1432-4692-975d-33719aadf098",
|
||
"sort_order": 2,
|
||
"rubric_text": "File `audit_log.xlsx` must exist and contain a header row with exactly these columns in order: 'Date/Time', 'CASE_ID', 'Action', 'File_Updated', 'Staff_User', 'Result', and exactly 5 data rows with CASE_ID 'Vasquez_03151971' and Staff_User 'Teresa Rodriguez' for all rows, with actions in chronological order: row1='Referral Received'/File='intake.xlsx'/Result='SUCCESS', row2='BVS Assigned'/File='intake.xlsx'/Result='SUCCESS', row3='BV Complete'/File='benefits.xlsx'/Result='SUCCESS', row4='Formulary Stop'/File=BLANK/Result='STOP', row5='PA Submitted'/File='auth.xlsx'/Result='SUCCESS'",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'audit_log.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx is empty'}\n\n def normalize_header(h):\n h = re.sub(r'\\s*\\(.*?\\)$', '', h).strip() # strip trailing parenthetical\n h = re.sub(r'(?i)date[_/]time', 'Date/Time', h) # accept Date_Time or Date/Time\n return h\n\n raw_headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n headers = [normalize_header(h) for h in raw_headers]\n\n expected_headers = ['Date/Time', 'CASE_ID', 'Action', 'File_Updated', 'Staff_User', 'Result']\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{raw_headers[i]}\"'}\n\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 5:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected 5 data rows in audit_log.xlsx, found {len(data_rows)}'}\n\n expected_rows = [\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'Referral Received', 'File_Updated': 'intake.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'BVS Assigned', 'File_Updated': 'intake.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'BV Complete', 'File_Updated': 'benefits.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'Formulary Stop', 'File_Updated': None, 'Staff_User': 'Teresa Rodriguez', 'Result': 'STOP'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'PA Submitted', 'File_Updated': 'auth.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n ]\n\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n\n def get(row, col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = row[idx]\n return str(v).strip() if v is not None else ''\n\n for ri, (row, exp) in enumerate(zip(data_rows, expected_rows)):\n for field, expected_val in exp.items():\n actual = get(row, field)\n if expected_val is None:\n if actual not in ('', 'None', None):\n return {'pass': False, 'score': 0.0, 'feedback': f'audit_log row {ri+1} field {field}: expected blank, got \"{actual}\"'}\n else:\n if actual is None or actual.lower() != expected_val.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'audit_log row {ri+1} field {field}: expected \"{expected_val}\", got \"{actual}\"'}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'audit_log.xlsx has correct headers, 5 data rows, all values match expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading audit_log.xlsx: {e}'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "1ef72364-b643-4193-8975-784de5f86738",
|
||
"sort_order": 3,
|
||
"rubric_text": "File `clinical.xlsx` must exist, contain exactly 2 rows (header + 1 data row), and the data row must have `Drug_Source` = `CareIG`. All other columns present in the header must have blank values in the data row. The header may contain any subset of the known clinical.xlsx columns (`Handoff_Status`, `PA_Number`, `PA_Expiration_Date`, `Drug_Order_Placed`, `Drug_Delivery_Confirmed`, `Drug_Lot_Number`, `Drug_NDC`, `NS_500mL_Qty_Ordered`, `Filter_Tubing_Ordered`, `Benadryl_Available`, `Supplies_Other`, `Supply_Checklist_Complete`, `Visit_Date`, `Visit_Time`, `Nurse_Assigned`, `Drug_Issue_Notes`) or none of them, but no columns outside this set (excluding `Drug_Source`) are permitted.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'clinical.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx is empty'}\n\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n\n allowed_cols = {\n 'drug_source', 'handoff_status', 'pa_number', 'pa_expiration_date',\n 'drug_order_placed', 'drug_delivery_confirmed', 'drug_lot_number', 'drug_ndc',\n 'ns_500ml_qty_ordered', 'filter_tubing_ordered', 'benadryl_available',\n 'supplies_other', 'supply_checklist_complete', 'visit_date', 'visit_time',\n 'nurse_assigned', 'drug_issue_notes'\n }\n\n unexpected = [h for h in headers if h.lower() not in allowed_cols]\n if unexpected:\n return {'pass': False, 'score': 0.0, 'feedback': f'clinical.xlsx contains unexpected columns: {unexpected}'}\n\n if 'drug_source' not in [h.lower() for h in headers]:\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx missing required column Drug_Source'}\n\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in clinical.xlsx, found {len(data_rows)}'}\n\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n\n drug_source_idx = col_idx['drug_source']\n drug_source_val = str(data[drug_source_idx]).strip() if data[drug_source_idx] is not None else ''\n if drug_source_val.lower() != 'careig':\n return {'pass': False, 'score': 0.0, 'feedback': f'Drug_Source expected \"CareIG\", got \"{drug_source_val}\"'}\n\n non_blank = []\n for i, val in enumerate(data):\n if i == drug_source_idx:\n continue\n if val is not None and str(val).strip() not in ('', 'None'):\n non_blank.append(f'{headers[i]}={val}')\n if non_blank:\n return {'pass': False, 'score': 0.0, 'feedback': f'clinical.xlsx has unexpected non-blank values: {non_blank}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'clinical.xlsx is valid: Drug_Source=CareIG, all other fields blank, no unexpected columns'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading clinical.xlsx: {e}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "c76850e3-11e3-4291-94ef-4c7dfd5dede2",
|
||
"sort_order": 4,
|
||
"rubric_text": "File `intake.xlsx` must exist, contain exactly 2 rows (header + 1 data row), have a header row with all 25 required columns in order: 'CASE_ID', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB', 'Patient_Address', 'Patient_Phone', 'Patient_Email', 'Physician_Name', 'Physician_NPI', 'Physician_Email', 'Diagnosis_ICD10', 'Drug_Name', 'Drug_Dose_Grams', 'Drug_Frequency', 'Primary_Insurance_Name', 'Primary_Member_ID', 'Primary_Group_Number', 'Primary_Payer_Phone', 'Secondary_Insurance_Name', 'Secondary_Member_ID', 'Contraindications', 'Consent_Status', 'Contact_Attempt_Log', 'BVS_Assigned', 'Financial_Hardship', and the data row must contain all required patient values",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'intake.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'intake.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'intake.xlsx is empty'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n expected_headers = [\n 'CASE_ID', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Patient_Address', 'Patient_Phone', 'Patient_Email', 'Physician_Name',\n 'Physician_NPI', 'Physician_Email', 'Diagnosis_ICD10', 'Drug_Name',\n 'Drug_Dose_Grams', 'Drug_Frequency', 'Primary_Insurance_Name',\n 'Primary_Member_ID', 'Primary_Group_Number', 'Primary_Payer_Phone',\n 'Secondary_Insurance_Name', 'Secondary_Member_ID', 'Contraindications',\n 'Consent_Status', 'Contact_Attempt_Log', 'BVS_Assigned', 'Financial_Hardship'\n ]\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{headers[i]}\"'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in intake.xlsx, found {len(data_rows)}'}\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n def get(col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = data[idx]\n return str(v).strip() if v is not None else ''\n checks = [\n ('CASE_ID', 'Vasquez_03151971'),\n ('Patient_Last_Name', 'Vasquez'),\n ('Patient_First_Name', 'Maria Elena'),\n ('Patient_DOB', '03/15/1971'),\n ('Patient_Address', '4821 SW 112th Avenue, Miami, FL 33165'),\n ('Patient_Phone', '(305) 555-4892'),\n ('Patient_Email', 'm.vasquez1971@gmail.com'),\n ('Physician_NPI', '1073541867'),\n ('Physician_Email', 'j.harrington@miamineurology.com'),\n ('Diagnosis_ICD10', 'D83.9'),\n ('Drug_Name', 'Privigen 10%'),\n ('Drug_Dose_Grams', '35'),\n ('Drug_Frequency', 'Every 4 weeks'),\n ('Primary_Insurance_Name', 'Aetna PPO'),\n ('Primary_Member_ID', 'AET847392018'),\n ('Primary_Group_Number', 'GRP29341'),\n ('Primary_Payer_Phone', '(800) 555-2940'),\n ('Consent_Status', 'Signed'),\n ]\n for col, expected in checks:\n val = get(col)\n if val is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column {col} not found in intake.xlsx'}\n if val.lower().replace(' ', '') != expected.lower().replace(' ', ''):\n return {'pass': False, 'score': 0.0, 'feedback': f'intake.xlsx column {col}: expected \"{expected}\", got \"{val}\"'}\n physician = get('Physician_Name')\n if 'harrington' not in physician.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Physician_Name expected to contain \"Harrington\", got \"{physician}\"'}\n for col in ('Secondary_Insurance_Name', 'Secondary_Member_ID'):\n val = get(col)\n if val.upper() not in ('', 'NONE', 'N/A'):\n return {'pass': False, 'score': 0.0, 'feedback': f'{col} expected blank or None, got \"{val}\"'}\n contra = get('Contraindications')\n full_contra = 'NKDA. Patient denies history of hyperprolinemia. No prior IVIG adverse reactions.'\n if contra.lower().replace(' ', '') != full_contra.lower().replace(' ', '') and contra.strip().upper() != 'NKDA':\n return {'pass': False, 'score': 0.0, 'feedback': f'Contraindications expected \"{full_contra}\" or \"NKDA\", got \"{contra}\"'}\n log = get('Contact_Attempt_Log')\n segments = [s.strip() for s in re.split(r'(?=\\b\\d{1,2}/\\d{1,2}/\\d{4}\\b)', log) if s.strip()]\n march_30 = next((s for s in segments if re.match(r'0?3/30/2026', s)), None)\n if march_30 is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log missing 03/30/2026 entry, got: \"{log}\"'}\n if '10:22' not in march_30:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log 03/30/2026 entry missing time 10:22, got: \"{march_30}\"'}\n if 'voicemail' not in march_30.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log 03/30/2026 entry missing \"Voicemail\" outcome, got: \"{march_30}\"'}\n march_31 = next((s for s in segments if re.match(r'0?3/31/2026', s)), None)\n if march_31 is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log missing 03/31/2026 entry, got: \"{log}\"'}\n if '14:05' not in march_31:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log 03/31/2026 entry missing time 14:05, got: \"{march_31}\"'}\n if 'reached' not in march_31.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log 03/31/2026 entry missing \"Reached\" outcome, got: \"{march_31}\"'}\n bvs = get('BVS_Assigned')\n if bvs.upper() != 'YES':\n return {'pass': False, 'score': 0.0, 'feedback': f'BVS_Assigned expected \"YES\", got \"{bvs}\"'}\n hardship = get('Financial_Hardship')\n if hardship.upper() not in ('', 'NONE', 'NO'):\n return {'pass': False, 'score': 0.0, 'feedback': f'Financial_Hardship expected blank or \"NO\", got \"{hardship}\"'}\n return {'pass': True, 'score': 1.0, 'feedback': 'intake.xlsx has correct headers and all required data values'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading intake.xlsx: {e}'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "04806b09-34d2-4eed-a61f-ad9ada2d4581",
|
||
"sort_order": 5,
|
||
"rubric_text": "File `auth.xlsx` must exist, contain exactly 2 rows (header + 1 data row), header row must contain all 18 required columns in order: 'CASE_ID', 'PA_Submission_Date', 'PA_Ref_Number', 'PA_Status', 'Last_Followup_Date', 'PA_Call_Log', 'PA_Number', 'PA_Approved_Drug', 'PA_Approved_Dose', 'PA_Approved_Frequency', 'PA_Effective_Date', 'PA_Expiration_Date', 'PA_Denial_Date', 'PA_Denial_Reason', 'Formulary_Exception_Notes', 'P2P_Available_Slots', 'P2P_Status', 'P2P_Request_Date', and the data row must contain: CASE_ID='Vasquez_03151971', PA_Submission_Date='04/07/2026', PA_Ref_Number='AET-PACONF-20260407-77291', PA_Status='Pending', Last_Followup_Date='04/07/2026', Formulary_Exception_Notes referencing Harrington/D83.9/Privigen",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'auth.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'auth.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'auth.xlsx is empty'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n expected_headers = [\n 'CASE_ID', 'PA_Submission_Date', 'PA_Ref_Number', 'PA_Status',\n 'Last_Followup_Date', 'PA_Call_Log', 'PA_Number', 'PA_Approved_Drug',\n 'PA_Approved_Dose', 'PA_Approved_Frequency', 'PA_Effective_Date',\n 'PA_Expiration_Date', 'PA_Denial_Date', 'PA_Denial_Reason',\n 'Formulary_Exception_Notes', 'P2P_Available_Slots', 'P2P_Status', 'P2P_Request_Date'\n ]\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{headers[i]}\"'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in auth.xlsx, found {len(data_rows)}'}\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n def get(col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = data[idx]\n return str(v).strip() if v is not None else ''\n checks = [\n ('CASE_ID', 'Vasquez_03151971'),\n ('PA_Submission_Date', '04/07/2026'),\n ('PA_Ref_Number', 'AET-PACONF-20260407-77291'),\n ('PA_Status', 'Pending'),\n ('Last_Followup_Date', '04/07/2026'),\n ]\n for col, expected in checks:\n val = get(col)\n if val is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column {col} not found in auth.xlsx'}\n if val.lower().replace(' ', '') != expected.lower().replace(' ', ''):\n return {'pass': False, 'score': 0.0, 'feedback': f'auth.xlsx column {col}: expected \"{expected}\", got \"{val}\"'}\n # Check Formulary_Exception_Notes contains key references\n notes = get('Formulary_Exception_Notes')\n if notes is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Formulary_Exception_Notes column not found'}\n required_in_notes = ['D83.9', 'Privigen']\n for term in required_in_notes:\n if term.lower() not in notes.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Formulary_Exception_Notes missing \"{term}\", got: \"{notes[:200]}\"'}\n if 'harrington' not in notes.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Formulary_Exception_Notes missing \"Harrington\", got: \"{notes[:200]}\"'}\n return {'pass': True, 'score': 1.0, 'feedback': 'auth.xlsx has correct headers and required data values including Formulary_Exception_Notes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading auth.xlsx: {e}'}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "3bf4fbaa-4f39-4bb9-8ee0-9b7c18fec8d9",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `mailbox.json`, there must be exactly 17 total emails across all folders",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided, cannot check mailbox.json'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n emails = data.get('emails', [])\n total = len(emails)\n if total != 17:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 17 emails in mailbox.json, found {total}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'mailbox.json contains exactly 17 emails'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {e}'}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1775287978014",
|
||
"sort_order": 7,
|
||
"rubric_text": "The `slack_data.json` file must contain exactly 30 total messages across all channels. No new Slack messages should have been posted by the model.",
|
||
"verifier_code": "import json\nfrom pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided, cannot check slack_data.json'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n messages = data.get('messages', {})\n total = sum(len(v) for v in messages.values())\n if total != 30:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 30 total Slack messages, found {total}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json contains exactly 30 messages across all channels'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {e}'}",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|