Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,58 @@
[
{
"id": "e82385e0-38ac-490b-8f93-5166cac146c9",
"sort_order": 0,
"rubric_text": "File `benefits.xlsx` must exist in the workspace and contain a header row with all 24 required columns in any order: 'Home_Infusion_Covered', 'Benefit_Type', 'PA_Required', 'PA_Phone', 'PA_Submission_Email', 'Specialty_Pharmacy_Required', 'Specialty_Pharmacy_Name', 'CareIG_Network_Status', 'Deductible_Individual', 'Deductible_Met_YTD', 'OOP_Max_Individual', 'OOP_Max_Met_YTD', 'Coinsurance_Pct', 'Copay_Amount', 'Plan_Year_Type', 'Payer_Type', 'Nursing_Code_Pref', 'Coverage_Effective_Date', 'Coverage_Term_Date', 'BV_Call_Date', 'BV_Rep_Name', 'BV_Call_Ref_Number', 'BV_Complete', 'Financial_Counseling_Log'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'benefits.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'benefits.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n expected_headers = [\n 'Home_Infusion_Covered', 'Benefit_Type', 'PA_Required', 'PA_Phone',\n 'PA_Submission_Email', 'Specialty_Pharmacy_Required', 'Specialty_Pharmacy_Name',\n 'CareIG_Network_Status', 'Deductible_Individual', 'Deductible_Met_YTD',\n 'OOP_Max_Individual', 'OOP_Max_Met_YTD', 'Coinsurance_Pct', 'Copay_Amount',\n 'Plan_Year_Type', 'Payer_Type', 'Nursing_Code_Pref', 'Coverage_Effective_Date',\n 'Coverage_Term_Date', 'BV_Call_Date', 'BV_Rep_Name', 'BV_Call_Ref_Number',\n 'BV_Complete', 'Financial_Counseling_Log'\n ]\n actual_headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n actual_headers_clean = [h for h in actual_headers if h]\n actual_lower = set(h.lower() for h in actual_headers_clean)\n missing = []\n found = []\n for exp in expected_headers:\n if exp.lower() in actual_lower:\n found.append(exp)\n else:\n missing.append(exp)\n if missing:\n return {'pass': False, 'score': len(found) / len(expected_headers), 'feedback': f'benefits.xlsx is missing {len(missing)} required header(s): {missing}. Found headers: {actual_headers_clean}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'benefits.xlsx has all 24 required headers. Found headers: {actual_headers_clean}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading benefits.xlsx: {e}'}\n",
"criterion_type": "expected_output"
},
{
"id": "e41ee32e-8d97-45f1-bd51-40c3ce243abd",
"sort_order": 1,
"rubric_text": "In file `benefits.xlsx`, the first data row must contain exactly two rows total (header + 1 data row) and the data row must contain: 'YES' for Home_Infusion_Covered, 'Medical' for Benefit_Type, 'YES' for PA_Required, '(800) 555-7221' for PA_Phone, 'pa@aetna.com' for PA_Submission_Email, 'NO' for Specialty_Pharmacy_Required, 'In-Network' for CareIG_Network_Status, '$1,500' for Deductible_Individual, '$750' for Deductible_Met_YTD, '$5,000' for OOP_Max_Individual, '$750' for OOP_Max_Met_YTD, '20%' for Coinsurance_Pct, '$0' for Copay_Amount, 'Calendar' for Plan_Year_Type, 'Commercial PPO' for Payer_Type, '99600' for Nursing_Code_Pref, '01/01/2026' for Coverage_Effective_Date, '12/31/2026' for Coverage_Term_Date, '04/02/2026' for BV_Call_Date, 'Marcus' or 'Marcus T.' for BV_Rep_Name, 'AET-BV-20260402-884471' for BV_Call_Ref_Number, 'YES' for BV_Complete, '04/03/2026 11:22' and 'Est OOP: $1,639' in Financial_Counseling_Log",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'benefits.xlsx'\n if not wb_path.exists():\n candidates = list(Path(workspace_path).glob('*benefits*.xlsx'))\n if candidates:\n wb_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'benefits.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected at least 2 rows (header + data) in benefits.xlsx, found {len(rows)}'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) < 1:\n return {'pass': False, 'score': 0.0, 'feedback': 'No data rows found in benefits.xlsx'}\n if len(data_rows) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in benefits.xlsx, found {len(data_rows)}'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n data = data_rows[0]\n\n def get_val(col_name):\n col_lower = col_name.lower().replace('_', '').replace(' ', '')\n for i, h in enumerate(headers):\n h_norm = h.lower().replace('_', '').replace(' ', '')\n if h_norm == col_lower:\n if i < len(data):\n return str(data[i]).strip() if data[i] is not None else ''\n return None\n\n def currency_eq(val, expected):\n def clean(s):\n return re.sub(r'[\\s,$%]', '', str(s).strip())\n return clean(val) == clean(expected)\n\n def loose_match(val, expected):\n v = val.strip().lower().replace(' ', '').replace(',', '')\n e = expected.strip().lower().replace(' ', '').replace(',', '')\n if v == e:\n return True\n if currency_eq(val, expected):\n return True\n if '%' in expected:\n num = expected.replace('%', '').strip()\n val_clean = val.replace('%', '').strip()\n try:\n if float(val_clean) == float(num) or float(val_clean) == float(num) / 100:\n return True\n except:\n pass\n return False\n\n def normalize_date(val, expected):\n # Strip time component if present\n val = re.split(r'\\s+', val.strip())[0]\n val_parts = re.split(r'[/\\-]', val.strip())\n if len(val_parts) == 3 and len(val_parts[0]) == 4:\n val = f'{val_parts[1]}/{val_parts[2]}/{val_parts[0]}'\n return val.lower().replace(' ', '') == expected.lower().replace(' ', '')\n\n errors = []\n checks = [\n ('Home_Infusion_Covered', 'YES'),\n ('Benefit_Type', 'Medical'),\n ('PA_Required', 'YES'),\n ('PA_Phone', '(800) 555-7221'),\n ('PA_Submission_Email', 'pa@aetna.com'),\n ('Specialty_Pharmacy_Required', 'NO'),\n ('CareIG_Network_Status', 'In-Network'),\n ('Deductible_Individual', '$1,500'),\n ('Deductible_Met_YTD', '$750'),\n ('OOP_Max_Individual', '$5,000'),\n ('OOP_Max_Met_YTD', '$750'),\n ('Coinsurance_Pct', '20%'),\n ('Copay_Amount', '$0'),\n ('Plan_Year_Type', 'Calendar'),\n ('Payer_Type', 'Commercial PPO'),\n ('Nursing_Code_Pref', '99600'),\n ('Coverage_Effective_Date', '01/01/2026'),\n ('Coverage_Term_Date', '12/31/2026'),\n ('BV_Call_Date', '04/02/2026'),\n ('BV_Call_Ref_Number', 'AET-BV-20260402-884471'),\n ('BV_Complete', 'YES'),\n ]\n for col, expected in checks:\n val = get_val(col)\n if val is None:\n errors.append(f'Column {col} not found in benefits.xlsx (headers found: {\", \".join(headers[:10])}...)')\n continue\n if not loose_match(val, expected):\n if col == 'PA_Phone':\n if re.sub(r'\\D', '', val) == re.sub(r'\\D', '', expected):\n continue\n if 'Date' in col or 'date' in col:\n if normalize_date(val, expected):\n continue\n errors.append(f'Column {col}: expected \"{expected}\", got \"{val}\"')\n\n rep = get_val('BV_Rep_Name')\n if rep is None:\n errors.append('Column BV_Rep_Name not found in benefits.xlsx')\n elif 'marcus' not in rep.lower():\n errors.append(f'BV_Rep_Name expected to contain \"Marcus\", got \"{rep}\"')\n\n log = get_val('Financial_Line truncated
"criterion_type": "expected_output"
},
{
"id": "910ad5ca-1432-4692-975d-33719aadf098",
"sort_order": 2,
"rubric_text": "File `audit_log.xlsx` must exist and contain a header row with exactly these columns in order: 'Date/Time', 'CASE_ID', 'Action', 'File_Updated', 'Staff_User', 'Result', and exactly 5 data rows with CASE_ID 'Vasquez_03151971' and Staff_User 'Teresa Rodriguez' for all rows, with actions in chronological order: row1='Referral Received'/File='intake.xlsx'/Result='SUCCESS', row2='BVS Assigned'/File='intake.xlsx'/Result='SUCCESS', row3='BV Complete'/File='benefits.xlsx'/Result='SUCCESS', row4='Formulary Stop'/File=BLANK/Result='STOP', row5='PA Submitted'/File='auth.xlsx'/Result='SUCCESS'",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'audit_log.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx is empty'}\n\n def normalize_header(h):\n h = re.sub(r'\\s*\\(.*?\\)$', '', h).strip() # strip trailing parenthetical\n h = re.sub(r'(?i)date[_/]time', 'Date/Time', h) # accept Date_Time or Date/Time\n return h\n\n raw_headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n headers = [normalize_header(h) for h in raw_headers]\n\n expected_headers = ['Date/Time', 'CASE_ID', 'Action', 'File_Updated', 'Staff_User', 'Result']\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{raw_headers[i]}\"'}\n\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 5:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected 5 data rows in audit_log.xlsx, found {len(data_rows)}'}\n\n expected_rows = [\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'Referral Received', 'File_Updated': 'intake.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'BVS Assigned', 'File_Updated': 'intake.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'BV Complete', 'File_Updated': 'benefits.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'Formulary Stop', 'File_Updated': None, 'Staff_User': 'Teresa Rodriguez', 'Result': 'STOP'},\n {'CASE_ID': 'Vasquez_03151971', 'Action': 'PA Submitted', 'File_Updated': 'auth.xlsx', 'Staff_User': 'Teresa Rodriguez', 'Result': 'SUCCESS'},\n ]\n\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n\n def get(row, col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = row[idx]\n return str(v).strip() if v is not None else ''\n\n for ri, (row, exp) in enumerate(zip(data_rows, expected_rows)):\n for field, expected_val in exp.items():\n actual = get(row, field)\n if expected_val is None:\n if actual not in ('', 'None', None):\n return {'pass': False, 'score': 0.0, 'feedback': f'audit_log row {ri+1} field {field}: expected blank, got \"{actual}\"'}\n else:\n if actual is None or actual.lower() != expected_val.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'audit_log row {ri+1} field {field}: expected \"{expected_val}\", got \"{actual}\"'}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'audit_log.xlsx has correct headers, 5 data rows, all values match expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading audit_log.xlsx: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "1ef72364-b643-4193-8975-784de5f86738",
"sort_order": 3,
"rubric_text": "File `clinical.xlsx` must exist, contain exactly 2 rows (header + 1 data row), and the data row must have `Drug_Source` = `CareIG`. All other columns present in the header must have blank values in the data row. The header may contain any subset of the known clinical.xlsx columns (`Handoff_Status`, `PA_Number`, `PA_Expiration_Date`, `Drug_Order_Placed`, `Drug_Delivery_Confirmed`, `Drug_Lot_Number`, `Drug_NDC`, `NS_500mL_Qty_Ordered`, `Filter_Tubing_Ordered`, `Benadryl_Available`, `Supplies_Other`, `Supply_Checklist_Complete`, `Visit_Date`, `Visit_Time`, `Nurse_Assigned`, `Drug_Issue_Notes`) or none of them, but no columns outside this set (excluding `Drug_Source`) are permitted.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'clinical.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx is empty'}\n\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n\n allowed_cols = {\n 'drug_source', 'handoff_status', 'pa_number', 'pa_expiration_date',\n 'drug_order_placed', 'drug_delivery_confirmed', 'drug_lot_number', 'drug_ndc',\n 'ns_500ml_qty_ordered', 'filter_tubing_ordered', 'benadryl_available',\n 'supplies_other', 'supply_checklist_complete', 'visit_date', 'visit_time',\n 'nurse_assigned', 'drug_issue_notes'\n }\n\n unexpected = [h for h in headers if h.lower() not in allowed_cols]\n if unexpected:\n return {'pass': False, 'score': 0.0, 'feedback': f'clinical.xlsx contains unexpected columns: {unexpected}'}\n\n if 'drug_source' not in [h.lower() for h in headers]:\n return {'pass': False, 'score': 0.0, 'feedback': 'clinical.xlsx missing required column Drug_Source'}\n\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in clinical.xlsx, found {len(data_rows)}'}\n\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n\n drug_source_idx = col_idx['drug_source']\n drug_source_val = str(data[drug_source_idx]).strip() if data[drug_source_idx] is not None else ''\n if drug_source_val.lower() != 'careig':\n return {'pass': False, 'score': 0.0, 'feedback': f'Drug_Source expected \"CareIG\", got \"{drug_source_val}\"'}\n\n non_blank = []\n for i, val in enumerate(data):\n if i == drug_source_idx:\n continue\n if val is not None and str(val).strip() not in ('', 'None'):\n non_blank.append(f'{headers[i]}={val}')\n if non_blank:\n return {'pass': False, 'score': 0.0, 'feedback': f'clinical.xlsx has unexpected non-blank values: {non_blank}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'clinical.xlsx is valid: Drug_Source=CareIG, all other fields blank, no unexpected columns'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading clinical.xlsx: {e}'}\n",
"criterion_type": "expected_output"
},
{
"id": "c76850e3-11e3-4291-94ef-4c7dfd5dede2",
"sort_order": 4,
"rubric_text": "File `intake.xlsx` must exist, contain exactly 2 rows (header + 1 data row), have a header row with all 25 required columns in order: 'CASE_ID', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB', 'Patient_Address', 'Patient_Phone', 'Patient_Email', 'Physician_Name', 'Physician_NPI', 'Physician_Email', 'Diagnosis_ICD10', 'Drug_Name', 'Drug_Dose_Grams', 'Drug_Frequency', 'Primary_Insurance_Name', 'Primary_Member_ID', 'Primary_Group_Number', 'Primary_Payer_Phone', 'Secondary_Insurance_Name', 'Secondary_Member_ID', 'Contraindications', 'Consent_Status', 'Contact_Attempt_Log', 'BVS_Assigned', 'Financial_Hardship', and the data row must contain all required patient values",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'intake.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'intake.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'intake.xlsx is empty'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n expected_headers = [\n 'CASE_ID', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Patient_Address', 'Patient_Phone', 'Patient_Email', 'Physician_Name',\n 'Physician_NPI', 'Physician_Email', 'Diagnosis_ICD10', 'Drug_Name',\n 'Drug_Dose_Grams', 'Drug_Frequency', 'Primary_Insurance_Name',\n 'Primary_Member_ID', 'Primary_Group_Number', 'Primary_Payer_Phone',\n 'Secondary_Insurance_Name', 'Secondary_Member_ID', 'Contraindications',\n 'Consent_Status', 'Contact_Attempt_Log', 'BVS_Assigned', 'Financial_Hardship'\n ]\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{headers[i]}\"'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in intake.xlsx, found {len(data_rows)}'}\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n def get(col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = data[idx]\n return str(v).strip() if v is not None else ''\n checks = [\n ('CASE_ID', 'Vasquez_03151971'),\n ('Patient_Last_Name', 'Vasquez'),\n ('Patient_First_Name', 'Maria Elena'),\n ('Patient_DOB', '03/15/1971'),\n ('Patient_Address', '4821 SW 112th Avenue, Miami, FL 33165'),\n ('Patient_Phone', '(305) 555-4892'),\n ('Patient_Email', 'm.vasquez1971@gmail.com'),\n ('Physician_NPI', '1073541867'),\n ('Physician_Email', 'j.harrington@miamineurology.com'),\n ('Diagnosis_ICD10', 'D83.9'),\n ('Drug_Name', 'Privigen 10%'),\n ('Drug_Dose_Grams', '35'),\n ('Drug_Frequency', 'Every 4 weeks'),\n ('Primary_Insurance_Name', 'Aetna PPO'),\n ('Primary_Member_ID', 'AET847392018'),\n ('Primary_Group_Number', 'GRP29341'),\n ('Primary_Payer_Phone', '(800) 555-2940'),\n ('Consent_Status', 'Signed'),\n ]\n for col, expected in checks:\n val = get(col)\n if val is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column {col} not found in intake.xlsx'}\n if val.lower().replace(' ', '') != expected.lower().replace(' ', ''):\n return {'pass': False, 'score': 0.0, 'feedback': f'intake.xlsx column {col}: expected \"{expected}\", got \"{val}\"'}\n physician = get('Physician_Name')\n if 'harrington' not in physician.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Physician_Name expected to contain \"Harrington\", got \"{physician}\"'}\n for col in ('Secondary_Insurance_Name', 'Secondary_Member_ID'):\n val = get(col)\n if val.upper() not in ('', 'NONE', 'N/A'):\n return {'pass': False, 'score': 0.0, 'feedback': f'{col} expected blank or None, got \"{val}\"'}\n contra = get('Contraindications')\n full_contra = 'NKDA. Patient denies history of hyperprolinemia. No prior IVIG adverse reactions.'\n if contra.lower().replace(' ', '') != full_contra.lower().replace(' ', '') and contra.strip().upper() != 'NKDA':\n return {'pass': False, 'score': 0.0, 'feedback': f'Contraindications expected \"{full_contra}\" or \"NKDA\", got \"{contra}\"'}\n log = get('Contact_Attempt_Log')\n segments = [s.strip() for s in re.split(r'(?=\\b\\d{1,2}/\\d{1,2}/\\d{4}\\b)', log) if s.strip()]\n march_30 = next((s for s in segments if re.match(r'0?3/30/2026', s)), None)\n if march_30 is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Log missing 03/30/2026 entry, got: \"{log}\"'}\n if '10:22' not in march_30:\n return {'pass': False, 'score': 0.0, 'feedback': f'Contact_Attempt_Line truncated
"criterion_type": "expected_output"
},
{
"id": "04806b09-34d2-4eed-a61f-ad9ada2d4581",
"sort_order": 5,
"rubric_text": "File `auth.xlsx` must exist, contain exactly 2 rows (header + 1 data row), header row must contain all 18 required columns in order: 'CASE_ID', 'PA_Submission_Date', 'PA_Ref_Number', 'PA_Status', 'Last_Followup_Date', 'PA_Call_Log', 'PA_Number', 'PA_Approved_Drug', 'PA_Approved_Dose', 'PA_Approved_Frequency', 'PA_Effective_Date', 'PA_Expiration_Date', 'PA_Denial_Date', 'PA_Denial_Reason', 'Formulary_Exception_Notes', 'P2P_Available_Slots', 'P2P_Status', 'P2P_Request_Date', and the data row must contain: CASE_ID='Vasquez_03151971', PA_Submission_Date='04/07/2026', PA_Ref_Number='AET-PACONF-20260407-77291', PA_Status='Pending', Last_Followup_Date='04/07/2026', Formulary_Exception_Notes referencing Harrington/D83.9/Privigen",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / 'auth.xlsx'\n if not wb_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'auth.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'auth.xlsx is empty'}\n headers = [str(c).strip() if c is not None else '' for c in rows[0]]\n expected_headers = [\n 'CASE_ID', 'PA_Submission_Date', 'PA_Ref_Number', 'PA_Status',\n 'Last_Followup_Date', 'PA_Call_Log', 'PA_Number', 'PA_Approved_Drug',\n 'PA_Approved_Dose', 'PA_Approved_Frequency', 'PA_Effective_Date',\n 'PA_Expiration_Date', 'PA_Denial_Date', 'PA_Denial_Reason',\n 'Formulary_Exception_Notes', 'P2P_Available_Slots', 'P2P_Status', 'P2P_Request_Date'\n ]\n for i, exp in enumerate(expected_headers):\n if i >= len(headers):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing header at position {i+1}: expected \"{exp}\"'}\n if headers[i].lower() != exp.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Header mismatch at pos {i+1}: expected \"{exp}\", got \"{headers[i]}\"'}\n data_rows = [r for r in rows[1:] if any(v is not None for v in r)]\n if len(data_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 data row in auth.xlsx, found {len(data_rows)}'}\n data = data_rows[0]\n col_idx = {h.lower(): i for i, h in enumerate(headers)}\n def get(col):\n idx = col_idx.get(col.lower())\n if idx is None:\n return None\n v = data[idx]\n return str(v).strip() if v is not None else ''\n checks = [\n ('CASE_ID', 'Vasquez_03151971'),\n ('PA_Submission_Date', '04/07/2026'),\n ('PA_Ref_Number', 'AET-PACONF-20260407-77291'),\n ('PA_Status', 'Pending'),\n ('Last_Followup_Date', '04/07/2026'),\n ]\n for col, expected in checks:\n val = get(col)\n if val is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column {col} not found in auth.xlsx'}\n if val.lower().replace(' ', '') != expected.lower().replace(' ', ''):\n return {'pass': False, 'score': 0.0, 'feedback': f'auth.xlsx column {col}: expected \"{expected}\", got \"{val}\"'}\n # Check Formulary_Exception_Notes contains key references\n notes = get('Formulary_Exception_Notes')\n if notes is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Formulary_Exception_Notes column not found'}\n required_in_notes = ['D83.9', 'Privigen']\n for term in required_in_notes:\n if term.lower() not in notes.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Formulary_Exception_Notes missing \"{term}\", got: \"{notes[:200]}\"'}\n if 'harrington' not in notes.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Formulary_Exception_Notes missing \"Harrington\", got: \"{notes[:200]}\"'}\n return {'pass': True, 'score': 1.0, 'feedback': 'auth.xlsx has correct headers and required data values including Formulary_Exception_Notes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading auth.xlsx: {e}'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "3bf4fbaa-4f39-4bb9-8ee0-9b7c18fec8d9",
"sort_order": 6,
"rubric_text": "In `mailbox.json`, there must be exactly 17 total emails across all folders",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided, cannot check mailbox.json'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n emails = data.get('emails', [])\n total = len(emails)\n if total != 17:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 17 emails in mailbox.json, found {total}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'mailbox.json contains exactly 17 emails'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {e}'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775287978014",
"sort_order": 7,
"rubric_text": "The `slack_data.json` file must contain exactly 30 total messages across all channels. No new Slack messages should have been posted by the model.",
"verifier_code": "import json\nfrom pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided, cannot check slack_data.json'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n messages = data.get('messages', {})\n total = sum(len(v) for v in messages.values())\n if total != 30:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 30 total Slack messages, found {total}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json contains exactly 30 messages across all channels'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {e}'}",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Oh SHOOT, I accidentally deleted four important patient files!! I need you to help me remake them fast. I don't remember which ones I deleted but they are all xlsx files for patient Maria Elena Vasquez. I think one of them was the audit log, ignore the Date/Time column on that one (but make sure entries are in chronological order, newest last) and put down me (Teresa Rodriguez) as the staff_user for all entries. I know that some of the other staff members might have completed some of the steps (and their names should be in the audit log), but I got verbal confirmation from Robert Nguyen that it is fine if my name is there. For any transcript timestamps, just put the time when the call was started. If you follow the SOP you should be able to rebuild the missing files, the last update for this case happened today at 2:15, the most recent email is about it. Also, the files you make will need ALL headers even if the values are blank, if the excel files don't have a template just write the headers in the order they appear in the sop. Oh and all .xlsx files must contain exactly one worksheet. Do not create, rename, or add additional sheets.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py