DerekSurge
66 lines
21 KiB
JSON
66 lines
21 KiB
JSON
[
|
||
{
|
||
"id": "76e6d4f9-971c-4a9e-af3a-90ebad31d1ff",
|
||
"sort_order": 0,
|
||
"rubric_text": "A folder named 'CASE-2026-00478' must exist in the workspace.",
|
||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n case_folder = Path(workspace_path) / 'CASE-2026-00478'\n if case_folder.exists() and case_folder.is_dir():\n return {'pass': True, 'score': 1.0, 'feedback': \"Folder 'CASE-2026-00478' exists in workspace.\"}\n return {'pass': False, 'score': 0.0, 'feedback': \"Folder 'CASE-2026-00478' not found in workspace.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "9c9243f4-674c-4f52-ab01-e744f345aef9",
|
||
"sort_order": 1,
|
||
"rubric_text": "Folder 'CASE-2026-00478' must contain 'Claims_History_Kovacs_James_44710006.pdf', 'EOC-MCA4471-PY2026.pdf', 'Kovacs_James_ClinicalPacket_CR-2026-044721.pdf', 'MCA_Denial_Letter_Kovacs_James_CR-2026-044721.pdf', and 'MP-019-v2.2-20260101.pdf'.",
|
||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n case_folder = Path(workspace_path) / 'CASE-2026-00478'\n if not case_folder.exists() or not case_folder.is_dir():\n return {'pass': False, 'score': 0.0, 'feedback': \"Folder 'CASE-2026-00478' does not exist.\"}\n\n required_files = [\n 'Claims_History_Kovacs_James_44710006.pdf',\n 'EOC-MCA4471-PY2026.pdf',\n 'Kovacs_James_ClinicalPacket_CR-2026-044721.pdf',\n 'MCA_Denial_Letter_Kovacs_James_CR-2026-044721.pdf',\n 'MP-019-v2.2-20260101.pdf',\n ]\n\n actual_files = [f.name for f in case_folder.iterdir() if f.is_file()]\n\n missing = [f for f in required_files if f not in actual_files]\n\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f\"Missing required files in 'CASE-2026-00478': {missing}. Files found: {actual_files}.\"}\n\n return {'pass': True, 'score': 1.0, 'feedback': f\"All required files present in 'CASE-2026-00478': {required_files}. Files found: {actual_files}.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "9aeac6a7-fef6-4523-9c07-d42bb426047d",
|
||
"sort_order": 2,
|
||
"rubric_text": "In file `appeals_log_master.xlsx`, Row 8 must contain Case ID 'CASE-2026-00478' (Column A), Member ID '44710006' (Column B), Group ID 'MCA-4471' (Column C), Plan Year '2026' (Column D), and Appeal Type 'Concurrent Care' (Column F), Total Dispute Value 'undetermined' (Column E), and Standard of Review 'Medical Necessity' (column H).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file broadly\n xlsx_files = list(Path(workspace_path).rglob('appeals_log_master.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': \"File 'appeals_log_master.xlsx' not found anywhere in workspace.\"}\n\n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n\n # Expected values by column letter per rubric:\n # A: Case ID 'CASE-2026-00478'\n # B: Member ID '44710006'\n # C: Group ID 'MCA-4471'\n # D: Plan Year '2026'\n # E: Total Dispute Value 'undetermined'\n # F: Appeal Type 'Concurrent Care'\n # H: Standard of Review 'Medical Necessity'\n expected = {\n 'A': ('Case ID', 'CASE-2026-00478', 1),\n 'B': ('Member ID', '44710006', 2),\n 'C': ('Group ID', 'MCA-4471', 3),\n 'D': ('Plan Year', '2026', 4),\n 'E': ('Total Dispute Value', 'undetermined', 5),\n 'F': ('Appeal Type', 'Concurrent Care', 6),\n 'H': ('Standard of Review', 'Medical Necessity', 8),\n }\n\n def check_row(sheet, row_num):\n failures = []\n for col_letter, (label, exp_val, col_idx) in expected.items():\n cell_val = sheet.cell(row=row_num, column=col_idx).value\n cell_str = str(cell_val).strip() if cell_val is not None else ''\n if exp_val.lower() not in cell_str.lower():\n failures.append(f\"Col {col_letter} ({label}): expected '{exp_val}', got '{cell_val}'\")\n return failures\n\n # Try row 8 on active sheet first\n failures = check_row(ws, 8)\n if not failures:\n return {'pass': True, 'score': 1.0, 'feedback': 'Row 8 contains correct Case ID (CASE-2026-00478), Member ID (44710006), Group ID (MCA-4471), Plan Year (2026), Total Dispute Value (undetermined), Appeal Type (Concurrent Care), and Standard of Review (Medical Necessity) in column H.'}\n\n # If row 8 didn't match, search all sheets and all rows for the case ID\n best_failures = failures\n best_row = 8\n best_sheet = ws.title\n\n for sheet_name in wb.sheetnames:\n sheet = wb[sheet_name]\n for row_num in range(1, sheet.max_row + 1):\n # Check column A for case ID\n cell_a = sheet.cell(row=row_num, column=1).value\n if cell_a is not None and 'CASE-2026-00478' in str(cell_a).upper():\n row_failures = check_row(sheet, row_num)\n if len(row_failures) < len(best_failures):\n best_failures = row_failures\n best_row = row_num\n best_sheet = sheet_name\n if not row_failures:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found correct data in sheet \"{sheet_name}\" row {row_num}: Case ID (CASE-2026-00478), Member ID (44710006), Group ID (MCA-4471), Plan Year (2026), Total Dispute Value (undetermined), Appeal Type (Concurrent Care), and Standard of Review (Medical Necessity) in column H.'}\n\n if best_failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Best match found in sheet \"{best_sheet}\" row {best_row}. Mismatches: ' + '; '.join(best_failures)}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'All expected values found for CASE-2026-00478.'}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "55b538b6-0a08-42fb-aad4-4ac14a0d3a56",
|
||
"sort_order": 3,
|
||
"rubric_text": "In file `appeals_log_master.xlsx`, Row 8 must contain Service Classification 'Inpatient' and 'SUD' and 'In-Network' (Column G) and Date of Receipt '2026-04-14 09:47:00' (Column I).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for appeals_log_master.xlsx\n xlsx_files = list(Path(workspace_path).glob('appeals_log_master.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).rglob('appeals_log_master.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': \"File 'appeals_log_master.xlsx' not found in workspace.\"}\n\n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n\n def check_row(row):\n failures = []\n\n # Column G: Service Classification - should contain inpatient, SUD, in-network keywords\n col_g_val = ws.cell(row=row, column=7).value\n col_g_str = str(col_g_val).lower() if col_g_val is not None else ''\n required_keywords = ['inpatient', 'sud', 'in-network']\n missing_keywords = []\n for kw in required_keywords:\n kw_lower = kw.lower()\n if kw_lower in col_g_str:\n continue\n alt = kw_lower.replace('-', ' ')\n if alt in col_g_str:\n continue\n alt2 = kw_lower.replace('-', '')\n if alt2 in col_g_str:\n continue\n missing_keywords.append(kw)\n if missing_keywords:\n failures.append(f\"Col G: missing keywords {missing_keywords} in '{col_g_val}'\")\n\n # Column I: Date of Receipt - should be 2026-04-14 09:47:00 (or close)\n col_i_val = ws.cell(row=row, column=9).value\n date_ok = False\n if col_i_val is not None:\n cell_str = str(col_i_val)\n # Check various representations\n if '2026-04-14' in cell_str:\n date_ok = True\n elif '04/14/2026' in cell_str:\n date_ok = True\n elif '14/04/2026' in cell_str:\n date_ok = True\n elif isinstance(col_i_val, datetime):\n if col_i_val.year == 2026 and col_i_val.month == 4 and col_i_val.day == 14:\n date_ok = True\n if not date_ok:\n failures.append(f\"Col I: expected date '2026-04-14 09:47:00', got '{col_i_val}'\")\n\n return failures\n\n # First try row 8\n failures = check_row(8)\n found_row = 8\n\n # If row 8 fails, scan other rows to be lenient\n if failures:\n best_failures = failures\n best_row = 8\n for r in range(2, ws.max_row + 1):\n if r == 8:\n continue\n row_failures = check_row(r)\n if not row_failures:\n found_row = r\n failures = row_failures\n break\n if len(row_failures) < len(best_failures):\n best_failures = row_failures\n best_row = r\n else:\n # No perfect match found, use best\n if len(best_failures) < len(failures):\n found_row = best_row\n failures = best_failures\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Row {found_row} mismatches: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row {found_row} contains correct Service Classification (Inpatient, SUD, In-Network) in Column G and Date of Receipt (2026-04-14) in Column I.'}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "46482065-82b3-40bc-8a6c-140a563c39d1",
|
||
"sort_order": 4,
|
||
"rubric_text": "In file appeals_log_master.xlsx, Row 8 must contain Reviewer Assigned 'Dr. Ontario' (Column P), Parity Flag 'Y' (Column Q), Case Status 'Intake' or 'With Reviewer' (Column W), and Coordinator 'K. Whitfield' (Column X).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('appeals_log_master.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).rglob('appeals_log_master.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': \"File 'appeals_log_master.xlsx' not found in workspace.\"}\n\n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n row = 8\n\n # Each entry: (column_index, list_of_acceptable_values)\n expected = {\n 'P': (16, ['Dr. Ontario']),\n 'Q': (17, ['Y']),\n 'W': (23, ['Intake', 'With Reviewer']),\n 'X': (24, ['K. Whitfield'])\n }\n\n failures = []\n for col_letter, (col_idx, exp_vals) in expected.items():\n cell_val = ws.cell(row=row, column=col_idx).value\n cell_str = str(cell_val).strip() if cell_val is not None else ''\n if not any(exp.lower() in cell_str.lower() for exp in exp_vals):\n failures.append(f\"Col {col_letter}: expected one of {exp_vals}, got '{cell_val}'\")\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 8 mismatches: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Row 8 contains correct Reviewer Assigned, Parity Flag, Case Status, and Coordinator.'}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776219626382",
|
||
"sort_order": 5,
|
||
"rubric_text": "In file `appeals_log_master.xlsx`, Row 8 Columns J and O contain the same formulas as in Row 7 in those respective columns, with Row 7 references substituted with Row 8 references.",
|
||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_path = Path(workspace_path) / \"appeals_log_master.xlsx\"\n if not xlsx_path.exists():\n matches = list(Path(workspace_path).rglob(\"appeals_log_master.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"appeals_log_master.xlsx not found\"}\n xlsx_path = matches[0]\n\n # Must use data_only=False — cells contain formulas, not cached values\n wb = openpyxl.load_workbook(str(xlsx_path), data_only=False)\n ws = wb.active\n\n checks = {\n \"J\": (10, \"+1\", \"Acknowledgment Due\"),\n \"O\": (15, \"+3\", \"Decision Due\"),\n }\n\n results = []\n overall_pass = True\n\n for col_letter, (col_idx, expected_offset, label) in checks.items():\n cell_val = ws.cell(row=8, column=col_idx).value\n\n if not isinstance(cell_val, str) or not cell_val.startswith(\"=\"):\n results.append(\n f\"Column {col_letter} ({label}): FAIL — expected a formula but found '{cell_val}'. \"\n f\"Pre-seeded formula must not be deleted or replaced with a static value.\"\n )\n overall_pass = False\n continue\n\n fu = cell_val.upper()\n has_row8_receipt = \"I8\" in fu\n has_cc_condition = \"CONCURRENT\" in fu\n has_offset = expected_offset in cell_val\n\n if has_row8_receipt and has_cc_condition and has_offset:\n results.append(\n f\"Column {col_letter} ({label}): PASS — formula present with I8 reference, \"\n f\"Concurrent Care condition, and {expected_offset} offset.\"\n )\n else:\n missing = []\n if not has_row8_receipt:\n missing.append(\"I8 reference\")\n if not has_cc_condition:\n missing.append(\"Concurrent Care condition\")\n if not has_offset:\n missing.append(f\"{expected_offset} offset\")\n results.append(\n f\"Column {col_letter} ({label}): FAIL — formula '{cell_val[:120]}' \"\n f\"missing: {', '.join(missing)}.\"\n )\n overall_pass = False\n\n wb.close()\n feedback = \"\\n\".join(results)\n return {\"pass\": overall_pass, \"score\": 1.0 if overall_pass else 0.0, \"feedback\": feedback}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776219727809",
|
||
"sort_order": 6,
|
||
"rubric_text": "No 'CASE' folders other than 'CASE-2026-00478' appear in the workspace.",
|
||
"verifier_code": "from pathlib import Path\nimport os\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n \n # Find all directories matching CASE-* pattern anywhere in the workspace\n case_folders = []\n for root, dirs, files in os.walk(workspace):\n for d in dirs:\n # Match folders that start with 'CASE' (case-insensitive)\n if re.match(r'^CASE', d, re.IGNORECASE):\n case_folders.append(d)\n \n # Remove duplicates\n case_folders = list(set(case_folders))\n \n # Check which ones are NOT 'CASE-2026-00478'\n allowed = 'CASE-2026-00478'\n unexpected = [f for f in case_folders if f != allowed]\n \n if not case_folders:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"No CASE folders found at all in the workspace. The rubric requires that no CASE folders other than 'CASE-2026-00478' appear, and none do. (Note: CASE-2026-00478 itself may or may not be present.)\"\n }\n \n if unexpected:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found unexpected CASE folder(s) in the workspace: {unexpected}. Only 'CASE-2026-00478' is allowed. All CASE folders found: {case_folders}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Only the allowed CASE folder 'CASE-2026-00478' was found in the workspace. All CASE folders: {case_folders}\"\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776219767187",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `appeals_log_master.xlsx`, no row after Row 8 is populated.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"appeals_log_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n ws = wb.active\n \n # Check all rows after row 8 for any populated cells\n populated_rows_after_8 = []\n for row_idx in range(9, ws.max_row + 1):\n for cell in ws[row_idx]:\n if cell.value is not None and str(cell.value).strip() != \"\":\n populated_rows_after_8.append(row_idx)\n break # no need to check more cells in this row\n \n if len(populated_rows_after_8) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No rows after Row 8 are populated in appeals_log_master.xlsx, as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Rows after Row 8 that are populated: {populated_rows_after_8}. Expected no rows after Row 8 to be populated.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776219797164",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `appeals_log_master.xlsx`, Row 8 has the following columns blank: Column T (Decision Date), Column U (Outcome), Column K (Acknowledgment Sent), and Column S (Related Case ID) and has the following field blank or containing the value \"N\": Column R (Compliance Escalation).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find appeals_log_master.xlsx - search recursively\n workspace = Path(workspace_path)\n candidates = list(workspace.rglob(\"appeals_log_master.xlsx\"))\n \n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'appeals_log_master.xlsx' not found anywhere in workspace: {workspace}\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening workbook: {e}\"}\n\n row = 8\n\n # Columns that must be blank\n must_be_blank = {\n \"T\": \"Decision Date\",\n \"U\": \"Outcome\",\n \"K\": \"Acknowledgment Sent\",\n \"S\": \"Related Case ID\"\n }\n\n # Column that must be blank or \"N\"\n blank_or_n = {\n \"R\": \"Compliance Escalation\"\n }\n\n total_checks = len(must_be_blank) + len(blank_or_n)\n passed_checks = 0\n issues = []\n details = []\n\n def is_blank(val):\n return val is None or (isinstance(val, str) and val.strip() == \"\")\n\n # Check columns that must be blank\n for col_letter, col_name in must_be_blank.items():\n cell = ws[f\"{col_letter}{row}\"]\n val = cell.value\n if is_blank(val):\n passed_checks += 1\n details.append(f\"{col_letter} ({col_name}) is blank as expected.\")\n else:\n issues.append(f\"{col_letter} ({col_name}) should be blank but has value '{val}'.\")\n\n # Check column(s) that must be blank or \"N\"\n for col_letter, col_name in blank_or_n.items():\n cell = ws[f\"{col_letter}{row}\"]\n val = cell.value\n if is_blank(val):\n passed_checks += 1\n details.append(f\"{col_letter} ({col_name}) is blank (acceptable).\")\n elif isinstance(val, str) and val.strip().upper() == \"N\":\n passed_checks += 1\n details.append(f\"{col_letter} ({col_name}) is 'N' (acceptable).\")\n else:\n issues.append(f\"{col_letter} ({col_name}) should be blank or 'N' but has value '{val}'.\")\n\n wb.close()\n\n score = passed_checks / total_checks if total_checks > 0 else 0.0\n passed = (passed_checks == total_checks)\n\n if passed:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All required columns in Row {row} meet the criteria. \" + \" \".join(details)\n }\n else:\n feedback = f\"Issues in Row {row}: \" + \" \".join(issues)\n if details:\n feedback += \" Passing checks: \" + \" \".join(details)\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": feedback\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|