DerekSurge
52 lines
19 KiB
JSON
52 lines
19 KiB
JSON
[
|
|
{
|
|
"id": "4afe5a92-12d2-4cf7-8cd8-a3966b2a227d",
|
|
"sort_order": 0,
|
|
"rubric_text": "In file `PBC_RA_Queue.xlsx`, cell F2 and F4 must be 'Complete' and Cell F3 must be 'In Progress'",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find the RA Queue xlsx file\n xlsx_files = list(Path(workspace_path).glob('*RA_Queue*'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in xlsx_files if 'queue' in f.name.lower() or 'ra' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No RA Queue xlsx file found in workspace'}\n\n file_path = xlsx_files[0]\n wb = openpyxl.load_workbook(file_path)\n ws = wb.active\n\n f2_val = ws['F2'].value\n f3_val = ws['F3'].value\n f4_val = ws['F4'].value\n\n issues = []\n score = 0.0\n total_checks = 3\n passed_checks = 0\n\n # Check F2 == 'Complete'\n f2_str = str(f2_val).strip() if f2_val is not None else ''\n if f2_str.lower() == 'complete':\n passed_checks += 1\n else:\n issues.append(f\"Cell F2: expected 'Complete', got '{f2_str}'\")\n\n # Check F3 == 'In Progress'\n f3_str = str(f3_val).strip() if f3_val is not None else ''\n if f3_str.lower() == 'in progress':\n passed_checks += 1\n else:\n issues.append(f\"Cell F3: expected 'In Progress', got '{f3_str}'\")\n\n # Check F4 == 'Complete'\n f4_str = str(f4_val).strip() if f4_val is not None else ''\n if f4_str.lower() == 'complete':\n passed_checks += 1\n else:\n issues.append(f\"Cell F4: expected 'Complete', got '{f4_str}'\")\n\n score = passed_checks / total_checks\n passed = passed_checks == total_checks\n\n if passed:\n feedback = f\"All checks passed in {file_path.name}: F2='{f2_str}', F3='{f3_str}', F4='{f4_str}'\"\n else:\n feedback = f\"File: {file_path.name}. Issues: {'; '.join(issues)}. Found F2='{f2_str}', F3='{f3_str}', F4='{f4_str}'\"\n\n return {'pass': passed, 'score': score, 'feedback': feedback}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "42dbc602-286d-481d-9d45-f5f864cff165",
|
|
"sort_order": 1,
|
|
"rubric_text": "In file `PBC_RA_Queue.xlsx`, cell E3 and E4 must be 'Level 2: High' and cell E2 must be 'Level 4: Low'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*RA_Queue*'))\n if not xlsx_files:\n all_xlsx = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in all_xlsx if 'queue' in f.name.lower() or 'ra' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No RA Queue xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n \n e2_val = str(ws['E2'].value or '').strip()\n e3_val = str(ws['E3'].value or '').strip()\n e4_val = str(ws['E4'].value or '').strip()\n \n issues = []\n found = []\n \n # E2 must be 'Level 4: Low' — both 'level 4' AND 'low' must be present\n if 'level 4' not in e2_val.lower() or 'low' not in e2_val.lower():\n issues.append(f'E2 expected \"Level 4: Low\" but got \"{e2_val}\"')\n else:\n found.append(f'E2=\"{e2_val}\"')\n \n # E3 must be 'Level 2: High' — both 'level 2' AND 'high' must be present\n if 'level 2' not in e3_val.lower() or 'high' not in e3_val.lower():\n issues.append(f'E3 expected \"Level 2: High\" but got \"{e3_val}\"')\n else:\n found.append(f'E3=\"{e3_val}\"')\n \n # E4 must be 'Level 2: High' — both 'level 2' AND 'high' must be present\n if 'level 2' not in e4_val.lower() or 'high' not in e4_val.lower():\n issues.append(f'E4 expected \"Level 2: High\" but got \"{e4_val}\"')\n else:\n found.append(f'E4=\"{e4_val}\"')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues) + f' in {xlsx_files[0].name}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': ', '.join(found) + f' - all risk levels correctly set in {xlsx_files[0].name}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "64b4e574-8b79-49cb-b8bb-96b70fd1bb3c",
|
|
"sort_order": 2,
|
|
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cell N14 must contain 'Written Off - RA Approved', cell N12 must contain 'Written Off', cell J12 must contain 175.25, cell L12 must contain 0, cell J14 must contain 816.5, cell L14 must contain 0, and cell N13 must contain 'In Recovery'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*Claim_Register*'))\n if not xlsx_files:\n all_xlsx = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in all_xlsx if 'claim' in f.name.lower() and 'register' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Claim Register xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n \n issues = []\n \n n12_val = str(ws['N12'].value or '').strip()\n if 'written off' not in n12_val.lower() or 'ra approved' in n12_val.lower():\n issues.append(f'N12 expected \"Written Off\" but got \"{n12_val}\"')\n \n j12_val = ws['J12'].value\n try:\n if float(j12_val) != 175.25:\n issues.append(f'J12 expected 175.25 but got {j12_val}')\n except (TypeError, ValueError):\n issues.append(f'J12 expected numeric 175.25 but got \"{j12_val}\"')\n \n l12_val = ws['L12'].value\n try:\n if float(l12_val) != 0:\n issues.append(f'L12 expected 0 but got {l12_val}')\n except (TypeError, ValueError):\n issues.append(f'L12 expected numeric 0 but got \"{l12_val}\"')\n \n n13_val = str(ws['N13'].value or '').strip()\n if 'in recovery' not in n13_val.lower():\n issues.append(f'N13 expected \"In Recovery\" but got \"{n13_val}\"')\n \n j14_val = ws['J14'].value\n try:\n if float(j14_val) != 816.5:\n issues.append(f'J14 expected 816.5 but got {j14_val}')\n except (TypeError, ValueError):\n issues.append(f'J14 expected numeric 816.5 but got \"{j14_val}\"')\n \n l14_val = ws['L14'].value\n try:\n if float(l14_val) != 0:\n issues.append(f'L14 expected 0 but got {l14_val}')\n except (TypeError, ValueError):\n issues.append(f'L14 expected numeric 0 but got \"{l14_val}\"')\n \n n14_val = str(ws['N14'].value or '').strip()\n if 'written off' not in n14_val.lower() or 'ra approved' not in n14_val.lower():\n issues.append(f'N14 expected \"Written Off - RA Approved\" but got \"{n14_val}\"')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues) + f' in {xlsx_files[0].name}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All Claim Register cells verified: N12=\"{n12_val}\", J12={j12_val}, L12={l12_val}, N13=\"{n13_val}\", J14={j14_val}, L14={l14_val}, N14=\"{n14_val}\" in {xlsx_files[0].name}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "e2e61714-832a-4dcd-b772-0946d02ed941",
|
|
"sort_order": 3,
|
|
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, cells H2='Written Off', I2='Write Off', H3='Escalated', I3='Appeal - Contractual Rate Dispute', H4='Written Off', and I4='Write Off' must all match expected values.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*Denial_Worklist*'))\n if not xlsx_files:\n all_xlsx = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in all_xlsx if 'denial' in f.name.lower() or 'worklist' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Denial Worklist xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n \n expected = {\n 'H2': 'written off',\n 'I2': 'write off',\n 'H3': 'escalated',\n 'I3': 'appeal - contractual rate dispute',\n 'H4': 'written off',\n 'I4': 'write off'\n }\n \n issues = []\n found = {}\n for cell_ref, exp_val in expected.items():\n actual = str(ws[cell_ref].value or '').strip().lower()\n found[cell_ref] = actual\n if actual != exp_val:\n issues.append(f'{cell_ref} expected \"{exp_val}\" but got \"{actual}\"')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues) + f' in {xlsx_files[0].name}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All Denial Worklist cells verified: {found} in {xlsx_files[0].name}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "b685ac26-6dc3-405a-9d4b-55f36e2d611f",
|
|
"sort_order": 4,
|
|
"rubric_text": "In Slack workspace, there must be exactly two messages total across all channels: one in #escalations with text containing 'RCM Manager', 'CLM-2026-0048', 'Payer Dispute', and 'Operational'; and one in #special-authorization with text containing 'CLM-2026-0049' and 'special action authorized'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check Slack data'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'slack_data.json not found at {slack_path}'}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n messages_by_channel = data.get('messages', {})\n channels = data.get('channels', {})\n \n name_to_id = {ch_data.get('name', '').lower(): ch_id for ch_id, ch_data in channels.items()}\n \n total_messages = sum(len(msgs) for msgs in messages_by_channel.values())\n all_messages = [\n (channels.get(ch_id, {}).get('name', ch_id), msg.get('text', ''))\n for ch_id, msgs in messages_by_channel.items()\n for msg in msgs\n ]\n \n def normalize(s):\n # strip square brackets so \"[RCM Manager]\" matches \"RCM Manager\"\n return s.replace('[', '').replace(']', '').strip().lower()\n \n issues = []\n \n if total_messages != 2:\n issues.append(f'Expected exactly 2 total Slack messages but found {total_messages}: {all_messages}')\n \n expected_escalation = 'RCM Manager - Claim ID: CLM-2026-0048; Reason: Payer Dispute; Escalation Category: Operational'\n escalations_id = name_to_id.get('escalations')\n escalations_msgs = messages_by_channel.get(escalations_id, []) if escalations_id else []\n if not escalations_msgs:\n for ch_id, msgs in messages_by_channel.items():\n if channels.get(ch_id, {}).get('name', '').lower() == 'escalations':\n escalations_msgs = msgs\n break\n \n escalation_found = any(\n normalize(msg.get('text', '')) == normalize(expected_escalation)\n for msg in escalations_msgs\n )\n if not escalation_found:\n issues.append(f'No #escalations message matching \"{expected_escalation}\" (brackets ignored). Found: {[m.get(\"text\") for m in escalations_msgs]}')\n \n expected_special_auth = 'James W. - CLM-2026-0049 special action authorized'\n special_auth_id = name_to_id.get('special-authorization')\n special_auth_msgs = messages_by_channel.get(special_auth_id, []) if special_auth_id else []\n if not special_auth_msgs:\n for ch_id, msgs in messages_by_channel.items():\n if 'special' in channels.get(ch_id, {}).get('name', '').lower() and 'auth' in channels.get(ch_id, {}).get('name', '').lower():\n special_auth_msgs = msgs\n break\n \n special_found = any(\n normalize(msg.get('text', '')) == normalize(expected_special_auth)\n for msg in special_auth_msgs\n )\n if not special_found:\n issues.append(f'No #special-authorization message matching \"{expected_special_auth}\" (brackets ignored). Found: {[m.get(\"text\") for m in special_auth_msgs]}')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Slack verified: #escalations match for CLM-2026-0048/Operational; #special-authorization match for CLM-2026-0049; total messages=2'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "6bc193bb-c945-4669-a1a4-f6c549ffd990",
|
|
"sort_order": 5,
|
|
"rubric_text": "In calendar_data.json, there must be exactly 15 calendar events total, and one event must have summary 'RCM Manager - Claim ID: CLM-2026-0048; Reason: Payer Dispute; Escalation Category: Operational' with start time of 3:15 PM on February 16, 2026, and an end time of 3:15 PM on February 16, 2026",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check calendar data'}\n \n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'calendar_data.json not found at {cal_path}'}\n \n with open(cal_path) as f:\n data = json.load(f)\n \n events_raw = data.get('events', {})\n if isinstance(events_raw, dict):\n events_list = list(events_raw.values())\n total_events = len(events_raw)\n elif isinstance(events_raw, list):\n events_list = events_raw\n total_events = len(events_raw)\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Unexpected events format: {type(events_raw)}'}\n \n issues = []\n \n if total_events != 15:\n issues.append(f'Expected exactly 15 calendar events but found {total_events}')\n \n # Accept the summary with or without square brackets around each filled-in field.\n # \\[? means \"an optional [ \" and \\]? means \"an optional ] \".\n # \\s* allows for any extra spaces.\n summary_pattern = re.compile(\n r'^\\s*'\n r'\\[?\\s*RCM Manager\\s*\\]?'\n r'\\s*-\\s*Claim ID:\\s*'\n r'\\[?\\s*CLM-2026-0048\\s*\\]?'\n r'\\s*;\\s*Reason:\\s*'\n r'\\[?\\s*Payer Dispute\\s*\\]?'\n r'\\s*;\\s*Escalation Category:\\s*'\n r'\\[?\\s*Operational\\s*\\]?'\n r'\\s*$',\n re.IGNORECASE\n )\n \n target_event = None\n for event in events_list:\n summary = str(event.get('summary', '')).strip()\n if summary_pattern.match(summary):\n target_event = event\n break\n \n if target_event is None:\n all_summaries = [str(e.get('summary', '')) for e in events_list]\n issues.append(\n 'No calendar event found with summary matching '\n '\"RCM Manager - Claim ID: CLM-2026-0048; Reason: Payer Dispute; '\n 'Escalation Category: Operational\" (square brackets around field values are allowed). '\n f'Found summaries: {all_summaries}'\n )\n else:\n start = str(target_event.get('start', ''))\n end = str(target_event.get('end', ''))\n \n def check_datetime(dt_str, label):\n dt_lower = dt_str.lower().strip()\n local_issues = []\n \n date_found = False\n if '2026-02-16' in dt_str:\n date_found = True\n elif '02/16/2026' in dt_str:\n date_found = True\n elif '2/16/2026' in dt_str:\n date_found = True\n elif 'feb' in dt_lower and '16' in dt_str and '2026' in dt_str:\n date_found = True\n elif '2026' in dt_str and '02' in dt_str and '16' in dt_str:\n date_found = True\n \n if not date_found:\n local_issues.append(f'{label} \"{dt_str}\" does not clearly indicate February 16, 2026')\n \n time_found = False\n if '15:15' in dt_str:\n time_found = True\n elif '3:15 pm' in dt_lower:\n time_found = True\n elif '3:15pm' in dt_lower:\n time_found = True\n elif re.search(r'3:15\\s*[pP]', dt_str):\n time_found = True\n \n if not time_found:\n local_issues.append(f'{label} \"{dt_str}\" does not clearly indicate 3:15 PM')\n \n return local_issues\n \n issues.extend(check_datetime(start, 'Event start time'))\n issues.extend(check_datetime(end, 'Event end time'))\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Calendar verified: {total_events} events found; escalation event found with valid summary, start and end at 3:15 PM Feb 16 2026'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "459b9697-95bd-4946-bad7-1e45a0cdbd94",
|
|
"sort_order": 6,
|
|
"rubric_text": "In mailbox.json, there must be no sent emails (folder='Sent') in the workspace.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox data'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n # If no mailbox file exists, there are no sent emails - pass\n return {'pass': True, 'score': 1.0, 'feedback': 'No mailbox.json found; no sent emails exist'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n \n if sent_emails:\n subjects = [e.get('subject', 'no subject') for e in sent_emails]\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(sent_emails)} sent email(s) but expected none. Subjects: {subjects}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'No sent emails found in mailbox. Total emails: {len(emails)}'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|