DerekSurge
108 lines
42 KiB
JSON
108 lines
42 KiB
JSON
[
|
|
{
|
|
"id": "be16e0cd-6bf1-4cdf-9cc8-09a89e7ae372",
|
|
"sort_order": 0,
|
|
"rubric_text": "appeals_log_master.xlsx must contain a new row for CASE-2026-00413 with correct intake fields: Member ID VA-88291-03, Group ID G-14827, Plan Year 2025, Appeal Type 'Concurrent Care', and Standard of Review 'Medical Necessity'",
|
|
"verifier_code": "\nimport openpyxl\nfrom pathlib import Path as _P\n\ndef _row(ws):\n fp = _P(ws) / 'appeals_log_master.xlsx'\n if not fp.exists():\n return None, None, 'appeals_log_master.xlsx not found in workspace'\n wb = openpyxl.load_workbook(fp, data_only=True)\n sh = wb.active\n hdrs = {str(c.value).strip(): c.column - 1 for c in sh[1] if c.value}\n for r in sh.iter_rows(min_row=2, values_only=True):\n if r and str(r[0]).strip() == 'CASE-2026-00413':\n return list(r), hdrs, None\n return None, hdrs, 'CASE-2026-00413 not found in appeals_log_master.xlsx'\n\ndef _g(row, hdrs, col):\n i = hdrs.get(col)\n return str(row[i]).strip() if i is not None and i < len(row) and row[i] is not None else ''\n\ndef verify(workspace_path, external_services_path=None):\n row, hdrs, err = _row(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n checks = [\n ('Member ID', 'VA-88291-03', 'VA-88291-03' in _g(row, hdrs, 'Member ID')),\n ('Group ID', 'G-14827', 'G-14827' in _g(row, hdrs, 'Group ID')),\n ('Plan Year', '2025', '2025' in _g(row, hdrs, 'Plan Year') and '2026' not in _g(row, hdrs, 'Plan Year')),\n ('Appeal Type', 'Concurrent Care', 'concurrent' in _g(row, hdrs, 'Appeal Type').lower()),\n ('Service Classification', 'Inpatient in-network (SUD)',\n 'sud' in _g(row, hdrs, 'Service Classification').lower()),\n ('Standard of Review','Medical Necessity', 'medical necessity' in _g(row, hdrs, 'Standard of Review').lower()),\n ]\n failures = [f'{lbl}: expected \"{exp}\", got \"{_g(row, hdrs, lbl)}\"' for lbl, exp, ok in checks if not ok]\n if failures:\n return {'pass': False, 'score': (6 - len(failures)) / 6,\n 'feedback': f'CASE-2026-00413 intake field errors: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'CASE-2026-00413 row has correct intake fields'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "36360d99-9b1e-4ef3-8989-8d9128d11420",
|
|
"sort_order": 1,
|
|
"rubric_text": "In appeals_log_master.xlsx, the Decision Due date for CASE-2026-00413 must be 01/10/2026, not 01/11/2026, correctly applying the concurrent care rule from §4.3.3 (earlier of 72 hours from receipt or termination date)",
|
|
"verifier_code": "\nimport openpyxl\nfrom pathlib import Path as _P\n\ndef _row(ws):\n fp = _P(ws) / 'appeals_log_master.xlsx'\n if not fp.exists():\n return None, None, 'appeals_log_master.xlsx not found in workspace'\n wb = openpyxl.load_workbook(fp, data_only=True)\n sh = wb.active\n hdrs = {str(c.value).strip(): c.column - 1 for c in sh[1] if c.value}\n for r in sh.iter_rows(min_row=2, values_only=True):\n if r and str(r[0]).strip() == 'CASE-2026-00413':\n return list(r), hdrs, None\n return None, hdrs, 'CASE-2026-00413 not found in appeals_log_master.xlsx'\n\ndef _g(row, hdrs, col):\n i = hdrs.get(col)\n return str(row[i]).strip() if i is not None and i < len(row) and row[i] is not None else ''\n\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n row, hdrs, err = _row(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n val = _g(row, hdrs, 'Decision Due')\n jan10 = bool(re.search(r'(01.10.2026|1.10.2026|2026-01-10|jan(uary)?.?10)', val, re.IGNORECASE))\n jan11 = bool(re.search(r'(01.11.2026|1.11.2026|2026-01-11|jan(uary)?.?11)', val, re.IGNORECASE))\n if jan11:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Decision Due is 01/11/2026 (72-hour rule incorrectly applied). '\n f'Correct deadline is 01/10/2026 (termination date is earlier). Got: \"{val}\"'}\n if jan10:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Decision Due correctly set to 01/10/2026. Got: \"{val}\"'}\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Decision Due does not contain 01/10/2026. Got: \"{val}\"'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "171391be-e85f-407e-8b23-2f23e6255072",
|
|
"sort_order": 2,
|
|
"rubric_text": "In appeals_log_master.xlsx, for CASE-2026-00413: Parity Flag must be Y, Compliance Escalation must be Y, and Reviewer Assigned must reference Dr. Ontario",
|
|
"verifier_code": "\nimport openpyxl\nfrom pathlib import Path as _P\n\ndef _row(ws):\n fp = _P(ws) / 'appeals_log_master.xlsx'\n if not fp.exists():\n return None, None, 'appeals_log_master.xlsx not found in workspace'\n wb = openpyxl.load_workbook(fp, data_only=True)\n sh = wb.active\n hdrs = {str(c.value).strip(): c.column - 1 for c in sh[1] if c.value}\n for r in sh.iter_rows(min_row=2, values_only=True):\n if r and str(r[0]).strip() == 'CASE-2026-00413':\n return list(r), hdrs, None\n return None, hdrs, 'CASE-2026-00413 not found in appeals_log_master.xlsx'\n\ndef _g(row, hdrs, col):\n i = hdrs.get(col)\n return str(row[i]).strip() if i is not None and i < len(row) and row[i] is not None else ''\n\ndef verify(workspace_path, external_services_path=None):\n row, hdrs, err = _row(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n failures = []\n if _g(row, hdrs, 'Parity Flag').upper() not in ('Y', 'YES'):\n failures.append(f'Parity Flag: expected Y, got \"{_g(row, hdrs, \"Parity Flag\")}\"')\n if not _g(row, hdrs, 'Compliance Escalation').upper().startswith('Y'):\n failures.append(f'Compliance Escalation: expected Y, got \"{_g(row, hdrs, \"Compliance Escalation\")}\"')\n if 'ontario' not in _g(row, hdrs, 'Reviewer Assigned').lower():\n failures.append(f'Reviewer Assigned: expected Dr. Ontario, got \"{_g(row, hdrs, \"Reviewer Assigned\")}\"')\n if failures:\n return {'pass': False, 'score': (3 - len(failures)) / 3,\n 'feedback': f'CASE-2026-00413 flag/reviewer errors: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'Parity Flag=Y, Compliance Escalation=Y, Reviewer=Dr. Ontario'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "ced4fefb-eefd-4242-b46d-e534f08b65c0",
|
|
"sort_order": 3,
|
|
"rubric_text": "An acknowledgment email following Template A-1 must have been sent to mtorres@desertspringsrecovery.com; the subject must match the Template A-1 format \"Mojave Crest Appeals Case #CASE-2026-00413 - Acknowledgment of Your Appeal\"; and the body must contain the required fields: Case Number, Member ID, Date Received, Appeal Type, and Decision Due By",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _mailbox(ext):\n if ext:\n p = _P(ext) / 'mailbox.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'mailbox.json not found in external_services_path'\n\ndef _sent(mbx):\n out = []\n for e in mbx.get('emails', []):\n folder = str(e.get('folder', '')).lower()\n frm = str(e.get('from_addr', '')).lower()\n if folder in ('sent', 'outbox') or 'mojavecrest.com' in frm:\n out.append(e)\n return out\n\ndef _find_ack(mbx):\n for e in _sent(mbx):\n to = str(e.get('to_addr', '')).lower()\n subj = str(e.get('subject', '')).lower()\n if 'mtorres@desertspringsrecovery.com' in to and 'case-2026-00413' in subj:\n return e\n return None\n\ndef verify(workspace_path, external_services_path=None):\n mbx, err = _mailbox(external_services_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n email = _find_ack(mbx)\n if email is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No sent email found to mtorres@desertspringsrecovery.com containing CASE-2026-00413'}\n failures = []\n subj = str(email.get('subject', ''))\n if 'Mojave Crest Appeals Case #CASE-2026-00413' not in subj or 'Acknowledgment' not in subj:\n failures.append(f'Subject does not match Template A-1 format. Got: \"{subj}\"')\n body = str(email.get('body_text', '') or email.get('body_html', '') or '').lower()\n for required in ['case number', 'member id', 'date received', 'appeal type', 'decision due']:\n if required not in body:\n failures.append(f'Required body field missing: \"{required}\"')\n if failures:\n return {'pass': False, 'score': (6 - len(failures)) / 6,\n 'feedback': f'Acknowledgment email Template A-1 issues: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'Acknowledgment email matches Template A-1 format with all required fields'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "7a85d37b-4ca6-48d5-91c8-7618e1930200",
|
|
"sort_order": 4,
|
|
"rubric_text": "The acknowledgment email must include the Expedited Acknowledgment Header Block with the \"--- EXPEDITED REVIEW ---\" marker, Expedited Basis referencing \"Concurrent Care\" and \"Automatic\", and Decision Due By of 01/10/2026",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _mailbox(ext):\n if ext:\n p = _P(ext) / 'mailbox.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'mailbox.json not found in external_services_path'\n\ndef _sent(mbx):\n out = []\n for e in mbx.get('emails', []):\n folder = str(e.get('folder', '')).lower()\n frm = str(e.get('from_addr', '')).lower()\n if folder in ('sent', 'outbox') or 'mojavecrest.com' in frm:\n out.append(e)\n return out\n\ndef _find_ack(mbx):\n for e in _sent(mbx):\n to = str(e.get('to_addr', '')).lower()\n subj = str(e.get('subject', '')).lower()\n if 'mtorres@desertspringsrecovery.com' in to and 'case-2026-00413' in subj:\n return e\n return None\n\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n mbx, err = _mailbox(external_services_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n email = _find_ack(mbx)\n if email is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Acknowledgment email to mtorres@desertspringsrecovery.com not found'}\n body = str(email.get('body_text', '') or email.get('body_html', '') or '')\n body_lower = body.lower()\n failures = []\n if 'expedited review' not in body_lower:\n failures.append('Missing \"--- EXPEDITED REVIEW ---\" header block')\n if 'expedited basis' not in body_lower:\n failures.append('Missing \"Expedited Basis:\" field')\n if 'concurrent care' not in body_lower or 'automatic' not in body_lower:\n failures.append('Expedited Basis does not reference \"Concurrent Care\" and \"Automatic\"')\n if 'decision due by' not in body_lower and 'decision due' not in body_lower:\n failures.append('Missing \"Decision Due By:\" field in header block')\n if not re.search(r'(01.10.2026|jan(uary)?.?10)', body_lower):\n failures.append('Decision Due By date 01/10/2026 not found in email body')\n if failures:\n return {'pass': False, 'score': (5 - len(failures)) / 5,\n 'feedback': f'Expedited header block issues: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'Expedited header block present with all required fields and correct values'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "3520e145-174b-4e55-8104-5b721b5adee3",
|
|
"sort_order": 5,
|
|
"rubric_text": "A Format B-1 Clinical Routing message must have been sent as a direct message to @aontario containing CASE-2026-00413 with all required fields (CLINICAL ROUTING header, Case #, Type, Service Classification, Standard of Review, Deadline, Parity Flag); and a corresponding post must exist in #appeals-urgent tagging @dwhitfield per §7.3 expedited requirement",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _slack(ext):\n if ext:\n p = _P(ext) / 'slack_data.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'slack_data.json not found in external_services_path'\n\ndef _msgs_in(data, channel_name):\n for cid, ch in data.get('channels', {}).items():\n if ch.get('name', '').lower() == channel_name.lower():\n return data.get('messages', {}).get(cid, [])\n return []\n\ndef _is_public(data, cid):\n ch = data.get('channels', {}).get(cid, {})\n return ch.get('is_channel', False) and not ch.get('is_private', False) and not ch.get('is_im', False)\n\ndef _dm_msgs_for(data, username):\n out = []\n for cid, ch in data.get('channels', {}).items():\n name = ch.get('name', '').lower()\n members = [str(m).lower() for m in ch.get('members', [])]\n is_dm = ch.get('is_im', False) or ch.get('is_mpim', False) or not ch.get('is_channel', True)\n if is_dm and (username.lower() in name or any(username.lower() in m for m in members)):\n out.extend(data.get('messages', {}).get(cid, []))\n return out\n\ndef verify(workspace_path, external_services_path=None):\n data, err = _slack(external_services_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n case_id = 'CASE-2026-00413'\n failures = []\n dm_msgs = _dm_msgs_for(data, 'aontario')\n case_dms = [str(m.get('text', '')) for m in dm_msgs if case_id in str(m.get('text', ''))]\n if not case_dms:\n failures.append('No DM to @aontario found containing CASE-2026-00413')\n else:\n combined = ' '.join(case_dms).lower()\n for field in ['clinical routing', 'case #', 'type', 'service classification',\n 'standard of review', 'deadline', 'parity flag']:\n if field not in combined:\n failures.append(f'B-1 DM missing required field: \"{field}\"')\n urgent_msgs = _msgs_in(data, 'appeals-urgent')\n dw_uid = next((uid for uid, u in data.get('users', {}).items()\n if u.get('name', '').lower() == 'dwhitfield'), None)\n def _tags_dwhitfield(text):\n t = str(text).lower()\n return 'dwhitfield' in t or (dw_uid is not None and f'<@{dw_uid.lower()}>' in t)\n if not any(case_id in str(m.get('text', '')) and _tags_dwhitfield(m.get('text', ''))\n for m in urgent_msgs):\n failures.append('No post in #appeals-urgent with CASE-2026-00413 tagging @dwhitfield')\n if failures:\n return {'pass': False, 'score': max(0, (9 - len(failures)) / 9),\n 'feedback': str(failures)}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'B-1 DM to @aontario has all required fields; #appeals-urgent post found'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "ea328263-a445-44c2-90f1-155b38d35957",
|
|
"sort_order": 6,
|
|
"rubric_text": "The Format B-1 Clinical Routing message must NOT have been posted to #appeals-clinical-queue for CASE-2026-00413; per §8.3, parity-flagged cases are routed exclusively to Dr. Ontario, not the standard clinical queue",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _slack(ext):\n if ext:\n p = _P(ext) / 'slack_data.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'slack_data.json not found in external_services_path'\n\ndef _msgs_in(data, channel_name):\n for cid, ch in data.get('channels', {}).items():\n if ch.get('name', '').lower() == channel_name.lower():\n return data.get('messages', {}).get(cid, [])\n return []\n\ndef _is_public(data, cid):\n ch = data.get('channels', {}).get(cid, {})\n return ch.get('is_channel', False) and not ch.get('is_private', False) and not ch.get('is_im', False)\n\ndef _dm_msgs_for(data, username):\n out = []\n for cid, ch in data.get('channels', {}).items():\n name = ch.get('name', '').lower()\n members = [str(m).lower() for m in ch.get('members', [])]\n is_dm = ch.get('is_im', False) or ch.get('is_mpim', False) or not ch.get('is_channel', True)\n if is_dm and (username.lower() in name or any(username.lower() in m for m in members)):\n out.extend(data.get('messages', {}).get(cid, []))\n return out\n\ndef verify(workspace_path, external_services_path=None):\n data, err = _slack(external_services_path)\n if err:\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; cannot verify (pass)'}\n queue_msgs = _msgs_in(data, 'appeals-clinical-queue')\n offenders = [str(m.get('text', ''))[:80] for m in queue_msgs\n if 'CASE-2026-00413' in str(m.get('text', ''))]\n if offenders:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'CASE-2026-00413 was posted to #appeals-clinical-queue — '\n 'parity-flagged cases must be routed exclusively to Dr. Ontario (§8.3)'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'CASE-2026-00413 correctly not posted to #appeals-clinical-queue'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "2738a3c1-3d45-47d1-ae19-d015bf746004",
|
|
"sort_order": 7,
|
|
"rubric_text": "A Format B-2 escalation message must have been sent as a direct message to @dwhitfield and also posted in #appeals-urgent, with all required fields (ESCALATION header, Case #, Type, Deadline, Trigger, Summary, My Recommendation, Requested Action) and Trigger listing all three applicable triggers in ascending numerical order: 8, 12, 13",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _slack(ext):\n if ext:\n p = _P(ext) / 'slack_data.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'slack_data.json not found in external_services_path'\n\ndef _msgs_in(data, channel_name):\n for cid, ch in data.get('channels', {}).items():\n if ch.get('name', '').lower() == channel_name.lower():\n return data.get('messages', {}).get(cid, [])\n return []\n\ndef _is_public(data, cid):\n ch = data.get('channels', {}).get(cid, {})\n return ch.get('is_channel', False) and not ch.get('is_private', False) and not ch.get('is_im', False)\n\ndef _dm_msgs_for(data, username):\n out = []\n for cid, ch in data.get('channels', {}).items():\n name = ch.get('name', '').lower()\n members = [str(m).lower() for m in ch.get('members', [])]\n is_dm = ch.get('is_im', False) or ch.get('is_mpim', False) or not ch.get('is_channel', True)\n if is_dm and (username.lower() in name or any(username.lower() in m for m in members)):\n out.extend(data.get('messages', {}).get(cid, []))\n return out\n\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n data, err = _slack(external_services_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n case_id = 'CASE-2026-00413'\n failures = []\n dm_msgs = _dm_msgs_for(data, 'dwhitfield')\n case_dms = [str(m.get('text', '')) for m in dm_msgs if case_id in str(m.get('text', ''))]\n if not case_dms:\n failures.append('No DM to @dwhitfield found containing CASE-2026-00413')\n else:\n combined = ' '.join(case_dms)\n combined_lower = combined.lower()\n for field in ['escalation', 'case #', 'type', 'deadline', 'trigger',\n 'summary', 'recommendation', 'requested action']:\n if field not in combined_lower:\n failures.append(f'B-2 DM missing required field: \"{field}\"')\n has_8 = bool(re.search(r'\\b8\\b', combined))\n has_12 = bool(re.search(r'\\b12\\b', combined))\n has_13 = bool(re.search(r'\\b13\\b', combined))\n if not (has_8 and has_12 and has_13):\n missing = [t for t, ok in [('8', has_8), ('12', has_12), ('13', has_13)] if not ok]\n failures.append(f'B-2 Trigger field missing trigger numbers: {missing}')\n elif not re.search(r'\\b8\\b.*\\b12\\b.*\\b13\\b', combined, re.DOTALL):\n failures.append('Triggers not in ascending order (expected 8, 12, 13)')\n urgent_msgs = _msgs_in(data, 'appeals-urgent')\n if not any(case_id in str(m.get('text', '')) for m in urgent_msgs):\n failures.append('No post in #appeals-urgent found for CASE-2026-00413')\n total = 10\n if failures:\n return {'pass': False, 'score': max(0, (total - len(failures)) / total),\n 'feedback': str(failures)}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'B-2 DM to @dwhitfield has all required fields with triggers 8, 12, 13 in order; #appeals-urgent post found'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "f28b4e47-7697-47d3-9679-65bfe7b8618f",
|
|
"sort_order": 8,
|
|
"rubric_text": "A Format B-3 Parity Flag Notification must have been sent as a direct message to @parity-committee containing CASE-2026-00413 with all required fields (PARITY FLAG header, Case #, Service Classification, Comparable M/S Classification, Flag Indicator, Decision Deadline)",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _slack(ext):\n if ext:\n p = _P(ext) / 'slack_data.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'slack_data.json not found in external_services_path'\n\ndef _msgs_in(data, channel_name):\n for cid, ch in data.get('channels', {}).items():\n if ch.get('name', '').lower() == channel_name.lower():\n return data.get('messages', {}).get(cid, [])\n return []\n\ndef _is_public(data, cid):\n ch = data.get('channels', {}).get(cid, {})\n return ch.get('is_channel', False) and not ch.get('is_private', False) and not ch.get('is_im', False)\n\ndef _dm_msgs_for(data, username):\n out = []\n for cid, ch in data.get('channels', {}).items():\n name = ch.get('name', '').lower()\n members = [str(m).lower() for m in ch.get('members', [])]\n is_dm = ch.get('is_im', False) or ch.get('is_mpim', False) or not ch.get('is_channel', True)\n if is_dm and (username.lower() in name or any(username.lower() in m for m in members)):\n out.extend(data.get('messages', {}).get(cid, []))\n return out\n\ndef verify(workspace_path, external_services_path=None):\n data, err = _slack(external_services_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n case_id = 'CASE-2026-00413'\n dm_msgs = _dm_msgs_for(data, 'parity-committee') + _dm_msgs_for(data, 'parity_committee')\n case_dms = [str(m.get('text', '')) for m in dm_msgs if case_id in str(m.get('text', ''))]\n if not case_dms:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No DM to @parity-committee found containing CASE-2026-00413'}\n combined = ' '.join(case_dms).lower()\n failures = []\n for field in ['parity flag', 'case #', 'service classification',\n 'comparable', 'flag indicator', 'decision deadline']:\n if field not in combined:\n failures.append(f'B-3 DM missing required field: \"{field}\"')\n if failures:\n return {'pass': False, 'score': (7 - len(failures)) / 7,\n 'feedback': f'B-3 format field errors: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'B-3 DM to @parity-committee found with all required fields'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "c467dc1e-3af0-4c9a-a6c5-0317dd23baee",
|
|
"sort_order": 9,
|
|
"rubric_text": "The Format B-3 parity flag must not have been posted in #appeals-parity or any other public channel; per §8.3, Format B-3 is a direct message to @parity-committee only",
|
|
"verifier_code": "\nimport json\nfrom pathlib import Path as _P\n\ndef _slack(ext):\n if ext:\n p = _P(ext) / 'slack_data.json'\n if p.exists():\n return json.loads(p.read_text()), None\n return None, 'slack_data.json not found in external_services_path'\n\ndef _msgs_in(data, channel_name):\n for cid, ch in data.get('channels', {}).items():\n if ch.get('name', '').lower() == channel_name.lower():\n return data.get('messages', {}).get(cid, [])\n return []\n\ndef _is_public(data, cid):\n ch = data.get('channels', {}).get(cid, {})\n return ch.get('is_channel', False) and not ch.get('is_private', False) and not ch.get('is_im', False)\n\ndef _dm_msgs_for(data, username):\n out = []\n for cid, ch in data.get('channels', {}).items():\n name = ch.get('name', '').lower()\n members = [str(m).lower() for m in ch.get('members', [])]\n is_dm = ch.get('is_im', False) or ch.get('is_mpim', False) or not ch.get('is_channel', True)\n if is_dm and (username.lower() in name or any(username.lower() in m for m in members)):\n out.extend(data.get('messages', {}).get(cid, []))\n return out\n\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n data, err = _slack(external_services_path)\n if err:\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; cannot verify (pass)'}\n case_id = 'CASE-2026-00413'\n violations = []\n # Only flag messages that look like actual B-3 format postings, not incidental parity mentions\n for cid, ch in data.get('channels', {}).items():\n if _is_public(data, cid):\n for msg in data.get('messages', {}).get(cid, []):\n text = str(msg.get('text', ''))\n # Only flag if message looks like an actual B-3: has PARITY FLAG as a standalone header line\n lines = [l.strip() for l in text.splitlines()]\n is_b3 = 'PARITY FLAG' in lines\n if case_id in text and is_b3:\n violations.append(f'#{ch.get(\"name\", cid)}: {text[:80]}')\n if violations:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Parity flag posted in public channel(s) — must be DM only (§8.3): {violations}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'Parity flag correctly not posted in any public channel'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "3cbd9388-dd37-427c-bcd2-840e1aae57a8",
|
|
"sort_order": 10,
|
|
"rubric_text": "A file named CASE-2026-00413_Determination_Memo.docx must exist in the workspace; the exact filename is required by the ACMS audit system per Appendix D",
|
|
"verifier_code": "\nfrom pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n exact_matches = list(Path(workspace_path).rglob('CASE-2026-00413_Determination_Memo.docx'))\n if exact_matches:\n return {'pass': True, 'score': 1.0,\n 'feedback': 'CASE-2026-00413_Determination_Memo.docx found in workspace'}\n others = list(Path(workspace_path).rglob('*Determination_Memo*'))\n if others:\n return {'pass': False, 'score': 0.5,\n 'feedback': f'Determination memo found but wrong filename: {[f.name for f in others]}. '\n 'Must be CASE-2026-00413_Determination_Memo.docx'}\n return {'pass': False, 'score': 0.0,\n 'feedback': 'CASE-2026-00413_Determination_Memo.docx not found in workspace'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "ec354567-16ef-4017-acf6-97a162d8c705",
|
|
"sort_order": 11,
|
|
"rubric_text": "In CASE-2026-00413_Determination_Memo.docx, all twelve required section headings from Appendix D must be present: Case Identification, Requestor and Standing, Disputed Service, Original Denial, Standard of Review Applied, Clinical Review Summary, Parity Analysis, Fraud and Mill Indicators, Timeline Log, Decision and Rationale, Authority, External Review Status",
|
|
"verifier_code": "\nimport zipfile, xml.etree.ElementTree as ET\nfrom pathlib import Path as _P\n\n_W = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'\n\ndef _paras(ws):\n # Prefer exact filename; fall back to any determination memo docx\n exact_matches = list(_P(ws).rglob('CASE-2026-00413_Determination_Memo.docx'))\n if exact_matches:\n fp = exact_matches[0]\n else:\n candidates = [f for f in _P(ws).rglob('*.docx')\n if 'determination' in f.name.lower() or 'memo' in f.name.lower()]\n if not candidates:\n return None, None, 'CASE-2026-00413_Determination_Memo.docx not found in workspace'\n fp = candidates[0]\n with zipfile.ZipFile(str(fp)) as z:\n with z.open('word/document.xml') as f:\n root = ET.parse(f).getroot()\n paras = []\n for p in root.iter(f'{{{_W}}}p'):\n style_el = p.find(f'.//{{{_W}}}pStyle')\n style = style_el.get(f'{{{_W}}}val', '') if style_el is not None else ''\n txt = ''.join(t.text for t in p.iter(f'{{{_W}}}t') if t.text).strip()\n if txt:\n paras.append({'text': txt, 'style': style})\n return paras, '\\n'.join(p['text'] for p in paras), None\n\ndef _field(text, label):\n import re\n m = re.search(rf'{re.escape(label)}\\s*:\\s*(.+)', text, re.IGNORECASE)\n return m.group(1).strip() if m else None\n\ndef verify(workspace_path, external_services_path=None):\n paras, text, err = _paras(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n text_lower = text.lower()\n required = [\n 'Case Identification',\n 'Requestor and Standing',\n 'Disputed Service',\n 'Original Denial',\n 'Standard of Review Applied',\n 'Clinical Review Summary',\n 'Parity Analysis',\n 'Fraud and Mill Indicators',\n 'Timeline Log',\n 'Decision and Rationale',\n 'Authority',\n 'External Review Status',\n ]\n missing = [h for h in required if h.lower() not in text_lower]\n if missing:\n return {'pass': False, 'score': (12 - len(missing)) / 12,\n 'feedback': f'Missing required section headings: {missing}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'All 12 required section headings found in memo'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "5c659570-ed30-4704-af7e-7305e449bac2",
|
|
"sort_order": 12,
|
|
"rubric_text": "In CASE-2026-00413_Determination_Memo.docx, section headings must have the Word paragraph style Heading 2 applied (not bold Normal text); the document title line must have Heading 1 applied; per Appendix D the ACMS parser identifies section boundaries by paragraph style tag, not visual appearance",
|
|
"verifier_code": "\nimport zipfile, xml.etree.ElementTree as ET\nfrom pathlib import Path as _P\n\n_W = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'\n\ndef _paras(ws):\n # Prefer exact filename; fall back to any determination memo docx\n exact_matches = list(_P(ws).rglob('CASE-2026-00413_Determination_Memo.docx'))\n if exact_matches:\n fp = exact_matches[0]\n else:\n candidates = [f for f in _P(ws).rglob('*.docx')\n if 'determination' in f.name.lower() or 'memo' in f.name.lower()]\n if not candidates:\n return None, None, 'CASE-2026-00413_Determination_Memo.docx not found in workspace'\n fp = candidates[0]\n with zipfile.ZipFile(str(fp)) as z:\n with z.open('word/document.xml') as f:\n root = ET.parse(f).getroot()\n paras = []\n for p in root.iter(f'{{{_W}}}p'):\n style_el = p.find(f'.//{{{_W}}}pStyle')\n style = style_el.get(f'{{{_W}}}val', '') if style_el is not None else ''\n txt = ''.join(t.text for t in p.iter(f'{{{_W}}}t') if t.text).strip()\n if txt:\n paras.append({'text': txt, 'style': style})\n return paras, '\\n'.join(p['text'] for p in paras), None\n\ndef _field(text, label):\n import re\n m = re.search(rf'{re.escape(label)}\\s*:\\s*(.+)', text, re.IGNORECASE)\n return m.group(1).strip() if m else None\n\ndef verify(workspace_path, external_services_path=None):\n paras, text, err = _paras(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n required_h2 = [\n 'Case Identification', 'Requestor and Standing', 'Disputed Service',\n 'Original Denial', 'Standard of Review Applied', 'Clinical Review Summary',\n 'Parity Analysis', 'Fraud and Mill Indicators', 'Timeline Log',\n 'Decision and Rationale', 'Authority', 'External Review Status',\n ]\n # Build lookup: heading text -> style\n style_map = {p['text'].strip().lower(): p['style'] for p in paras}\n failures = []\n # Check document title has Heading 1\n title_paras = [p for p in paras if 'CASE-2026-00413' in p['text'] and 'Determination Memo' in p['text']]\n if not title_paras:\n failures.append('Document title line (CASE-2026-00413 - Case Determination Memo) not found')\n elif not any('heading1' in p['style'].lower().replace(' ','') or p['style'] == '1' for p in title_paras):\n failures.append(f'Document title has style \"{title_paras[0][\"style\"]}\" instead of Heading 1')\n # Check each section heading has Heading 2\n for h in required_h2:\n style = style_map.get(h.lower(), '')\n if not ('heading2' in style.lower().replace(' ','') or style == '2'):\n failures.append(f'\"{h}\" has style \"{style}\" instead of Heading 2')\n if failures:\n return {'pass': False, 'score': max(0, (13 - len(failures)) / 13),\n 'feedback': f'Heading style errors: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'Document title has Heading 1; all section headings have Heading 2 style'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "c427b970-9568-4d37-9f8e-d03ad311d8f4",
|
|
"sort_order": 13,
|
|
"rubric_text": "In CASE-2026-00413_Determination_Memo.docx, key fields must have correct values: Plan Year = 2025, Appeal Type = Concurrent Care, Decision Deadline = 01/10/2026, Standard = Medical Necessity, Parity Flag Raised at Intake = YES, Within Coordinator Authority = NO, Supervisor Approval Required = YES, Total Dispute Value = $85,000, AOR on File = N/A, Appeal Mill Flag = NO, and EOC Provision Cited must reference EOC_G-14827_2025",
|
|
"verifier_code": "\nimport zipfile, xml.etree.ElementTree as ET\nfrom pathlib import Path as _P\n\n_W = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'\n\ndef _paras(ws):\n # Prefer exact filename; fall back to any determination memo docx\n exact_matches = list(_P(ws).rglob('CASE-2026-00413_Determination_Memo.docx'))\n if exact_matches:\n fp = exact_matches[0]\n else:\n candidates = [f for f in _P(ws).rglob('*.docx')\n if 'determination' in f.name.lower() or 'memo' in f.name.lower()]\n if not candidates:\n return None, None, 'CASE-2026-00413_Determination_Memo.docx not found in workspace'\n fp = candidates[0]\n with zipfile.ZipFile(str(fp)) as z:\n with z.open('word/document.xml') as f:\n root = ET.parse(f).getroot()\n paras = []\n for p in root.iter(f'{{{_W}}}p'):\n style_el = p.find(f'.//{{{_W}}}pStyle')\n style = style_el.get(f'{{{_W}}}val', '') if style_el is not None else ''\n txt = ''.join(t.text for t in p.iter(f'{{{_W}}}t') if t.text).strip()\n if txt:\n paras.append({'text': txt, 'style': style})\n return paras, '\\n'.join(p['text'] for p in paras), None\n\ndef _field(text, label):\n import re\n m = re.search(rf'{re.escape(label)}\\s*:\\s*(.+)', text, re.IGNORECASE)\n return m.group(1).strip() if m else None\n\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n paras, text, err = _paras(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n def f(label): return _field(text, label)\n checks = [\n ('Plan Year', lambda v: v is not None and '2025' in v and '2026' not in v),\n ('Appeal Type', lambda v: v is not None and 'concurrent' in v.lower()),\n ('Decision Deadline', lambda v: v is not None and bool(re.search(r'01.10.2026|jan(uary)?.?10', v, re.IGNORECASE))),\n ('Standard', lambda v: v is not None and 'medical necessity' in v.lower()),\n ('Parity Flag Raised at Intake', lambda v: v is not None and v.upper() == 'YES'),\n ('Within Coordinator Authority', lambda v: v is not None and v.upper() == 'NO'),\n ('Supervisor Approval Required', lambda v: v is not None and v.upper() == 'YES'),\n ('Total Dispute Value', lambda v: v is not None and '85,000' in v),\n ('AOR on File', lambda v: v is not None and v.upper() in ('N/A', 'NA')),\n ('Appeal Mill Flag', lambda v: v is not None and v.upper() == 'NO'),\n ('EOC Provision Cited', lambda v: v is not None and '2025' in v and 'G-14827' in v.upper()),\n ]\n failures = [f'{lbl}: got \"{f(lbl)}\"' for lbl, test in checks if not test(f(lbl))]\n if failures:\n return {'pass': False, 'score': (11 - len(failures)) / 11,\n 'feedback': f'Memo key field errors: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'All memo key fields have correct values'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "94be666f-4e28-4cf6-a741-d856b92f9ab1",
|
|
"sort_order": 14,
|
|
"rubric_text": "In CASE-2026-00413_Determination_Memo.docx: all narrative fields must be marked PENDING in capitals; all YES/NO fields must contain exactly YES or NO (not y/n, Yes/No, or other variants); and all date-only fields must use MM/DD/YYYY format",
|
|
"verifier_code": "\nimport zipfile, xml.etree.ElementTree as ET\nfrom pathlib import Path as _P\n\n_W = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'\n\ndef _paras(ws):\n # Prefer exact filename; fall back to any determination memo docx\n exact_matches = list(_P(ws).rglob('CASE-2026-00413_Determination_Memo.docx'))\n if exact_matches:\n fp = exact_matches[0]\n else:\n candidates = [f for f in _P(ws).rglob('*.docx')\n if 'determination' in f.name.lower() or 'memo' in f.name.lower()]\n if not candidates:\n return None, None, 'CASE-2026-00413_Determination_Memo.docx not found in workspace'\n fp = candidates[0]\n with zipfile.ZipFile(str(fp)) as z:\n with z.open('word/document.xml') as f:\n root = ET.parse(f).getroot()\n paras = []\n for p in root.iter(f'{{{_W}}}p'):\n style_el = p.find(f'.//{{{_W}}}pStyle')\n style = style_el.get(f'{{{_W}}}val', '') if style_el is not None else ''\n txt = ''.join(t.text for t in p.iter(f'{{{_W}}}t') if t.text).strip()\n if txt:\n paras.append({'text': txt, 'style': style})\n return paras, '\\n'.join(p['text'] for p in paras), None\n\ndef _field(text, label):\n import re\n m = re.search(rf'{re.escape(label)}\\s*:\\s*(.+)', text, re.IGNORECASE)\n return m.group(1).strip() if m else None\n\nimport re\n\nNARRATIVE_FIELDS = [\n 'Basis for Standing',\n 'Denial Reasoning Summary',\n 'Clinical Finding Summary',\n 'Flag Indicator Summary',\n 'Parity Finding Summary',\n]\n# Rationale appears twice (Standard of Review and Decision sections); check both\nRATIONALE_LABEL = 'Rationale'\n\nYESNO_FIELDS = [\n 'Standing Confirmed',\n 'Parity Flag Raised at Intake',\n 'Parity Review Committee Consulted',\n 'Parity Committee Concurrence Received',\n 'Within Coordinator Authority',\n 'Supervisor Approval Required',\n 'Appeal Mill Flag',\n]\n\nDATE_FIELDS = [\n 'Date of Receipt',\n 'Decision Deadline',\n]\n\ndef verify(workspace_path, external_services_path=None):\n paras, text, err = _paras(workspace_path)\n if err:\n return {'pass': False, 'score': 0.0, 'feedback': err}\n failures = []\n def f(label): return _field(text, label)\n\n # Narrative fields must be PENDING\n for label in NARRATIVE_FIELDS:\n val = f(label)\n if val is None:\n failures.append(f'{label}: field not found')\n elif val.upper() != 'PENDING':\n failures.append(f'{label}: expected PENDING, got \"{val[:60]}\"')\n # Rationale — may appear multiple times; all occurrences must be PENDING\n for m in re.finditer(r'Rationale\\s*:\\s*(.+)', text, re.IGNORECASE):\n val = m.group(1).strip()\n if val.upper() != 'PENDING':\n failures.append(f'Rationale: expected PENDING, got \"{val[:60]}\"')\n\n # YES/NO fields must be exactly YES or NO in capitals\n for label in YESNO_FIELDS:\n val = f(label)\n if val is None:\n continue # absence caught by section headings verifier\n if val not in ('YES', 'NO', 'PENDING', 'N/A'):\n failures.append(f'{label}: YES/NO field must be YES, NO, PENDING, or N/A — got \"{val}\"')\n\n # Date fields must match MM/DD/YYYY\n date_re = re.compile(r'^\\d{2}/\\d{2}/\\d{4}')\n for label in DATE_FIELDS:\n val = f(label)\n if val is None:\n failures.append(f'{label}: field not found')\n elif not date_re.match(val):\n failures.append(f'{label}: expected MM/DD/YYYY, got \"{val}\"')\n\n total = len(NARRATIVE_FIELDS) + 2 + len(YESNO_FIELDS) + len(DATE_FIELDS)\n if failures:\n return {'pass': False, 'score': max(0, (total - len(failures)) / total),\n 'feedback': f'Memo format errors: {failures}'}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'All narrative fields PENDING, YES/NO fields correctly capitalized, dates in MM/DD/YYYY'}\n",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|