87 lines
42 KiB
JSON
87 lines
42 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "5ed40139-37bd-4cb5-a0e3-7f15639947ea",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In `calendar.json` (or `calendar_data.json`), an event must exist with summary containing 'IVIG Infusion' and 'Thompson, Michael (CASE-1001)', start datetime 2026-04-14T12:00, end datetime 2026-04-14T15:00, and description containing 'Jane Smith'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services path first, then workspace\n calendar_data = None\n checked_paths = []\n \n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n for fname in ['calendar.json', 'calendar_data.json']:\n p = Path(base) / fname\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n calendar_data = json.load(f)\n break\n if calendar_data is not None:\n break\n \n if calendar_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No calendar file found. Checked: {checked_paths}\"}\n \n # Get events - handle both formats\n events = []\n if isinstance(calendar_data, dict):\n if 'events' in calendar_data:\n ev = calendar_data['events']\n if isinstance(ev, dict):\n events = list(ev.values())\n elif isinstance(ev, list):\n events = ev\n elif isinstance(calendar_data, list):\n events = calendar_data\n elif isinstance(calendar_data, list):\n events = calendar_data\n \n for event in events:\n summary = str(event.get('summary', ''))\n start = str(event.get('start', ''))\n end = str(event.get('end', ''))\n description = str(event.get('description', ''))\n \n if 'IVIG Infusion' not in summary:\n continue\n if 'Thompson, Michael (CASE-1001)' not in summary and ('Thompson' not in summary or 'CASE-1001' not in summary):\n continue\n if '2026-04-14' not in start or '12:00' not in start:\n continue\n if '2026-04-14' not in end or '15:00' not in end:\n continue\n if 'Jane Smith' not in description:\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching calendar event: summary='{summary}', start='{start}', end='{end}', description contains 'Jane Smith'\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No calendar event found with IVIG Infusion, Thompson/CASE-1001 in summary, start 2026-04-14T12:00, end 2026-04-14T15:00, and Jane Smith in description. Found {len(events)} events total.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "20c387f6-003a-4dfa-9e45-098f82a1ec9f",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In `mailbox.json`, a sent email must exist to 'j.smith@careig.com', with subject containing 'VISIT ASSIGNMENT', 'CASE-1001', and '2026-04-14', and body containing: '742 Evergreen Terrace, Beverly Hills, CA 90212', 'Privigen 10%', '30g', 'LOT-PV-88421', '49281-0396-05', '30 mL/hr', 'Diphenhydramine 75mg', '(310) 555-0147', and 'None' (contraindications).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_data = None\n checked_paths = []\n\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'mailbox.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n mailbox_data = json.load(f)\n break\n\n if mailbox_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found. Checked: {checked_paths}\"}\n\n emails = mailbox_data.get('emails', [])\n\n required_body_strings = [\n '742 Evergreen Terrace, Beverly Hills, CA 90212',\n 'Privigen 10%',\n '30g',\n 'LOT-PV-88421',\n '49281-0396-05',\n '30 mL/hr',\n 'Diphenhydramine 75mg',\n '(310) 555-0147',\n 'None'\n ]\n\n subject_keywords = ['VISIT ASSIGNMENT', 'CASE-1001', '2026-04-14']\n\n candidate_emails = []\n for email in emails:\n to_addr = str(email.get('to_addr', '') or '').lower()\n # Also check 'to' field as key name may vary\n if not to_addr:\n to_addr = str(email.get('to', '') or '').lower()\n subject = str(email.get('subject', '') or '')\n body = str(email.get('body_text', '') or '')\n # Also check 'body' field\n if not body:\n body = str(email.get('body', '') or '')\n\n if 'j.smith@careig.com' not in to_addr:\n continue\n\n # Check subject keywords (case-insensitive for leniency)\n subject_upper = subject.upper()\n missing_subj = [kw for kw in subject_keywords if kw.upper() not in subject_upper]\n if missing_subj:\n candidate_emails.append({'email': email, 'missing_subject': missing_subj, 'missing_body': []})\n continue\n\n # Check body strings\n missing = [s for s in required_body_strings if s not in body]\n if missing:\n # Try case-insensitive check for some fields as fallback\n body_lower = body.lower()\n still_missing = []\n for s in missing:\n if s.lower() not in body_lower:\n still_missing.append(s)\n if still_missing:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found email to j.smith@careig.com with correct subject but body missing: {still_missing}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found matching visit assignment email to j.smith@careig.com with all required body fields.\"}\n\n if candidate_emails:\n details = candidate_emails[0]\n return {\"pass\": False, \"score\": 0.25, \"feedback\": f\"Found email to j.smith@careig.com but subject missing keywords: {details['missing_subject']}. Subject was: {details['email'].get('subject', '')}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email found to j.smith@careig.com with subject containing VISIT ASSIGNMENT, CASE-1001, and 2026-04-14. Total emails checked: {len(emails)}. Checked paths: {checked_paths}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "3a738752-7d69-40ea-b89f-e314fda21f71",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In `mailbox.json`, an email must exist to 'm.thompson@email.com' with subject containing 'CareIG', 'IVIG Infusion Appointment', and '2026-04-14', and body containing 'Michael', '2026-04-14', '12:00 pm' (or '12:00 PM'), 'Duration: 3 hours', and signed off with 'Angela Torres'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_data = None\n checked_paths = []\n \n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'mailbox.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n mailbox_data = json.load(f)\n break\n \n if mailbox_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found. Checked: {checked_paths}\"}\n \n emails = mailbox_data.get('emails', [])\n \n for email in emails:\n to_addr = str(email.get('to_addr', '')).lower()\n subject = str(email.get('subject', ''))\n body = str(email.get('body_text', ''))\n body_lower = body.lower()\n \n if 'm.thompson@email.com' not in to_addr:\n continue\n if 'CareIG' not in subject:\n continue\n if 'IVIG Infusion Appointment' not in subject:\n continue\n if '2026-04-14' not in subject:\n continue\n \n missing = []\n if 'Michael' not in body:\n missing.append('Michael')\n if '2026-04-14' not in body:\n missing.append('2026-04-14')\n if '12:00 pm' not in body_lower:\n missing.append('12:00 pm')\n if 'Duration: 3 hours' not in body and 'duration: 3 hours' not in body_lower:\n missing.append('Duration: 3 hours')\n if 'Angela Torres' not in body:\n missing.append('Angela Torres')\n \n if missing:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found email to m.thompson@email.com with correct subject but body missing: {missing}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found matching patient appointment email to m.thompson@email.com with all required fields including Angela Torres sign-off.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email found to m.thompson@email.com with subject containing CareIG, IVIG Infusion Appointment, and 2026-04-14.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "4cf02e38-3598-405b-9ae2-2bf6ca6bfcbb",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `slack_data.json`, a message must exist in channel #clinical-scheduling containing 'SCHEDULED', 'CASE-1001', 'Date: 2026-04-14', and 'Nurse: Jane Smith'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_data = None\n checked_paths = []\n \n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'slack_data.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n slack_data = json.load(f)\n break\n \n if slack_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found. Checked: {checked_paths}\"}\n \n channels = slack_data.get('channels', {})\n messages = slack_data.get('messages', {})\n \n # Find #clinical-scheduling channel ID\n target_channel_id = None\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and cdata.get('name', '').lower() in ['clinical-scheduling', 'clinical_scheduling']:\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n # Try searching all channels by name\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and 'clinical' in cdata.get('name', '').lower():\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #clinical-scheduling channel in slack_data.json\"}\n \n channel_messages = messages.get(target_channel_id, [])\n \n required_strings = ['SCHEDULED', 'CASE-1001', 'Date: 2026-04-14', 'Nurse: Jane Smith']\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n if all(s in text for s in required_strings):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching message in #clinical-scheduling with all required strings: {required_strings}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #clinical-scheduling (channel {target_channel_id}) with all of: {required_strings}. Found {len(channel_messages)} messages total.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d54967ec-461c-45c3-8db2-9c8e7479f6d7",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "File `claim_CASE-1001.xlsx` must exist in the workspace with exactly 1 data row (2 rows including header), containing the required claim fields: A2=CASE-1001, B2=2026-03-15, C2=BlueCross Commercial PPO, D2=J1459, E2=59, G2=99600, H2=1, I2=99603, J2=2, K2=J7030, L2=2, M2=J7050, N2=1, O2=J0171, P2=2, S2=A4221, T2=1, U2=A4222, V2=1, W2=D80.1, X2=BC-2026-44821, Y2=1122334455, Z2=1234567893, AA2=47-3821056, AB2=333600000X, AC2=7, AD2=Standard, AE2=4153.75, AF2=2026-04-02, AH2=Submitted.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n claim_file = Path(workspace_path) / 'claim_CASE-1001.xlsx'\n if not claim_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"claim_CASE-1001.xlsx not found in workspace\"}\n \n try:\n wb = openpyxl.load_workbook(claim_file)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open claim_CASE-1001.xlsx: {e}\"}\n \n data_rows = [r for r in ws.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r)]\n if len(data_rows) > 1:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"claim_CASE-1001.xlsx has {len(data_rows)} data rows, expected exactly 1\"}\n if len(data_rows) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"claim_CASE-1001.xlsx has no data rows\"}\n \n def cell_val(col_letter, row=2):\n val = ws[f'{col_letter}{row}'].value\n if val is None:\n return ''\n s = str(val).strip()\n # Handle Excel formula cells: =59, =\"Submitted\", =(59*58)+...\n if s.startswith('='):\n inner = s[1:]\n # Strip surrounding quotes: =\"Submitted\" -> Submitted\n if (inner.startswith('\"') and inner.endswith('\"')) or (inner.startswith(\"'\") and inner.endswith(\"'\")):\n return inner[1:-1]\n # Try to evaluate simple numeric expressions\n try:\n result = float(eval(inner))\n # Return as int string if whole number\n if result == int(result):\n return str(int(result))\n return str(result)\n except:\n pass\n return inner\n return s\n \n def check_val(col, expected, actual=None):\n if actual is None:\n actual = cell_val(col)\n if str(expected).lower() == str(actual).lower():\n return True\n try:\n exp_num = float(str(expected).replace(',', ''))\n act_num = float(str(actual).replace(',', ''))\n return abs(exp_num - act_num) <= abs(exp_num) * 0.01 + 0.01\n except:\n pass\n expected_str = str(expected).replace('-', '').replace('/', '')\n actual_str = str(actual).replace('-', '').replace('/', '')\n if expected_str == actual_str:\n return True\n return False\n \n checks = [\n ('A', 'CASE-1001'),\n ('B', '2026-03-15'),\n ('C', 'BlueCross Commercial PPO'),\n ('D', 'J1459'),\n ('E', '59'),\n ('G', '99600'),\n ('H', '1'),\n ('I', '99603'),\n ('J', '2'),\n ('K', 'J7030'),\n ('L', '2'),\n ('M', 'J7050'),\n ('N', '1'),\n ('O', 'J0171'),\n ('P', '2'),\n ('S', 'A4221'),\n ('T', '1'),\n ('U', 'A4222'),\n ('V', '1'),\n ('W', 'D80.1'),\n ('X', 'BC-2026-44821'),\n ('Y', '1122334455'),\n ('Z', '1234567893'),\n ('AA', '47-3821056'),\n ('AB', '333600000X'),\n ('AC', '7'),\n ('AD', 'Standard'),\n ('AE', '4153.75'),\n ('AF', '2026-04-02'),\n ('AH', 'Submitted'),\n ]\n \n failures = []\n for col, expected in checks:\n actual = cell_val(col)\n if not check_val(col, expected, actual):\n failures.append(f\"{col}2: expected '{expected}', got '{actual}'\")\n \n if failures:\n return {\"pass\": False, \"score\": max(0.0, 1.0 - len(failures)/len(checks)), \"feedback\": f\"claim_CASE-1001.xlsx has {len(failures)} incorrect values: {failures}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"claim_CASE-1001.xlsx exists with 1 data row and all required field values are correct.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d1d818b0-885b-49f0-a57c-decf3fa517c5",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "A claim PDF file named `claim_20260315.pdf` or `claim_2026-03-15.pdf` must exist in the workspace, containing Drug_Units=59, Premed_Units_1=2, Nursing_Units_Addl=2, and Total_Billed=4153.75.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\nimport subprocess\n\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n # Locate the claim PDF - accept any of the three canonical filename variants\n candidates = (\n list(ws.glob('claim_20260315.pdf'))\n + list(ws.glob('claim_2026-03-15.pdf'))\n + list(ws.glob('claim_2026_03_15.pdf'))\n )\n if not candidates:\n all_pdfs = list(ws.glob('*.pdf'))\n candidates = [\n p for p in all_pdfs\n if any(s in p.name for s in ('20260315', '2026-03-15', '2026_03_15'))\n ]\n\n if not candidates:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': 'No claim PDF found matching claim_20260315.pdf or claim_2026-03-15.pdf',\n }\n\n pdf_file = candidates[0]\n\n if pdf_file.stat().st_size < 100:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'PDF {pdf_file.name} exists but is too small '\n f'({pdf_file.stat().st_size} bytes) — likely empty'\n ),\n }\n\n # --- Text extraction (try three backends in order) ---\n text = None\n\n try:\n result = subprocess.run(\n ['pdftotext', str(pdf_file), '-'],\n capture_output=True, text=True, timeout=10,\n )\n if result.returncode == 0 and result.stdout.strip():\n text = result.stdout\n except (FileNotFoundError, subprocess.TimeoutExpired):\n pass\n\n if text is None:\n try:\n import pdfplumber\n with pdfplumber.open(pdf_file) as pdf:\n extracted = '\\n'.join(page.extract_text() or '' for page in pdf.pages)\n if extracted.strip():\n text = extracted\n except Exception:\n pass\n\n if text is None:\n try:\n from PyPDF2 import PdfReader\n reader = PdfReader(str(pdf_file))\n extracted = '\\n'.join(page.extract_text() or '' for page in reader.pages)\n if extracted.strip():\n text = extracted\n except Exception:\n pass\n\n # --- CRITICAL FIX: unreadable PDF must FAIL, not pass ---\n # A corrupt or image-only PDF cannot be submitted as a valid claim document.\n # Existence alone does not satisfy the criterion; content must be verifiable.\n if not text or not text.strip():\n return {\n 'pass': False,\n 'score': 0.2,\n 'feedback': (\n f'PDF {pdf_file.name} exists ({pdf_file.stat().st_size} bytes) but text '\n 'could not be extracted — PDF may be corrupt, image-only, or malformed. '\n 'Cannot verify required field values '\n '(Drug_Units, Nursing_Units_Addl, Premed_Units_1, Total_Billed).'\n ),\n }\n\n # --- Content checks ---\n # Each check uses two patterns in priority order:\n # 1. The explicit field-label header near the value (handles spreadsheet-to-PDF exports)\n # 2. The billing/J-code near the unit count (handles tabular layouts)\n # Overly-generic patterns (bare numbers or drug-name proximity alone) are excluded\n # to prevent false positives from coincidental text matches.\n checks = [\n (\n 'Drug_Units=59',\n [\n r'Drug.{0,15}Units[^\\n]{0,80}\\b59\\b', # column header + value\n r'J1459\\b[^\\n]{0,60}\\b59\\b', # J-code + unit count, same line\n ],\n ),\n (\n 'Nursing_Units_Addl=2',\n [\n r'Nursing.{0,15}Units.{0,15}Addl[^\\n]{0,80}\\b2\\b', # column header + value\n r'99603\\b[^\\n]{0,60}\\b2\\b', # CPT code + unit count\n ],\n ),\n (\n 'Premed_Units_1=2',\n [\n r'Premed.{0,15}Units.{0,15}1[^\\n]{0,80}\\b2\\b', # column hea
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ac019917-5e98-4414-b8ca-f9938a2644f2",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In mailbox.json, a sent email must exist to 'claims@bluecross-ppo.example.com', with attachment 'claim_20260315.pdf' or 'claim_2026-03-15.pdf', and subject containing 'CLAIM SUBMISSION', 'CareIG NPI: 1234567893', 'Thompson', 'MemberID: BCX-998712', and 'DOS: 2026-03-15'.\r\n\r\n\r\n",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_data = None\n checked_paths = []\n\n # Search for mailbox.json in external_services_path first, then workspace_path\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'mailbox.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n mailbox_data = json.load(f)\n break\n\n if mailbox_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found. Checked: {checked_paths}\"}\n\n emails = mailbox_data.get('emails', [])\n\n required_subject_parts = ['CLAIM SUBMISSION', 'CareIG NPI: 1234567893', 'Thompson', 'MemberID: BCX-998712', 'DOS: 2026-03-15']\n\n best_partial_subject = None\n best_partial_attachment = None\n\n for email in emails:\n to_addr = str(email.get('to_addr', '') or '').lower()\n subject = str(email.get('subject', '') or '')\n attachments = email.get('attachments', [])\n\n # Check recipient - must be claims@bluecross-ppo.example.com\n if 'claims@bluecross-ppo.example.com' not in to_addr:\n continue\n\n # Check subject contains all required parts (case-insensitive)\n subject_upper = subject.upper()\n missing_subj = [s for s in required_subject_parts if s.upper() not in subject_upper]\n if missing_subj:\n if best_partial_subject is None or len(missing_subj) < len(best_partial_subject['missing']):\n best_partial_subject = {'missing': missing_subj, 'subject': subject, 'attachments': attachments}\n continue\n\n # Check attachment\n has_attachment = False\n att_list = []\n if isinstance(attachments, list):\n att_list = attachments\n elif isinstance(attachments, str):\n att_list = [attachments]\n\n for att in att_list:\n att_str = str(att).lower()\n if 'claim_20260315' in att_str or 'claim_2026-03-15' in att_str or 'claim_2026_03_15' in att_str:\n has_attachment = True\n break\n\n if not has_attachment:\n # Remember this near-match but keep searching for a fully correct email\n if best_partial_attachment is None:\n best_partial_attachment = {'subject': subject, 'attachments': attachments}\n continue\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching claim submission email to claims@bluecross-ppo.example.com with correct subject containing all required parts and correct attachment. Subject: {subject}\"}\n\n # No fully matching email found; report the best near-miss\n if best_partial_attachment:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found matching claim submission email to claims@bluecross-ppo.example.com with correct subject, but missing expected claim PDF attachment (expected 'claim_20260315.pdf' or 'claim_2026-03-15.pdf'). Attachments found: {best_partial_attachment['attachments']}\"}\n\n if best_partial_subject:\n return {\"pass\": False, \"score\": 0.25, \"feedback\": f\"Found email to claims@bluecross-ppo.example.com but subject is missing required parts: {best_partial_subject['missing']}. Subject found: {best_partial_subject['subject']}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email found to 'claims@bluecross-ppo.example.com' with required subject parts: {required_subject_parts}. Total emails in mailbox: {len(emails)}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "3ab1d233-20a9-47a8-a76f-86965431234b",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In `slack_data.json`, a message must exist in channel #billing containing 'CLAIM SUBMITTED', 'CASE-1001', 'DOS: 2026-03-15' (or '03/15/2026'), 'Payer: BlueCross Commercial PPO', and 'Billed: $4,153.75' (or '4153.75').",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_data = None\n checked_paths = []\n \n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'slack_data.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n slack_data = json.load(f)\n break\n \n if slack_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found. Checked: {checked_paths}\"}\n \n channels = slack_data.get('channels', {})\n messages = slack_data.get('messages', {})\n \n # Find #billing channel ID\n target_channel_id = None\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and cdata.get('name', '').lower() == 'billing':\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #billing channel in slack_data.json\"}\n \n channel_messages = messages.get(target_channel_id, [])\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n \n if 'CLAIM SUBMITTED' not in text:\n continue\n if 'CASE-1001' not in text:\n continue\n if 'BlueCross Commercial PPO' not in text:\n continue\n \n # Check DOS date in multiple formats\n dos_ok = 'DOS: 2026-03-15' in text or '03/15/2026' in text or 'DOS: 03-15-2026' in text or '2026-03-15' in text\n if not dos_ok:\n continue\n \n # Check billed amount in multiple formats\n billed_ok = '4,153.75' in text or '4153.75' in text\n if not billed_ok:\n continue\n \n if 'Payer: BlueCross Commercial PPO' not in text and 'BlueCross Commercial PPO' not in text:\n continue\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching billing message in #billing channel with all required strings.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #billing channel (id={target_channel_id}) with all required strings. Found {len(channel_messages)} messages total.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "fd361d7d-da84-49ea-8644-ff6d841b2c41",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "File `audit_log.xlsx` must exist in the workspace with a row dated 2026-04-02 containing: CASE_ID=CASE-1001, Action=Claim Submitted, File_Updated=claim_CASE-1001.xlsx, Field_Updated containing Drug_Units/Nursing_Units_Addl/Premed_Units_1/Claim_Frequency_Code/Total_Billed/Submission_Date, and New_Value containing 59/2/2/7/4153.75/2026-04-02. ",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n audit_file = Path(workspace_path) / 'audit_log.xlsx'\n if not audit_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"audit_log.xlsx not found in workspace\"}\n \n try:\n wb = openpyxl.load_workbook(audit_file)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open audit_log.xlsx: {e}\"}\n \n # Get header row\n headers = []\n for cell in ws[1]:\n headers.append(str(cell.value).strip() if cell.value is not None else '')\n \n def get_col_index(name):\n for i, h in enumerate(headers):\n if h.lower() == name.lower():\n return i\n for i, h in enumerate(headers):\n if name.lower() in h.lower():\n return i\n return None\n \n # Count data rows\n data_rows = []\n for row in ws.iter_rows(min_row=2, values_only=True):\n if any(c is not None for c in row):\n data_rows.append(row)\n \n # Find the required columns\n col_date = get_col_index('date')\n col_case = get_col_index('CASE_ID')\n col_action = get_col_index('Action')\n col_file = get_col_index('File_Updated')\n col_field = get_col_index('Field_Updated')\n col_newval = get_col_index('New_Value')\n \n missing_cols = []\n if col_date is None: missing_cols.append('date')\n if col_case is None: missing_cols.append('CASE_ID')\n if col_action is None: missing_cols.append('Action')\n if col_file is None: missing_cols.append('File_Updated')\n if col_field is None: missing_cols.append('Field_Updated')\n if col_newval is None: missing_cols.append('New_Value')\n \n if missing_cols:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"audit_log.xlsx missing columns: {missing_cols}. Found headers: {headers}\"}\n \n required_fields = ['Drug_Units', 'Nursing_Units_Addl', 'Premed_Units_1', 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date']\n required_values = ['59', '2', '2', '7', '4153.75', '2026-04-02']\n \n def normalize_cell(val):\n if val is None:\n return ''\n if isinstance(val, datetime):\n return val.strftime('%Y-%m-%d')\n s = str(val).strip()\n if re.match(r'^\\d+\\.0$', s):\n s = s[:-2]\n return s\n \n def date_matches(date_str):\n date_str = date_str.strip()\n if not date_str:\n return False\n if '2026-04-02' in date_str:\n return True\n if '04/02/2026' in date_str:\n return True\n if '04-02-2026' in date_str:\n return True\n if '2026/04/02' in date_str:\n return True\n if 'Apr' in date_str and '2026' in date_str and '2' in date_str:\n return True\n return False\n \n def value_present(val_str, check_val):\n if check_val in val_str:\n return True\n if check_val == '2026-04-02':\n if '04/02/2026' in val_str or '04-02-2026' in val_str or '2026/04/02' in val_str:\n return True\n if check_val == '4153.75':\n if '4153.75' in val_str or '4,153.75' in val_str or '$4153.75' in val_str or '$4,153.75' in val_str:\n return True\n if check_val in ('59', '2', '7'):\n parts = re.split(r'[/,;|\\s]+', val_str)\n for part in parts:\n cleaned = part.strip().rstrip('.0')\n if cleaned == check_val:\n return True\n return False\n \n best_partial_score = 0.0\n best_partial_feedback = \"\"\n \n for row in data_rows:\n def rv(idx):\n if idx is None or idx >= len(row):\n return ''\n return normalize_cell(row[idx])\n \n date_val = rv(col_date)\n case_val = rv(c
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "09c9969a-db48-43bb-bd64-6d83e68b0396",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "File `clinical_CASE-1001.xlsx` must exist in the workspace with Visit_Date=2026-04-14, Visit_Time=12:00, and Nurse_Assigned=Jane Smith.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n clinical_file = Path(workspace_path) / 'clinical_CASE-1001.xlsx'\n if not clinical_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"clinical_CASE-1001.xlsx not found in workspace\"}\n \n try:\n wb = openpyxl.load_workbook(clinical_file)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open clinical_CASE-1001.xlsx: {e}\"}\n \n # Get header row\n headers = []\n for cell in ws[1]:\n headers.append(str(cell.value).strip().lower() if cell.value is not None else '')\n \n def get_col_index(name):\n for i, h in enumerate(headers):\n if h == name.lower():\n return i\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return None\n \n col_date = get_col_index('visit_date')\n col_time = get_col_index('visit_time')\n col_nurse = get_col_index('nurse_assigned')\n \n missing_cols = []\n if col_date is None: missing_cols.append('Visit_Date')\n if col_time is None: missing_cols.append('Visit_Time')\n if col_nurse is None: missing_cols.append('Nurse_Assigned')\n \n if missing_cols:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"clinical_CASE-1001.xlsx missing columns: {missing_cols}. Found headers: {headers}\"}\n \n # Find row with CASE-1001 data (check all data rows)\n for row in ws.iter_rows(min_row=2, values_only=True):\n if not any(c is not None for c in row):\n continue\n \n def rv(idx):\n if idx is None or idx >= len(row):\n return ''\n return str(row[idx]).strip() if row[idx] is not None else ''\n \n date_val = rv(col_date)\n time_val = rv(col_time)\n nurse_val = rv(col_nurse)\n \n date_ok = '2026-04-14' in date_val or '04/14/2026' in date_val or '04-14-2026' in date_val\n time_ok = '12:00' in time_val\n nurse_ok = 'Jane Smith' in nurse_val\n \n if date_ok and time_ok and nurse_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"clinical_CASE-1001.xlsx has correct values: Visit_Date='{date_val}', Visit_Time='{time_val}', Nurse_Assigned='{nurse_val}'\"}\n \n # Find best partial match for feedback\n for row in ws.iter_rows(min_row=2, values_only=True):\n if not any(c is not None for c in row):\n continue\n def rv(idx):\n if idx is None or idx >= len(row): return ''\n return str(row[idx]).strip() if row[idx] is not None else ''\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"clinical_CASE-1001.xlsx first data row: Visit_Date='{rv(col_date)}', Visit_Time='{rv(col_time)}', Nurse_Assigned='{rv(col_nurse)}'. Expected: Visit_Date=2026-04-14, Visit_Time=12:00, Nurse_Assigned=Jane Smith\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"clinical_CASE-1001.xlsx has no data rows\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e436fe95-d992-440f-aa87-32a99876cbe2",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "In `mailbox.json`, exactly 17 emails must exist in the emails array.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_data = None\n checked_paths = []\n \n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n p = Path(base) / 'mailbox.json'\n checked_paths.append(str(p))\n if p.exists():\n with open(p, 'r') as f:\n mailbox_data = json.load(f)\n break\n \n if mailbox_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found. Checked: {checked_paths}\"}\n \n emails = mailbox_data.get('emails', [])\n count = len(emails)\n \n if count == 17:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json contains exactly 17 emails.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json contains {count} emails, expected exactly 17.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775136857045",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "In `slack_data.json`, exactly 31 messages must exist across all channels.",
|
||
|
|
"verifier_code": "import json\nimport os\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n \n # Try every possible location\n search_paths = []\n if external_services_path:\n search_paths.append(os.path.join(str(external_services_path), \"final\", \"slack_data.json\"))\n search_paths.append(os.path.join(str(external_services_path), \"slack_data.json\"))\n if workspace_path:\n search_paths.append(os.path.join(str(workspace_path), \"slack_data.json\"))\n \n for p in search_paths:\n try:\n if os.path.isfile(p):\n slack_path = p\n break\n except Exception:\n continue\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json. Searched: \" + str(search_paths)}\n \n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Error reading \" + str(slack_path) + \": \" + str(e)}\n \n total_messages = 0\n channel_details = {}\n \n messages = slack_data.get(\"messages\", {})\n channels_meta = slack_data.get(\"channels\", {})\n \n if isinstance(messages, dict):\n for ch_id, msg_list in messages.items():\n if isinstance(msg_list, list):\n count = len(msg_list)\n ch_name = ch_id\n if isinstance(channels_meta.get(ch_id), dict):\n ch_name = channels_meta[ch_id].get(\"name\", ch_id)\n channel_details[ch_name] = count\n total_messages += count\n \n if total_messages == 31:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found exactly 31 messages. Breakdown: \" + str(channel_details)}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Expected 31 messages, found \" + str(total_messages) + \". Breakdown: \" + str(channel_details)}",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|