122 lines
46 KiB
JSON
122 lines
46 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "e5f1fe58-b919-47b1-94b9-edc95fe25ffb",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In file `audit_log.xlsx` in folder `Rodriguez_07221975`, there must be exactly 2 rows of data (excluding headers).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Rodriguez_07221975'\n if not folder.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'Folder Rodriguez_07221975 not found in workspace'}\n \n audit_file = folder / 'audit_log.xlsx'\n if not audit_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in Rodriguez_07221975 folder'}\n \n try:\n wb = openpyxl.load_workbook(audit_file)\n ws = wb.active\n # Count non-empty rows excluding header\n data_rows = []\n for row in ws.iter_rows(min_row=2, values_only=True):\n if any(cell is not None for cell in row):\n data_rows.append(row)\n \n if len(data_rows) == 2:\n return {'pass': True, 'score': 1.0, 'feedback': f'audit_log.xlsx in Rodriguez_07221975 has exactly 2 data rows as expected.'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'audit_log.xlsx in Rodriguez_07221975 has {len(data_rows)} data rows, expected exactly 2.'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading audit_log.xlsx: {e}'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "a10085f6-f540-40ad-9068-d6722d443148",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#intake-alerts` must contain a message with exactly the text: `[UNREACHABLE] Rodriguez_07221975 — 3 failed attempts`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport string\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n\n def normalize_dashes(s):\n # Normalize em-dash (—), en-dash (–), and figure dash (‒) to plain hyphen\n return s.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-')\n\n def strip_terminal_punct(s):\n return s.rstrip(string.punctuation + ' ')\n\n try:\n with open(slack_file, 'r') as f:\n data = json.load(f)\n\n target_text = '[UNREACHABLE] Rodriguez_07221975 — 3 failed attempts'\n target_normalized = normalize_dashes(target_text)\n\n # Find #intake-alerts channel\n channels = data.get('channels', {})\n intake_alerts_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'intake-alerts':\n intake_alerts_id = cid\n break\n\n if intake_alerts_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Channel #intake-alerts not found in slack_data.json'}\n\n messages = data.get('messages', {}).get(intake_alerts_id, [])\n for msg in messages:\n msg_text = msg.get('text', '') if isinstance(msg, dict) else str(msg)\n msg_normalized = strip_terminal_punct(normalize_dashes(msg.get('text', '')))\n if msg_normalized == strip_terminal_punct(normalize_dashes(target_text)):\n return {'pass': True, 'score': 1.0, 'feedback': 'Found expected [UNREACHABLE] message in #intake-alerts for Rodriguez_07221975'}\n\n return {'pass': False, 'score': 0.0, 'feedback': f'No message containing \"{target_text}\" found in #intake-alerts'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {e}'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d58b3ad1-6004-4cc4-9330-157577be57bf",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be a sent email to `rcho@greenmedclinic.org` with subject `Unable to Contact Patient | CASE_ID: Rodriguez_07221975` and sign-off by `Jake Morrow`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json, re, unicodedata\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_file = Path(external_services_path) / 'mailbox.json'\n if not mailbox_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n try:\n with open(mailbox_file, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {e}'}\n\n def normalize(s):\n s = unicodedata.normalize('NFKC', s)\n s = s.lower()\n s = s.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-')\n s = s.replace('\\u2018', \"'\").replace('\\u2019', \"'\").replace('\\u201c', '\"').replace('\\u201d', '\"')\n s = re.sub(r'\\s+', ' ', s)\n return s.strip()\n\n expected_to = 'rcho@greenmedclinic.org'\n expected_subject = 'Unable to Contact Patient | CASE_ID: Rodriguez_07221975'\n expected_signoff = 'Jake Morrow'\n\n emails = data.get('emails', [])\n for email in emails:\n to_addr = email.get('to_addr', '')\n subject = email.get('subject', '')\n folder = email.get('folder', '')\n body = email.get('body_text', '')\n\n to_match = expected_to.lower() in to_addr.lower()\n subject_match = normalize(subject) == normalize(expected_subject)\n folder_match = folder.strip().lower() == 'sent'\n\n if to_match and subject_match and folder_match:\n if normalize(expected_signoff) in normalize(body):\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Sent email to {expected_to} has correct subject and is signed by {expected_signoff}.'}\n else:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Email found with correct recipient/subject/folder, but sign-off \"{expected_signoff}\" not present in body.'}\n\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No sent email found to {expected_to} with subject \"{expected_subject.strip()}\" in Sent folder.'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "8261add0-78e4-4cb0-ab85-de30da8108c8",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In file `intake.xlsx` in folder `Webb_03151968`, row 2 must contain `Gammagard Liquid 10%` in column `Drug_Name` and `35` in column `Drug_Dose_Grams`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the folder Webb_03151968 anywhere in the workspace\n workspace = Path(workspace_path)\n \n # Search for the folder\n folder = None\n if (workspace / 'Webb_03151968').exists():\n folder = workspace / 'Webb_03151968'\n else:\n # Try glob search\n candidates = list(workspace.rglob('Webb_03151968'))\n if candidates:\n folder = candidates[0]\n \n if folder is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Folder Webb_03151968 not found in workspace'}\n \n # Find intake.xlsx\n intake_file = folder / 'intake.xlsx'\n if not intake_file.exists():\n # Try case-insensitive search\n candidates = list(folder.glob('*.xlsx'))\n intake_file = None\n for c in candidates:\n if c.stem.lower() == 'intake':\n intake_file = c\n break\n if intake_file is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'intake.xlsx not found in Webb_03151968 folder'}\n \n try:\n wb = openpyxl.load_workbook(intake_file)\n ws = wb.active\n \n # Get headers from row 1\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n \n drug_name_col = headers.get('Drug_Name')\n drug_dose_col = headers.get('Drug_Dose_Grams')\n \n # Try case-insensitive header matching if not found\n if drug_name_col is None:\n for h, idx in headers.items():\n if h.lower().replace(' ', '_') == 'drug_name':\n drug_name_col = idx\n break\n if drug_dose_col is None:\n for h, idx in headers.items():\n if h.lower().replace(' ', '_') == 'drug_dose_grams':\n drug_dose_col = idx\n break\n \n if drug_name_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column Drug_Name not found in intake.xlsx. Found headers: {list(headers.keys())}'}\n if drug_dose_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Column Drug_Dose_Grams not found in intake.xlsx. Found headers: {list(headers.keys())}'}\n \n # Check row 2\n rows = list(ws.iter_rows(min_row=2, max_row=2, values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 not found in intake.xlsx'}\n row2 = rows[0]\n \n drug_name_val = str(row2[drug_name_col - 1]).strip() if row2[drug_name_col - 1] is not None else ''\n drug_dose_val = str(row2[drug_dose_col - 1]).strip() if row2[drug_dose_col - 1] is not None else ''\n \n # Check Drug_Name contains 'Gammagard Liquid 10%' (case-insensitive, loose matching)\n # Accept variations like 'Gammagard Liquid 10%', 'gammagard liquid 10%', etc.\n if drug_name_val.lower() != 'gammagard liquid 10%':\n return {'pass': False, 'score': 0.0, 'feedback': f'Drug_Name in row 2 is \"{drug_name_val}\", expected \"Gammagard Liquid 10%\"'}\n \n # Check Drug_Dose_Grams is 35 (with tolerance)\n try:\n dose_num = float(drug_dose_val)\n except ValueError:\n return {'pass': False, 'score': 0.0, 'feedback': f'Drug_Dose_Grams in row 2 is \"{drug_dose_val}\", expected 35 (not a valid number)'}\n \n if abs(dose_num - 35) > 0.5:\n return {'pass': False, 'score': 0.0, 'feedback': f'Drug_Dose_Grams in row 2 is {dose_num}, expected 35'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'intake.xlsx row 2 correctly has Drug_Name=\"{drug_name_val}\" and Drug_Dose_Grams=\"{drug_dose_val}\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading i
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "073e198a-45db-4ce0-b86f-cf595d03870c",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In file `auth.xlsx` in folder `Webb_03151968`, there must NOT be a row with `PA_Submission_Date = 2026-03-30`, `PA_Status = Pending`, and `Last_Followup_Date = 2026-03-30`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport io, zipfile\nimport openpyxl\nfrom datetime import date, datetime\n\nCASE = 'Webb_03151968'\nFILENAME = 'auth.xlsx'\n\ndef _resolve_workbook(workspace_path, case, filename):\n \"\"\"Read case/filename from the unzipped folder; else from a zip\n (canonical <case>.zip preferred); else return (None, None) -> fail closed.\"\"\"\n root = Path(workspace_path)\n direct = root / case / filename\n if direct.exists():\n return openpyxl.load_workbook(direct), str(direct)\n zips = []\n canonical = root / (case + '.zip')\n if canonical.exists():\n zips.append(canonical)\n for z in sorted(root.rglob('*.zip')):\n if z not in zips:\n zips.append(z)\n for zp in zips:\n try:\n with zipfile.ZipFile(zp) as zf:\n members = [m for m in zf.namelist() if '__MACOSX' not in m and not m.endswith('/')]\n for m in members:\n if m.endswith(case + '/' + filename):\n return openpyxl.load_workbook(io.BytesIO(zf.read(m))), zp.name + '::' + m\n if zp.stem == case:\n for m in members:\n if m.rsplit('/', 1)[-1] == filename:\n return openpyxl.load_workbook(io.BytesIO(zf.read(m))), zp.name + '::' + m\n except zipfile.BadZipFile:\n continue\n return None, None\n\ndef verify(workspace_path, external_services_path=None):\n wb, src = _resolve_workbook(workspace_path, CASE, FILENAME)\n if wb is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'{CASE}/{FILENAME} could not be read from an unzipped folder or any zip; cannot verify constraint.'}\n try:\n ws = wb.active\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n pa_sub_col = headers.get('PA_Submission_Date')\n pa_status_col = headers.get('PA_Status')\n last_followup_col = headers.get('Last_Followup_Date')\n if pa_sub_col is None or pa_status_col is None or last_followup_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Required columns (PA_Submission_Date/PA_Status/Last_Followup_Date) not all present in {src}; cannot verify constraint.'}\n target_date = '2026-03-30'\n def normalize_date(v):\n if v is None:\n return ''\n if isinstance(v, (date, datetime)):\n return v.strftime('%Y-%m-%d')\n return str(v).strip()[:10]\n for row in ws.iter_rows(min_row=2, values_only=True):\n sub_str = normalize_date(row[pa_sub_col - 1])\n followup_str = normalize_date(row[last_followup_col - 1])\n status_val = row[pa_status_col - 1]\n status_str = str(status_val).strip() if status_val is not None else ''\n if sub_str == target_date and status_str.lower() == 'pending' and followup_str == target_date:\n return {'pass': False, 'score': 0.0, 'feedback': f'{src} contains a row with PA_Submission_Date=2026-03-30, PA_Status=Pending, Last_Followup_Date=2026-03-30 - this should NOT be present'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Verified {src}: forbidden PA submission row (2026-03-30 / Pending) not present.'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading {src}: {e}'}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "5f202b29-1414-4ed7-a976-7dd7d4833b4c",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#intake-team` must NOT contain a message with text `[PA SUBMITTED] Webb_03151968 ",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'external_services_path is None; cannot find forbidden message, constraint satisfied'}\n \n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; constraint satisfied by absence'}\n \n try:\n with open(slack_file, 'r') as f:\n data = json.load(f)\n \n forbidden_prefix = '[PA SUBMITTED] Webb_03151968'\n \n channels = data.get('channels', {})\n intake_team_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'intake-team':\n intake_team_id = cid\n break\n \n if intake_team_id is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'Channel #intake-team not found; constraint satisfied'}\n \n messages = data.get('messages', {}).get(intake_team_id, [])\n for msg in messages:\n msg_text = msg.get('text', '')\n if forbidden_prefix in msg_text:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found forbidden [PA SUBMITTED] message for Webb_03151968 in #intake-team: \"{msg_text}\"'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No forbidden [PA SUBMITTED] message found in #intake-team for Webb_03151968'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {e}'}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d66d819e-c468-4049-b40b-5c2776ad92e9",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must NOT be a sent email to `pa.submissions@bcpremier.com`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'external_services_path is None; cannot find forbidden email, constraint satisfied'}\n \n mailbox_file = Path(external_services_path) / 'mailbox.json'\n if not mailbox_file.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; constraint satisfied by absence'}\n \n try:\n with open(mailbox_file, 'r') as f:\n data = json.load(f)\n \n forbidden_to = 'pa.submissions@bcpremier.com'\n \n emails = data.get('emails', [])\n for email in emails:\n to_addr = email.get('to_addr', '')\n if forbidden_to.lower() in to_addr.lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'Found forbidden email sent to pa.submissions@bcpremier.com — PA submission should not have been made for Webb case'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No email sent to pa.submissions@bcpremier.com; PA submission correctly not made'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {e}'}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "fe00dac5-ed07-45ee-b0cf-a02f5b2c6d22",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#intake-team` must NOT contain any message with text containing `[FORMULARY ISSUE]`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'external_services_path is None; cannot find forbidden message, constraint satisfied'}\n \n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; constraint satisfied by absence'}\n \n try:\n with open(slack_file, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n intake_team_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'intake-team':\n intake_team_id = cid\n break\n \n if intake_team_id is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'Channel #intake-team not found; constraint satisfied'}\n \n messages = data.get('messages', {}).get(intake_team_id, [])\n for msg in messages:\n if '[FORMULARY ISSUE]' in msg.get('text', ''):\n return {'pass': False, 'score': 0.0, 'feedback': f'Found forbidden [FORMULARY ISSUE] message in #intake-team: \"{msg.get(\"text\", \"\")}\"'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No [FORMULARY ISSUE] message found in #intake-team'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {e}'}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "db76f824-2752-4a4d-a1e5-04d28a3833a9",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In file `audit_log.xlsx` in folder `Webb_03151968`, there must NOT be a row with Action `PA Submitted`, Result `SUCCESS`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport io, zipfile\nimport openpyxl\nfrom datetime import date, datetime\n\nCASE = 'Webb_03151968'\nFILENAME = 'audit_log.xlsx'\n\ndef _resolve_workbook(workspace_path, case, filename):\n \"\"\"Read case/filename from the unzipped folder; else from a zip\n (canonical <case>.zip preferred); else return (None, None) -> fail closed.\"\"\"\n root = Path(workspace_path)\n direct = root / case / filename\n if direct.exists():\n return openpyxl.load_workbook(direct), str(direct)\n zips = []\n canonical = root / (case + '.zip')\n if canonical.exists():\n zips.append(canonical)\n for z in sorted(root.rglob('*.zip')):\n if z not in zips:\n zips.append(z)\n for zp in zips:\n try:\n with zipfile.ZipFile(zp) as zf:\n members = [m for m in zf.namelist() if '__MACOSX' not in m and not m.endswith('/')]\n for m in members:\n if m.endswith(case + '/' + filename):\n return openpyxl.load_workbook(io.BytesIO(zf.read(m))), zp.name + '::' + m\n if zp.stem == case:\n for m in members:\n if m.rsplit('/', 1)[-1] == filename:\n return openpyxl.load_workbook(io.BytesIO(zf.read(m))), zp.name + '::' + m\n except zipfile.BadZipFile:\n continue\n return None, None\n\ndef verify(workspace_path, external_services_path=None):\n wb, src = _resolve_workbook(workspace_path, CASE, FILENAME)\n if wb is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'{CASE}/{FILENAME} could not be read from an unzipped folder or any zip; cannot verify constraint.'}\n try:\n ws = wb.active\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n action_col = headers.get('Action')\n result_col = headers.get('Result')\n if action_col is None or result_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Required columns (Action/Result) not present in {src}; cannot verify constraint.'}\n for row_idx, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2):\n row_action = str(row[action_col - 1]).strip() if row[action_col - 1] is not None else ''\n row_result = str(row[result_col - 1]).strip() if row[result_col - 1] is not None else ''\n if row_action.lower() == 'pa submitted' and row_result.upper() == 'SUCCESS':\n return {'pass': False, 'score': 0.0, 'feedback': f'{src} contains forbidden row at row {row_idx}: Action=\"{row_action}\", Result=\"{row_result}\". The rubric states there must NOT be a row with Action PA Submitted and Result SUCCESS.'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Verified {src}: no row with Action \"PA Submitted\" and Result \"SUCCESS\".'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading {src}: {e}'}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "915c6380-12dd-4c0c-8577-976644b028c0",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In file `audit_log.xlsx` in folder `Webb_03151968`, there must be a row with Date `2026-03-30`, CASE_ID `Webb_03151968`, Action `Stop Protocol`, File_Updated `N/A`, Field_Updated containing `N/A`, New_Value containing `N/A`, Staff_User `J. Morrow`, Result `STOP - invalid file`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import date, datetime\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Webb_03151968'\n if not folder.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'Folder Webb_03151968 not found in workspace'}\n \n audit_file = folder / 'audit_log.xlsx'\n if not audit_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in Webb_03151968 folder'}\n \n try:\n wb = openpyxl.load_workbook(audit_file)\n ws = wb.active\n \n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n \n date_col = headers.get('Date')\n case_col = headers.get('CASE_ID')\n action_col = headers.get('Action')\n file_updated_col = headers.get('File_Updated')\n field_updated_col = headers.get('Field_Updated')\n new_value_col = headers.get('New_Value')\n staff_col = headers.get('Staff_User')\n result_col = headers.get('Result')\n \n def normalize_date(v):\n if v is None:\n return ''\n if isinstance(v, (date, datetime)):\n return v.strftime('%Y-%m-%d')\n return str(v).strip()[:10]\n \n def get_val(row, col):\n if col is None:\n return ''\n return str(row[col - 1]).strip() if row[col - 1] is not None else ''\n \n for row in ws.iter_rows(min_row=2, values_only=True):\n row_date = normalize_date(row[date_col - 1]) if date_col else ''\n row_case = get_val(row, case_col)\n row_action = get_val(row, action_col)\n row_file = get_val(row, file_updated_col)\n row_field = get_val(row, field_updated_col)\n row_new_val = get_val(row, new_value_col)\n row_staff = get_val(row, staff_col)\n row_result = get_val(row, result_col)\n \n date_match = row_date == '2026-03-30'\n case_match = row_case == 'Webb_03151968'\n action_match = row_action.lower() == 'stop protocol'\n file_match = row_file.upper() == 'N/A'\n field_match = 'N/A' in row_field or row_field.upper() == 'N/A'\n new_val_match = 'N/A' in row_new_val or row_new_val.upper() == 'N/A'\n staff_match = row_staff == 'J. Morrow'\n result_match = 'STOP' in row_result.upper() and 'INVALID' in row_result.upper()\n \n if date_match and case_match and action_match and file_match and field_match and new_val_match and staff_match and result_match:\n return {'pass': True, 'score': 1.0, 'feedback': 'Found required Stop Protocol row in audit_log.xlsx for Webb_03151968 with all expected fields'}\n \n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx in Webb_03151968 does not contain the required Stop Protocol row with Date=2026-03-30, CASE_ID=Webb_03151968, Action=Stop Protocol, Staff_User=J. Morrow, Result=STOP - invalid file'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading audit_log.xlsx: {e}'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775161894799",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In the file `intake.xlsx` in the folder `Webb_03151968`, row 2 must NOT contain `Octagam` in the `Drug_Name` column or `40` in the `Drug_Dose_Grams` column.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for intake.xlsx in Webb_03151968 folder\n file_path = Path(workspace_path) / \"Webb_03151968\" / \"intake.xlsx\"\n if not file_path.exists():\n # Try case-insensitive search\n found = list(Path(workspace_path).rglob(\"*Webb*03151968*\"))\n matching = None\n for d in found:\n if d.is_dir():\n candidate = d / \"intake.xlsx\"\n if candidate.exists():\n matching = candidate\n break\n if matching is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find intake.xlsx in Webb_03151968 folder. Searched in {workspace_path}. Found dirs: {found}\"}\n file_path = matching\n\n try:\n wb = openpyxl.load_workbook(file_path)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open {file_path}: {e}\"}\n\n # Find column indices by header names in row 1\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n\n drug_name_col = None\n drug_dose_col = None\n for h, idx in headers.items():\n if h.lower().replace(' ', '_') == 'drug_name' or h.lower() == 'drug_name' or h == 'Drug_Name':\n drug_name_col = idx\n if h.lower().replace(' ', '_') == 'drug_dose_grams' or h.lower() == 'drug_dose_grams' or h == 'Drug_Dose_Grams':\n drug_dose_col = idx\n\n # If we can't find headers by name, also try checking all cells in row 2\n row2_values = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[2]]\n\n has_octagam = False\n has_40 = False\n\n if drug_name_col is not None:\n val = ws.cell(row=2, column=drug_name_col).value\n if val is not None and str(val).strip().lower() == 'octagam':\n has_octagam = True\n\n if drug_dose_col is not None:\n val = ws.cell(row=2, column=drug_dose_col).value\n if val is not None:\n try:\n numeric_val = float(str(val).strip())\n if abs(numeric_val - 40.0) < 0.5:\n has_40 = True\n except ValueError:\n if str(val).strip() == '40':\n has_40 = True\n\n feedback_parts = []\n feedback_parts.append(f\"Headers found: {headers}\")\n feedback_parts.append(f\"Row 2 values: {row2_values}\")\n feedback_parts.append(f\"Drug_Name col: {drug_name_col}, Drug_Dose_Grams col: {drug_dose_col}\")\n feedback_parts.append(f\"Has Octagam in Drug_Name: {has_octagam}\")\n feedback_parts.append(f\"Has 40 in Drug_Dose_Grams: {has_40}\")\n\n # The rubric says row 2 must NOT contain Octagam in Drug_Name OR 40 in Drug_Dose_Grams\n # This means NEITHER should be present (interpreting \"or\" as logical or: not(A or B) = not A and not B)\n if has_octagam or has_40:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Row 2 still contains forbidden values. Octagam found: {has_octagam}, 40 found: {has_40}. {'; '.join(feedback_parts)}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS: Row 2 does not contain Octagam in Drug_Name or 40 in Drug_Dose_Grams. {'; '.join(feedback_parts)}\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775169232979",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In slack_data.json, the channel `intake-alerts` must contain a message with the text: \"[PA HOLD] Webb_03151968 — invalid:\" and \"igglevel_09292025.pdf\" or \"Most recent IgG lab result\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"slack_data.json not found in external_services_path\"}\n\n def normalize_dashes(s):\n # Treat em-dash, en-dash, figure-dash all as plain hyphen\n return s.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-')\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n # Find the intake-alerts channel id\n channels = slack_data.get('channels', {})\n intake_alerts_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'intake-alerts':\n intake_alerts_id = cid\n break\n\n if intake_alerts_id is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Could not find #intake-alerts channel in slack_data.json. \"\n f\"Available channels: {[v.get('name') for v in channels.values()]}\"}\n\n messages = slack_data.get('messages', {}).get(intake_alerts_id, [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Found #intake-alerts channel but no messages in it.\"}\n\n # Required prefix (must be present)\n required_prefix = normalize_dashes('[PA HOLD] Webb_03151968 — invalid:')\n # Alternatives (at least one must be present)\n alt_filename = 'igglevel_09292025.pdf'\n alt_phrase = 'Most recent IgG lab result'\n\n for msg in messages:\n msg_text = msg.get('text', '') if isinstance(msg, dict) else str(msg)\n msg_norm = normalize_dashes(msg_text)\n\n has_prefix = required_prefix in msg_norm\n has_alt = (alt_filename in msg_norm) or (alt_phrase.lower() in msg_norm.lower())\n\n if has_prefix and has_alt:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found matching [PA HOLD] message in #intake-alerts: '{msg_text}'\"}\n\n all_texts = [msg.get('text', '') if isinstance(msg, dict) else str(msg) for msg in messages]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": (f\"Did not find a message in #intake-alerts containing \"\n f\"'{required_prefix}' AND ('{alt_filename}' OR '{alt_phrase}'). \"\n f\"Messages found: {all_texts}\")}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775169561964",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be exactly 1 email in the Sent folder.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check both workspace and external services paths for mailbox.json\n mailbox_path = None\n \n # First check external_services_path\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in workspace or external services path.\"}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 'emails' to be a list, got {type(emails).__name__}.\"}\n \n sent_emails = [e for e in emails if isinstance(e, dict) and e.get(\"folder\", \"\").strip().lower() == \"sent\"]\n count = len(sent_emails)\n \n if count == 1:\n subject = sent_emails[0].get(\"subject\", \"(no subject)\")\n to_addr = sent_emails[0].get(\"to_addr\", \"(unknown)\")\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Exactly 1 email found in Sent folder. Subject: '{subject}', To: '{to_addr}'.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 email in Sent folder, but found {count}.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775175785924",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In the file `audit_log.xlsx` in folder `Webb_03151968`, there must be exactly 4 rows of data (excluding headers).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for audit_log.xlsx in the Webb_03151968 folder\n target_file = Path(workspace_path) / \"Webb_03151968\" / \"audit_log.xlsx\"\n \n # Also check case-insensitive and common variations\n if not target_file.exists():\n # Try to find it with glob\n candidates = list(Path(workspace_path).rglob(\"**/Webb_03151968/**/audit_log.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).rglob(\"**/Webb*/**/audit_log.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).rglob(\"**/audit_log.xlsx\"))\n if candidates:\n target_file = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File 'audit_log.xlsx' not found in folder 'Webb_03151968' or anywhere in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n # Count rows with data (excluding the header row)\n # We consider a row to have data if at least one cell in it is not None\n total_rows = 0\n header_skipped = False\n for row in ws.iter_rows(min_row=1, values_only=True):\n # Check if the row has any non-None value\n if any(cell is not None for cell in row):\n if not header_skipped:\n header_skipped = True\n continue # skip header\n total_rows += 1\n \n wb.close()\n \n if total_rows == 4:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 4 data rows (excluding headers) in 'audit_log.xlsx' within 'Webb_03151968'. File: {target_file}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 4 data rows (excluding headers) in 'audit_log.xlsx' within 'Webb_03151968', but found {total_rows} data rows. File: {target_file}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading 'audit_log.xlsx': {str(e)}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775176091454",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "In the workspace, there must be a folder titled `Webb_03151968`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n target_folder = ws / \"Webb_03151968\"\n if target_folder.is_dir():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found folder 'Webb_03151968' at {target_folder}.\"}\n # Also check case-insensitively and with minor variations just in case\n for item in ws.iterdir():\n if item.is_dir() and item.name.lower() == \"webb_03151968\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found folder '{item.name}' (case-insensitive match) at {item}.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Folder 'Webb_03151968' was not found in workspace. Directories found: {[d.name for d in ws.iterdir() if d.is_dir()]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775176111983",
|
|||
|
|
"sort_order": 15,
|
|||
|
|
"rubric_text": "In the workspace, there must be a folder titled `Rodriguez_07221975`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_folder = Path(workspace_path) / 'Rodriguez_07221975'\n if target_folder.is_dir():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found the required folder 'Rodriguez_07221975' at {target_folder}.\"}\n else:\n # Also check subdirectories in case it was created nested\n matches = list(Path(workspace_path).rglob('Rodriguez_07221975'))\n dirs = [m for m in matches if m.is_dir()]\n if dirs:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found the required folder 'Rodriguez_07221975' at {dirs[0]}.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a folder titled 'Rodriguez_07221975' anywhere in the workspace ({workspace_path}). Contents of workspace root: {[p.name for p in Path(workspace_path).iterdir()]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "2963fce0-442e-4466-8b4d-5888d08b8397",
|
|||
|
|
"sort_order": 16,
|
|||
|
|
"rubric_text": "In mailbox.json, the body with the sent email subject `Unable to Contact Patient | CASE_ID: Rodriguez_07221975` must contain the following in the same order: `Dear Dr. Robert Cho,` or `Dear Dr. Cho,` `We received a referral for Carmen Rodriguez (DOB: 07221975).`, `After 3 contact attempts, we have been unable to reach the patient`, `at the number on file: (555) 618-9043.`, `Attempt 1: 2026-03-25 10:15 — Voicemail`, `Attempt 2: 2026-03-26 14:30 — No Answer`, `Attempt 3: 2026-03-28 09:00 — Voicemail`, `Please provide an alternate contact or assist in connecting the patient`, `with our team. We cannot proceed until contact is established.`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json, re, unicodedata\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_file = Path(external_services_path) / 'mailbox.json'\n if not mailbox_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n try:\n with open(mailbox_file, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {e}'}\n\n def normalize(s):\n s = unicodedata.normalize('NFKC', s)\n s = s.lower()\n s = s.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-')\n s = s.replace('\\u2018', \"'\").replace('\\u2019', \"'\").replace('\\u201c', '\"').replace('\\u201d', '\"')\n s = re.sub(r'\\s+', ' ', s)\n return s.strip()\n\n expected_subject = 'Unable to Contact Patient | CASE_ID: Rodriguez_07221975'\n\n # Ordered list of required fragments. The salutation accepts either form.\n fragments = [\n (\"Salutation\", [\"Dear Dr. Robert Cho,\", \"Dear Dr. Cho,\"]),\n (\"Referral line\", [\"We received a referral for Carmen Rodriguez (DOB: 07221975).\"]),\n (\"Contact attempts\", [\"After 3 contact attempts, we have been unable to reach the patient\"]),\n (\"Phone number\", [\"at the number on file: (555) 618-9043.\"]),\n (\"Attempt 1\", [\"Attempt 1: 2026-03-25 10:15 — Voicemail\"]),\n (\"Attempt 2\", [\"Attempt 2: 2026-03-26 14:30 — No Answer\"]),\n (\"Attempt 3\", [\"Attempt 3: 2026-03-28 09:00 — Voicemail\"]),\n (\"Closing request\", [\"Please provide an alternate contact or assist in connecting the patient\"]),\n (\"Closing statement\", [\"with our team. We cannot proceed until contact is established.\"]),\n ]\n\n emails = data.get('emails', [])\n for email in emails:\n subject = email.get('subject', '')\n folder = email.get('folder', '')\n body = email.get('body_text', '')\n\n subject_match = normalize(subject) == normalize(expected_subject)\n folder_match = folder.strip().lower() == 'sent'\n\n if subject_match and folder_match:\n body_n = normalize(body)\n\n # Walk through fragments in order. For each one, find the earliest\n # accepted variant at or after the current cursor position.\n cursor = 0\n missing = []\n out_of_order = []\n for label, variants in fragments:\n best_pos = -1\n for v in variants:\n pos = body_n.find(normalize(v), cursor)\n if pos != -1 and (best_pos == -1 or pos < best_pos):\n best_pos = pos\n if best_pos == -1:\n # Check if it exists anywhere (before the cursor) to distinguish\n # \"missing\" from \"out of order\".\n anywhere = any(normalize(v) in body_n for v in variants)\n if anywhere:\n out_of_order.append(label)\n else:\n missing.append(label)\n else:\n cursor = best_pos + 1\n\n total = len(fragments)\n passed = total - len(missing) - len(out_of_order)\n score = round(passed / total, 2)\n\n if not missing and not out_of_order:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'Sent email with correct subject contains all {total} required body fragments in the correct order.'}\n else:\n problems = []\n if missing:\n problems.append(f'missing: {missing}')\n if out_of
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
}
|
|||
|
|
]
|