Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,128 @@
|
||||
[
|
||||
{
|
||||
"id": "292496ce-fecf-4a85-a8ac-244d4df88aae",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `claim.xlsx`, the row where `DOS` = `02/14/2026` must have `Claim_Status` = `Denied`, `ERA_Received_Date` = `04/08/2026` or 8 April 2026, `Payment_Posted_Date` blank/null, `Benefit_Tier_Billed` = `Medical`, `Paid_Amount` = `0` and `Claim_Frequency_Code` = `1`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find claim.xlsx - be flexible with naming\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n # Try glob for any xlsx with 'claim' in the name\n candidates = list(Path(workspace_path).glob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n # Also try recursively\n candidates = list(Path(workspace_path).rglob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n headers_upper = [h.upper().replace(' ', '_') for h in headers]\n\n # Find columns flexibly\n dos_col = None\n status_col = None\n era_received_col = None\n payment_posted_col = None\n tier_col = None\n freq_col = None\n paid_amount_col = None\n\n for i, h in enumerate(headers_upper):\n if dos_col is None and 'DOS' in h:\n dos_col = i\n if status_col is None and ('CLAIM_STATUS' in h or ('CLAIM' in h and 'STATUS' in h) or h == 'STATUS' or h == 'CLAIM_STATUS'):\n status_col = i\n if era_received_col is None and ('ERA' in h and 'RECEIVED' in h):\n era_received_col = i\n if payment_posted_col is None and ('PAYMENT_POSTED' in h or ('PAYMENT' in h and 'POSTED' in h)):\n payment_posted_col = i\n if tier_col is None and ('BENEFIT_TIER' in h or ('TIER' in h and 'BILLED' in h) or ('BENEFIT' in h and 'TIER' not in h)):\n tier_col = i\n if tier_col is None and 'BENEFIT' in h:\n tier_col = i\n if freq_col is None and ('FREQUENCY' in h or 'FREQ' in h):\n freq_col = i\n if paid_amount_col is None and ('PAID_AMOUNT' in h or ('PAID' in h and 'AMOUNT' in h) or h == 'PAID_AMOUNT'):\n paid_amount_col = i\n\n if dos_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'DOS column not found. Headers: {headers}'}\n\n # Find the target row with DOS = 02/14/2026\n target_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n dos_val = row[dos_col]\n if dos_val is None:\n continue\n dos_str = str(dos_val).strip()\n match = False\n if '02/14/2026' in dos_str or '2/14/2026' in dos_str or '2026-02-14' in dos_str:\n match = True\n if isinstance(dos_val, datetime.datetime):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n elif isinstance(dos_val, datetime.date):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n if match:\n target_row = row\n break\n\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=02/14/2026 found in claim.xlsx'}\n\n issues = []\n details = []\n\n # Check Claim_Status = Denied\n if status_col is not None:\n status_val = str(target_row[status_col]).strip().lower() if target_row[status_col] is not None else ''\n if 'denied' in status_val or 'deny' in status_val or 'denial' in status_val:\n details.append(f'Claim_Status={target_row[status_col]}')\n else:\n issues.append(f'Claim_Status expected Denied, got: {target_row[status_col]}')\n else:\n issues.append(f'Claim_Status column not found. Headers: {headers}')\n\n # Check ERA_Received_Date = 04/08/2026 or 8 April 2026\n if era_received_col is not None:\n era_val = target_row[era_received_col]\n era_match = False\n if era_val is not None:\n if isinstance(era_val, (datetime.datetime, datetime.date)):\n if era_val.year == 2026 and era_val.month == 4 and era_val.day == 8:\n era_match = True\n else:\n era_str = str(era_val).strip()\n if '04/08/2026' in era_str or '4/8/2026' in era_str or '2026-04-08' in era_str or '8 April 2026' in era_str or 'April 8' in era_str or '08-Apr' in era_str or '8-Apr' in era_str or '04-08-2026' in era_str:\n era_match = True\n if era_match:\n details.append(f'ERA_Received_Date={era_val}')\n else:\n issues.append(f'ERA_Received_Date expected 04/08/2026 (8 April 2026), got: {era_val}')\n else:\n issues.append(f'ERA_Received_Date column not found. Headers: {headers}')\n\n # Check Payment_Posted_Date blank/null\n if payment_posted_col is not None:\n payment_val = target_Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "826aeb1f-eea5-4a46-b9d1-3bb715ba3013",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In file `claim.xlsx`, total row count must be exactly 5 (1 header + 4 data rows).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n \n row_count = sum(1 for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row))\n \n if row_count == 5:\n return {'pass': True, 'score': 1.0, 'feedback': f'claim.xlsx has exactly 5 rows (1 header + 4 data rows)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'claim.xlsx has {row_count} non-empty rows, expected 5 (1 header + 4 data rows)'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f7996998-b356-45fc-a368-c4ad88757797",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "A file named `appeal_L1_02142026.pdf` must exist in the workspace.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n all_files = [f for f in ws.rglob('*') if f.is_file()]\n\n # --- Pass 1: exact stem match, PDF only ---\n target_stem = 'appeal_l1_02142026'\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n if f.stem.lower() == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found appeal_L1_02142026.pdf at {f.relative_to(ws)}'\n }\n\n # --- Pass 2: normalised separators (hyphens → underscores), PDF only ---\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n normalised = f.stem.lower().replace('-', '_').replace(' ', '_')\n if normalised == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found matching PDF with alternate separators: '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Pass 3: component match, PDF only ---\n date_variants = ['02142026', '02_14_2026', '02-14-2026']\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n n = f.name.lower()\n if 'appeal' in n and 'l1' in n and any(d in n for d in date_variants):\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found appeal letter PDF (component match): '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Diagnosis: did the model produce the file in a non-PDF format? ---\n non_pdf_matches = [\n f for f in all_files\n if f.suffix.lower() != '.pdf'\n and 'appeal' in f.name.lower()\n and 'l1' in f.name.lower()\n and any(d in f.name.lower() for d in date_variants)\n ]\n if non_pdf_matches:\n names = [f.name for f in non_pdf_matches]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'Appeal letter found but not in PDF format: {names}. '\n f'The criterion requires a .pdf file. '\n f'Rename or re-export the file as appeal_L1_02142026.pdf.'\n )\n }\n\n all_names = [f.name for f in all_files]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No appeal_L1_02142026.pdf found in workspace. '\n f'Expected a .pdf file whose name contains \"appeal\", \"l1\", '\n f'and date \"02142026\". '\n f'Files present: {all_names}'\n )\n }",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f7446601-8f7d-4712-9097-7620699d893c",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json` (external services), exactly 1 email must exist in the `Sent` folder with `to_addr` = `appeals@silverpath-claims.com`, with subject containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `SPA-00291847`, `DOS:`, `02/14/2026`, `Claim:`, `CLM-REF-20260218-9931`, `CARC:`, `CO-97`, and attachments containing exactly `appeal_L1_02142026.pdf`, `claim_02142026_Holloway_SilverPath.pdf`, `pa_approval_SilverPath_Holloway.pdf`, `lmn_DrOsei_Holloway.pdf`, `era_02142026_SilverPath.pdf`, and filename containing `visit_note`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find sent emails to the appeals address (case-insensitive)\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@silverpath-claims.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent email to appeals@silverpath-claims.com found'}\n\n if len(target_emails) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 sent email to appeals@silverpath-claims.com, found {len(target_emails)}'}\n\n email = target_emails[0]\n\n # Check subject for required substrings\n subject = str(email.get('subject', ''))\n required_substrings = [\n 'APPEAL L1',\n 'CareIG',\n 'Holloway',\n 'MemberID:',\n 'SPA-00291847',\n 'DOS:',\n '02/14/2026',\n 'Claim:',\n 'CLM-REF-20260218-9931',\n 'CARC:',\n 'CO-97'\n ]\n\n # Case-sensitive IDs that must match exactly\n exact_match_required = {'SPA-00291847', 'CLM-REF-20260218-9931', 'CO-97', '02/14/2026'}\n\n missing = []\n for s in required_substrings:\n if s in subject:\n continue\n # For non-exact fields, try case-insensitive\n if s not in exact_match_required and s.upper() in subject.upper():\n continue\n missing.append(s)\n\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Email subject missing required substrings: {missing}. Subject was: {subject}'}\n\n # Check attachments\n attachments = email.get('attachments', [])\n attach_names = []\n for a in attachments:\n if isinstance(a, str):\n attach_names.append(a)\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', str(a)))\n attach_names.append(name)\n else:\n attach_names.append(str(a))\n\n attach_names_lower = [n.lower() for n in attach_names]\n\n # Required exact attachment filenames\n required_exact_attachments = [\n 'appeal_L1_02142026.pdf',\n 'claim_02142026_Holloway_SilverPath.pdf',\n 'pa_approval_SilverPath_Holloway.pdf',\n 'lmn_DrOsei_Holloway.pdf',\n 'era_02142026_SilverPath.pdf'\n ]\n\n missing_attachments = []\n for req in required_exact_attachments:\n # Check case-insensitive\n found = any(req.lower() in n for n in attach_names_lower)\n if not found:\n missing_attachments.append(req)\n\n if missing_attachments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required attachments: {missing_attachments}. Found attachments: {attach_names}'}\n\n # Check for an attachment whose filename contains 'visit_note' (case-insensitive)\n found_visit_note = any('visit_note' in n.lower() for n in attach_names)\n if not found_visit_note:\n return {'pass': False, 'score': 0.0, 'feedback': f'No attachment with filename containing \"visit_note\" found. Found attachments: {attach_names}'}\n\n # Total expected: 5 exact + 1 visit_note = 6\n # Be lenient: as long as all required ones are present, pass even if there are extras\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Sent email to appeals@silverpath-claims.com found with correct subject substrings and all required attachments. Subject: {subject}. Attachments: {attach_names}'\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "74e31c33-9667-455a-8fd5-dd989a7231eb",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json` (external services), no email in any folder must have `to_addr` containing `billing@silverpath-claims.com` with a date of `2026-04-01` or later.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n \n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n cutoff = datetime(2026, 4, 1)\n \n violations = []\n for e in emails:\n to_addr = str(e.get('to_addr', ''))\n if 'billing@silverpath-claims.com' in to_addr.lower():\n date_str = str(e.get('date', ''))\n try:\n email_date = datetime.fromisoformat(date_str[:10])\n if email_date >= cutoff:\n violations.append(f\"Subject: {e.get('subject', 'N/A')}, Date: {date_str}\")\n except Exception:\n pass\n \n if violations:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found emails to billing@silverpath-claims.com on/after 2026-04-01: {violations}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No emails sent to billing@silverpath-claims.com on or after 2026-04-01 (no corrected claim submitted to payer)'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "bce26050-60a1-4108-8bf3-836752efd837",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "File `appeals_log.xlsx` must contain exactly 4 rows (1 header + 3 data rows), with exactly 1 row having `DOS` = `02/14/2026`, and that row must have `CARC_Code` = `CO-97`, `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, `Appeal_Status` = `Submitted`, `Payer` = `SilverPath Medicare Advantage`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) != 4:\n return {'pass': False, 'score': 0.0, 'feedback': f'appeals_log.xlsx has {len(rows)} rows, expected 4 (1 header + 3 data rows)'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n carc_col = col_idx('CARC')\n level_col = col_idx('LEVEL')\n subdate_col = col_idx('SUBMITTED_DATE') or col_idx('SUBMIT')\n status_col = col_idx('STATUS')\n payer_col = col_idx('PAYER')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '02/14/2026' in dos_val or '2026-02-14' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 row with DOS=02/14/2026 in appeals_log.xlsx, found {len(target_rows)}'}\n \n row = target_rows[0]\n issues = []\n \n if carc_col is not None:\n carc_val = str(row[carc_col]).strip() if row[carc_col] is not None else ''\n if 'CO-97' not in carc_val.upper() and 'CO97' not in carc_val.upper():\n issues.append(f'CARC_Code expected CO-97, got: {carc_val}')\n else:\n issues.append('CARC_Code column not found')\n \n if level_col is not None:\n level_val = str(row[level_col]).strip().upper() if row[level_col] is not None else ''\n if 'L1' not in level_val:\n issues.append(f'Appeal_Level expected L1, got: {row[level_col]}')\n else:\n issues.append('Appeal_Level column not found')\n \n if subdate_col is not None:\n date_val = str(row[subdate_col]).strip() if row[subdate_col] is not None else ''\n if '04/08/2026' not in date_val and '2026-04-08' not in date_val:\n issues.append(f'Appeal_Submitted_Date expected 04/08/2026, got: {date_val}')\n else:\n issues.append('Appeal_Submitted_Date column not found')\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if payer_col is not None:\n payer_val = str(row[payer_col]).strip() if row[payer_col] is not None else ''\n if 'SILVERPATH' not in payer_val.upper():\n issues.append(f'Payer expected SilverPath Medicare Advantage, got: {payer_val}')\n else:\n issues.append('Payer column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=02/14/2026 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx has 4 rows, DOS=02/14/2026 row has CARC_Code=CO-97, Appeal_Level=L1, Appeal_Submitted_Date=04/08/2026, Appeal_Status=Submitted, Payer=SilverPath Medicare Advantage'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "3e721c8d-53cd-47f9-a9a0-19515432c546",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `appeals_log.xlsx`, the row where `DOS` = `12/09/2025` must have `Appeal_Status` = `Submitted` and `Followup_Log` that is non-blank and contains a date matching `04/08/2026` or `8 April 2026`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx has no data rows'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n status_col = col_idx('STATUS')\n followup_col = col_idx('FOLLOWUP') or col_idx('FOLLOW_UP') or col_idx('FOLLOW')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '12/09/2025' in dos_val or '2025-12-09' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=12/09/2025 found in appeals_log.xlsx'}\n \n row = target_rows[0]\n issues = []\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if followup_col is not None:\n followup_val = str(row[followup_col]).strip() if row[followup_col] is not None else ''\n if followup_val in ('', 'None', 'nan', 'NaN', 'NULL'):\n issues.append('Followup_Log is blank')\n else:\n has_date = '04/08/2026' in followup_val or '2026-04-08' in followup_val or '8 april 2026' in followup_val.lower() or 'april 8, 2026' in followup_val.lower()\n if not has_date:\n issues.append(f'Followup_Log does not contain date 04/08/2026 or 8 April 2026. Value: {followup_val}')\n else:\n issues.append('Followup_Log column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=12/09/2025 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx DOS=12/09/2025 row has Appeal_Status=Submitted and Followup_Log containing 04/08/2026'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "fc648229-7c0f-4abc-9ba6-d8d05942a39a",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `mailbox.json` (external services), exactly 3 emails must exist in the `Sent` folder with `to_addr` = `appeals@bluecrest-health.com`. The 2 of the emails must have subjects containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `BCR-00184523`, `DOS:`, `12/09/2025`, `Claim:`, `CLM-REF-20251215-6612`, `CARC:`, `PI-204`, email attachment name = `appeal_L1_12092025.pdf`, email body containing `BlueCrest Commercial PPO — Provider Appeals Department`, `Member ID: BCR-00184523`, `DOS: 12/09/2025`, `Claim Reference: CLM-REF-20251215-6612`, `Billed: $5,840.00`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find all sent emails to appeals@bluecrest-health.com\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@bluecrest-health.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails to appeals@bluecrest-health.com, found {len(target_emails)}'}\n\n # Required substrings in the subject (case-insensitive, whitespace-flexible)\n required_subject_substrings = [\n 'APPEAL L1', 'CareIG', 'Holloway', 'MemberID:',\n 'BCR-00184523', 'DOS:', '12/09/2025', 'Claim:',\n 'CLM-REF-20251215-6612', 'CARC:', 'PI-204'\n ]\n\n # Required body substrings\n required_body_substrings = [\n 'BlueCrest Commercial PPO',\n 'Provider Appeals Department',\n 'Member ID: BCR-00184523',\n 'DOS: 12/09/2025',\n 'Claim Reference: CLM-REF-20251215-6612',\n 'Billed: $5,840.00'\n ]\n\n required_attachment = 'appeal_L1_12092025.pdf'\n\n def normalize(text):\n \"\"\"Normalize whitespace for flexible matching.\"\"\"\n return re.sub(r'\\s+', ' ', str(text)).strip()\n\n def check_subject(subject):\n subj_norm = normalize(subject).lower()\n # Also try without spaces around colons\n subj_no_space = re.sub(r'\\s*:\\s*', ':', subj_norm)\n missing = []\n for s in required_subject_substrings:\n s_lower = s.lower()\n s_no_space = re.sub(r'\\s*:\\s*', ':', s_lower)\n # Try multiple matching strategies\n if s_lower in subj_norm:\n continue\n if s_no_space in subj_no_space:\n continue\n # Try without space: \"MemberID:\" might appear as \"Member ID:\" or \"MemberID:\"\n if s == 'MemberID:':\n if 'memberid:' in subj_norm or 'member id:' in subj_norm or 'memberid :' in subj_norm:\n continue\n missing.append(s)\n return missing\n\n def check_body(body):\n body_norm = normalize(body)\n missing = []\n for s in required_body_substrings:\n # Case-insensitive check with normalized whitespace\n s_norm = normalize(s)\n if s_norm.lower() in body_norm.lower():\n continue\n # Try with em-dash variants\n s_variants = [\n s_norm,\n s_norm.replace('\\u2014', '-'),\n s_norm.replace('\\u2014', '--'),\n s_norm.replace('-', '\\u2014'),\n ]\n found = False\n for variant in s_variants:\n if variant.lower() in body_norm.lower():\n found = True\n break\n if not found:\n missing.append(s)\n return missing\n\n def get_attachment_names(attachments):\n names = []\n for a in (attachments or []):\n if isinstance(a, str):\n names.append(a.strip())\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', a.get('file_name', '')))\n names.append(str(name).strip())\n else:\n names.append(str(a).strip())\n return names\n\n def check_attachment(email_obj):\n att_names = get_attachment_names(email_obj.get('attachments', []))\n for name in att_names:\n if name.lower() == required_attachment.lower():\n return True\n return False\n\n # Find emails that match all criteria\n fully_matching = []\n partial_details = []\n\n for email in target_emails:\n subject = str(email.get('subject', ''))\n body = str(email.get('body_text', email.get('body', '')))\n\n subj_missing = check_subject(subject)\n body_missing = check_body(body)\n has_attachment = check_attachment(email)\n\n if not subj_missing and not body_missing and has_attachment:\n fully_matching.append(email)\n else:\n partial_details.append({\n 'subject': subject[:100],\n 'subj_missing': subj_missing,\n 'body_missing': body_missing,\n 'has_attachment': has_attachment\n })\nLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "52595972-13bc-49f5-b35b-4674e117b8ff",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `slack_data.json` (external services), channel `billing-appeals` must contain a message with text containing all of: `APPEAL FILED L1`, `Holloway_DOB_03141961`, `02/14/2026`, `SilverPath`, `CO-97`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check Slack'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n \n with open(slack_path, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n messages = data.get('messages', {})\n \n # Find #billing-appeals channel (be flexible with naming)\n billing_appeals_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if ch_name == 'billingappeals':\n billing_appeals_id = ch_id\n break\n \n if billing_appeals_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Channel #billing-appeals not found in slack_data.json. Available channels: {[ch.get(\"name\") for ch in channels.values()]}'}\n \n channel_messages = messages.get(billing_appeals_id, [])\n \n # Required content pieces to find in a message\n # Be loose: case-insensitive where appropriate, flexible date formats, flexible separators\n required_checks = {\n 'APPEAL FILED L1': lambda t: 'appeal' in t.lower() and ('filed' in t.lower() or 'file' in t.lower()) and ('l1' in t.lower() or 'level 1' in t.lower() or 'level1' in t.lower()),\n 'Holloway_DOB_03141961': lambda t: 'holloway' in t.lower() and ('03141961' in t.replace('/', '').replace('-', '').replace(' ', '') or '03/14/1961' in t or '03-14-1961' in t or '3/14/1961' in t),\n '02/14/2026': lambda t: any(d in t for d in ['02/14/2026', '2/14/2026', '02-14-2026', '2-14-2026', '2026-02-14', '02142026']),\n 'SilverPath': lambda t: 'silverpath' in t.lower().replace(' ', '').replace('-', '').replace('_', ''),\n 'CO-97': lambda t: 'co-97' in t.lower() or 'co 97' in t.lower() or 'co97' in t.lower(),\n }\n \n best_match_count = 0\n best_match_text = ''\n matched_keys_best = []\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n \n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n \n if len(matched_keys) == len(required_checks):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found matching message in #billing-appeals containing all required elements: {text[:300]}'}\n \n # Also check all messages across all channels in case channel name matching failed\n for ch_id, ch_msgs in messages.items():\n if ch_id == billing_appeals_id:\n continue\n for msg in ch_msgs:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n # Note: don't pass here since it's wrong channel, but track for feedback\n \n missing = [k for k in required_checks if k not in matched_keys_best]\n return {\n 'pass': False,\n 'score': best_match_count / len(required_checks) * 0.5, # partial credit\n 'feedback': f'No message in #billing-appeals found containing all required elements. '\n f'Best match had {best_match_count}/{len(required_checks)} elements. '\n f'Missing: {missing}. '\n f'Found {len(channel_messages)} messages in #billing-appeals. '\n f'Best matching text: \"{best_match_text[:300]}\"'\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "29097e98-d53d-4af6-ac8a-8df8f3ec442e",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `audit_log.xlsx`, total row count must be exactly 37 (1 header + 36 data rows).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find audit_log.xlsx - be flexible with location\n audit_path = Path(workspace_path) / 'audit_log.xlsx'\n if not audit_path.exists():\n # Try glob search\n candidates = list(Path(workspace_path).rglob('audit_log.xlsx'))\n if candidates:\n audit_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n\n try:\n wb = openpyxl.load_workbook(audit_path)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = wb.active\n\n # Collect all non-empty rows\n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n\n total_row_count = len(rows)\n\n # Rubric says exactly 37 rows (1 header + 36 data rows)\n if total_row_count != 37:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'audit_log.xlsx has {total_row_count} non-empty rows, expected exactly 37 (1 header + 36 data rows).'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'audit_log.xlsx has exactly {total_row_count} non-empty rows (1 header + 36 data rows), as required.'\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775206985975",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "File `DENIAL_CO97_02142026.xlsx` must exist and contain exactly 2 rows (1 header + 1 data row) with the following values in the data row: `CASE_ID` = `Holloway_DOB_03141961`, `CARC_Code` = `CO-97`, `CARC_Description` = `Benefit not assigned`, `DOS` = `02/14/2026`, `Payer` = `SilverPath Medicare Advantage`, `Denied_Code` containing `J1560`, `99600`, `99603`, `J7030`, `J0171`, `A4221`, and `Denied_Amount` = `6140.00` (numerical or string). It must also have `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, and `Appeal_Status` = `Submitted`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with flexible matching\n target_file = None\n exact_path = Path(workspace_path) / \"DENIAL_CO97_02142026.xlsx\"\n if exact_path.exists():\n target_file = exact_path\n else:\n # Try glob for similar filenames\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*02142026*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*denial*co*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*denial*co*97*.xlsx\"))\n if not candidates:\n # Even more lenient: any xlsx with denial and 97\n candidates = list(Path(workspace_path).glob(\"**/*enial*97*.xlsx\"))\n if not candidates:\n # Try any xlsx with denial in name\n candidates = list(Path(workspace_path).glob(\"**/*enial*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ENIAL*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n # List all xlsx files to help debug\n all_xlsx = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File DENIAL_CO97_02142026.xlsx not found in workspace. Found xlsx files: {[str(f.name) for f in all_xlsx]}\"}\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n ws = wb.active\n # If active sheet is empty, try other sheets\n if ws.max_row < 2:\n for sheet_name in wb.sheetnames:\n candidate = wb[sheet_name]\n if candidate.max_row >= 2:\n ws = candidate\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n rows = list(ws.iter_rows(values_only=False))\n if len(rows) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 rows (1 header + 1 data), but found {len(rows)} rows.\"}\n\n # Be lenient: ignore completely empty rows beyond row 2\n non_empty = [r for r in rows if any(c.value is not None for c in r)]\n if len(non_empty) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n if len(non_empty) > 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n\n # Build header map from first non-empty row\n header_row = non_empty[0]\n headers = {}\n for idx, cell in enumerate(header_row):\n if cell.value is not None:\n headers[str(cell.value).strip()] = idx\n\n # Helper to find a column by flexible name matching\n def find_col(possible_names):\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if h_lower == pn_lower:\n return idx\n # Also try substring matching\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if pn_lower in h_lower or h_lower in pn_lower:\n return idx\n return None\n\n data_row = non_empty[1]\n\n def cell_val(row, col_idx):\n if col_idx is None:\n return None\n if col_idx < len(row):\n return row[col_idx].value\n return None\n\n def normalize_str(v):\n if v is None:\n return \"\"\n return str(v).strip()\n\n def normalize_date(v):\n \"\"\"Try to normalize a date value to MM/DD/YYYY for comparison.\"\"\"\n if v is None:\n return \"\"\n if isinstance(v, (datetime.datetime, datetime.date)):\n return v.strftime(\"%m/%d/%Y\")\n s = str(v).strip()\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\", \"%d/%m/%Y\"]:\n try:\n dt = datetime.datetime.strptime(s, fmt)\n return dt.strftime(\"Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775268135625",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "File `era_02142026_SilverPath.pdf` must exist in the workspace.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"era_02142026_SilverPath.pdf\"\n if target_file.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' exists at {target_file}.\"}\n else:\n # Also do a case-insensitive search as a generous fallback\n found = [f for f in Path(workspace_path).rglob(\"*\") if f.name.lower() == \"era_02142026_silverpath.pdf\"]\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File found (case-insensitive match): {found[0]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' was NOT found in the workspace at '{workspace_path}'. Files present: {[f.name for f in Path(workspace_path).rglob('*') if f.is_file()]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775268234332",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "File `statement_02142026.pdf` must not exist in the workspace.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"statement_02142026.pdf\"\n if target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'statement_02142026.pdf' still exists in the workspace at '{target_file}'. It should have been removed.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'statement_02142026.pdf' does not exist in the workspace, as expected.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775268384702",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `mailbox.json`, an email with the following details must not exist: `to_addr` = `m.holloway@emailhost.com`, subject containing `CareIG Statement of Account` and `02/14/2026` (or the equivalent date in any format).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the absence of a specific email.\n # The mailbox.json could be in the workspace or in external_services_path.\n \n mailbox_path = None\n \n # Check external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json found at all, the email trivially doesn't exist -> PASS\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n # Define date patterns that all represent February 14, 2026\n date_patterns = [\n r'02/14/2026',\n r'02-14-2026',\n r'2/14/2026',\n r'2-14-2026',\n r'14/02/2026',\n r'14-02-2026',\n r'2026-02-14',\n r'2026/02/14',\n r'Feb(?:ruary)?\\s*14,?\\s*2026',\n r'14\\s*Feb(?:ruary)?,?\\s*2026',\n ]\n \n def subject_contains_date(subject):\n for pat in date_patterns:\n if re.search(pat, subject, re.IGNORECASE):\n return True\n return False\n \n violating_emails = []\n \n for email in emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n subject = email.get(\"subject\", \"\")\n \n # Check to_addr matches\n if to_addr != \"m.holloway@emailhost.com\":\n continue\n \n # Check subject contains \"CareIG Statement of Account\" (case-insensitive)\n if \"careig statement of account\" not in subject.lower():\n continue\n \n # Check subject contains the date 02/14/2026 in any format\n if subject_contains_date(subject):\n violating_emails.append(email)\n \n if len(violating_emails) > 0:\n details = []\n for e in violating_emails:\n details.append(f\" - email_id={e.get('email_id','?')}, to={e.get('to_addr','?')}, subject='{e.get('subject','?')}', folder={e.get('folder','?')}\")\n detail_str = \"\\n\".join(details)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) that should NOT exist (to=m.holloway@emailhost.com, subject containing 'CareIG Statement of Account' and date 02/14/2026):\\n{detail_str}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No email found in mailbox.json with to_addr='m.holloway@emailhost.com' and subject containing both 'CareIG Statement of Account' and the date 02/14/2026.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775268649908",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "The file `appeal_L1_02142026.pdf` must contain the following:\n- Date: `04/08/2026`\n- Patient name: `Margaret` `Holloway`\n- Member ID: `SPA-00291847`\n- Date of Service (DOS): `02/14/2026`\n- Claim #: `CLM-REF-20260218-9931`\n- CARC: `CO-97`\n- Billed amount: `$6,140.00`\n- The letter body must contain: `#SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026` and `D83.9`.\n- Contracted rate: '$4,800.00'",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n pdf_file = None\n\n # Search for the file - try exact name and case-insensitive\n target_name = \"appeal_L1_02142026.pdf\"\n candidates = list(workspace.rglob(\"*.pdf\"))\n for c in candidates:\n if c.name.lower() == target_name.lower():\n pdf_file = c\n break\n\n # Also try broader matching in case of slight name variations\n if pdf_file is None:\n for c in candidates:\n if \"appeal\" in c.name.lower() and \"02142026\" in c.name:\n pdf_file = c\n break\n\n if pdf_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{target_name}' not found in workspace. PDF files found: {[str(c.relative_to(workspace)) for c in candidates]}\"}\n\n # Extract text from PDF\n text = \"\"\n try:\n import pdfplumber\n with pdfplumber.open(str(pdf_file)) as pdf:\n for page in pdf.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e1:\n try:\n import PyPDF2\n with open(str(pdf_file), 'rb') as f:\n reader = PyPDF2.PdfReader(f)\n for page in reader.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read PDF. pdfplumber error: {e1}; PyPDF2 error: {e2}\"}\n\n if not text.strip():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PDF file exists but no text could be extracted.\"}\n\n # Normalize text for searching\n text_lower = text.lower()\n # Also create a version with collapsed whitespace for matching longer strings\n text_collapsed = re.sub(r'\\s+', ' ', text).strip()\n text_collapsed_lower = text_collapsed.lower()\n\n checks = []\n total = 0\n passed = 0\n\n # Helper function for flexible matching\n def check_present(label, patterns, required=True):\n nonlocal total, passed\n total += 1\n for pattern in patterns:\n if pattern.lower() in text_collapsed_lower:\n passed += 1\n checks.append(f\"PASS: {label} - found '{pattern}'\")\n return True\n checks.append(f\"FAIL: {label} - none of {patterns} found in PDF text\")\n return False\n\n # 1. Date = 04/08/2026\n check_present(\"Date 04/08/2026\", [\n \"04/08/2026\", \"04-08-2026\", \"april 8, 2026\", \"april 08, 2026\",\n \"4/8/2026\", \"8 april 2026\", \"08 april 2026\"\n ])\n\n # 2. Margaret\n check_present(\"Patient first name 'Margaret'\", [\"margaret\"])\n\n # 3. Holloway\n check_present(\"Patient last name 'Holloway'\", [\"holloway\"])\n\n # 4. Member ID SPA-00291847\n check_present(\"Member ID SPA-00291847\", [\n \"spa-00291847\", \"spa00291847\", \"spa 00291847\"\n ])\n\n # 5. DOS 02/14/2026\n check_present(\"DOS 02/14/2026\", [\n \"02/14/2026\", \"02-14-2026\", \"2/14/2026\", \"february 14, 2026\",\n \"february 14 2026\", \"14 february 2026\"\n ])\n\n # 6. Claim # CLM-REF-20260218-9931\n check_present(\"Claim # CLM-REF-20260218-9931\", [\n \"clm-ref-20260218-9931\", \"clm ref 20260218 9931\"\n ])\n\n # 7. CARC CO-97\n check_present(\"CARC CO-97\", [\n \"co-97\", \"co 97\", \"co97\"\n ])\n\n # 8. Billed $6,140.00\n check_present(\"Billed $6,140.00\", [\n \"$6,140.00\", \"$6140.00\", \"6,140.00\", \"6140.00\", \"$ 6,140.00\", \"$6,140\"\n ])\n\n # 9. Letter body: #SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026\n # Check for the key components since exact whitespace/formatting may vary\n spa_pa_found = False\n spa_patterns = [\"spa-pa-2026-3302\"]\n for p in spa_patterns:\n if p in text_collapsed_lower:\n spa_pa_found = True\n break\n\n octagam_found = False\n octagam_patterns = [\"octagam 10% 40g\", \"octagam 10% 40 g\", \"octagam10% 40g\", \"octagam 10%, 40g\", \"octagam 10% 40g\"]\n for p in octagam_patterns:\n if p in text_collapsed_lower:\n octagam_found = True\n break\n # Also try regex for more flexible matching\n if not octagam_found:\n if re.search(r'octagam\\s*10\\s*%\\s*,?\\s*40\\s*g', text_collapsed_lower):\n octagam_found = True\n\n dates_found = False\n start_date_found = any(d in text_collapsed for d in [\"01/28/2026\", \"1/28/2026\", \"01-28-2026\"]) or \\\n any(d in text_collapsed_lower for d in [\"january 28, 2026\", \"january 28 2026Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775269539963",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `audit_log.xlsx`, rows 36 and 37 should have: `CASE_ID` = `Holloway_DOB_03141961`, the date in `Datetime` = `04/08/2026`, `Action` = `Denial Received — CO-97` (case-insensitive) for row 36, and `Action` = `Appeal Submitted` (case-insensitive) for row 37.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"audit_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening audit_log.xlsx: {e}\"}\n\n # Find column indices by header row (row 1)\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip().upper()] = col_idx\n\n # Try to locate relevant columns\n case_id_col = None\n datetime_col = None\n action_col = None\n\n for key, idx in headers.items():\n if 'CASE' in key and 'ID' in key:\n case_id_col = idx\n if key == 'DATETIME': # FIX: was ('DATE' in key or 'TIME' in key or key == 'DATETIME')\n datetime_col = idx # 'DATE' in key matches FILE_UPDATED and FIELD_UPDATED,\n if key == 'ACTION': # overwriting datetime_col with the wrong column index.\n action_col = idx # Exact match eliminates the substring collision.\n\n if case_id_col is None or datetime_col is None or action_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find required columns. Headers found: {headers}\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_case_id(val):\n normed = normalize(val).lower()\n return normed == \"holloway_dob_03141961\"\n\n def check_date(val):\n if val is None:\n return False\n import datetime as dt\n if isinstance(val, (dt.datetime, dt.date)):\n d = val.date() if isinstance(val, dt.datetime) else val\n return d.month == 4 and d.day == 8 and d.year == 2026\n s = normalize(val)\n patterns = [\n r'0?4[/\\-]0?8[/\\-]2026',\n r'2026[/\\-]0?4[/\\-]0?8',\n r'April\\s+8,?\\s+2026',\n r'8\\s+April\\s+2026',\n r'04082026',\n ]\n for p in patterns:\n if re.search(p, s, re.IGNORECASE):\n return True\n return False\n\n def check_action_denial(val):\n normed = normalize(val).lower()\n normed_dashes = re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\-]+', '-', normed)\n if 'denial received' in normed_dashes and ('co-97' in normed_dashes or 'co 97' in normed_dashes or 'co97' in normed_dashes):\n return True\n return False\n\n def check_action_appeal(val):\n normed = normalize(val).lower()\n return 'appeal submitted' in normed\n\n # Check row 36\n row36_case = ws.cell(row=36, column=case_id_col).value\n row36_date = ws.cell(row=36, column=datetime_col).value\n row36_action = ws.cell(row=36, column=action_col).value\n\n # Check row 37\n row37_case = ws.cell(row=37, column=case_id_col).value\n row37_date = ws.cell(row=37, column=datetime_col).value\n row37_action = ws.cell(row=37, column=action_col).value\n\n issues = []\n passes = []\n\n if check_case_id(row36_case):\n passes.append(f\"Row 36 CASE_ID: '{row36_case}' matches\")\n else:\n issues.append(f\"Row 36 CASE_ID: expected 'Holloway_DOB_03141961', got '{row36_case}'\")\n\n if check_date(row36_date):\n passes.append(f\"Row 36 Datetime: '{row36_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 36 Datetime: expected date 04/08/2026, got '{row36_date}'\")\n\n if check_action_denial(row36_action):\n passes.append(f\"Row 36 Action: '{row36_action}' matches 'Denial Received — CO-97'\")\n else:\n issues.append(f\"Row 36 Action: expected 'Denial Received — CO-97', got '{row36_action}'\")\n\n if check_case_id(row37_case):\n passes.append(f\"Row 37 CASE_ID: '{row37_case}' matches\")\n else:\n issues.append(f\"Row 37 CASE_ID: expected 'Holloway_DOB_03141961', got '{row37_case}'\")\n\n if check_date(row37_date):\n passes.append(f\"Row 37 Datetime: '{row37_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 37 Datetime: expected date 04/08/2026, got '{row37_date}'\")\n\n if check_action_appeal(row37_action):\n passes.append(f\"Row 37 Action: '{row37_action}' matches 'Appeal Submitted'\")\n else:\n issues.append(f\"Row 37 Action: expected 'Appeal Submitted', got '{row37_action}'\")\n\n total_checks = 6\n passed_checks = total_checks - len(issues)\n score = passed_checks / total_checks\n\n if len(issues) == 0:\n return {\"pLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775271368334",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must contain 2 messages with text containing all of: `APPEAL`, `L1`, `Holloway_DOB_03141961`, `12/09/2025`, `BlueCrest Commercial PPO`, `PI-204` and with exactly one message containing `FOLLOW-UP`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_path}\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n if not isinstance(slack_data, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json top-level is not a dict.\"}\n\n # Collect all messages from billing-appeals channel using multiple possible structures\n all_channel_messages = []\n\n channels_map = slack_data.get(\"channels\", {})\n messages_map = slack_data.get(\"messages\", {})\n\n # Standard format: channels dict keyed by channel ID, messages dict keyed by channel ID\n if isinstance(channels_map, dict) and isinstance(messages_map, dict):\n for ch_id, ch_info in channels_map.items():\n ch_name = \"\"\n if isinstance(ch_info, dict):\n ch_name = ch_info.get(\"name\", \"\") or ch_info.get(\"channel_name\", \"\") or \"\"\n elif isinstance(ch_info, str):\n ch_name = ch_info\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n msgs = messages_map.get(ch_id, [])\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n # Also check if messages_map is keyed by channel name directly\n if not all_channel_messages:\n for key, msgs in messages_map.items():\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n\n # Fallback: channels is a dict with channel names as keys containing messages\n if not all_channel_messages and isinstance(channels_map, dict):\n for ch_name, ch_data in channels_map.items():\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n all_channel_messages.extend(ch_data[\"messages\"])\n elif isinstance(ch_data, list):\n all_channel_messages.extend(ch_data)\n break\n\n # Fallback: channels is a list of channel objects\n if not all_channel_messages and isinstance(channels_map, list):\n for ch in channels_map:\n if isinstance(ch, dict):\n ch_name = ch.get(\"name\", \"\") or ch.get(\"channel_name\", \"\")\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n all_channel_messages.extend(ch.get(\"messages\", []))\n break\n\n # Fallback: top-level keys are channel names\n if not all_channel_messages:\n for key in slack_data:\n if key in (\"channels\", \"messages\", \"users\", \"counters\"):\n continue\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n val = slack_data[key]\n if isinstance(val, dict) and \"messages\" in val:\n all_channel_messages.extend(val[\"messages\"])\n elif isinstance(val, list):\n all_channel_messages.extend(val)\n break\n\n if not all_channel_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find any messages in #billing-appeals channel in slack_data.json. Top-level keys: {list(slack_data.keys())}\"\n }\n\n def get_text(msg):\n if isinstance(msg, dict):\n return msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"content\", \"\") or msg.get(\"body\", \"\") or \"\"\n elif isinstance(msg, str):\n return msg\n return \"\"\n\n def term_in_text(term, text):\n \"\"\"Check if a required term appears in text, with flexible matching.\"\"\"\n text_upper = text.upper()\n term_upper = term.upper()\n if term_upper in text_upper:\n return True\n # Flexible matching for specific terms\n if term == \"Holloway_DOB_03141961\":\n pattern = re.compile(r\"holloway[\\s_\\-]*dob[\\s_\\-]*0?3[\\s_\\-/]*14[\\s_\\-/]*1961\", re.IGNORECASE)\Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775274130585",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must have a message containing: `[DENIAL]`, `Holloway_DOB_03141961`, and `CO-97`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_file = Path(external_services_path) / \"slack_data.json\"\n if not slack_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_file}\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n # According to the documented Slack data format:\n # \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } }\n # \"messages\": { \"C001\": [ { \"text\": \"...\", ... } ] }\n\n # Step 1: Find the channel ID for billing-appeals\n channels = slack_data.get(\"channels\", {})\n messages_by_channel = slack_data.get(\"messages\", {})\n\n billing_appeals_channel_id = None\n\n if isinstance(channels, dict):\n for cid, cinfo in channels.items():\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\":\n billing_appeals_channel_id = cid\n break\n # Also check loosely\n if \"billing\" in cname and \"appeal\" in cname:\n billing_appeals_channel_id = cid\n break\n elif isinstance(channels, list):\n for cinfo in channels:\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\" or (\"billing\" in cname and \"appeal\" in cname):\n billing_appeals_channel_id = cinfo.get(\"id\", cname)\n break\n\n # Collect message texts from the identified channel\n message_texts = []\n\n if billing_appeals_channel_id and isinstance(messages_by_channel, dict):\n channel_msgs = messages_by_channel.get(billing_appeals_channel_id, [])\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for key in [\"text\", \"message\", \"content\", \"body\"]:\n if key in msg and isinstance(msg[key], str):\n message_texts.append(msg[key])\n elif isinstance(msg, str):\n message_texts.append(msg)\n\n # Also try direct key lookup by name in messages dict (fallback)\n if not message_texts and isinstance(messages_by_channel, dict):\n for key in messages_by_channel:\n key_lower = key.lower().replace(\"#\", \"\").strip()\n if key_lower == \"billing-appeals\" or (\"billing\" in key_lower and \"appeal\" in key_lower):\n channel_msgs = messages_by_channel[key]\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for mkey in [\"text\", \"message\", \"content\", \"body\"]:\n if mkey in msg and isinstance(msg[mkey], str):\n message_texts.append(msg[mkey])\n elif isinstance(msg, str):\n message_texts.append(msg)\n break\n\n # Fallback: recursively search the entire structure for billing-appeals messages\n if not message_texts:\n def find_channel_messages(data):\n results = []\n if isinstance(data, dict):\n for key, value in data.items():\n key_lower = key.lower().replace('#', '').replace(' ', '-')\n if 'billing' in key_lower and 'appeal' in key_lower:\n results.extend(_extract_texts(value))\n else:\n results.extend(find_channel_messages(value))\n elif isinstance(data, list):\n for item in data:\n if isinstance(item, dict):\n name = (item.get('name', '') or item.get('channel_name', '') or '').lower().replace('#', '').replace(' ', '-')\n if 'billing' in name and 'appeal' in name:\n results.extend(_extract_texts(item.get('messages', item)))\n else:\n results.extend(find_channel_messages(item))\n return results\n\n def _extract_texts(obj):\n texts = []\n if isinstance(obj, lLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
Date: Wednesday, 8 April 2026 - Morning shift
|
||||
It's the start of your morning shift as billing specialist - work through your inbox and the active case files, and take care of anything that needs actioning today in line with the CareIG SOP. All relevant files are available in your workspace.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user