Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,37 @@
[
{
"id": "495d3ee8-810b-47fd-8706-2838d3b884ca",
"sort_order": 0,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cells for CLM-1008 (Harold Jensen) in row 9 must have: M9=210.00 (Allowed Amount), N9=180.00 (Paid Amount), O9=70.00 (Contractual Adj), P9=30.00 (Patient Rep), Q9='Paid - Full' (Claim Status), R9='Posted' (Payment Posting Status), Y9='20260315_BCBS' (ERA Batch ID)",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find the claim register file - be loose with naming\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n # Try any xlsx file\n for f in xlsx_files:\n if 'claim' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'No claim register xlsx file found in workspace. Files found: ' + str([f.name for f in xlsx_files])}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n # First, try to find the row for CLM-1008 / Harold Jensen dynamically\n target_row = 9 # default from rubric\n for row in range(1, ws.max_row + 1):\n for col in range(1, min(ws.max_column + 1, 10)):\n cell_val = ws.cell(row=row, column=col).value\n if cell_val is not None and 'CLM-1008' in str(cell_val).upper().replace(' ', ''):\n target_row = row\n break\n \n issues = []\n found_values = {}\n \n # Helper to get cell value by column letter and row\n def get_cell(col_letter, row):\n return ws[f'{col_letter}{row}'].value\n \n # Check M: Allowed Amount = 210.00\n m_val = get_cell('M', target_row)\n try:\n m_num = float(m_val) if m_val is not None else None\n if m_num is None or abs(m_num - 210.00) > 210.00 * 0.02:\n issues.append(f'M{target_row} (Allowed Amount) expected 210.00, got {m_val}')\n else:\n found_values['M'] = m_num\n except (TypeError, ValueError):\n issues.append(f'M{target_row} (Allowed Amount) expected 210.00, got {m_val}')\n \n # Check N: Paid Amount = 180.00\n n_val = get_cell('N', target_row)\n try:\n n_num = float(n_val) if n_val is not None else None\n if n_num is None or abs(n_num - 180.00) > 180.00 * 0.02:\n issues.append(f'N{target_row} (Paid Amount) expected 180.00, got {n_val}')\n else:\n found_values['N'] = n_num\n except (TypeError, ValueError):\n issues.append(f'N{target_row} (Paid Amount) expected 180.00, got {n_val}')\n \n # Check O: Contractual Adj = 70.00\n o_val = get_cell('O', target_row)\n try:\n o_num = float(o_val) if o_val is not None else None\n if o_num is None or abs(o_num - 70.00) > 70.00 * 0.02:\n issues.append(f'O{target_row} (Contractual Adj) expected 70.00, got {o_val}')\n else:\n found_values['O'] = o_num\n except (TypeError, ValueError):\n issues.append(f'O{target_row} (Contractual Adj) expected 70.00, got {o_val}')\n \n # Check P: Patient Rep = 30.00\n p_val = get_cell('P', target_row)\n try:\n p_num = float(p_val) if p_val is not None else None\n if p_num is None or abs(p_num - 30.00) > 30.00 * 0.02:\n issues.append(f'P{target_row} (Patient Rep) expected 30.00, got {p_val}')\n else:\n found_values['P'] = p_num\n except (TypeError, ValueError):\n issues.append(f'P{target_row} (Patient Rep) expected 30.00, got {p_val}')\n \n # Check Q: Claim Status = 'Paid - Full'\n q_val = get_cell('Q', target_row)\n q_str = str(q_val).strip() if q_val is not None else ''\n if q_val is None or ('paid' not in q_str.lower() or 'full' not in q_str.lower()):\n issues.append(f'Q{target_row} (Claim Status) expected \"Paid - Full\", got {q_val}')\n else:\n found_values['Q'] = q_str\n \n # Check R: Payment Posting Status = 'Posted'\n r_val = get_cell('R', target_row)\n r_str = str(r_val).strip() if r_val is not None else ''\n if r_val is None or 'posted' not in r_str.lower():\n issues.append(f'R{target_row} (Payment Posting Status) expected \"Posted\", got {r_val}')\n else:\n found_values['R'] = r_str\n \n # Check Y: ERA Batch ID = '20260315_BCBS'\n y_val = get_cell('Y', target_row)\n y_str = str(y_val).strip() if y_val is not None else ''\n if y_val is None or y_str != '20260315_BCBS':\n issues.append(f'Y{target_row} (ERA Batch ID) expected \"20260315_BCBS\", got {y_val}')\n else:\n found_values['Y'] = y_str\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': f'CLM-1008 (Harold Jensen) row {tarLine truncated
"criterion_type": "expected_output"
},
{
"id": "b1182ec4-1939-43e2-8f0e-7a882d3911c4",
"sort_order": 1,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cells for CLM-1033 (Elaine Foster) in row 34 must have: M34=165.00 (Allowed Amount), N34=142.00 (Paid Amount), O34=58.00 (Contractual Adj), P34=20.00 (Patient Rep), Q34='Written Off' (Claim Status), R34='Posted' (Payment Posting Status), Y34='20260315_BCBS' (ERA Batch ID)",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Search for claim register file - be loose with naming\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n # Also try exact name\n if not target_file:\n exact = Path(workspace_path) / 'PBC_Claim_Register.xlsx'\n if exact.exists():\n target_file = exact\n if not target_file:\n # Try any xlsx\n for f in xlsx_files:\n if 'claim' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': f'No claim register xlsx file found in workspace. Files found: {[f.name for f in xlsx_files]}'}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n # Try to find the correct row for CLM-1033 dynamically\n target_row = 34\n # Scan for CLM-1033 in case it's not exactly row 34\n for row in range(1, ws.max_row + 1):\n for col in range(1, min(ws.max_column + 1, 10)):\n cell_val = ws.cell(row=row, column=col).value\n if cell_val is not None and 'CLM-1033' in str(cell_val):\n target_row = row\n break\n \n issues = []\n \n # Check M: Allowed Amount = 165.00\n m_val = ws.cell(row=target_row, column=13).value # M column = 13\n try:\n m_num = float(m_val) if m_val is not None else None\n if m_num is None or abs(m_num - 165.00) > 165.00 * 0.02:\n issues.append(f'M{target_row} (Allowed Amount) expected 165.00, got {m_val}')\n except (TypeError, ValueError):\n issues.append(f'M{target_row} (Allowed Amount) expected 165.00, got {m_val}')\n \n # Check N: Paid Amount = 142.00\n n_val = ws.cell(row=target_row, column=14).value # N column = 14\n try:\n n_num = float(n_val) if n_val is not None else None\n if n_num is None or abs(n_num - 142.00) > 142.00 * 0.02:\n issues.append(f'N{target_row} (Paid Amount) expected 142.00, got {n_val}')\n except (TypeError, ValueError):\n issues.append(f'N{target_row} (Paid Amount) expected 142.00, got {n_val}')\n \n # Check O: Contractual Adj = 58.00\n o_val = ws.cell(row=target_row, column=15).value # O column = 15\n try:\n o_num = float(o_val) if o_val is not None else None\n if o_num is None or abs(o_num - 58.00) > 58.00 * 0.02:\n issues.append(f'O{target_row} (Contractual Adj) expected 58.00, got {o_val}')\n except (TypeError, ValueError):\n issues.append(f'O{target_row} (Contractual Adj) expected 58.00, got {o_val}')\n \n # Check P: Patient Rep = 20.00\n p_val = ws.cell(row=target_row, column=16).value # P column = 16\n try:\n p_num = float(p_val) if p_val is not None else None\n if p_num is None or abs(p_num - 20.00) > 20.00 * 0.02:\n issues.append(f'P{target_row} (Patient Rep) expected 20.00, got {p_val}')\n except (TypeError, ValueError):\n issues.append(f'P{target_row} (Patient Rep) expected 20.00, got {p_val}')\n \n # Check Q: Claim Status = 'Written Off'\n q_val = ws.cell(row=target_row, column=17).value # Q column = 17\n if q_val is None or ('written' not in str(q_val).lower() or 'off' not in str(q_val).lower()):\n issues.append(f'Q{target_row} (Claim Status) expected \"Written Off\", got {q_val}')\n \n # Check R: Payment Posting Status = 'Posted' (rubric says 'Posted')\n r_val = ws.cell(row=target_row, column=18).value # R column = 18\n if r_val is None or 'posted' not in str(r_val).lower():\n issues.append(f'R{target_row} (Payment Posting Status) expected \"Posted\", got {r_val}')\n \n # Check Y: ERA Batch ID = '20260315_BCBS'\n y_val = ws.cell(row=target_row, column=25).value # Y column = 25\n y_str = str(y_val).strip() if y_val is not None else ''\n if y_val is None or y_str != '20260315_BCBS':\n issues.append(f'Y{target_row} (ERA Batch ID) expected \"20260315_BCBS\", got {y_val}')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': f'CLM-1033 (Elaine Foster) row {target_row} issues in {target_file.name}: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'CLM-1033 (Elaine Foster) row {target_row} verified in {target_file.name}: M=165.00, N=142.00, O=58.00, P=20.00, Q={q_val}, R={r_val}, Y={y_val}'}\n except ExceptLine truncated
"criterion_type": "expected_output"
},
{
"id": "4ff463fc-9192-4f9a-a68b-878c966f5510",
"sort_order": 2,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cell Z17 (Notes for CLM-1016, Sarah Mitchell) must contain text indicating a clawback of $30.00 applied on 2026-03-15 via batch 20260315_BCBS (e.g., 'Clawback applied 2026-03-15: $30.00 via 20260315_BCBS.')",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n xlsx_files_all = list(Path(workspace_path).glob('**/*.xlsx'))\n for f in xlsx_files_all:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'No claim register xlsx file found in workspace'}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n z17 = ws['Z17'].value\n if z17 is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Z17 (Notes for CLM-1016 Sarah Mitchell) is empty, expected clawback note'}\n \n z17_str = str(z17).lower()\n \n issues = []\n if 'clawback' not in z17_str:\n issues.append('missing keyword \"clawback\"')\n if '30' not in z17_str:\n issues.append('missing amount \"30\"')\n if '2026-03-15' not in z17_str and '20260315' not in z17_str:\n issues.append('missing date \"2026-03-15\" or \"20260315\"')\n if '20260315_bcbs' not in z17_str:\n issues.append('missing batch ID \"20260315_BCBS\"')\n \n if issues:\n joiner = '; '\n return {'pass': False, 'score': 0.0, 'feedback': f'Z17 clawback note issues: {joiner.join(issues)}. Found: \"{z17}\"'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Z17 clawback note verified for CLM-1016 (Sarah Mitchell): \"{z17}\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading xlsx file {target_file.name}: {str(e)}'}\n",
"criterion_type": "expected_output"
},
{
"id": "e87a6e28-1fa3-4df9-b6fd-d13b2aeeb509",
"sort_order": 3,
"rubric_text": "In `mailbox.json`, there must be exactly 3 emails in the Sent folder",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n count = len(sent_emails)\n \n if count != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails in mailbox.json, found {count}. Sent email subjects: {[e.get(\"subject\", \"\") for e in sent_emails]}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Found exactly 3 sent emails in mailbox.json. Subjects: {[e.get(\"subject\", \"\") for e in sent_emails]}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {str(e)}'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775866205434",
"sort_order": 4,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, the financial and status fields for CLM-1016 must remain unchanged from their original values: L17=220 (Billed Amount), M17=162 (Allowed Amount), N17=172 (Paid Amount), O17=58 (Contractual Adj), P17=20 (Patient Resp), Q17='Paid - Full' (Claim Status), R17='Reconciled' (Payment Posting Status), S17=empty (CO Code), U17=empty (Recovery Action), V17=empty (Recovery Status), Y17='20260120_BCBS' (ERA Batch ID).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening workbook: {e}\"}\n\n row = 17\n issues = []\n details = []\n\n def check_numeric(cell_ref, expected, label):\n val = ws[cell_ref].value\n if val is None:\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got empty/None\")\n return\n try:\n num_val = float(val)\n except (ValueError, TypeError):\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got non-numeric '{val}'\")\n return\n if abs(num_val - expected) > 0.01:\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got {num_val}\")\n else:\n details.append(f\"{cell_ref} ({label}): {num_val} == {expected} OK\")\n\n def check_string(cell_ref, expected, label):\n val = ws[cell_ref].value\n if val is None:\n val_str = \"\"\n else:\n val_str = str(val).strip()\n if val_str.lower() != expected.lower():\n issues.append(f\"{cell_ref} ({label}): expected '{expected}', got '{val_str}'\")\n else:\n details.append(f\"{cell_ref} ({label}): '{val_str}' OK\")\n\n def check_empty(cell_ref, label):\n val = ws[cell_ref].value\n if val is not None and str(val).strip() != \"\":\n issues.append(f\"{cell_ref} ({label}): expected empty, got '{val}'\")\n else:\n details.append(f\"{cell_ref} ({label}): empty OK\")\n\n # First verify row 17 actually contains CLM-1016 (likely in an early column)\n # We'll check common columns A-D for the claim ID\n claim_found = False\n for col_letter in ['A', 'B', 'C', 'D', 'E']:\n cell_val = ws[f\"{col_letter}{row}\"].value\n if cell_val is not None and 'CLM-1016' in str(cell_val).strip().upper():\n claim_found = True\n break\n if not claim_found:\n # Still proceed but note it\n details.append(f\"Note: CLM-1016 identifier not found in columns A-E of row {row}, but checking specified cells anyway.\")\n\n # Numeric checks\n check_numeric(f\"L{row}\", 220, \"Billed Amount\")\n check_numeric(f\"M{row}\", 162, \"Allowed Amount\")\n check_numeric(f\"N{row}\", 172, \"Paid Amount\")\n check_numeric(f\"O{row}\", 58, \"Contractual Adj\")\n check_numeric(f\"P{row}\", 20, \"Patient Resp\")\n\n # String checks\n check_string(f\"Q{row}\", \"Paid - Full\", \"Claim Status\")\n check_string(f\"R{row}\", \"Reconciled\", \"Payment Posting Status\")\n check_string(f\"Y{row}\", \"20260120_BCBS\", \"ERA Batch ID\")\n\n # Empty checks\n check_empty(f\"S{row}\", \"CO Code\")\n check_empty(f\"U{row}\", \"Recovery Action\")\n check_empty(f\"V{row}\", \"Recovery Status\")\n\n wb.close()\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"CLM-1016 fields were modified. Issues: \" + \"; \".join(issues) + \". Verified: \" + \"; \".join(details)\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"All CLM-1016 fields remain unchanged. Details: \" + \"; \".join(details)\n }\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Hi, it's 16 March 2026. Can you deal with the unread email in my inbox, as per the processes in the SOP (PBC_SOP_RCM024_Rev3_28.pdf). I've already downloaded its attachment. Thanks.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py