Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,51 @@
[
{
"id": "07a5184d-ae31-4c36-a902-61b22ad416ce",
"sort_order": 0,
"rubric_text": "In file `benefits.xlsx`, the BV_Complete field must be 'YES' and the Financial_Counseling_Log field must contain '03/26/2026 14:36—SUCCESS'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find benefits.xlsx (case-insensitive)\n benefits_files = list(Path(workspace_path).glob('**/benefits.xlsx'))\n if not benefits_files:\n benefits_files = list(Path(workspace_path).glob('**/Benefits.xlsx'))\n if not benefits_files:\n benefits_files = list(Path(workspace_path).glob('**/*.xlsx'))\n benefits_files = [f for f in benefits_files if 'benefit' in f.name.lower()]\n if not benefits_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No benefits.xlsx file found in workspace\"}\n \n wb = openpyxl.load_workbook(benefits_files[0])\n \n bv_complete_found = False\n fc_log_found = False\n bv_complete_value = None\n fc_log_value = None\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Collect all cell values into a grid\n all_values = []\n for row in ws.iter_rows():\n row_vals = []\n for c in row:\n row_vals.append(str(c.value).strip() if c.value is not None else '')\n all_values.append(row_vals)\n \n # Find header positions\n bv_col = None\n fc_col = None\n header_row_idx = None\n \n for i, row_vals in enumerate(all_values):\n for j, val in enumerate(row_vals):\n upper_val = val.upper().replace(' ', '_')\n if upper_val == 'BV_COMPLETE':\n bv_col = j\n if header_row_idx is None:\n header_row_idx = i\n if upper_val == 'FINANCIAL_COUNSELING_LOG':\n fc_col = j\n if header_row_idx is None:\n header_row_idx = i\n \n if header_row_idx is not None:\n for row_vals in all_values[header_row_idx + 1:]:\n if bv_col is not None and bv_col < len(row_vals):\n cell_val = row_vals[bv_col].strip()\n if cell_val.upper() == 'YES':\n bv_complete_found = True\n bv_complete_value = cell_val\n if fc_col is not None and fc_col < len(row_vals):\n cell_val = row_vals[fc_col].strip()\n if cell_val:\n fc_log_value = cell_val\n # Check for the specific log entry: '03/26/2026 14:36—SUCCESS'\n # Be lenient: accept various dash types (em dash, en dash, hyphen) and slight formatting\n normalized = cell_val.upper()\n # Check contains SUCCESS and the date/time components\n has_success = 'SUCCESS' in normalized\n has_date = '03/26/2026' in normalized or '3/26/2026' in normalized\n has_time = '14:36' in normalized\n if has_success and has_date and has_time:\n fc_log_found = True\n \n # Also do a broad fallback scan: look at column-header mapping approach\n if not bv_complete_found or not fc_log_found:\n # Scan using openpyxl column-header tracking\n col_headers = {}\n for row in ws.iter_rows():\n for cell in row:\n if cell.value is not None:\n val_upper = str(cell.value).strip().upper().replace(' ', '_')\n if val_upper == 'BV_COMPLETE':\n col_headers[('BV_COMPLETE', cell.column)] = cell.row\n if val_upper == 'FINANCIAL_COUNSELING_LOG':\n col_headers[('FINANCIAL_COUNSELING_LOG', cell.column)] = cell.row\n \n for row in ws.iter_rows():\n for cell in row:\n if cell.value is None:\n continue\n val = str(cell.value).strip()\n \n if not bv_complete_found and val.upper() == 'YES':\n if ('BV_COMPLETE', cell.column) in col_headers:\n if cell.row > col_headers[('BV_COMPLETE', cell.column)]:\n bv_complete_found = True\n bv_complete_value = val\n \n if not fc_log_found:\n if ('FINANCIAL_COUNSELING_LOG', cell.column) in col_headers:\n if cell.row > col_headers[('FINANCIAL_COUNSELING_LOG', cell.column)]:\n normalized = val.upper()\n has_success = 'SUCCESS' in normalized\n has_date = '03/26/2026' in normalized or '3/26/2026' in normalized\n has_time = '14:36' in normaliLine truncated
"criterion_type": "expected_output"
},
{
"id": "5a2c25a5-749f-4525-9c7f-8bd381963cae",
"sort_order": 1,
"rubric_text": "In file `intake.xlsx`, the Financial_Hardship field must NOT be set to 'YES'",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n intake_files = list(Path(workspace_path).glob('**/intake.xlsx'))\n if not intake_files:\n intake_files = list(Path(workspace_path).glob('**/Intake.xlsx'))\n if not intake_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No intake.xlsx file found in workspace\"}\n \n wb = openpyxl.load_workbook(intake_files[0])\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n all_rows = []\n for row in ws.iter_rows():\n row_vals = [str(c.value).strip() if c.value is not None else '' for c in row]\n all_rows.append(row_vals)\n \n # Find header row and Financial_Hardship column\n header_row_idx = None\n fh_col = None\n for i, row_vals in enumerate(all_rows):\n for j, val in enumerate(row_vals):\n if 'FINANCIAL_HARDSHIP' in val.upper().replace(' ', '_'):\n header_row_idx = i\n fh_col = j\n break\n if header_row_idx is not None:\n break\n \n if header_row_idx is None or fh_col is None:\n # Check if it appears as a row label (vertical layout)\n for row_vals in all_rows:\n for j, val in enumerate(row_vals):\n if 'FINANCIAL_HARDSHIP' in val.upper().replace(' ', '_'):\n # Check adjacent cell\n next_val = row_vals[j+1] if j+1 < len(row_vals) else ''\n if next_val.upper() == 'YES':\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Financial_Hardship is set to 'YES' in intake.xlsx\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Financial_Hardship is not set to 'YES' in intake.xlsx (value: '{next_val}')\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Financial_Hardship field not found in intake.xlsx (assumed not set to YES)\"}\n \n for row_vals in all_rows[header_row_idx+1:]:\n if fh_col < len(row_vals):\n val = row_vals[fh_col].upper().strip()\n if val == 'YES':\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Financial_Hardship is set to 'YES' in intake.xlsx\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Financial_Hardship is not set to 'YES' in intake.xlsx\"}",
"criterion_type": "incorrect_behavior"
},
{
"id": "19bd2c51-9cf4-431c-bcf2-1414221735b6",
"sort_order": 2,
"rubric_text": "In slack_data.json, there must be no message in any channel containing '[FINANCIAL HARDSHIP]', no message containing '[BV_PARTIAL]', and no messages in the #compliance channel",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n \n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_file, 'r') as f:\n data = json.load(f)\n \n messages = data.get('messages', {})\n channels = data.get('channels', {})\n \n # Find compliance channel ID\n compliance_channel_id = None\n for ch_id, ch_info in channels.items():\n if ch_info.get('name', '').lower() == 'compliance':\n compliance_channel_id = ch_id\n break\n \n financial_hardship_found = []\n bv_partial_found = []\n compliance_messages = []\n \n for channel_id, msgs in messages.items():\n for msg in msgs:\n text = msg.get('text', '') or ''\n if '[FINANCIAL HARDSHIP]' in text.upper():\n financial_hardship_found.append(f\"Channel {channel_id}: {text[:100]}\")\n if '[BV_PARTIAL]' in text.upper():\n bv_partial_found.append(f\"Channel {channel_id}: {text[:100]}\")\n if channel_id == compliance_channel_id:\n compliance_messages.append(text[:100])\n \n issues = []\n if financial_hardship_found:\n issues.append(f\"Found [FINANCIAL HARDSHIP] messages: {financial_hardship_found}\")\n if bv_partial_found:\n issues.append(f\"Found [BV_PARTIAL] messages: {bv_partial_found}\")\n if compliance_channel_id and compliance_messages:\n issues.append(f\"Found {len(compliance_messages)} message(s) in #compliance channel\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Slack constraint violations: \" + \"; \".join(issues)}\n \n compliance_status = f\"#compliance channel has no messages\" if compliance_channel_id else \"#compliance channel not found (no messages possible)\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No [FINANCIAL HARDSHIP] messages, no [BV_PARTIAL] messages, and {compliance_status}\"}",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775062985592",
"sort_order": 3,
"rubric_text": "File `audit_log.xlsx` must exist in the workspace.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / \"audit_log.xlsx\"\n if target.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'audit_log.xlsx' exists at {target}.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'audit_log.xlsx' was not found in the workspace ({workspace_path}).\"}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775063074284",
"sort_order": 4,
"rubric_text": "File `audit_log.xlsx` must contain the following headers in any sheet: 'Date', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB', 'Action', 'File_Updated', 'Fields_Updated', 'New_Value', 'Staff_User', 'Result' (or 'Result (SUCCESS/FAILED/STOP—reason)').",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"audit_log.xlsx\"\n if not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'audit_log.xlsx' not found in workspace at {workspace_path}.\"}\n\n try:\n wb = openpyxl.load_workbook(target_file)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'audit_log.xlsx': {e}\"}\n\n # Expected headers per rubric:\n # 'Date', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n # 'Action', 'File_Updated', 'Fields_Updated', 'New_Value', 'Staff_User',\n # 'Result' (or 'Result (SUCCESS/FAILED/STOP—reason)')\n #\n # We accept underscores or spaces interchangeably and are case-insensitive.\n expected_headers = [\n ('Date', ['date']),\n ('Patient_Last_Name', ['patient last name', 'patient_last_name']),\n ('Patient_First_Name', ['patient first name', 'patient_first_name']),\n ('Patient_DOB', ['patient dob', 'patient_dob']),\n ('Action', ['action']),\n ('File_Updated', ['file_updated', 'file updated']),\n ('Fields_Updated', ['fields_updated', 'fields updated']),\n ('New_Value', ['new_value', 'new value']),\n ('Staff_User', ['staff_user', 'staff user']),\n ('Result', None), # Special handling: any header starting with 'result'\n ]\n\n def normalize(s):\n if s is None:\n return None\n return ' '.join(s.strip().lower().split())\n\n def check_headers(ws):\n actual_headers_raw = []\n for cell in ws[1]:\n val = cell.value\n if val is not None:\n actual_headers_raw.append(str(val).strip())\n else:\n actual_headers_raw.append(None)\n\n actual_normalized = [normalize(h) for h in actual_headers_raw]\n\n missing = []\n for display_name, variants in expected_headers:\n found = False\n for act in actual_normalized:\n if act is None:\n continue\n if display_name == 'Result':\n if act == 'result' or act.startswith('result'):\n found = True\n break\n else:\n act_underscored = act.replace(' ', '_')\n act_spaced = act.replace('_', ' ')\n for v in variants:\n v_underscored = v.replace(' ', '_')\n v_spaced = v.replace('_', ' ')\n if act in (v, v_underscored, v_spaced) or act_underscored in (v, v_underscored) or act_spaced in (v, v_spaced):\n found = True\n break\n if found:\n break\n if not found:\n missing.append(display_name)\n\n return missing, actual_headers_raw\n\n # Check ALL sheets\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n missing, actual_headers_raw = check_headers(ws)\n if not missing:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All required headers found in 'audit_log.xlsx' (sheet '{sheet_name}'). Headers found: {actual_headers_raw}.\"\n }\n\n # If we get here, no sheet had all headers. Report missing from active sheet.\n ws = wb.active\n missing, actual_headers_raw = check_headers(ws)\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Missing headers in 'audit_log.xlsx': {missing}. Checked all sheets ({wb.sheetnames}). Headers found in active sheet (raw): {actual_headers_raw}.\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775063145806",
"sort_order": 5,
"rubric_text": "In `audit_log.xlsx`, there must be a row in any sheet with Date=03/26/2026, Action='Financial Counseling Complete', File_Updated='Benefits.xlsx', Fields_Updated containing both 'Financial_Counseling_Log' and 'BV_Complete', New_Value containing both '03/26/2026 14:36—SUCCESS' and 'YES', Result='SUCCESS', Patient_Last _Name='Lopez', Patient_First_Name='Kevin', and Patient_DOB='11091990'. For Patient_DOB, any separators (e.g., slashes, dashes, dots) are a failure; it must be exactly '11091990'.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n target_file = ws / \"audit_log.xlsx\"\n\n if not target_file.exists():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n f\"File 'audit_log.xlsx' not found in workspace. \"\n f\"Files present: {[f.name for f in ws.iterdir()]}\"\n ),\n }\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open audit_log.xlsx: {e}\"}\n\n # Search all sheets\n for sheet_name in wb.sheetnames:\n ws_sheet = wb[sheet_name]\n\n # Build header map from row 1\n headers = {}\n for col_idx, cell in enumerate(ws_sheet[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip().lower()] = col_idx\n\n if not headers:\n continue\n\n # ---------------------------------------------------------------------------\n # find_col uses exact normalized matching (with underscore/space\n # interchangeability). This avoids 'date' matching 'file_updated' etc.\n # ---------------------------------------------------------------------------\n def find_col(possible_names):\n for name in possible_names:\n nl = name.lower().strip()\n nl_u = nl.replace(\" \", \"_\")\n nl_s = nl.replace(\"_\", \" \")\n for h, idx in headers.items():\n h_u = h.replace(\" \", \"_\")\n h_s = h.replace(\"_\", \" \")\n if h in (nl, nl_u, nl_s) or h_u in (nl, nl_u, nl_s) or h_s in (nl, nl_u, nl_s):\n return idx\n return None\n\n col_date = find_col([\"date\"])\n col_action = find_col([\"action\"])\n col_file_updated = find_col([\"file_updated\", \"file updated\"])\n col_fields_updated = find_col([\"fields_updated\", \"fields updated\"])\n col_new_value = find_col([\"new_value\", \"new value\"])\n col_result = find_col([\"result\"])\n col_last_name = find_col([\"patient last name\", \"patient_last_name\", \"patient last name\", \"patient_last _name\"])\n col_first_name = find_col([\"patient first name\", \"patient_first_name\"])\n col_dob = find_col([\"patient_dob\", \"patient dob\"])\n\n # Iterate data rows (row 2 onward)\n for row_idx in range(2, ws_sheet.max_row + 1):\n\n def get_val(col, _row=row_idx):\n if col is None:\n return None\n v = ws_sheet.cell(row=_row, column=col).value\n if v is None:\n return \"\"\n return str(v).strip()\n\n date_val = get_val(col_date)\n action_val = get_val(col_action)\n file_updated_val = get_val(col_file_updated)\n fields_updated_val = get_val(col_fields_updated)\n new_value_val = get_val(col_new_value)\n result_val = get_val(col_result)\n last_name_val = get_val(col_last_name)\n first_name_val = get_val(col_first_name)\n dob_val = get_val(col_dob)\n\n # ── Date: must be 03/26/2026 ────────────────────────────────────────\n date_ok = False\n if col_date:\n raw_cell = ws_sheet.cell(row=row_idx, column=col_date).value\n if isinstance(raw_cell, (datetime.datetime, datetime.date)):\n if raw_cell.month == 3 and raw_cell.day == 26 and raw_cell.year == 2026:\n date_ok = True\n if not date_ok:\n date_norm = (date_val or \"\").replace(\"-\", \"/\").strip()\n if \"03/26/2026\" in date_norm or \"3/26/2026\" in date_norm:\n date_ok = True\n else:\n date_ok = True # no date column found; be lenient\n\n if not date_ok:\n continue\n\n # ── Action: must contain 'financial counseling complete' ────────────\n if not action_val or \"financial counseling complete\" not in action_val.lower():\n continue\n\n # ── File_Updated: must reference benefits.xlsx ──────────────────────\n if not file_updated_val or \"benefits\" not in file_updated_val.lower():\n continue\n\n # ── Fields_Updated: must contain both column names ───Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775063351791",
"sort_order": 6,
"rubric_text": "In `audit_log.xlsx`, there must be exactly one row of data (in any sheet), excluding the header row.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"audit_log.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'audit_log.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'audit_log.xlsx': {e}\"}\n \n total_data_rows = 0\n details = []\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n all_rows = list(ws.iter_rows(values_only=True))\n \n # Filter out completely empty rows\n non_empty_rows = [row for row in all_rows if any(cell is not None and str(cell).strip() != '' for cell in row)]\n \n if len(non_empty_rows) == 0:\n details.append(f\"Sheet '{sheet_name}': no non-empty rows.\")\n continue\n \n # First non-empty row is assumed to be the header\n header_row = non_empty_rows[0]\n data_rows = non_empty_rows[1:]\n num_data = len(data_rows)\n total_data_rows += num_data\n details.append(f\"Sheet '{sheet_name}': header={header_row}, {num_data} data row(s): {data_rows}\")\n \n detail_str = \" | \".join(details)\n \n if total_data_rows == 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"audit_log.xlsx has exactly 1 data row (excluding headers) across all sheets. {detail_str}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"audit_log.xlsx has {total_data_rows} total data row(s) across all sheets (expected exactly 1). {detail_str}\"}\n",
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,3 @@
I just called a patient a few minutes ago as part of the process for Patient Financial Counseling. I've provided you with the transcript of this phone call.
Something came up, so I'd like you to do the few things I left undone after the phone call. Follow the instructions found in the standard procedure document for what remains to be done in section 7.3 only. Make sure no globally defined stops apply, and if any do, follow procedure accordingly. You can also assume that the coverage term covers the patient regardless of the dates for coverage in benefits.xlsx. I've already handled escalations to verify the patient's required payment in the transcript is correct and you do not need to flag that as a problem.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py