Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,191 @@
[
{
"id": "a017242d-7557-4fc5-b933-13e3a0a45e9b",
"sort_order": 0,
"rubric_text": "In file mailbox.json (in `external_data/final`), There must be a sent email to david.park@pathfinderbilling.com with exact subject \"Pattern Recurrence Alert: Dr. Sarah Chen — CO-16 — 5 Instances\" and exact body (no partial match accepted): \"Pattern Recurrence Report — 04/24/2026\n\nThis is Maria Santos, Resolution Auditor, reporting a pattern recurrence for Dr. Sarah Chen, NPI: 1234567890.\n\nDenial Code: CO-16 — RARC: N382 — 5 instances identified.\n\nAffected Claims:\n\n1. Claim ID: CLM-2026-0234 — Patient ID: PAT-1008 — DOS: 01/15/2026 — CPT: 99214 — Status: Resubmitted\n\n2. Claim ID: CLM-2026-0356 — Patient ID: PAT-1012 — DOS: 01/28/2026 — CPT: 99213 — Status: Resubmitted\n\n3. Claim ID: CLM-2026-0489 — Patient ID: PAT-1019 — DOS: 02/10/2026 — CPT: 99214 — Status: Resubmitted\n\n4. Claim ID: CLM-2026-0612 — Patient ID: PAT-1027 — DOS: 02/22/2026 — CPT: 99213 — Status: Resubmitted\n\n5. Claim ID: CLM-2026-0892 — Patient ID: PAT-1042 — DOS: 03/15/2026 — CPT: 99214 — Status: Resubmitted\n\nAssessment: This pattern suggests a systemic submission error. Requesting management review to determine whether a corrective action plan or direct clinic intervention is warranted.\n\nEscalation Log Entry ID: ESC-2026-0008. Escalation Category: Operational. Escalation Reason: Pattern Recurrence.\" where \"ESC-2026-0008\" could be \"ESC-2026-0009\"",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external services path.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = []\n if isinstance(mailbox_data, list):\n emails = mailbox_data\n elif isinstance(mailbox_data, dict):\n if \"emails\" in mailbox_data:\n emails = mailbox_data[\"emails\"]\n elif \"sent\" in mailbox_data:\n emails = mailbox_data[\"sent\"]\n else:\n for key, val in mailbox_data.items():\n if isinstance(val, list) and len(val) > 0:\n emails = val\n break\n\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n\n def normalize_dashes(s):\n s = str(s)\n s = s.replace('\\u2013', '\\u2014')\n s = s.replace(' -- ', ' \\u2014 ')\n s = s.replace('--', '\\u2014')\n s = s.replace(' - ', ' \\u2014 ')\n return s\n\n def normalize(s):\n s = normalize_dashes(str(s))\n return re.sub(r'\\s+', ' ', s).strip().lower()\n\n expected_subject = \"Pattern Recurrence Alert: Dr. Sarah Chen \\u2014 CO-16 \\u2014 5 Instances\"\n\n # Required content checks — all must be present in body\n required_claim_ids = [\n \"CLM-2026-0234\",\n \"CLM-2026-0356\",\n \"CLM-2026-0489\",\n \"CLM-2026-0612\",\n \"CLM-2026-0892\",\n ]\n required_phrases = [\n \"Maria Santos\",\n \"Dr. Sarah Chen\",\n \"NPI: 1234567890\",\n \"CO-16\",\n \"N382\",\n \"Pattern Recurrence\",\n \"Operational\",\n ]\n esc_pattern = re.compile(r'ESC-2026-00\\d{2}')\n\n to_addr = \"david.park@pathfinderbilling.com\"\n from_addr = \"maria.santos@pathfinderbilling.com\"\n\n best_feedback = f\"No email found to {to_addr}.\"\n best_score = 0.0\n\n for email in emails:\n if not isinstance(email, dict):\n continue\n\n # Check recipient\n e_to = None\n for field in [\"to_addr\", \"to\", \"recipient\", \"to_address\", \"to_email\", \"recipients\"]:\n if field in email and email[field]:\n e_to = email[field]\n break\n if e_to is None:\n continue\n e_to_str = \" \".join(str(x) for x in e_to) if isinstance(e_to, list) else str(e_to)\n if to_addr not in e_to_str.lower():\n continue\n\n # Check sender (allow missing/unknown sender for sent-folder emails)\n e_from = \"\"\n for field in [\"from_addr\", \"from\", \"sender\", \"from_address\", \"from_email\"]:\n if field in email and email[field]:\n e_from = str(email[field]).strip().lower()\n break\n e_folder = str(email.get(\"folder\", \"\")).strip().lower()\n if e_from and from_addr not in e_from and e_folder != \"sent\":\n continue\n\n # Check subject (collapse whitespace + normalize dashes, case-insensitive)\n e_subject = str(email.get(\"subject\", \"\"))\n subject_ok = normalize(e_subject) == normalize(expected_subject)\n\n # Get body\n e_body = \"\"\n for field in [\"body_text\", \"body\", \"content\", \"text\", \"message\", \"html\", \"body_html\"]:\n if field in email and email[field]:\n e_body = str(email[field])\n break\n\n body_norm = normalize(e_body)\n\n # Check all required claim IDs\n missing_claims = [c for c in required_claim_ids if c.lower() not in body_norm]\n\n # Check all required phrases\n missing_phrases = [p for p in required_phrases if p.lower() not in body_norm]\n\n # Check ESC ID present\n esc_ok = bool(esc_pattern.search(e_body))\n\n all_claims_ok = len(missing_claims) == 0\n all_phrases_ok = len(missing_phrases) == 0\n\n if subject_ok and all_claims_ok and all_phrases_ok and esc_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found matching Pattern Recurrence email to david.park with correct subject and all required content.\"}\n\n # Partial scoring for feedback\n total_checks = len(required_claim_ids) + len(required_phrases) + 2 # +subject, +esc\n passed_checks = (\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705367514",
"sort_order": 1,
"rubric_text": "In `mailbox.json`, there must be exactly three sent emails in total.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox.json)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'emails' key is missing or not a list in mailbox.json.\"}\n\n sent_emails = [e for e in emails if isinstance(e, dict) and str(e.get(\"folder\", \"\")).strip().lower() == \"sent\"]\n count = len(sent_emails)\n\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_emails]\n subjects_str = \"; \".join(subjects)\n\n if count == 3:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 3 sent emails as expected. Subjects: {subjects_str}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 3 sent emails, but found {count}. Sent email subjects: {subjects_str}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775705423565",
"sort_order": 2,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be a table with exactly 40 total rows (including the header row). No leniency, must be exactly 40.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Claim_Register.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'PBC_Claim_Register.xlsx': {e}\"}\n \n target = 40\n \n # Check all sheets for a table with exactly 40 rows\n results = []\n for sheet_name in wb.sheetnames:\n sws = wb[sheet_name]\n # Count rows that have at least one non-empty cell\n non_empty_rows = 0\n for row in sws.iter_rows(min_row=1, max_row=sws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n max_row = sws.max_row\n results.append((sheet_name, non_empty_rows, max_row))\n \n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has exactly {non_empty_rows} non-empty rows (including header), which meets the requirement of exactly 40 total rows.\"\n }\n if max_row == target and non_empty_rows != target:\n # Also accept if max_row is exactly 40 (some cells might appear empty but row exists)\n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has max_row={max_row} matching exactly 40 total rows.\"\n }\n \n # If no sheet matched exactly, provide detailed failure feedback\n details = \"; \".join([f\"Sheet '{s}': {nr} non-empty rows, max_row={mr}\" for s, nr, mr in results])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx does not have exactly 40 total rows (including header) on any sheet. Found: {details}. Expected exactly 40 rows.\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705456779",
"sort_order": 3,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0845\", \"PAT-1021\", \"Jennifer Walsh\", \"03/10/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99213\", \"J06.9\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$180.00\", \"$142.00\", \"$38.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0845\",\n \"PAT-1021\",\n \"Jennifer Walsh\",\n \"03/10/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99213\",\n \"J06.9\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Paid - Full\",\n \"$180.00\",\n \"$142.00\",\n \"$38.00\",\n \"$0.00\",\n \"Reconciled\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, stripping $ and commas.\"\"\"\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual_val)\n\n norm_actual = normalize(actual_val)\n if norm_actual is None:\n return False\n\n norm_expected = expected_val.strip()\n\n # Direct case-insensitive match\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Strip $ for comparison\n if norm_actual.replace('$', '').strip().lower() == norm_expected.replace('$', '').strip().lower():\n return True\n\n # Try numeric comparison with tolerance\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0 and abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n if exp_num == 0 and abs(act_num) < 0.01:\n return True\n\n # Date flexibility: 03/10/2026 vs 2026-03-10 etc.\n if norm_expected == \"03/10/2026\":\n date_patterns = [\"03/10/2026\", \"3/10/2026\", \"2026-03-10\", \"2026/03/10\", \"03-10-2026\"]\n if norm_actual in date_patterns:\n return True\n # openpyxl might return a datetime\n import datetime\n if isinstance(actual_val, datetime.datetime) or isinstance(actual_val, datetime.date):\n try:\n if actual_val.month == 3 and actual_val.day == 10 and actual_val.year == 2026:\n return True\n except:\n pass\n\n # \"1\" could also be stored as integer 1\n if norm_expected == \"1\" and norm_actual in [\"1\", \"1.0\"]:\n return True\n\n return False\n\n num_expected = len(expected)\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n cells = [cell.value for cell in row]\n # We need to find the starting column. The row might have extra columns.\n # Try each possible starting position.\n for start_col in range(max(1, len(cells) - num_expected + 1)):\n subset = cells[start_col:start_col + num_expected]\n if len(subset) < num_expected:\n # Pad with None\n subset = subset + [None] * (num_expected - len(subset))\n\n match_count = 0\n mismatches = []\n for i, (exp_v, act_v) in enumerate(zip(expected, subset)):\n if values_match(exp_v, act_v):\n match_count += 1\n else:\n mismatches.append((i, exp_v, normalize(act_v)))\n\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row_idx\n best_mismatches = mismatches\n\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705468789",
"sort_order": 4,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-0878\", \"PAT-1035\", \"David Kim\", \"03/12/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"99214\", \"M54.5\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$285.00\", \"$228.00\", \"$57.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected values for a row (columns A through W = 23 columns)\n expected = [\n \"CLM-2026-0878\", # A\n \"PAT-1035\", # B\n \"David Kim\", # C\n \"03/12/2026\", # D\n \"Dr. James Miller\",# E\n \"2345678901\", # F\n None, # G - BLANK\n \"BlueCross TX\", # H\n \"BCTX1\", # I\n \"99214\", # J\n \"M54.5\", # K\n None, # L - BLANK\n \"1\", # M\n None, # N - BLANK\n \"Paid - Full\", # O\n \"$285.00\", # P\n \"$228.00\", # Q\n \"$57.00\", # R\n \"$0.00\", # S\n \"Reconciled\", # T\n None, # U - BLANK\n None, # V - BLANK\n None, # W - BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def money_val(s):\n \"\"\"Extract numeric value from money string like $285.00\"\"\"\n if s is None:\n return None\n cleaned = re.sub(r'[^\\d.\\-]', '', str(s))\n try:\n return float(cleaned)\n except:\n return None\n\n def vals_match(actual, expected_val):\n a = normalize(actual)\n e = normalize(expected_val)\n # Both blank\n if a is None and e is None:\n return True\n # One blank, other not\n if a is None or e is None:\n return False\n # Direct case-insensitive match\n if a.lower() == e.lower():\n return True\n # Try numeric / money comparison with tolerance\n ma = money_val(a)\n me = money_val(e)\n if ma is not None and me is not None:\n if me == 0:\n return abs(ma) < 0.01\n return abs(ma - me) / max(abs(me), 0.001) <= 0.05\n # Date flexibility: 03/12/2026 vs 2026-03-12 etc.\n # Try removing leading zeros and comparing\n a_stripped = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_stripped = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_stripped.lower() == e_stripped.lower():\n return True\n # Date: check if it's a datetime object rendered differently\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n try:\n formatted = actual.strftime('%m/%d/%Y')\n if formatted == e:\n return True\n except:\n pass\n return False\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [row[i].value if i < len(row) else None for i in range(23)]\n # Quick check: does column A match the claim ID?\n if normalize(cells[0]) is None:\n continue\n if normalize(cells[0]).upper() != \"CLM-2026-0878\":\n continue\n # Detailed check\n match_count = 0\n mismatches = []\n for idx in range(23):\n if vals_match(cells[idx] if idx < len(cells) else None, expected[idx]):\n match_count += 1\n else:\n mismatches.append((idx, normalize(cells[idx] if idx < len(cells) else None), normalize(expected[idx])))\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row[0].row\n best_mismatches = mismatches\n if match_count == 23:\n found = True\n break\n\n wb.close()\n\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row for CLM-2026-0878 with all 23 expected column values.\"}\n\n if best_row_num is not None:\n mismatch_details = \"; \".join([f\"Col {m[0]+1}: got '{m[1]}' expected '{m[2]}'\" for m in best_mismatches])\n return {\"pass\": False, \"score\": best_match_count / 23.0, \"feedback\": f\"Found row for CLM-2026-0878 at row {best_row_num} but {len(best_mismatches)} column(s) differ: {mismatch_details}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row with Claim ID 'CLM-2026-0878' found in PBC_Claim_Register.xlsx.\"}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705484020",
"sort_order": 5,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$285.00\", \"$0.00\", \"$285.00\", \"$0.00\", BLANK, BLANK, \"CO-16\", \"N382\".",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n target_files = list(Path(workspace_path).glob(\"**/PBC_Claim_Register.xlsx\"))\n if not target_files:\n # Also try case-insensitive\n target_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n target_files = [f for f in target_files if 'claim_register' in f.name.lower()]\n if not target_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n target_file = target_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$285.00\",\n \"$0.00\",\n \"$285.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-16\",\n \"N382\"\n ]\n\n # Column index 14 (0-based) is the status column where \"Submitted\" is also acceptable\n status_col_idx = 14\n\n def normalize(val):\n \"\"\"Normalize a cell value to a stripped string, or None if blank.\"\"\"\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def values_match(actual, expected_val, col_idx=None):\n \"\"\"Loosely compare an actual cell value against an expected value.\"\"\"\n actual_n = normalize(actual)\n if expected_val is None:\n # Expect blank\n return actual_n is None\n if actual_n is None:\n return False\n exp_str = str(expected_val).strip()\n\n # Direct case-insensitive match\n if actual_n.lower() == exp_str.lower():\n return True\n\n # Special handling for status column: accept \"Submitted\" as well\n if col_idx == status_col_idx:\n if actual_n.lower() in [\"resubmitted\", \"submitted\"]:\n return True\n\n # Try removing $ and commas for currency comparison\n def strip_currency(s):\n return s.replace('$', '').replace(',', '').strip()\n act_clean = strip_currency(actual_n)\n exp_clean = strip_currency(exp_str)\n if act_clean == exp_clean:\n return True\n\n # Try numeric comparison with tolerance\n try:\n act_num = float(act_clean)\n exp_num = float(exp_clean)\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n except (ValueError, TypeError):\n pass\n\n # Date flexibility: 03/15/2026 vs 2026-03-15 etc.\n date_formats = ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d']\n exp_date = None\n act_date = None\n for fmt in date_formats:\n try:\n exp_date = datetime.datetime.strptime(exp_str, fmt).date()\n break\n except ValueError:\n continue\n if exp_date:\n for fmt in date_formats:\n try:\n act_date = datetime.datetime.strptime(actual_n, fmt).date()\n break\n except ValueError:\n continue\n if act_date is None and isinstance(actual, datetime.datetime):\n act_date = actual.date()\n if act_date is None and isinstance(actual, datetime.date):\n act_date = actual\n if act_date is not None and act_date == exp_date:\n return True\n\n # Check if expected is \"1\" and actual could be 1 or 1.0\n if exp_str == \"1\":\n try:\n if float(actual_n) == 1.0:\n return True\n except (ValueError, TypeError):\n pass\n\n return False\n\n num_expected = len(expected)\n best_match_count = 0\n best_row_idx = None\n best_mismatches = []\n\n # Check all sheets\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n if len(cells) < num_expected:\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705498691",
"sort_order": 6,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-1104\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29881\", \"M23.211\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$2,200.00\", \"$1,850.00\", \"$350.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected row values in column order\n expected = [\n \"CLM-2026-1104\", # col 1\n \"PAT-1058\", # col 2\n \"Linda Park\", # col 3\n \"03/20/2026\", # col 4\n \"Dr. James Miller\",# col 5\n \"2345678901\", # col 6\n None, # col 7 BLANK\n \"BlueCross TX\", # col 8\n \"BCTX1\", # col 9\n \"29881\", # col 10\n \"M23.211\", # col 11\n None, # col 12 BLANK\n \"1\", # col 13\n None, # col 14 BLANK\n \"Paid - Full\", # col 15\n \"$2,200.00\", # col 16\n \"$1,850.00\", # col 17\n \"$350.00\", # col 18\n \"$0.00\", # col 19\n \"Reconciled\", # col 20\n None, # col 21 BLANK\n None, # col 22 BLANK\n None, # col 23 BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string like $2,200.00 or 2200\"\"\"\n if s is None:\n return None\n s = str(s).strip()\n # Remove dollar sign, commas, spaces\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def values_match(actual, expected_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual)\n \n act = normalize(actual)\n exp = normalize(expected_val)\n \n if act is None:\n return False\n \n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n \n # Handle date: expected 03/20/2026\n if '/' in exp:\n # Try to match dates loosely\n # actual could be a datetime object\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n dt = actual\n formatted = dt.strftime('%m/%d/%Y')\n if formatted == exp:\n return True\n # Also try without leading zeros\n formatted2 = f\"{dt.month}/{dt.day}/{dt.year}\"\n if formatted2 == exp:\n return True\n # String comparison already done above\n return False\n \n # Numeric comparison with tolerance\n exp_num = parse_number(exp)\n act_num = parse_number(act)\n if exp_num is not None and act_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 0.001) < 0.05:\n return True\n \n # Try comparing as plain numbers (e.g., \"1\" vs 1.0)\n try:\n if float(act) == float(exp):\n return True\n except (ValueError, TypeError):\n pass\n \n return False\n\n # Search all rows for a matching row\n num_expected = len(expected)\n best_match_count = 0\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n # Find the row that starts with CLM-2026-1104\n first_val = normalize(cells[0]) if len(cells) > 0 else None\n if first_val is None or 'CLM-2026-1104' not in first_val.upper().replace(' ', ''):\n # Also check if it might be in the row somewhere\n found_clm = False\n for c in cells:\n cn = normalize(c)\n if cn and 'CLM-2026-1104' in cn.upper().replace(' ', ''):\n found_clm = True\n break\n if not found_clm:\n continue\n\n # We found a candidate row. Check all expected columns.\n match_count = 0\n mismatches = []\n for i, exp_val in enumerate(expected):\n if i < len(cells):\n actual_val = cells[i]\n else:\n actual_val = None\n \n if values_match(actual_val, exp_val):\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705525107",
"sort_order": 7,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values [No partial matches acceptable, must match every value]: \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$1,850.00\", \"$0.00\", \"$1,850.00\", \"$0.00\", BLANK, BLANK, \"CO-97\", \"N520\".",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*.xlsx\")) + list(workspace.glob(\"*claim_register*.xlsx\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx and no alternative claim register file found in {workspace}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # Expected values in order across columns\n expected = [\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$1,850.00\",\n \"$0.00\",\n \"$1,850.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-97\",\n \"N520\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def is_blank(val):\n \"\"\"Check if a value is effectively blank/empty.\"\"\"\n if val is None:\n return True\n s = str(val).strip()\n if s == '' or s.lower() == 'none' or s == '0' or s == '0.0' or s == '$0.00':\n return True\n return False\n\n def values_match(actual_val, expected_val, col_index):\n act = normalize(actual_val)\n exp = normalize(expected_val)\n\n # Both blank\n if exp is None:\n return is_blank(actual_val)\n\n if act is None:\n return False\n\n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n\n # Special handling for column 14 (index 14): accept both Resubmitted and Submitted\n if col_index == 14:\n if act.lower() in ('resubmitted', 'submitted'):\n return True\n\n # Try numeric comparison with tolerance\n act_num = parse_number(act)\n exp_num = parse_number(exp)\n if act_num is not None and exp_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n\n # Currency: strip $ and commas and compare\n act_clean = re.sub(r'[\\$,\\s]', '', act)\n exp_clean = re.sub(r'[\\$,\\s]', '', exp)\n if act_clean == exp_clean:\n return True\n\n # Date flexibility\n date_patterns_act = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', act)\n date_patterns_exp = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', exp)\n if date_patterns_act and date_patterns_exp:\n def norm_date(parts):\n a, b, c = parts\n if len(a) == 4:\n return (b, c, a)\n else:\n return (a, b, c)\n if norm_date(date_patterns_act[0]) == norm_date(date_patterns_exp[0]):\n return True\n\n # Handle datetime objects from openpyxl\n if isinstance(actual_val, (datetime.datetime, datetime.date)):\n formatted = actual_val.strftime(\"%m/%d/%Y\")\n if formatted == exp:\n return True\n formatted3 = f\"{actual_val.month}/{actual_val.day}/{actual_val.year}\"\n if formatted3 == exp:\n return True\n formatted4 = f\"{actual_val.month:02d}/{actual_val.day:02d}/{actual_val.year}\"\n if formatted4 == exp:\n return True\n\n # Try comparing just digits for things like NPI numbers\n act_digits = re.sub(r'\\D', '', act)\n exp_digits = re.sub(r'\\D', '', exp)\n if act_digits and exp_digits and act_digits == exp_digits:\n return True\n\n return False\n\n # Search all sheets\n best_match_count = 0\n best_mismatches = []\n best_row_vals = []\n\n for ws_naLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705535872",
"sort_order": 8,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable, must match every value]: \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"Dr. Amanda Foster\", \"3456789012\", BLANK, \"BlueCross TX\", \"BCTX1\", \"27447\", \"M17.11\", BLANK, \"1\", BLANK, \"In Recovery\", \"$7,500.00\", \"$4,200.00\", \"$3,300.00\", \"$0.00\", \"Held - Contractual Variance\", BLANK, \"CO-45\", \"N130\".",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob in case of slight name differences\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*\")) + list(workspace.glob(\"*claim_register*\")) + list(workspace.glob(\"*ClaimRegister*\")) + list(workspace.glob(\"*claim*register*\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n # Try any xlsx candidate\n for c in candidates:\n if c.suffix == '.xlsx':\n file_path = c\n break\n if not file_path.exists():\n # Try any xlsx at all\n all_xlsx = list(workspace.glob(\"*.xlsx\"))\n for f in all_xlsx:\n if 'claim' in f.name.lower():\n file_path = f\n break\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx in {workspace}. Files present: {[f.name for f in workspace.iterdir()]}\"}\n\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n # Expected values in column order. None means BLANK.\n expected = [\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"Dr. Amanda Foster\",\n \"3456789012\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"27447\",\n \"M17.11\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"In Recovery\",\n \"$7,500.00\",\n \"$4,200.00\",\n \"$3,300.00\",\n \"$0.00\",\n \"Held - Contractual Variance\",\n None, # BLANK\n \"CO-45\",\n \"N130\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, removing $, commas, etc.\"\"\"\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def match_cell(expected_val, actual_val):\n \"\"\"Check if actual_val matches expected_val with reasonable flexibility.\"\"\"\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n # Both blank\n if e is None and a is None:\n return True\n # One blank, other not\n if e is None or a is None:\n return False\n\n # Direct case-insensitive match\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison with tolerance\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(a_num - e_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(a_num - e_num) / abs(a_num) <= 0.05:\n return True\n\n # Date flexibility: compare 03/08/2026 with various date formats\n date_formats = [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%m/%d/%y\"]\n e_date = None\n a_date = None\n for fmt in date_formats:\n try:\n e_date = datetime.datetime.strptime(e, fmt).date()\n break\n except ValueError:\n pass\n # actual might be a datetime object already\n if isinstance(actual_val, datetime.datetime):\n a_date = actual_val.date()\n elif isinstance(actual_val, datetime.date) and not isinstance(actual_val, datetime.datetime):\n a_date = actual_val\n else:\n for fmt in date_formats:\n try:\n a_date = datetime.datetime.strptime(a, fmt).date()\n break\n except ValueError:\n pass\n if e_date is not None and a_date is not None:\n if e_date == a_date:\n return True\n\n\n return False\n\n # Search all rows for a matching row\n found = False\n best_match_count = 0\n best_row_idx = None\n best_row_vals = None\n num_expected = len(expected)\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n row_vals = [cell.value for cell in row]\n # We need at least num_expected columns; pad if shorter\n while len(row_vals) < num_expected:\n row_vals.append(None)\n\n # First, find the row by the claim ID CLM-2026-0967Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705546419",
"sort_order": 9,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable]: \"CLM-2026-1106\" (or \"CLM-2026-1107\"), \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"7\", \"CLM-2026-0892\", \"Submitted\", \"$285.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob in case of path variations\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Claim_Register.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # The rubric says CLM-2026-1106 or CLM-2026-1107 for column 1\n expected_values = [\n [\"CLM-2026-1106\", \"CLM-2026-1107\"], # accept either claim ID\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"7\",\n \"CLM-2026-0892\",\n \"Submitted\",\n \"$285.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def single_value_match(expected_str, actual):\n \"\"\"Check if a single expected string matches the actual value.\"\"\"\n if expected_str is None:\n return is_blank(actual)\n\n norm_actual = normalize(actual)\n norm_expected = normalize(expected_str)\n\n if norm_actual is None:\n return False\n\n # Direct string match (case-insensitive)\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Whitespace-insensitive\n if norm_expected.lower().replace(' ', '') == norm_actual.lower().replace(' ', ''):\n return True\n\n # Try numeric comparison for currency values\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0:\n if abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n # Also check exact equality for non-zero\n if exp_num == act_num:\n return True\n\n # Date matching - be very flexible\n if '/' in norm_expected:\n # Parse the expected date\n try:\n exp_parts = norm_expected.split('/')\n exp_month = int(exp_parts[0])\n exp_day = int(exp_parts[1])\n exp_year = int(exp_parts[2])\n except:\n exp_month = exp_day = exp_year = None\n\n if exp_month is not None:\n # If actual is a datetime object\n if hasattr(actual, 'year') and hasattr(actual, 'month') and hasattr(actual, 'day'):\n if actual.month == exp_month and actual.day == exp_day and actual.year == exp_year:\n return True\n\n # Try parsing the string form\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%m/%d/%y', '%Y/%m/%d']:\n try:\n dt = datetime.datetime.strptime(norm_actual, fmt)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Try iso format\n try:\n dt = datetime.datetime.fromisoformat(norm_actual)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Number as string comparison (e.g., \"7\" vs 7)\n try:\n ef = float(norm_expected)\n af = float(norm_actual)\n if ef == af:\n return True\n except:\n pass\n\n # Also try int comparison (\"7\" vs \"7.0\")\n try:\n ei = intLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705558604",
"sort_order": 10,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-1107\" (or \"CLM-2026-1106\"), \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", \"LT\", \"7\", \"CLM-2026-1105\", \"Submitted\", \"$1,850.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n file_path = None\n for f in Path(workspace_path).glob(\"**/*Claim_Register*\"):\n if f.suffix == '.xlsx':\n file_path = f\n break\n if file_path is None:\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # Acceptable claim IDs for the first column per the rubric\n acceptable_claim_ids = [\"CLM-2026-1107\", \"CLM-2026-1106\"]\n\n expected = [\n None, # placeholder for claim ID - checked separately\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n \"LT\",\n \"7\",\n \"CLM-2026-1105\",\n \"Submitted\",\n \"$1,850.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip()\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n if e is None and a is None:\n return True\n if e is None or a is None:\n return False\n\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(e_num - a_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(e_num - a_num) / abs(a_num) <= 0.05:\n return True\n\n # Date comparison\n if '/' in e:\n try:\n expected_date = datetime.datetime.strptime(e, '%m/%d/%Y').date()\n if hasattr(actual_val, 'date'):\n if actual_val.date() == expected_date:\n return True\n elif hasattr(actual_val, 'year'):\n if actual_val == expected_date:\n return True\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y']:\n try:\n if datetime.datetime.strptime(a, fmt).date() == expected_date:\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n\n # NPI / numeric string cleaning\n e_clean = re.sub(r'[^\\w.]', '', e)\n a_clean = re.sub(r'[^\\w.]', '', a)\n if e_clean.lower() == a_clean.lower():\n return True\n\n return False\n\n def is_acceptable_claim_id(val):\n n = normalize(val)\n if n is None:\n return False\n for cid in acceptable_claim_ids:\n if n.upper() == cid.upper():\n return True\n return False\n\n best_mismatches = None\n best_sheet = None\n best_claim_id = None\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n\n # Find a cell that matches one of the acceptable claim IDs\n start_idx = None\n matched_claim_id = None\n for i, c in enumerate(cells):\n if is_acceptable_claim_id(c):\n start_idx = i\n matched_claim_id = normalize(c)\n break\n if start_idx is None:\n continue\n\n # Pad cells if needed\n if start_idx + len(expected) > len(cells):\n cells = cells + [None] * (start_idx + len(expected) - len(cells))\n\n all_match = True\n mismatches = []\n for j, exp_val in enumerate(expLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705566070",
"sort_order": 11,
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be no duplicate rows or `Claim_ID`s.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n rows = list(ws.iter_rows(min_row=1, values_only=True))\n if len(rows) < 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File has 0 or 1 data rows, so no duplicates possible.\"}\n\n header = [str(c).strip().lower() if c is not None else \"\" for c in rows[0]]\n\n # Find Claim_ID column\n claim_id_col = None\n for i, h in enumerate(header):\n if h in (\"claim_id\", \"claimid\", \"claim id\"):\n claim_id_col = i\n break\n\n # Check for duplicate Claim_IDs\n duplicate_claim_ids = []\n if claim_id_col is not None:\n seen_ids = {}\n for row_idx, row in enumerate(rows[1:], start=2):\n val = row[claim_id_col]\n if val is None:\n continue\n val_str = str(val).strip()\n if val_str == \"\":\n continue\n if val_str in seen_ids:\n duplicate_claim_ids.append((val_str, seen_ids[val_str], row_idx))\n else:\n seen_ids[val_str] = row_idx\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a 'Claim_ID' column in header: {header}\"}\n\n # Check for fully duplicate rows\n def row_key(row):\n return tuple(str(c).strip() if c is not None else \"\" for c in row)\n\n seen_rows = {}\n duplicate_rows = []\n for row_idx, row in enumerate(rows[1:], start=2):\n rk = row_key(row)\n # Skip completely empty rows\n if all(v == \"\" for v in rk):\n continue\n if rk in seen_rows:\n duplicate_rows.append((row_idx, seen_rows[rk]))\n else:\n seen_rows[rk] = row_idx\n\n issues = []\n if duplicate_claim_ids:\n for cid, first_row, dup_row in duplicate_claim_ids:\n issues.append(f\"Duplicate Claim_ID '{cid}' found in rows {first_row} and {dup_row}\")\n if duplicate_rows:\n for dup_row, first_row in duplicate_rows:\n issues.append(f\"Duplicate row found: rows {first_row} and {dup_row} are identical\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicates found in PBC_Claim_Register.xlsx: \" + \"; \".join(issues)\n }\n\n total_data_rows = sum(1 for row in rows[1:] if not all((str(c).strip() if c is not None else \"\") == \"\" for c in row))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Claim_IDs found in PBC_Claim_Register.xlsx. {total_data_rows} unique data rows checked.\"\n }\n\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Claim_Register.xlsx: {str(e)}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775705648199",
"sort_order": 12,
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be a table with 11 total rows (including the header row).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (rows that have at least one non-None cell)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Sheet '{sheet_name}' in PBC_Denial_Worklist.xlsx has exactly 11 non-empty rows (including header), as required.\"\n }\n \n # If no sheet matched exactly, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with 11 total rows (including header) in PBC_Denial_Worklist.xlsx. Found: {'; '.join(details)}\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705668317",
"sort_order": 13,
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0028\" (where 28 can be 29 or 30), \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"BlueCross TX\", \"CO-16\", \"N382\", \"Technical/Administrative\", \"Correct and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and BLANK (empty cell) in consecutive columns.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob to be lenient\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Denial_Worklist*.xlsx\"))\n if not candidates:\n candidates = list(workspace.glob(\"**/*.xlsx\"))\n \n file_path = None\n for c in candidates:\n if \"denial\" in c.name.lower() or \"worklist\" in c.name.lower() or \"pbc\" in c.name.lower():\n file_path = c\n break\n if file_path is None and candidates:\n file_path = candidates[0]\n if file_path is None:\n file_path = workspace / \"PBC_Denial_Worklist.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found. Searched in {workspace}. No matching xlsx files found.\"}\n\n # The rubric says DEN-2026-0028 where 28 can be 29 or 30.\n # We'll also accept a broader range (28-31) to be lenient.\n # All other values are fixed.\n expected_values = [\n \"DEN-2026-XXXX\", # special handling\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"BlueCross TX\",\n \"CO-16\",\n \"N382\",\n \"Technical/Administrative\",\n \"Correct and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def den_id_matches(actual_val):\n \"\"\"Check if actual matches DEN-2026-00XX where XX is 28-31 (lenient range)\"\"\"\n if actual_val is None:\n return False\n a = str(actual_val).strip().upper()\n # Accept DEN-2026-0028 through DEN-2026-0031\n pattern = r'^DEN-2026-00(2[89]|3[01])$'\n return bool(re.match(pattern, a))\n\n def values_match(actual, expected):\n if expected is None:\n return is_blank(actual)\n if actual is None:\n return False\n a = normalize(actual)\n e = normalize(expected)\n if a is None:\n return False\n # Case-insensitive comparison\n if a.lower() == e.lower():\n return True\n # Try date variants: strip leading zeros\n a_clean = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_clean = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_clean.lower() == e_clean.lower():\n return True\n # Also handle date formats like 2026-03-15 vs 03/15/2026\n # Try converting common date patterns\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\"),\n ]\n def to_canonical_date(s):\n for pat, fn in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fn(m)\n return None\n \n a_date = to_canonical_date(a)\n e_date = to_canonical_date(e)\n if a_date and e_date and a_date == e_date:\n return True\n \n # Handle BCBS TX variants\n if e.lower() == \"bluecross tx\":\n variants = [\"bluecross tx\", \"bcbs tx\", \"bcbstx\", \"blue cross tx\", \"bluecrosstx\", \"bcbs-tx\"]\n if a.lower().replace(\" \", \"\") in [v.replace(\" \", \"\") for v in variants]:\n return True\n \n return False\n\n rows_to_check = list(range(1, ws.max_row + 1)) if ws.max_row and ws.max_row <= 50 else list(range(1, 51))\n # Prioritize rows 9-11 as stated in rubric, but check all rows for leniency\n priority_rows = [9, 10, 11]\n all_rows = priority_rows + [r for r in rows_to_check if r not in priority_rows]\n\n num_expected = len(expected_values)\n best_match_count = 0\n best_row = None\n best_mismatches = []\n\n for row_num in all_rows:\n row_cells = []\n for col in range(1, max(ws.max_column + 1 if ws.max_column else 1, 20)):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n for start_col_idx in range(len(row_cells) - num_expected + 1):\n segment = row_cells[start_col_idx:start_col_idx + num_expected]\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705689978",
"sort_order": 14,
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0029\" (where 29 can be 28 or 30), \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"BlueCross TX\", \"CO-97\", \"N520\", \"Technical/Administrative\", \"Add Modifier and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob\n fp = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n fp = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n \n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {fp}\"}\n\n try:\n wb = openpyxl.load_workbook(fp, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # The rubric says DEN-2026-0029 where 29 can be 28 or 30\n # So we accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n # We also accept variants like DEN-2026-0035 or other denial IDs with flexible matching\n expected_values = [\n \"DEN-2026-FLEX\", # special handling\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"BlueCross TX\",\n \"CO-97\",\n \"N520\",\n \"Technical/Administrative\",\n \"Add Modifier and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n try:\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n except Exception:\n pass\n s = str(val).strip()\n return s\n\n def match_den_id(actual_str):\n \"\"\"Match DEN-2026-0028, DEN-2026-0029, or DEN-2026-0030 (flexible)\"\"\"\n norm = actual_str.strip().upper()\n # Accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n if re.match(r'^DEN-2026-00(28|29|30)$', norm):\n return True\n # Also accept any DEN-2026-XXXX pattern as a looser fallback\n # But primary match is the three specific ones\n return False\n\n def match_date(actual_str, expected_str):\n \"\"\"Flexible date matching\"\"\"\n date_pat = re.compile(r'^(\\d{1,2})/(\\d{1,2})/(\\d{4})$')\n m_a = date_pat.match(actual_str)\n m_e = date_pat.match(expected_str)\n if m_a and m_e:\n return (int(m_a.group(1)) == int(m_e.group(1)) and\n int(m_a.group(2)) == int(m_e.group(2)) and\n m_a.group(3) == m_e.group(3))\n return False\n\n def match(actual, expected, idx):\n norm_actual = normalize(actual)\n \n # Special case: DEN ID (first column)\n if idx == 0:\n return match_den_id(norm_actual)\n \n # Special case: BLANK (last column)\n if expected is None:\n return norm_actual == \"\" or norm_actual.lower() == \"none\"\n \n norm_expected = str(expected).strip()\n \n # Case-insensitive comparison\n if norm_actual.lower() == norm_expected.lower():\n return True\n \n # Date matching\n if match_date(norm_actual, norm_expected):\n return True\n \n # Contains check as fallback for things like \"BlueCross TX\" vs \"BCBS TX\" etc.\n if norm_expected.lower() in norm_actual.lower() or norm_actual.lower() in norm_expected.lower():\n return True\n \n # Also try matching with flexible whitespace/case\n if re.sub(r'\\s+', ' ', norm_actual.lower()) == re.sub(r'\\s+', ' ', norm_expected.lower()):\n return True\n \n return False\n\n for row_num in [9, 10, 11]:\n row_cells = []\n for col in range(1, ws.max_column + 2): # +2 for safety to check blank\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n # Try to match expected_values against consecutive columns starting from various offsets\n for start_col in range(min(5, len(row_cells))):\n all_match = True\n mismatches = []\n for i in range(len(expected_values)):\n col_idx = start_col + i\n if col_idx >= len(row_cells):\n actual_val = None\n else:\n actual_val = row_cells[col_idx]\n if not match(actual_val, expected_values[i], i):\n all_match = False\n mismatches.append((i, expected_values[i], normalize(actual_val)))\n if all_match:\n vals = [normalize(row_cells[start_col + i]) if (start_col + i) < len(row_cells) else \"\" for i in range(len(expected_values))]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {row_num} contains all expected values starting at column {start_col+1}. Values: {vals}\"}\n\n # If no exact matLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705707748",
"sort_order": 15,
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the following exact values: \"DEN-2026-0030\" (where 30 can be 29 or 38), \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"BlueCross TX\", \"CO-45\", \"N130\", \"Contractual/Payer Error\", \"Appeal - Contractual Rate Dispute\", \"Escalated\" [\"Appealed\" also acceptable], \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n filepath = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n filepath = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if fp.exists():\n filepath = fp\n\n if filepath is None or not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(filepath)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # The DEN ID can be DEN-2026-0029, DEN-2026-0030, DEN-2026-0038, DEN-2026-0039\n valid_den_ids = [\"den-2026-0029\", \"den-2026-0030\", \"den-2026-0038\", \"den-2026-0039\"]\n\n # Expected values for columns after DEN ID (indices 1..12)\n # Index 0: DEN ID (special)\n # Index 1: CLM-2026-0967\n # Index 2: PAT-1063\n # Index 3: Robert Daniels\n # Index 4: 03/08/2026\n # Index 5: BlueCross TX\n # Index 6: CO-45\n # Index 7: N130\n # Index 8: Contractual/Payer Error\n # Index 9: Appeal - Contractual Rate Dispute\n # Index 10: Escalated or Appealed\n # Index 11: Maria Santos\n # Index 12: 04/24/2026\n # Index 13: BLANK\n\n expected_values = [\n None, # placeholder for DEN ID\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"BlueCross TX\",\n \"CO-45\",\n \"N130\",\n \"Contractual/Payer Error\",\n \"Appeal - Contractual Rate Dispute\",\n \"Escalated\", # also accept \"Appealed\"\n \"Maria Santos\",\n \"04/24/2026\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n return s\n\n def norm_date(s):\n \"\"\"Normalize date string to MM/DD/YYYY format.\"\"\"\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\")\n ]\n for pat, fmt in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fmt(m)\n return s.strip()\n\n def cell_to_normalized_string(actual):\n \"\"\"Convert a cell value to a normalized comparable string.\"\"\"\n if actual is None:\n return \"\"\n if isinstance(actual, (datetime.datetime, datetime.date)):\n d = actual if isinstance(actual, datetime.date) else actual.date()\n return f\"{d.month:02d}/{d.day:02d}/{d.year}\"\n return str(actual).strip()\n\n def values_match(actual, expected, col_index):\n a = cell_to_normalized_string(actual).lower()\n e = expected.lower().strip()\n # Special handling for column 10 (Escalated / Appealed)\n if col_index == 10:\n return a in [\"escalated\", \"appealed\"]\n if a == e:\n return True\n # Try date normalization\n if norm_date(a) == norm_date(e):\n return True\n # Loose matching: strip punctuation differences\n a_clean = re.sub(r'[\\s\\-/]+', '', a)\n e_clean = re.sub(r'[\\s\\-/]+', '', e)\n if a_clean == e_clean:\n return True\n return False\n\n def is_blank(val):\n return val is None or str(val).strip() == \"\"\n\n rows_to_check = list(range(2, ws.max_row + 1)) # Check all data rows, but prioritize 9-11\n priority_rows = [9, 10, 11]\n other_rows = [r for r in rows_to_check if r not in priority_rows]\n ordered_rows = priority_rows + other_rows\n\n best_score = 0\n best_details = []\n best_row = None\n\n for row_num in ordered_rows:\n if row_num > ws.max_row:\n continue\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n if len(row_cells) < len(expected_values):\n continue\n\n all_match = True\n match_details = []\n matched_count = 0\n\n # Check column 1 (DEN ID)\n col0_val = cell_to_normalized_string(row_cells[0]).lower()\n if col0_val in valid_den_ids:\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' matches DEN ID pattern\")\n matched_count += 1\n else:\n all_match = False\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' not in valid DEN IDs {valid_den_ids}\")\n\n # Check remaininLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705711177",
"sort_order": 16,
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be no duplicate rows or Denial IDs.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if f.name.lower() == \"pbc_denial_worklist.xlsx\":\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n rows = []\n header_row = None\n denial_id_col = None\n \n for i, row in enumerate(ws.iter_rows(values_only=True)):\n # Skip completely empty rows\n if all(cell is None or str(cell).strip() == '' for cell in row):\n continue\n if header_row is None:\n header_row = tuple(str(c).strip() if c is not None else '' for c in row)\n # Find the Denial ID column (be flexible with naming)\n for col_idx, h in enumerate(header_row):\n h_lower = h.lower().strip()\n if 'denial' in h_lower and 'id' in h_lower:\n denial_id_col = col_idx\n break\n elif h_lower in ('denialid', 'denial_id', 'denial id'):\n denial_id_col = col_idx\n break\n continue\n # Normalize cell values for comparison\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else ''\n for cell in row\n )\n rows.append((i + 1, normalized, row)) # 1-indexed row number\n \n if not rows:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PBC_Denial_Worklist.xlsx has no data rows (only header or empty), so no duplicates.\"}\n \n issues = []\n \n # Check for duplicate full rows\n seen_rows = {}\n duplicate_rows = []\n for row_num, normalized, raw in rows:\n if normalized in seen_rows:\n duplicate_rows.append((row_num, seen_rows[normalized]))\n else:\n seen_rows[normalized] = row_num\n \n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {dup_row} is a duplicate of row {orig_row}\"\n for dup_row, orig_row in duplicate_rows\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate row(s): {dup_details}\")\n \n # Check for duplicate Denial IDs\n if denial_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for row_num, normalized, raw in rows:\n denial_id = normalized[denial_id_col] if denial_id_col < len(normalized) else ''\n if denial_id and denial_id in seen_ids:\n duplicate_ids.append((row_num, denial_id, seen_ids[denial_id]))\n elif denial_id:\n seen_ids[denial_id] = row_num\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Row {dup_row} has Denial ID '{did}' which duplicates row {orig_row}\"\n for dup_row, did, orig_row in duplicate_ids\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Denial ID(s): {dup_id_details}\")\n else:\n # If we can't find a Denial ID column, note it but don't fail on that alone\n pass\n \n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx has duplicates. \" + \" | \".join(issues)\n }\n \n denial_id_info = f\" Denial ID column found at index {denial_id_col} ('{header_row[denial_id_col]}').\" if denial_id_col is not None else \" No Denial ID column detected (checked row uniqueness only).\"\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx contains {len(rows)} data rows with no duplicate rows or Denial IDs.{denial_id_info}\"\n }\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Denial_Worklist.xlsx: {str(e)}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775705797706",
"sort_order": 17,
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be a table with 5 total rows (including the header row).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 5 rows (1 header + 4 data)\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows: a row is non-empty if at least one cell has a value\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 5:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 5 total rows (including header) in sheet '{sheet_name}' of PBC_Escalation_Log.xlsx.\"\n }\n \n # If no sheet had exactly 5 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with exactly 5 total rows (including header) in PBC_Escalation_Log.xlsx. Found: {'; '.join(details)}\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705829052",
"sort_order": 18,
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the following exact values: \"ESC-2026-0008\" (or \"ESC-2026-0009\"), \"04/24/2026\", \"CLM-2026-0234, CLM-2026-0356, CLM-2026-0489, CLM-2026-0612, CLM-2026-0892\" (in any order), \"Operational\", \"Pattern Recurrence\", \"Maria Santos\", \"David Park\", \"Open\", and a BLANK cell.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file with glob to be lenient\n xlsx_files = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if not xlsx_files:\n # Try case-insensitive\n xlsx_files = [f for f in Path(workspace_path).glob(\"**/*.xlsx\") if \"escalation\" in f.name.lower()]\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Escalation_Log.xlsx not found in workspace.\"}\n\n file_path = xlsx_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n expected_claims = {\"CLM-2026-0234\", \"CLM-2026-0356\", \"CLM-2026-0489\", \"CLM-2026-0612\", \"CLM-2026-0892\"}\n # Accept ESC-2026-0008 or ESC-2026-0009 (or nearby numbers to be generous)\n valid_esc_ids = {\"ESC-2026-0008\", \"ESC-2026-0009\", \"ESC-2026-0010\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_row(row_num):\n row_vals = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_vals.append(cell_val)\n\n row_strs = [normalize(v) for v in row_vals]\n feedback_parts = []\n score = 0\n total_checks = 8 # 8 required fields\n\n # Check ESC ID\n found_esc = False\n for s in row_strs:\n for esc_id in valid_esc_ids:\n if esc_id in s:\n found_esc = True\n break\n if found_esc:\n break\n if found_esc:\n score += 1\n feedback_parts.append(\"Found valid ESC ID\")\n else:\n feedback_parts.append(f\"Missing ESC ID (expected one of {valid_esc_ids})\")\n\n # Check date 04/24/2026 - be flexible with date formats\n found_date = False\n for idx, s in enumerate(row_strs):\n if not s:\n continue\n if \"04/24/2026\" in s or \"4/24/2026\" in s or \"2026-04-24\" in s:\n found_date = True\n break\n raw_val = row_vals[idx]\n if hasattr(raw_val, 'strftime'):\n try:\n formatted = raw_val.strftime(\"%m/%d/%Y\")\n if formatted == \"04/24/2026\":\n found_date = True\n break\n except:\n pass\n if hasattr(raw_val, 'year'):\n try:\n if raw_val.year == 2026 and raw_val.month == 4 and raw_val.day == 24:\n found_date = True\n break\n except:\n pass\n if found_date:\n score += 1\n feedback_parts.append(\"Found date 04/24/2026\")\n else:\n feedback_parts.append(\"Missing date 04/24/2026\")\n\n # Check claims - they can be in any order, possibly in one cell or across cells\n found_claims = set()\n for s in row_strs:\n for claim in expected_claims:\n if claim in s:\n found_claims.add(claim)\n if found_claims == expected_claims:\n score += 1\n feedback_parts.append(\"Found all 5 claims\")\n else:\n missing = expected_claims - found_claims\n feedback_parts.append(f\"Missing claims: {missing}\")\n\n # Check Operational\n found_operational = any(\"operational\" in s.lower() for s in row_strs)\n if found_operational:\n score += 1\n feedback_parts.append(\"Found 'Operational'\")\n else:\n feedback_parts.append(\"Missing 'Operational'\")\n\n # Check Pattern Recurrence\n found_pattern = any(\"pattern recurrence\" in s.lower() for s in row_strs)\n if found_pattern:\n score += 1\n feedback_parts.append(\"Found 'Pattern Recurrence'\")\n else:\n feedback_parts.append(\"Missing 'Pattern Recurrence'\")\n\n # Check Maria Santos\n found_maria = any(\"maria santos\" in s.lower() for s in row_strs)\n if found_maria:\n score += 1\n feedback_parts.append(\"Found 'Maria Santos'\")\n else:\n feedback_parts.append(\"Missing 'Maria Santos'\")\n\n # Check David Park\n found_david = any(\"david park\" in s.lower() for s in row_strs)\n if found_david:\n score += 1\n feedback_parts.append(\"Found 'David Park'\")\n else:\n feedback_parts.append(\"Missing 'David Park'\")\n\n # Check Open\n found_open = any(s.Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705849430",
"sort_order": 19,
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the exact values: \"ESC-2026-0009\" (or \"ESC-2026-0008\"), \"04/24/2026\", \"CLM-2026-0967\", \"Financial\", \"Payer Dispute\", \"Maria Santos\", \"David Park\", \"Open\", and BLANK (empty cell).",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for leniency\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Try exact path\n fp = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if fp.exists():\n file_path = fp\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # The rubric says ESC-2026-0009 or ESC-2026-0008, but we'll accept a range of ESC IDs\n valid_esc_ids = [\"esc-2026-0008\", \"esc-2026-0009\", \"esc-2026-0010\", \"esc-2026-0011\"]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def check_date(val):\n \"\"\"Check if the value represents 04/24/2026\"\"\"\n if isinstance(val, datetime):\n return val.month == 4 and val.day == 24 and val.year == 2026\n s = normalize(val).lower()\n date_patterns = [\"04/24/2026\", \"4/24/2026\", \"2026-04-24\", \"04-24-2026\", \"april 24, 2026\", \"apr 24, 2026\"]\n return s in date_patterns\n\n def check_blank(val):\n return val is None or str(val).strip() == \"\"\n\n # Search rows 4-10 for leniency (rubric says 4 or 5, but be generous)\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n \n row_strs = [normalize(c).lower() for c in row_cells]\n \n # Check if key identifiers are present in the row\n has_esc = any(any(esc_id in s for esc_id in valid_esc_ids) for s in row_strs)\n has_clm = any(\"clm-2026-0967\" in s for s in row_strs)\n has_financial = any(\"financial\" in s for s in row_strs)\n has_payer = any(\"payer dispute\" in s for s in row_strs)\n has_maria = any(\"maria santos\" in s or \"maria\" in s for s in row_strs)\n has_david = any(\"david park\" in s or \"david\" in s for s in row_strs)\n has_open = any(s == \"open\" for s in row_strs)\n \n # Check date\n has_date = any(check_date(c) for c in row_cells)\n \n checks = {\n \"ESC ID\": has_esc,\n \"CLM-2026-0967\": has_clm,\n \"Financial\": has_financial,\n \"Payer Dispute\": has_payer,\n \"Maria Santos\": has_maria,\n \"David Park\": has_david,\n \"Open\": has_open,\n \"Date 04/24/2026\": has_date\n }\n \n if all(checks.values()):\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row {row_num} in PBC_Escalation_Log.xlsx contains all expected escalation values. Row values: {row_strs}\"\n }\n \n # Gather debug info\n debug = []\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n row_strs = [normalize(c) for c in row_cells]\n if any(s.strip() for s in row_strs): # Only show non-empty rows\n debug.append(f\"Row {row_num}: {row_strs}\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No row (4-10) in PBC_Escalation_Log.xlsx contains the expected escalation entry. Expected ESC-2026-0008/0009, 04/24/2026, CLM-2026-0967, Financial, Payer Dispute, Maria Santos, David Park, Open. Found rows: {'; '.join(debug)}\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705853789",
"sort_order": 20,
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be no duplicate rows or Escalation IDs",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if \"escalation\" in f.name.lower() and \"log\" in f.name.lower():\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {file_path}: {e}\"}\n\n issues = []\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else \"\"\n for cell in row\n )\n rows.append(normalized)\n\n if not rows:\n continue\n\n header = rows[0]\n data_rows = rows[1:] if len(rows) > 1 else []\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(cell != \"\" for cell in r)]\n\n # Check for duplicate full rows\n seen_rows = set()\n duplicate_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n duplicate_rows.append((i + 2, r))\n else:\n seen_rows.add(r)\n\n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {idx}: {row}\" for idx, row in duplicate_rows[:5]\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate data row(s) in sheet '{sheet_name}'. Examples: {dup_details}\")\n\n # Check for duplicate Escalation IDs\n # Find the column index for Escalation ID (flexible matching)\n esc_id_col = None\n for col_idx, col_name in enumerate(header):\n if col_name and (\"escalation\" in col_name and \"id\" in col_name):\n esc_id_col = col_idx\n break\n \n if esc_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for i, r in enumerate(data_rows):\n if esc_id_col < len(r):\n esc_id = r[esc_id_col]\n if esc_id and esc_id != \"\":\n if esc_id in seen_ids:\n duplicate_ids.append((i + 2, esc_id, seen_ids[esc_id]))\n else:\n seen_ids[esc_id] = i + 2\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Escalation ID '{eid}' in rows {first_row} and {dup_row}\" \n for dup_row, eid, first_row in duplicate_ids[:5]\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Escalation ID(s) in sheet '{sheet_name}'. Examples: {dup_id_details}\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicate issues found in PBC_Escalation_Log.xlsx: \" + \" | \".join(issues)\n }\n\n total_sheets = len(wb.sheetnames)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Escalation IDs found in PBC_Escalation_Log.xlsx across {total_sheets} sheet(s).\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775705926737",
"sort_order": 21,
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be a table with 11 total rows (including the header row).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for the file with case-insensitive matching\n target = None\n ws_path = Path(workspace_path)\n \n # Try exact name first\n exact = ws_path / \"PBC_RA_Queue.xlsx\"\n if exact.exists():\n target = exact\n else:\n # Try case-insensitive glob\n for f in ws_path.glob(\"*.xlsx\"):\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target = f\n break\n \n if target is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File 'PBC_RA_Queue.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open '{target.name}': {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (a row is non-empty if at least one cell has a value)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 11 total rows (including header) in sheet '{sheet_name}' of '{target.name}'.\"\n }\n \n # If no sheet had exactly 11 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected 11 total rows (including header) in 'PBC_RA_Queue.xlsx', but found: {'; '.join(details)}.\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705946912",
"sort_order": 22,
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0031\" (where 31 can be 32 or 33), \"04/24/2026\", \"CLM-2026-0892\", \"PAT-1042\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the file with case-insensitive search\n file_path = None\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n file_path = c\n break\n if file_path is None:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n for f in xlsx_files:\n if \"ra_queue\" in f.name.lower() or \"raqueue\" in f.name.lower():\n file_path = f\n break\n if file_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in {workspace_path}. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n # The rubric says the RA ID can be RA-2026-0031, 0032, or 0033\n valid_ra_ids = [\"ra-2026-0031\", \"ra-2026-0032\", \"ra-2026-0033\"]\n\n # Core required values that MUST be present:\n # RA ID (flexible from list above)\n # Date: 04/24/2026\n # Claim: CLM-2026-0892\n # Patient: PAT-1042\n # Task: ERA Processing\n # Priority: [Any value is acceptable] - we won't check this\n # Assigned: Maria Santos\n #\n # Status: \"Resolved\" or \"Open\" or BLANK\n # Resolution Date: \"04/24/2026\" or BLANK\n\n core_required = [\n \"04/24/2026\", # date\n \"CLM-2026-0892\", # claim id\n \"PAT-1042\", # patient id\n \"ERA Processing\", # task type\n \"Maria Santos\", # assigned to\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def values_match(cell_val, expected_val):\n n_cell = normalize(cell_val).lower()\n n_exp = expected_val.strip().lower()\n if n_cell == n_exp:\n return True\n # Handle date variations\n if n_exp == \"04/24/2026\":\n if n_cell in [\"4/24/2026\", \"04/24/2026\", \"2026-04-24\", \"04-24-2026\", \"4-24-2026\"]:\n return True\n # Handle minor whitespace/case differences\n if n_cell.replace(\" \", \"\") == n_exp.replace(\" \", \"\"):\n return True\n # Partial matching for things like \"era processing\"\n if n_exp in n_cell or n_cell in n_exp:\n if len(n_exp) > 3 and len(n_cell) > 3:\n return True\n return False\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == \"\" or s.lower() == \"none\"\n\n # Check rows 9, 10, 11 first, then expand search\n priority_rows = [9, 10, 11]\n check_rows = list(range(2, min(ws.max_row + 1, 50)))\n all_rows = priority_rows + [r for r in check_rows if r not in priority_rows]\n\n best_match_count = 0\n best_row_info = \"\"\n\n for row_num in all_rows:\n row_values = []\n for col in range(1, ws.max_column + 1):\n row_values.append(ws.cell(row=row_num, column=col).value)\n\n # Check RA ID\n ra_id_found = False\n for cv in row_values:\n nv = normalize(cv).lower().strip()\n if nv in valid_ra_ids:\n ra_id_found = True\n break\n\n # Check core required values\n core_matched = []\n for exp in core_required:\n found = False\n for cv in row_values:\n if values_match(cv, exp):\n found = True\n break\n core_matched.append(found)\n\n total_core = sum(core_matched) + (1 if ra_id_found else 0)\n if total_core > best_match_count:\n best_match_count = total_core\n best_row_info = f\"Row {row_num}: {[normalize(v) for v in row_values]}\"\n\n # Need RA ID + all core required values (6 total)\n if ra_id_found and all(core_matched):\n # Check status - rubric says \"Resolved\" and \"04/24/2026\" for last two,\n # but also says they could be \"Open\", BLANK instead\n has_resolved = False\n has_open = False\n status_val = \"unknown\"\n\n for cv in row_values:\n nv = normalize(cv).lower()\n if nv == \"resolved\":\n has_resolved = True\n status_val = \"Resolved\"\n if nv == \"open\":\n has_open = True\n status_val = \"Open\"\n\n #Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705963295",
"sort_order": 23,
"rubric_text": "In `PBC_RA_Queue.xlsx`, row 9, 10, or 11 must contain the exact values: \"RA-2026-0032\" (where 32 can be 31 or 33), \"04/24/2026\", \"CLM-2026-1105\", \"PAT-1058\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\n\ndef normalize(val):\n \"\"\"Normalize a cell value to a comparable string.\"\"\"\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n\ndef values_match(actual, expected):\n \"\"\"Loosely compare two values.\"\"\"\n a = normalize(actual)\n e = expected.strip()\n # Direct match (case-insensitive)\n if a.lower() == e.lower():\n return True\n # Try date parsing: expected is MM/DD/YYYY\n if '/' in e:\n try:\n exp_date = datetime.strptime(e, \"%m/%d/%Y\")\n # Try parsing actual in various formats\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\"]:\n try:\n act_date = datetime.strptime(a, fmt)\n if act_date.date() == exp_date.date():\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n return False\n\n\ndef ra_id_match(actual):\n \"\"\"Check if a cell value matches RA-2026-00{31,32,33}.\"\"\"\n a = normalize(actual).upper()\n return a in [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n\ndef verify(workspace_path, external_services_path=None):\n ws_path = Path(workspace_path)\n # Try case-insensitive search for file\n target_file = None\n for f in ws_path.iterdir():\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target_file = f\n break\n\n if target_file is None:\n # Also try glob\n candidates = list(ws_path.glob(\"*RA_Queue*\")) + list(ws_path.glob(\"*ra_queue*\")) + list(ws_path.glob(\"*Ra_Queue*\")) + list(ws_path.glob(\"*Queue*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n # The core required values (excluding RA ID which has special matching,\n # excluding priority which accepts any value,\n # and excluding status/completion date which are flexible)\n core_expected = [\n \"04/24/2026\",\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"ERA Processing\",\n \"Maria Santos\",\n ]\n\n # Search rows 8-12 to be generous (rubric says 9, 10, or 11)\n search_rows = list(range(8, 13))\n\n for row_num in search_rows:\n row_cells = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_cells.append(cell_val)\n\n # Check if RA ID matches\n ra_found = False\n for cell in row_cells:\n if ra_id_match(cell):\n ra_found = True\n break\n\n if not ra_found:\n continue\n\n # Check if all core expected values can be found in this row\n all_core_found = True\n matched_details = []\n matched_details.append(\" RA ID found (one of RA-2026-0031/32/33)\")\n\n for exp in core_expected:\n found = False\n for i, cell in enumerate(row_cells):\n if values_match(cell, exp):\n found = True\n matched_details.append(f\" Expected '{exp}' -> found '{normalize(cell)}' in column {i+1}\")\n break\n if not found:\n all_core_found = False\n matched_details.append(f\" Expected '{exp}' -> NOT FOUND\")\n break\n\n if all_core_found:\n # Status can be \"Resolved\", \"Open\", or blank - check if present\n status_info = \"\"\n for i, cell in enumerate(row_cells):\n n = normalize(cell).lower()\n if n in [\"resolved\", \"open\"]:\n status_info = f\"Status='{normalize(cell)}' in column {i+1}\"\n break\n if not status_info:\n status_info = \"Status is blank or not found (acceptable)\"\n\n detail = \"\\n\".join(matched_details)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All expected values found in row {row_num} of PBC_RA_Queue.xlsx:\\n{detail}\\n {status_info}\\nPriority: any value accepted.\"\n }\n\n # If not found, provide diagnostic info\n diag = []\n for row_num in search_rows:\n row_vals = []\n for col in range(1, ws.max_column + 1Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705979579",
"sort_order": 24,
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0033\" (where 33 can be 32 or 31), \"04/24/2026\", \"CLM-2026-0967\", \"PAT-1063\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Open\", and a BLANK cell (in the appropriate columns).",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n if not file_path.exists():\n # Try case-insensitive glob\n candidates = list(Path(workspace_path).glob(\"PBC_RA_Queue*\")) + list(Path(workspace_path).glob(\"PBC_Ra_Queue*\")) + list(Path(workspace_path).glob(\"pbc_ra_queue*\"))\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n acceptable_ra_ids = [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n # Expected fixed values (excluding RA ID, Date, and Priority which are handled separately)\n # The rubric says: RA ID, Date, CLM-2026-0967, PAT-1063, ERA Processing, [Any priority], Maria Santos, Open, BLANK\n expected_fixed = {\n \"CLM\": \"CLM-2026-0967\",\n \"PAT\": \"PAT-1063\",\n \"TYPE\": \"ERA Processing\",\n \"ASSIGNEE\": \"Maria Santos\",\n \"STATUS\": \"Open\"\n }\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def clean(s):\n return s.lower().replace(\" \", \"\").replace(\"-\", \"\").replace(\"/\", \"\").replace(\"_\", \"\")\n\n def match_date(actual):\n a_clean = normalize(actual)\n # Try multiple date formats\n try:\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%B %d, %Y\", \"%m/%d/%y\"]:\n try:\n parsed = datetime.strptime(a_clean, fmt)\n if parsed.month == 4 and parsed.day == 24 and parsed.year == 2026:\n return True\n except:\n pass\n except:\n pass\n # Fallback: clean comparison\n if clean(a_clean) == clean(\"04/24/2026\"):\n return True\n if clean(a_clean) == \"04242026\":\n return True\n return False\n\n def match_ra_id(actual):\n a = normalize(actual)\n if not a:\n return False\n for rid in acceptable_ra_ids:\n if clean(a) == clean(rid):\n return True\n if a.upper().replace(\"-\", \"\").replace(\" \", \"\") == rid.upper().replace(\"-\", \"\"):\n return True\n # Regex: RA-2026-003[1-3] with flexible separators\n pattern = r'(?i)ra[\\-_ ]?2026[\\-_ ]?0{0,2}3[123]'\n if re.search(pattern, a):\n return True\n return False\n\n def fuzzy_match(expected, actual):\n e_norm = normalize(expected)\n a_norm = normalize(actual)\n if not a_norm and not e_norm:\n return True\n if clean(e_norm) == clean(a_norm):\n return True\n if e_norm.lower().strip() == a_norm.lower().strip():\n return True\n return False\n\n def check_row(row_values):\n \"\"\"Check if a row contains all expected values. Returns (True, msg) or (False, msg).\"\"\"\n # Strategy 1: Ordered match\n # Expected column order: A=RA_ID, B=Date, C=CLM, D=PAT, E=Type, F=Priority, G=Assignee, H=Status, I=blank(notes)\n if len(row_values) >= 8:\n ordered_ok = True\n issues = []\n # Column A (index 0): RA ID\n if not match_ra_id(row_values[0]):\n ordered_ok = False\n issues.append(f\"Col A: expected RA-2026-003[1-3], got '{normalize(row_values[0])}'\")\n # Column B (index 1): Date\n if ordered_ok and not match_date(row_values[1]):\n ordered_ok = False\n issues.append(f\"Col B: expected 04/24/2026, got '{normalize(row_values[1])}'\")\n # Column C (index 2): CLM-2026-0967\n if ordered_ok and not fuzzy_match(\"CLM-2026-0967\", row_values[2]):\n ordered_ok = False\n issues.append(f\"Col C: expected CLM-2026-0967, got '{normalize(row_values[2])}'\")\n # Column D (index 3): PAT-1063\n if ordered_ok and not fuzzy_match(\"PAT-1063\", row_values[3]):\n ordered_ok = False\n issues.append(f\"Col D: expected PAT-1063, got '{normalize(row_values[3])}'\")\n # Column E (index 4): ERA Processing\n if ordered_ok and not fuzzy_match(\"ERA Processing\", row_values[4]):\n ordered_ok = False\n issues.append(f\"Col E: expected ERALine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775705982031",
"sort_order": 25,
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be no duplicate rows or Queue Entry IDs.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n # Also try case-insensitive search\n if not target.exists():\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n target = c\n break\n\n if not target.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in workspace. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = []\n for cell in row:\n if isinstance(cell, str):\n normalized.append(cell.strip().lower())\n elif cell is None:\n normalized.append(None)\n else:\n normalized.append(cell)\n rows.append(tuple(normalized))\n\n if len(rows) <= 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(rows)} row(s); no duplicates possible.\"}\n\n header = rows[0]\n data_rows = rows[1:]\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(c is not None for c in r)]\n\n issues = []\n\n # Check 1: Duplicate entire rows\n seen_rows = set()\n dup_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n dup_rows.append((i + 2, r)) # +2 for 1-indexed + header\n else:\n seen_rows.add(r)\n\n if dup_rows:\n dup_info = \"; \".join([f\"Row {idx}\" for idx, vals in dup_rows[:5]])\n issues.append(f\"Found {len(dup_rows)} duplicate row(s). Examples: {dup_info}\")\n\n # Check 2: Duplicate Queue Entry IDs\n # Try to find a column that looks like a Queue Entry ID\n qe_col_idx = None\n if header:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue entry id' in h_lower or 'queueentryid' in h_lower or 'queue_entry_id' in h_lower or h_lower == 'id' or 'entry id' in h_lower:\n qe_col_idx = idx\n break\n # If not found, also check for columns starting with 'queue'\n if qe_col_idx is None:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue' in h_lower and 'id' in h_lower:\n qe_col_idx = idx\n break\n\n if qe_col_idx is not None:\n seen_ids = {}\n dup_ids = []\n for i, r in enumerate(data_rows):\n if qe_col_idx < len(r):\n val = r[qe_col_idx]\n if val is not None:\n val_key = str(val).strip().lower()\n if val_key in seen_ids:\n dup_ids.append((i + 2, val))\n else:\n seen_ids[val_key] = i + 2\n if dup_ids:\n dup_id_info = \"; \".join([f\"Row {idx}: ID={val}\" for idx, val in dup_ids[:5]])\n issues.append(f\"Found {len(dup_ids)} duplicate Queue Entry ID(s). Examples: {dup_id_info}\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx has duplicates: \" + \" | \".join(issues)}\n else:\n id_note = f\" Queue Entry ID column found at index {qe_col_idx}, all unique.\" if qe_col_idx is not None else \" No Queue Entry ID column identified; only checked full row uniqueness.\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(data_rows)} data rows with no duplicate rows or Queue Entry IDs.{id_note}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775706048803",
"sort_order": 26,
"rubric_text": "In `slack_data.json` (in `external_data/final`), there must be exactly 3 total messages in #general and exactly 8 total messages in #rcm-team.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n # Try multiple possible locations for slack_data.json\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n \n # Also check workspace-based paths\n wp = Path(workspace_path)\n candidates.append(wp / \"external_data\" / \"final\" / \"slack_data.json\")\n candidates.append(wp / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find slack_data.json. Checked: {[str(c) for c in candidates]}\"}\n \n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n \n messages = data.get(\"messages\", {})\n \n # Find the general channel and rcm-team channel\n # First, map channel names to IDs from channels info\n channels = data.get(\"channels\", {})\n general_id = None\n rcm_id = None\n for ch_id, ch_info in channels.items():\n name = ch_info.get(\"name\", \"\").lower()\n if name == \"general\":\n general_id = ch_id\n elif name == \"rcm-team\":\n rcm_id = ch_id\n \n # Count messages\n general_msgs = []\n rcm_msgs = []\n \n # Try by channel ID first, then by channel name as key\n for key, msg_list in messages.items():\n if key == general_id or key.lower() == \"general\":\n general_msgs = msg_list\n elif key == rcm_id or key.lower() == \"rcm-team\":\n rcm_msgs = msg_list\n \n general_count = len(general_msgs)\n rcm_count = len(rcm_msgs)\n \n feedback_parts = []\n passed = True\n score = 0.0\n \n if general_count == 3:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). FAIL.\")\n passed = False\n \n if rcm_count == 8:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). FAIL.\")\n passed = False\n \n return {\"pass\": passed, \"score\": score, \"feedback\": \" | \".join(feedback_parts)}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Hey, it's Maria. If you'll notice from my email I have a batch that I need completed today, April 24, 2026. Can you use the SOP and process the ERA for me? You can go ahead and do workflows even if they're technically for other roles.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py