DerekSurge
192 lines
149 KiB
JSON
192 lines
149 KiB
JSON
[
|
|
{
|
|
"id": "a017242d-7557-4fc5-b933-13e3a0a45e9b",
|
|
"sort_order": 0,
|
|
"rubric_text": "In file mailbox.json (in `external_data/final`), There must be a sent email to david.park@pathfinderbilling.com with exact subject \"Pattern Recurrence Alert: Dr. Sarah Chen — CO-16 — 5 Instances\" and exact body (no partial match accepted): \"Pattern Recurrence Report — 04/24/2026\n\nThis is Maria Santos, Resolution Auditor, reporting a pattern recurrence for Dr. Sarah Chen, NPI: 1234567890.\n\nDenial Code: CO-16 — RARC: N382 — 5 instances identified.\n\nAffected Claims:\n\n1. Claim ID: CLM-2026-0234 — Patient ID: PAT-1008 — DOS: 01/15/2026 — CPT: 99214 — Status: Resubmitted\n\n2. Claim ID: CLM-2026-0356 — Patient ID: PAT-1012 — DOS: 01/28/2026 — CPT: 99213 — Status: Resubmitted\n\n3. Claim ID: CLM-2026-0489 — Patient ID: PAT-1019 — DOS: 02/10/2026 — CPT: 99214 — Status: Resubmitted\n\n4. Claim ID: CLM-2026-0612 — Patient ID: PAT-1027 — DOS: 02/22/2026 — CPT: 99213 — Status: Resubmitted\n\n5. Claim ID: CLM-2026-0892 — Patient ID: PAT-1042 — DOS: 03/15/2026 — CPT: 99214 — Status: Resubmitted\n\nAssessment: This pattern suggests a systemic submission error. Requesting management review to determine whether a corrective action plan or direct clinic intervention is warranted.\n\nEscalation Log Entry ID: ESC-2026-0008. Escalation Category: Operational. Escalation Reason: Pattern Recurrence.\" where \"ESC-2026-0008\" could be \"ESC-2026-0009\"",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external services path.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = []\n if isinstance(mailbox_data, list):\n emails = mailbox_data\n elif isinstance(mailbox_data, dict):\n if \"emails\" in mailbox_data:\n emails = mailbox_data[\"emails\"]\n elif \"sent\" in mailbox_data:\n emails = mailbox_data[\"sent\"]\n else:\n for key, val in mailbox_data.items():\n if isinstance(val, list) and len(val) > 0:\n emails = val\n break\n\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n\n def normalize_dashes(s):\n s = str(s)\n s = s.replace('\\u2013', '\\u2014')\n s = s.replace(' -- ', ' \\u2014 ')\n s = s.replace('--', '\\u2014')\n s = s.replace(' - ', ' \\u2014 ')\n return s\n\n def normalize(s):\n s = normalize_dashes(str(s))\n return re.sub(r'\\s+', ' ', s).strip().lower()\n\n expected_subject = \"Pattern Recurrence Alert: Dr. Sarah Chen \\u2014 CO-16 \\u2014 5 Instances\"\n\n # Required content checks — all must be present in body\n required_claim_ids = [\n \"CLM-2026-0234\",\n \"CLM-2026-0356\",\n \"CLM-2026-0489\",\n \"CLM-2026-0612\",\n \"CLM-2026-0892\",\n ]\n required_phrases = [\n \"Maria Santos\",\n \"Dr. Sarah Chen\",\n \"NPI: 1234567890\",\n \"CO-16\",\n \"N382\",\n \"Pattern Recurrence\",\n \"Operational\",\n ]\n esc_pattern = re.compile(r'ESC-2026-00\\d{2}')\n\n to_addr = \"david.park@pathfinderbilling.com\"\n from_addr = \"maria.santos@pathfinderbilling.com\"\n\n best_feedback = f\"No email found to {to_addr}.\"\n best_score = 0.0\n\n for email in emails:\n if not isinstance(email, dict):\n continue\n\n # Check recipient\n e_to = None\n for field in [\"to_addr\", \"to\", \"recipient\", \"to_address\", \"to_email\", \"recipients\"]:\n if field in email and email[field]:\n e_to = email[field]\n break\n if e_to is None:\n continue\n e_to_str = \" \".join(str(x) for x in e_to) if isinstance(e_to, list) else str(e_to)\n if to_addr not in e_to_str.lower():\n continue\n\n # Check sender (allow missing/unknown sender for sent-folder emails)\n e_from = \"\"\n for field in [\"from_addr\", \"from\", \"sender\", \"from_address\", \"from_email\"]:\n if field in email and email[field]:\n e_from = str(email[field]).strip().lower()\n break\n e_folder = str(email.get(\"folder\", \"\")).strip().lower()\n if e_from and from_addr not in e_from and e_folder != \"sent\":\n continue\n\n # Check subject (collapse whitespace + normalize dashes, case-insensitive)\n e_subject = str(email.get(\"subject\", \"\"))\n subject_ok = normalize(e_subject) == normalize(expected_subject)\n\n # Get body\n e_body = \"\"\n for field in [\"body_text\", \"body\", \"content\", \"text\", \"message\", \"html\", \"body_html\"]:\n if field in email and email[field]:\n e_body = str(email[field])\n break\n\n body_norm = normalize(e_body)\n\n # Check all required claim IDs\n missing_claims = [c for c in required_claim_ids if c.lower() not in body_norm]\n\n # Check all required phrases\n missing_phrases = [p for p in required_phrases if p.lower() not in body_norm]\n\n # Check ESC ID present\n esc_ok = bool(esc_pattern.search(e_body))\n\n all_claims_ok = len(missing_claims) == 0\n all_phrases_ok = len(missing_phrases) == 0\n\n if subject_ok and all_claims_ok and all_phrases_ok and esc_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found matching Pattern Recurrence email to david.park with correct subject and all required content.\"}\n\n # Partial scoring for feedback\n total_checks = len(required_claim_ids) + len(required_phrases) + 2 # +subject, +esc\n passed_checks = (\n (1 if subject_ok else 0)\n + (len(required_claim_ids) - len(missing_claims))\n + (len(required_phrases) - len(missing_phrases))\n + (1 if esc_ok else 0)\n )\n score = 0.1 + 0.9 * (passed_checks / total_checks)\n\n issues = []\n if not subject_ok:\n issues.append(f\"wrong subject (got: '{e_subject[:80]}')\")\n if missing_claims:\n issues.append(f\"missing claim IDs: {missing_claims}\")\n if missing_phrases:\n issues.append(f\"missing phrases: {missing_phrases}\")\n if not esc_ok:\n issues.append(\"no ESC-2026-00XX ID found in body\")\n\n feedback = f\"Email to david.park found but failed checks — {'; '.join(issues)}.\"\n\n if score > best_score:\n best_score = score\n best_feedback = feedback\n\n return {\"pass\": False, \"score\": best_score, \"feedback\": best_feedback}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705367514",
|
|
"sort_order": 1,
|
|
"rubric_text": "In `mailbox.json`, there must be exactly three sent emails in total.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox.json)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'emails' key is missing or not a list in mailbox.json.\"}\n\n sent_emails = [e for e in emails if isinstance(e, dict) and str(e.get(\"folder\", \"\")).strip().lower() == \"sent\"]\n count = len(sent_emails)\n\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_emails]\n subjects_str = \"; \".join(subjects)\n\n if count == 3:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 3 sent emails as expected. Subjects: {subjects_str}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 3 sent emails, but found {count}. Sent email subjects: {subjects_str}\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1775705423565",
|
|
"sort_order": 2,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be a table with exactly 40 total rows (including the header row). No leniency, must be exactly 40.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Claim_Register.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'PBC_Claim_Register.xlsx': {e}\"}\n \n target = 40\n \n # Check all sheets for a table with exactly 40 rows\n results = []\n for sheet_name in wb.sheetnames:\n sws = wb[sheet_name]\n # Count rows that have at least one non-empty cell\n non_empty_rows = 0\n for row in sws.iter_rows(min_row=1, max_row=sws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n max_row = sws.max_row\n results.append((sheet_name, non_empty_rows, max_row))\n \n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has exactly {non_empty_rows} non-empty rows (including header), which meets the requirement of exactly 40 total rows.\"\n }\n if max_row == target and non_empty_rows != target:\n # Also accept if max_row is exactly 40 (some cells might appear empty but row exists)\n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has max_row={max_row} matching exactly 40 total rows.\"\n }\n \n # If no sheet matched exactly, provide detailed failure feedback\n details = \"; \".join([f\"Sheet '{s}': {nr} non-empty rows, max_row={mr}\" for s, nr, mr in results])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx does not have exactly 40 total rows (including header) on any sheet. Found: {details}. Expected exactly 40 rows.\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705456779",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0845\", \"PAT-1021\", \"Jennifer Walsh\", \"03/10/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99213\", \"J06.9\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$180.00\", \"$142.00\", \"$38.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0845\",\n \"PAT-1021\",\n \"Jennifer Walsh\",\n \"03/10/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99213\",\n \"J06.9\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Paid - Full\",\n \"$180.00\",\n \"$142.00\",\n \"$38.00\",\n \"$0.00\",\n \"Reconciled\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, stripping $ and commas.\"\"\"\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual_val)\n\n norm_actual = normalize(actual_val)\n if norm_actual is None:\n return False\n\n norm_expected = expected_val.strip()\n\n # Direct case-insensitive match\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Strip $ for comparison\n if norm_actual.replace('$', '').strip().lower() == norm_expected.replace('$', '').strip().lower():\n return True\n\n # Try numeric comparison with tolerance\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0 and abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n if exp_num == 0 and abs(act_num) < 0.01:\n return True\n\n # Date flexibility: 03/10/2026 vs 2026-03-10 etc.\n if norm_expected == \"03/10/2026\":\n date_patterns = [\"03/10/2026\", \"3/10/2026\", \"2026-03-10\", \"2026/03/10\", \"03-10-2026\"]\n if norm_actual in date_patterns:\n return True\n # openpyxl might return a datetime\n import datetime\n if isinstance(actual_val, datetime.datetime) or isinstance(actual_val, datetime.date):\n try:\n if actual_val.month == 3 and actual_val.day == 10 and actual_val.year == 2026:\n return True\n except:\n pass\n\n # \"1\" could also be stored as integer 1\n if norm_expected == \"1\" and norm_actual in [\"1\", \"1.0\"]:\n return True\n\n return False\n\n num_expected = len(expected)\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n cells = [cell.value for cell in row]\n # We need to find the starting column. The row might have extra columns.\n # Try each possible starting position.\n for start_col in range(max(1, len(cells) - num_expected + 1)):\n subset = cells[start_col:start_col + num_expected]\n if len(subset) < num_expected:\n # Pad with None\n subset = subset + [None] * (num_expected - len(subset))\n\n match_count = 0\n mismatches = []\n for i, (exp_v, act_v) in enumerate(zip(expected, subset)):\n if values_match(exp_v, act_v):\n match_count += 1\n else:\n mismatches.append((i, exp_v, normalize(act_v)))\n\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row_idx\n best_mismatches = mismatches\n\n if match_count == num_expected:\n found = True\n break\n if found:\n break\n\n if found:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found matching row at row {best_row_num} in PBC_Claim_Register.xlsx with all {num_expected} expected values.\"\n }\n else:\n detail = \"\"\n if best_row_num:\n detail = f\" Best partial match at row {best_row_num} with {best_match_count}/{num_expected} matches. Mismatches: \"\n for idx, exp, act in best_mismatches[:5]:\n detail += f\"[col {idx}: expected={exp!r}, actual={act!r}] \"\n if len(best_mismatches) > 5:\n detail += f\"... and {len(best_mismatches)-5} more.\"\n return {\n \"pass\": False,\n \"score\": best_match_count / num_expected if num_expected > 0 else 0.0,\n \"feedback\": f\"No row in PBC_Claim_Register.xlsx fully matches the expected values.{detail}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705468789",
|
|
"sort_order": 4,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-0878\", \"PAT-1035\", \"David Kim\", \"03/12/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"99214\", \"M54.5\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$285.00\", \"$228.00\", \"$57.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected values for a row (columns A through W = 23 columns)\n expected = [\n \"CLM-2026-0878\", # A\n \"PAT-1035\", # B\n \"David Kim\", # C\n \"03/12/2026\", # D\n \"Dr. James Miller\",# E\n \"2345678901\", # F\n None, # G - BLANK\n \"BlueCross TX\", # H\n \"BCTX1\", # I\n \"99214\", # J\n \"M54.5\", # K\n None, # L - BLANK\n \"1\", # M\n None, # N - BLANK\n \"Paid - Full\", # O\n \"$285.00\", # P\n \"$228.00\", # Q\n \"$57.00\", # R\n \"$0.00\", # S\n \"Reconciled\", # T\n None, # U - BLANK\n None, # V - BLANK\n None, # W - BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def money_val(s):\n \"\"\"Extract numeric value from money string like $285.00\"\"\"\n if s is None:\n return None\n cleaned = re.sub(r'[^\\d.\\-]', '', str(s))\n try:\n return float(cleaned)\n except:\n return None\n\n def vals_match(actual, expected_val):\n a = normalize(actual)\n e = normalize(expected_val)\n # Both blank\n if a is None and e is None:\n return True\n # One blank, other not\n if a is None or e is None:\n return False\n # Direct case-insensitive match\n if a.lower() == e.lower():\n return True\n # Try numeric / money comparison with tolerance\n ma = money_val(a)\n me = money_val(e)\n if ma is not None and me is not None:\n if me == 0:\n return abs(ma) < 0.01\n return abs(ma - me) / max(abs(me), 0.001) <= 0.05\n # Date flexibility: 03/12/2026 vs 2026-03-12 etc.\n # Try removing leading zeros and comparing\n a_stripped = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_stripped = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_stripped.lower() == e_stripped.lower():\n return True\n # Date: check if it's a datetime object rendered differently\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n try:\n formatted = actual.strftime('%m/%d/%Y')\n if formatted == e:\n return True\n except:\n pass\n return False\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [row[i].value if i < len(row) else None for i in range(23)]\n # Quick check: does column A match the claim ID?\n if normalize(cells[0]) is None:\n continue\n if normalize(cells[0]).upper() != \"CLM-2026-0878\":\n continue\n # Detailed check\n match_count = 0\n mismatches = []\n for idx in range(23):\n if vals_match(cells[idx] if idx < len(cells) else None, expected[idx]):\n match_count += 1\n else:\n mismatches.append((idx, normalize(cells[idx] if idx < len(cells) else None), normalize(expected[idx])))\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row[0].row\n best_mismatches = mismatches\n if match_count == 23:\n found = True\n break\n\n wb.close()\n\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row for CLM-2026-0878 with all 23 expected column values.\"}\n\n if best_row_num is not None:\n mismatch_details = \"; \".join([f\"Col {m[0]+1}: got '{m[1]}' expected '{m[2]}'\" for m in best_mismatches])\n return {\"pass\": False, \"score\": best_match_count / 23.0, \"feedback\": f\"Found row for CLM-2026-0878 at row {best_row_num} but {len(best_mismatches)} column(s) differ: {mismatch_details}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row with Claim ID 'CLM-2026-0878' found in PBC_Claim_Register.xlsx.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705484020",
|
|
"sort_order": 5,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$285.00\", \"$0.00\", \"$285.00\", \"$0.00\", BLANK, BLANK, \"CO-16\", \"N382\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n target_files = list(Path(workspace_path).glob(\"**/PBC_Claim_Register.xlsx\"))\n if not target_files:\n # Also try case-insensitive\n target_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n target_files = [f for f in target_files if 'claim_register' in f.name.lower()]\n if not target_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n target_file = target_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$285.00\",\n \"$0.00\",\n \"$285.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-16\",\n \"N382\"\n ]\n\n # Column index 14 (0-based) is the status column where \"Submitted\" is also acceptable\n status_col_idx = 14\n\n def normalize(val):\n \"\"\"Normalize a cell value to a stripped string, or None if blank.\"\"\"\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def values_match(actual, expected_val, col_idx=None):\n \"\"\"Loosely compare an actual cell value against an expected value.\"\"\"\n actual_n = normalize(actual)\n if expected_val is None:\n # Expect blank\n return actual_n is None\n if actual_n is None:\n return False\n exp_str = str(expected_val).strip()\n\n # Direct case-insensitive match\n if actual_n.lower() == exp_str.lower():\n return True\n\n # Special handling for status column: accept \"Submitted\" as well\n if col_idx == status_col_idx:\n if actual_n.lower() in [\"resubmitted\", \"submitted\"]:\n return True\n\n # Try removing $ and commas for currency comparison\n def strip_currency(s):\n return s.replace('$', '').replace(',', '').strip()\n act_clean = strip_currency(actual_n)\n exp_clean = strip_currency(exp_str)\n if act_clean == exp_clean:\n return True\n\n # Try numeric comparison with tolerance\n try:\n act_num = float(act_clean)\n exp_num = float(exp_clean)\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n except (ValueError, TypeError):\n pass\n\n # Date flexibility: 03/15/2026 vs 2026-03-15 etc.\n date_formats = ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d']\n exp_date = None\n act_date = None\n for fmt in date_formats:\n try:\n exp_date = datetime.datetime.strptime(exp_str, fmt).date()\n break\n except ValueError:\n continue\n if exp_date:\n for fmt in date_formats:\n try:\n act_date = datetime.datetime.strptime(actual_n, fmt).date()\n break\n except ValueError:\n continue\n if act_date is None and isinstance(actual, datetime.datetime):\n act_date = actual.date()\n if act_date is None and isinstance(actual, datetime.date):\n act_date = actual\n if act_date is not None and act_date == exp_date:\n return True\n\n # Check if expected is \"1\" and actual could be 1 or 1.0\n if exp_str == \"1\":\n try:\n if float(actual_n) == 1.0:\n return True\n except (ValueError, TypeError):\n pass\n\n return False\n\n num_expected = len(expected)\n best_match_count = 0\n best_row_idx = None\n best_mismatches = []\n\n # Check all sheets\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n if len(cells) < num_expected:\n continue\n for offset in range(len(cells) - num_expected + 1):\n match_count = 0\n mismatches = []\n for i in range(num_expected):\n if values_match(cells[offset + i], expected[i], col_idx=i):\n match_count += 1\n else:\n mismatches.append((i, expected[i], normalize(cells[offset + i])))\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_idx = row[0].row\n best_mismatches = mismatches\n if match_count == num_expected:\n wb.close()\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row at row {row[0].row} in sheet '{sheet_name}' of {target_file.name} with all {num_expected} expected values.\"}\n\n wb.close()\n if best_row_idx is not None:\n mismatch_details = \"; \".join([f\"col {m[0]+1}: expected '{m[1]}' got '{m[2]}'\" for m in best_mismatches[:10]])\n return {\"pass\": False, \"score\": best_match_count / num_expected, \"feedback\": f\"Best partial match at row {best_row_idx}: {best_match_count}/{num_expected} values matched. Mismatches: {mismatch_details}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No row found in {target_file.name} matching the expected 23-column pattern.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705498691",
|
|
"sort_order": 6,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-1104\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29881\", \"M23.211\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$2,200.00\", \"$1,850.00\", \"$350.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected row values in column order\n expected = [\n \"CLM-2026-1104\", # col 1\n \"PAT-1058\", # col 2\n \"Linda Park\", # col 3\n \"03/20/2026\", # col 4\n \"Dr. James Miller\",# col 5\n \"2345678901\", # col 6\n None, # col 7 BLANK\n \"BlueCross TX\", # col 8\n \"BCTX1\", # col 9\n \"29881\", # col 10\n \"M23.211\", # col 11\n None, # col 12 BLANK\n \"1\", # col 13\n None, # col 14 BLANK\n \"Paid - Full\", # col 15\n \"$2,200.00\", # col 16\n \"$1,850.00\", # col 17\n \"$350.00\", # col 18\n \"$0.00\", # col 19\n \"Reconciled\", # col 20\n None, # col 21 BLANK\n None, # col 22 BLANK\n None, # col 23 BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string like $2,200.00 or 2200\"\"\"\n if s is None:\n return None\n s = str(s).strip()\n # Remove dollar sign, commas, spaces\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def values_match(actual, expected_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual)\n \n act = normalize(actual)\n exp = normalize(expected_val)\n \n if act is None:\n return False\n \n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n \n # Handle date: expected 03/20/2026\n if '/' in exp:\n # Try to match dates loosely\n # actual could be a datetime object\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n dt = actual\n formatted = dt.strftime('%m/%d/%Y')\n if formatted == exp:\n return True\n # Also try without leading zeros\n formatted2 = f\"{dt.month}/{dt.day}/{dt.year}\"\n if formatted2 == exp:\n return True\n # String comparison already done above\n return False\n \n # Numeric comparison with tolerance\n exp_num = parse_number(exp)\n act_num = parse_number(act)\n if exp_num is not None and act_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 0.001) < 0.05:\n return True\n \n # Try comparing as plain numbers (e.g., \"1\" vs 1.0)\n try:\n if float(act) == float(exp):\n return True\n except (ValueError, TypeError):\n pass\n \n return False\n\n # Search all rows for a matching row\n num_expected = len(expected)\n best_match_count = 0\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n # Find the row that starts with CLM-2026-1104\n first_val = normalize(cells[0]) if len(cells) > 0 else None\n if first_val is None or 'CLM-2026-1104' not in first_val.upper().replace(' ', ''):\n # Also check if it might be in the row somewhere\n found_clm = False\n for c in cells:\n cn = normalize(c)\n if cn and 'CLM-2026-1104' in cn.upper().replace(' ', ''):\n found_clm = True\n break\n if not found_clm:\n continue\n\n # We found a candidate row. Check all expected columns.\n match_count = 0\n mismatches = []\n for i, exp_val in enumerate(expected):\n if i < len(cells):\n actual_val = cells[i]\n else:\n actual_val = None\n \n if values_match(actual_val, exp_val):\n match_count += 1\n else:\n mismatches.append(f\"Col {i+1}: expected '{exp_val}', got '{actual_val}'\")\n \n if match_count == num_expected:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row with all {num_expected} values matching for CLM-2026-1104.\"}\n \n if match_count > best_match_count:\n best_match_count = match_count\n best_mismatches = mismatches\n\n if best_match_count > 0:\n pct = best_match_count / num_expected\n return {\"pass\": False, \"score\": pct, \"feedback\": f\"Found row CLM-2026-1104 but only {best_match_count}/{num_expected} columns match. Mismatches: {'; '.join(best_mismatches)}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row with CLM-2026-1104 found in PBC_Claim_Register.xlsx.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705525107",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values [No partial matches acceptable, must match every value]: \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$1,850.00\", \"$0.00\", \"$1,850.00\", \"$0.00\", BLANK, BLANK, \"CO-97\", \"N520\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*.xlsx\")) + list(workspace.glob(\"*claim_register*.xlsx\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx and no alternative claim register file found in {workspace}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # Expected values in order across columns\n expected = [\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$1,850.00\",\n \"$0.00\",\n \"$1,850.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-97\",\n \"N520\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def is_blank(val):\n \"\"\"Check if a value is effectively blank/empty.\"\"\"\n if val is None:\n return True\n s = str(val).strip()\n if s == '' or s.lower() == 'none' or s == '0' or s == '0.0' or s == '$0.00':\n return True\n return False\n\n def values_match(actual_val, expected_val, col_index):\n act = normalize(actual_val)\n exp = normalize(expected_val)\n\n # Both blank\n if exp is None:\n return is_blank(actual_val)\n\n if act is None:\n return False\n\n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n\n # Special handling for column 14 (index 14): accept both Resubmitted and Submitted\n if col_index == 14:\n if act.lower() in ('resubmitted', 'submitted'):\n return True\n\n # Try numeric comparison with tolerance\n act_num = parse_number(act)\n exp_num = parse_number(exp)\n if act_num is not None and exp_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n\n # Currency: strip $ and commas and compare\n act_clean = re.sub(r'[\\$,\\s]', '', act)\n exp_clean = re.sub(r'[\\$,\\s]', '', exp)\n if act_clean == exp_clean:\n return True\n\n # Date flexibility\n date_patterns_act = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', act)\n date_patterns_exp = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', exp)\n if date_patterns_act and date_patterns_exp:\n def norm_date(parts):\n a, b, c = parts\n if len(a) == 4:\n return (b, c, a)\n else:\n return (a, b, c)\n if norm_date(date_patterns_act[0]) == norm_date(date_patterns_exp[0]):\n return True\n\n # Handle datetime objects from openpyxl\n if isinstance(actual_val, (datetime.datetime, datetime.date)):\n formatted = actual_val.strftime(\"%m/%d/%Y\")\n if formatted == exp:\n return True\n formatted3 = f\"{actual_val.month}/{actual_val.day}/{actual_val.year}\"\n if formatted3 == exp:\n return True\n formatted4 = f\"{actual_val.month:02d}/{actual_val.day:02d}/{actual_val.year}\"\n if formatted4 == exp:\n return True\n\n # Try comparing just digits for things like NPI numbers\n act_digits = re.sub(r'\\D', '', act)\n exp_digits = re.sub(r'\\D', '', exp)\n if act_digits and exp_digits and act_digits == exp_digits:\n return True\n\n return False\n\n # Search all sheets\n best_match_count = 0\n best_mismatches = []\n best_row_vals = []\n\n for ws_name in wb.sheetnames:\n ws = wb[ws_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n # Pad cells if needed\n while len(cells) < len(expected) + 10:\n cells.append(None)\n\n # Check if first expected value (CLM-2026-1105) is in the row\n found_start = False\n for i, cell in enumerate(cells):\n n = normalize(cell)\n if n is not None and 'CLM-2026-1105' in n.upper():\n found_start = True\n break\n if not found_start:\n continue\n\n # Try different start offsets in case there's an index column\n for offset in range(0, min(10, len(cells))):\n if offset + len(expected) > len(cells):\n break\n # Quick check: first cell at this offset should be CLM-2026-1105\n first_norm = normalize(cells[offset])\n if first_norm is None or 'CLM-2026-1105' not in first_norm.upper():\n continue\n match_count = 0\n mismatches = []\n for j, exp_val in enumerate(expected):\n actual = cells[offset + j]\n if values_match(actual, exp_val, j):\n match_count += 1\n else:\n mismatches.append((j, exp_val, normalize(actual)))\n if match_count == len(expected):\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found matching row for CLM-2026-1105 in sheet '{ws_name}' with all 23 expected values matching correctly.\"\n }\n if match_count > best_match_count:\n best_match_count = match_count\n best_mismatches = mismatches\n best_row_vals = [normalize(c) for c in cells[:30]]\n\n # Provide diagnostics\n diagnostics = \"\"\n for ws_name in wb.sheetnames:\n ws = wb[ws_name]\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, max_row=ws.max_row), start=1):\n cells = [cell.value for cell in row]\n for cell in cells:\n n = normalize(cell)\n if n is not None and 'CLM-2026-1105' in str(n).upper():\n diagnostics = f\"Found CLM-2026-1105 in sheet '{ws_name}' row {row_idx}, but values didn't fully match ({best_match_count}/23 matched). Mismatches: {best_mismatches}. Row values: {[normalize(c) for c in cells[:25]]}\"\n break\n if diagnostics:\n break\n\n if not diagnostics:\n diagnostics = \"No row containing 'CLM-2026-1105' was found in any sheet.\"\n\n return {\n \"pass\": False,\n \"score\": best_match_count / 23.0 if best_match_count > 0 else 0.0,\n \"feedback\": f\"Required row not found in PBC_Claim_Register.xlsx. {diagnostics}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705535872",
|
|
"sort_order": 8,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable, must match every value]: \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"Dr. Amanda Foster\", \"3456789012\", BLANK, \"BlueCross TX\", \"BCTX1\", \"27447\", \"M17.11\", BLANK, \"1\", BLANK, \"In Recovery\", \"$7,500.00\", \"$4,200.00\", \"$3,300.00\", \"$0.00\", \"Held - Contractual Variance\", BLANK, \"CO-45\", \"N130\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob in case of slight name differences\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*\")) + list(workspace.glob(\"*claim_register*\")) + list(workspace.glob(\"*ClaimRegister*\")) + list(workspace.glob(\"*claim*register*\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n # Try any xlsx candidate\n for c in candidates:\n if c.suffix == '.xlsx':\n file_path = c\n break\n if not file_path.exists():\n # Try any xlsx at all\n all_xlsx = list(workspace.glob(\"*.xlsx\"))\n for f in all_xlsx:\n if 'claim' in f.name.lower():\n file_path = f\n break\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx in {workspace}. Files present: {[f.name for f in workspace.iterdir()]}\"}\n\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n # Expected values in column order. None means BLANK.\n expected = [\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"Dr. Amanda Foster\",\n \"3456789012\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"27447\",\n \"M17.11\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"In Recovery\",\n \"$7,500.00\",\n \"$4,200.00\",\n \"$3,300.00\",\n \"$0.00\",\n \"Held - Contractual Variance\",\n None, # BLANK\n \"CO-45\",\n \"N130\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, removing $, commas, etc.\"\"\"\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def match_cell(expected_val, actual_val):\n \"\"\"Check if actual_val matches expected_val with reasonable flexibility.\"\"\"\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n # Both blank\n if e is None and a is None:\n return True\n # One blank, other not\n if e is None or a is None:\n return False\n\n # Direct case-insensitive match\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison with tolerance\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(a_num - e_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(a_num - e_num) / abs(a_num) <= 0.05:\n return True\n\n # Date flexibility: compare 03/08/2026 with various date formats\n date_formats = [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%m/%d/%y\"]\n e_date = None\n a_date = None\n for fmt in date_formats:\n try:\n e_date = datetime.datetime.strptime(e, fmt).date()\n break\n except ValueError:\n pass\n # actual might be a datetime object already\n if isinstance(actual_val, datetime.datetime):\n a_date = actual_val.date()\n elif isinstance(actual_val, datetime.date) and not isinstance(actual_val, datetime.datetime):\n a_date = actual_val\n else:\n for fmt in date_formats:\n try:\n a_date = datetime.datetime.strptime(a, fmt).date()\n break\n except ValueError:\n pass\n if e_date is not None and a_date is not None:\n if e_date == a_date:\n return True\n\n\n return False\n\n # Search all rows for a matching row\n found = False\n best_match_count = 0\n best_row_idx = None\n best_row_vals = None\n num_expected = len(expected)\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n row_vals = [cell.value for cell in row]\n # We need at least num_expected columns; pad if shorter\n while len(row_vals) < num_expected:\n row_vals.append(None)\n\n # First, find the row by the claim ID CLM-2026-0967\n has_claim_id = False\n start_col = None\n for ci, cv in enumerate(row_vals):\n if normalize(cv) is not None and \"CLM-2026-0967\" in str(cv).strip():\n has_claim_id = True\n start_col = ci\n break\n\n if not has_claim_id:\n continue\n\n # Now check all expected values starting from start_col\n sub_row = row_vals[start_col:start_col + num_expected]\n while len(sub_row) < num_expected:\n sub_row.append(None)\n\n match_count = 0\n mismatches = []\n for i, (ev, av) in enumerate(zip(expected, sub_row)):\n if match_cell(ev, av):\n match_count += 1\n else:\n mismatches.append((i, ev, normalize(av)))\n\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_idx = row_idx\n best_row_vals = sub_row\n\n if match_count == num_expected:\n found = True\n break\n\n wb.close()\n\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row for CLM-2026-0967 at row {best_row_idx} with all {num_expected} expected values.\"}\n\n if best_row_idx is not None:\n # Re-check mismatches for reporting\n sub_row = best_row_vals\n mismatches = []\n for i, (ev, av) in enumerate(zip(expected, sub_row)):\n if not match_cell(ev, av):\n mismatches.append((i, ev, normalize(av)))\n mismatch_details = \"; \".join([f\"col {m[0]}: expected='{m[1]}' got='{m[2]}'\" for m in mismatches[:10]])\n return {\"pass\": False, \"score\": best_match_count / num_expected, \"feedback\": f\"Row for CLM-2026-0967 found at row {best_row_idx} but only {best_match_count}/{num_expected} values match. Mismatches: {mismatch_details}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row containing CLM-2026-0967 found in PBC_Claim_Register.xlsx.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705546419",
|
|
"sort_order": 9,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable]: \"CLM-2026-1106\" (or \"CLM-2026-1107\"), \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"7\", \"CLM-2026-0892\", \"Submitted\", \"$285.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob in case of path variations\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Claim_Register.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # The rubric says CLM-2026-1106 or CLM-2026-1107 for column 1\n expected_values = [\n [\"CLM-2026-1106\", \"CLM-2026-1107\"], # accept either claim ID\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"7\",\n \"CLM-2026-0892\",\n \"Submitted\",\n \"$285.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def single_value_match(expected_str, actual):\n \"\"\"Check if a single expected string matches the actual value.\"\"\"\n if expected_str is None:\n return is_blank(actual)\n\n norm_actual = normalize(actual)\n norm_expected = normalize(expected_str)\n\n if norm_actual is None:\n return False\n\n # Direct string match (case-insensitive)\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Whitespace-insensitive\n if norm_expected.lower().replace(' ', '') == norm_actual.lower().replace(' ', ''):\n return True\n\n # Try numeric comparison for currency values\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0:\n if abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n # Also check exact equality for non-zero\n if exp_num == act_num:\n return True\n\n # Date matching - be very flexible\n if '/' in norm_expected:\n # Parse the expected date\n try:\n exp_parts = norm_expected.split('/')\n exp_month = int(exp_parts[0])\n exp_day = int(exp_parts[1])\n exp_year = int(exp_parts[2])\n except:\n exp_month = exp_day = exp_year = None\n\n if exp_month is not None:\n # If actual is a datetime object\n if hasattr(actual, 'year') and hasattr(actual, 'month') and hasattr(actual, 'day'):\n if actual.month == exp_month and actual.day == exp_day and actual.year == exp_year:\n return True\n\n # Try parsing the string form\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%m/%d/%y', '%Y/%m/%d']:\n try:\n dt = datetime.datetime.strptime(norm_actual, fmt)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Try iso format\n try:\n dt = datetime.datetime.fromisoformat(norm_actual)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Number as string comparison (e.g., \"7\" vs 7)\n try:\n ef = float(norm_expected)\n af = float(norm_actual)\n if ef == af:\n return True\n except:\n pass\n\n # Also try int comparison (\"7\" vs \"7.0\")\n try:\n ei = int(float(norm_expected))\n ai = int(float(norm_actual))\n if ei == ai and float(norm_expected) == ei: # only if expected is integer\n return True\n except:\n pass\n\n return False\n\n def values_match(expected, actual):\n \"\"\"expected can be a string, None, or a list of acceptable strings.\"\"\"\n if isinstance(expected, list):\n return any(single_value_match(e, actual) for e in expected)\n else:\n return single_value_match(expected, actual)\n\n def expected_display(expected):\n if isinstance(expected, list):\n return \" or \".join([repr(e) for e in expected])\n return repr(expected)\n\n best_match_count = 0\n best_row_idx = None\n best_mismatches = []\n total = len(expected_values)\n\n # Iterate through all rows in all sheets\n for sheet in wb.sheetnames:\n ws = wb[sheet]\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=True), start=1):\n row_list = list(row)\n # Extend row if shorter than expected\n while len(row_list) < total:\n row_list.append(None)\n\n match_count = 0\n mismatches = []\n for col_idx, exp in enumerate(expected_values):\n actual_val = row_list[col_idx] if col_idx < len(row_list) else None\n if values_match(exp, actual_val):\n match_count += 1\n else:\n mismatches.append((col_idx + 1, expected_display(exp), normalize(actual_val)))\n\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_idx = row_idx\n best_mismatches = mismatches\n\n if match_count == total:\n wb.close()\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row at row {row_idx} in sheet '{sheet}' with all {total} expected values.\"}\n\n wb.close()\n\n # Allow 1-2 minor mismatches to still pass with reduced score\n if best_match_count >= total - 2 and best_match_count >= total * 0.9:\n mismatch_details = \"; \".join([f\"Col {c}: expected {e} got '{a}'\" for c, e, a in best_mismatches])\n return {\"pass\": True, \"score\": best_match_count / total, \"feedback\": f\"Best match at row {best_row_idx}: {best_match_count}/{total} values match. Minor mismatches: {mismatch_details}\"}\n\n if best_row_idx:\n mismatch_details = \"; \".join([f\"Col {c}: expected {e} got '{a}'\" for c, e, a in best_mismatches[:10]])\n return {\"pass\": False, \"score\": best_match_count / total, \"feedback\": f\"No exact matching row found. Best match at row {best_row_idx} with {best_match_count}/{total} values matching. Mismatches: {mismatch_details}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No rows found in PBC_Claim_Register.xlsx that match the expected values.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705558604",
|
|
"sort_order": 10,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-1107\" (or \"CLM-2026-1106\"), \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", \"LT\", \"7\", \"CLM-2026-1105\", \"Submitted\", \"$1,850.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n file_path = None\n for f in Path(workspace_path).glob(\"**/*Claim_Register*\"):\n if f.suffix == '.xlsx':\n file_path = f\n break\n if file_path is None:\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # Acceptable claim IDs for the first column per the rubric\n acceptable_claim_ids = [\"CLM-2026-1107\", \"CLM-2026-1106\"]\n\n expected = [\n None, # placeholder for claim ID - checked separately\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n \"LT\",\n \"7\",\n \"CLM-2026-1105\",\n \"Submitted\",\n \"$1,850.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip()\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n if e is None and a is None:\n return True\n if e is None or a is None:\n return False\n\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(e_num - a_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(e_num - a_num) / abs(a_num) <= 0.05:\n return True\n\n # Date comparison\n if '/' in e:\n try:\n expected_date = datetime.datetime.strptime(e, '%m/%d/%Y').date()\n if hasattr(actual_val, 'date'):\n if actual_val.date() == expected_date:\n return True\n elif hasattr(actual_val, 'year'):\n if actual_val == expected_date:\n return True\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y']:\n try:\n if datetime.datetime.strptime(a, fmt).date() == expected_date:\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n\n # NPI / numeric string cleaning\n e_clean = re.sub(r'[^\\w.]', '', e)\n a_clean = re.sub(r'[^\\w.]', '', a)\n if e_clean.lower() == a_clean.lower():\n return True\n\n return False\n\n def is_acceptable_claim_id(val):\n n = normalize(val)\n if n is None:\n return False\n for cid in acceptable_claim_ids:\n if n.upper() == cid.upper():\n return True\n return False\n\n best_mismatches = None\n best_sheet = None\n best_claim_id = None\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n\n # Find a cell that matches one of the acceptable claim IDs\n start_idx = None\n matched_claim_id = None\n for i, c in enumerate(cells):\n if is_acceptable_claim_id(c):\n start_idx = i\n matched_claim_id = normalize(c)\n break\n if start_idx is None:\n continue\n\n # Pad cells if needed\n if start_idx + len(expected) > len(cells):\n cells = cells + [None] * (start_idx + len(expected) - len(cells))\n\n all_match = True\n mismatches = []\n for j, exp_val in enumerate(expected):\n actual = cells[start_idx + j]\n if j == 0:\n # Already verified claim ID\n continue\n if not values_match(exp_val, actual):\n all_match = False\n mismatches.append(f\"Column {j+1}: expected '{exp_val}', got '{actual}'\")\n\n if all_match:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row with {matched_claim_id} in sheet '{sheet_name}' with all expected values.\"}\n else:\n if best_mismatches is None or len(mismatches) < len(best_mismatches):\n best_mismatches = mismatches\n best_sheet = sheet_name\n best_claim_id = matched_claim_id\n\n if best_mismatches is not None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found row with {best_claim_id} in sheet '{best_sheet}' but mismatches: {'; '.join(best_mismatches)}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row containing an acceptable claim ID (CLM-2026-1107 or CLM-2026-1106) found in any sheet of PBC_Claim_Register.xlsx.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705566070",
|
|
"sort_order": 11,
|
|
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be no duplicate rows or `Claim_ID`s.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n rows = list(ws.iter_rows(min_row=1, values_only=True))\n if len(rows) < 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File has 0 or 1 data rows, so no duplicates possible.\"}\n\n header = [str(c).strip().lower() if c is not None else \"\" for c in rows[0]]\n\n # Find Claim_ID column\n claim_id_col = None\n for i, h in enumerate(header):\n if h in (\"claim_id\", \"claimid\", \"claim id\"):\n claim_id_col = i\n break\n\n # Check for duplicate Claim_IDs\n duplicate_claim_ids = []\n if claim_id_col is not None:\n seen_ids = {}\n for row_idx, row in enumerate(rows[1:], start=2):\n val = row[claim_id_col]\n if val is None:\n continue\n val_str = str(val).strip()\n if val_str == \"\":\n continue\n if val_str in seen_ids:\n duplicate_claim_ids.append((val_str, seen_ids[val_str], row_idx))\n else:\n seen_ids[val_str] = row_idx\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a 'Claim_ID' column in header: {header}\"}\n\n # Check for fully duplicate rows\n def row_key(row):\n return tuple(str(c).strip() if c is not None else \"\" for c in row)\n\n seen_rows = {}\n duplicate_rows = []\n for row_idx, row in enumerate(rows[1:], start=2):\n rk = row_key(row)\n # Skip completely empty rows\n if all(v == \"\" for v in rk):\n continue\n if rk in seen_rows:\n duplicate_rows.append((row_idx, seen_rows[rk]))\n else:\n seen_rows[rk] = row_idx\n\n issues = []\n if duplicate_claim_ids:\n for cid, first_row, dup_row in duplicate_claim_ids:\n issues.append(f\"Duplicate Claim_ID '{cid}' found in rows {first_row} and {dup_row}\")\n if duplicate_rows:\n for dup_row, first_row in duplicate_rows:\n issues.append(f\"Duplicate row found: rows {first_row} and {dup_row} are identical\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicates found in PBC_Claim_Register.xlsx: \" + \"; \".join(issues)\n }\n\n total_data_rows = sum(1 for row in rows[1:] if not all((str(c).strip() if c is not None else \"\") == \"\" for c in row))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Claim_IDs found in PBC_Claim_Register.xlsx. {total_data_rows} unique data rows checked.\"\n }\n\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Claim_Register.xlsx: {str(e)}\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1775705648199",
|
|
"sort_order": 12,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be a table with 11 total rows (including the header row).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (rows that have at least one non-None cell)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Sheet '{sheet_name}' in PBC_Denial_Worklist.xlsx has exactly 11 non-empty rows (including header), as required.\"\n }\n \n # If no sheet matched exactly, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with 11 total rows (including header) in PBC_Denial_Worklist.xlsx. Found: {'; '.join(details)}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705668317",
|
|
"sort_order": 13,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0028\" (where 28 can be 29 or 30), \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"BlueCross TX\", \"CO-16\", \"N382\", \"Technical/Administrative\", \"Correct and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and BLANK (empty cell) in consecutive columns.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob to be lenient\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Denial_Worklist*.xlsx\"))\n if not candidates:\n candidates = list(workspace.glob(\"**/*.xlsx\"))\n \n file_path = None\n for c in candidates:\n if \"denial\" in c.name.lower() or \"worklist\" in c.name.lower() or \"pbc\" in c.name.lower():\n file_path = c\n break\n if file_path is None and candidates:\n file_path = candidates[0]\n if file_path is None:\n file_path = workspace / \"PBC_Denial_Worklist.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found. Searched in {workspace}. No matching xlsx files found.\"}\n\n # The rubric says DEN-2026-0028 where 28 can be 29 or 30.\n # We'll also accept a broader range (28-31) to be lenient.\n # All other values are fixed.\n expected_values = [\n \"DEN-2026-XXXX\", # special handling\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"BlueCross TX\",\n \"CO-16\",\n \"N382\",\n \"Technical/Administrative\",\n \"Correct and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def den_id_matches(actual_val):\n \"\"\"Check if actual matches DEN-2026-00XX where XX is 28-31 (lenient range)\"\"\"\n if actual_val is None:\n return False\n a = str(actual_val).strip().upper()\n # Accept DEN-2026-0028 through DEN-2026-0031\n pattern = r'^DEN-2026-00(2[89]|3[01])$'\n return bool(re.match(pattern, a))\n\n def values_match(actual, expected):\n if expected is None:\n return is_blank(actual)\n if actual is None:\n return False\n a = normalize(actual)\n e = normalize(expected)\n if a is None:\n return False\n # Case-insensitive comparison\n if a.lower() == e.lower():\n return True\n # Try date variants: strip leading zeros\n a_clean = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_clean = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_clean.lower() == e_clean.lower():\n return True\n # Also handle date formats like 2026-03-15 vs 03/15/2026\n # Try converting common date patterns\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\"),\n ]\n def to_canonical_date(s):\n for pat, fn in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fn(m)\n return None\n \n a_date = to_canonical_date(a)\n e_date = to_canonical_date(e)\n if a_date and e_date and a_date == e_date:\n return True\n \n # Handle BCBS TX variants\n if e.lower() == \"bluecross tx\":\n variants = [\"bluecross tx\", \"bcbs tx\", \"bcbstx\", \"blue cross tx\", \"bluecrosstx\", \"bcbs-tx\"]\n if a.lower().replace(\" \", \"\") in [v.replace(\" \", \"\") for v in variants]:\n return True\n \n return False\n\n rows_to_check = list(range(1, ws.max_row + 1)) if ws.max_row and ws.max_row <= 50 else list(range(1, 51))\n # Prioritize rows 9-11 as stated in rubric, but check all rows for leniency\n priority_rows = [9, 10, 11]\n all_rows = priority_rows + [r for r in rows_to_check if r not in priority_rows]\n\n num_expected = len(expected_values)\n best_match_count = 0\n best_row = None\n best_mismatches = []\n\n for row_num in all_rows:\n row_cells = []\n for col in range(1, max(ws.max_column + 1 if ws.max_column else 1, 20)):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n for start_col_idx in range(len(row_cells) - num_expected + 1):\n segment = row_cells[start_col_idx:start_col_idx + num_expected]\n match_count = 0\n mismatches = []\n for i, (act, exp) in enumerate(zip(segment, expected_values)):\n if i == 0:\n # Special handling for DEN ID\n if den_id_matches(act):\n match_count += 1\n else:\n mismatches.append((i, \"DEN-2026-00(28-31)\", act))\n else:\n if values_match(act, exp):\n match_count += 1\n else:\n mismatches.append((i, exp, act))\n \n if match_count == num_expected:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row {row_num} contains all expected values starting at column {start_col_idx + 1}. DEN ID: {segment[0]}\"\n }\n \n if match_count > best_match_count:\n best_match_count = match_count\n best_row = row_num\n best_mismatches = mismatches\n\n # Diagnostic info\n diag_lines = []\n for row_num in priority_rows:\n row_vals = []\n for col in range(1, min(ws.max_column + 1 if ws.max_column else 1, 20)):\n row_vals.append(repr(ws.cell(row=row_num, column=col).value))\n diag_lines.append(f\"Row {row_num}: {row_vals}\")\n\n mismatch_info = \"\"\n if best_row is not None:\n mismatch_info = f\"\\nBest partial match: Row {best_row} with {best_match_count}/{num_expected} matches. Mismatches: {best_mismatches}\"\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected values not found in PBC_Denial_Worklist.xlsx. Row contents (rows 9-11):\\n\" + \"\\n\".join(diag_lines) + mismatch_info\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705689978",
|
|
"sort_order": 14,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0029\" (where 29 can be 28 or 30), \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"BlueCross TX\", \"CO-97\", \"N520\", \"Technical/Administrative\", \"Add Modifier and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob\n fp = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n fp = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n \n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {fp}\"}\n\n try:\n wb = openpyxl.load_workbook(fp, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # The rubric says DEN-2026-0029 where 29 can be 28 or 30\n # So we accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n # We also accept variants like DEN-2026-0035 or other denial IDs with flexible matching\n expected_values = [\n \"DEN-2026-FLEX\", # special handling\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"BlueCross TX\",\n \"CO-97\",\n \"N520\",\n \"Technical/Administrative\",\n \"Add Modifier and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n try:\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n except Exception:\n pass\n s = str(val).strip()\n return s\n\n def match_den_id(actual_str):\n \"\"\"Match DEN-2026-0028, DEN-2026-0029, or DEN-2026-0030 (flexible)\"\"\"\n norm = actual_str.strip().upper()\n # Accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n if re.match(r'^DEN-2026-00(28|29|30)$', norm):\n return True\n # Also accept any DEN-2026-XXXX pattern as a looser fallback\n # But primary match is the three specific ones\n return False\n\n def match_date(actual_str, expected_str):\n \"\"\"Flexible date matching\"\"\"\n date_pat = re.compile(r'^(\\d{1,2})/(\\d{1,2})/(\\d{4})$')\n m_a = date_pat.match(actual_str)\n m_e = date_pat.match(expected_str)\n if m_a and m_e:\n return (int(m_a.group(1)) == int(m_e.group(1)) and\n int(m_a.group(2)) == int(m_e.group(2)) and\n m_a.group(3) == m_e.group(3))\n return False\n\n def match(actual, expected, idx):\n norm_actual = normalize(actual)\n \n # Special case: DEN ID (first column)\n if idx == 0:\n return match_den_id(norm_actual)\n \n # Special case: BLANK (last column)\n if expected is None:\n return norm_actual == \"\" or norm_actual.lower() == \"none\"\n \n norm_expected = str(expected).strip()\n \n # Case-insensitive comparison\n if norm_actual.lower() == norm_expected.lower():\n return True\n \n # Date matching\n if match_date(norm_actual, norm_expected):\n return True\n \n # Contains check as fallback for things like \"BlueCross TX\" vs \"BCBS TX\" etc.\n if norm_expected.lower() in norm_actual.lower() or norm_actual.lower() in norm_expected.lower():\n return True\n \n # Also try matching with flexible whitespace/case\n if re.sub(r'\\s+', ' ', norm_actual.lower()) == re.sub(r'\\s+', ' ', norm_expected.lower()):\n return True\n \n return False\n\n for row_num in [9, 10, 11]:\n row_cells = []\n for col in range(1, ws.max_column + 2): # +2 for safety to check blank\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n # Try to match expected_values against consecutive columns starting from various offsets\n for start_col in range(min(5, len(row_cells))):\n all_match = True\n mismatches = []\n for i in range(len(expected_values)):\n col_idx = start_col + i\n if col_idx >= len(row_cells):\n actual_val = None\n else:\n actual_val = row_cells[col_idx]\n if not match(actual_val, expected_values[i], i):\n all_match = False\n mismatches.append((i, expected_values[i], normalize(actual_val)))\n if all_match:\n vals = [normalize(row_cells[start_col + i]) if (start_col + i) < len(row_cells) else \"\" for i in range(len(expected_values))]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {row_num} contains all expected values starting at column {start_col+1}. Values: {vals}\"}\n\n # If no exact match, report best partial match\n best_row = None\n best_matched = 0\n best_mismatches = []\n best_start = 0\n for row_num in [9, 10, 11]:\n row_cells = []\n for col in range(1, ws.max_column + 2):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n for start_col in range(min(5, len(row_cells))):\n matched = 0\n mismatches = []\n for i in range(len(expected_values)):\n col_idx = start_col + i\n actual_val = row_cells[col_idx] if col_idx < len(row_cells) else None\n if match(actual_val, expected_values[i], i):\n matched += 1\n else:\n exp_display = expected_values[i] if expected_values[i] is not None else \"BLANK\"\n if i == 0:\n exp_display = \"DEN-2026-0028/0029/0030\"\n mismatches.append((i+1, exp_display, normalize(actual_val)))\n if matched > best_matched:\n best_matched = matched\n best_row = row_num\n best_mismatches = mismatches\n best_start = start_col\n\n detail = \"; \".join([f\"col {m[0]}: expected '{m[1]}' got '{m[2]}'\" for m in best_mismatches[:8]])\n return {\"pass\": False, \"score\": best_matched / len(expected_values), \"feedback\": f\"No matching row found in rows 9-11. Best match: row {best_row} with {best_matched}/{len(expected_values)} values matched. Mismatches: {detail}\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705707748",
|
|
"sort_order": 15,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the following exact values: \"DEN-2026-0030\" (where 30 can be 29 or 38), \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"BlueCross TX\", \"CO-45\", \"N130\", \"Contractual/Payer Error\", \"Appeal - Contractual Rate Dispute\", \"Escalated\" [\"Appealed\" also acceptable], \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n filepath = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n filepath = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if fp.exists():\n filepath = fp\n\n if filepath is None or not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(filepath)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # The DEN ID can be DEN-2026-0029, DEN-2026-0030, DEN-2026-0038, DEN-2026-0039\n valid_den_ids = [\"den-2026-0029\", \"den-2026-0030\", \"den-2026-0038\", \"den-2026-0039\"]\n\n # Expected values for columns after DEN ID (indices 1..12)\n # Index 0: DEN ID (special)\n # Index 1: CLM-2026-0967\n # Index 2: PAT-1063\n # Index 3: Robert Daniels\n # Index 4: 03/08/2026\n # Index 5: BlueCross TX\n # Index 6: CO-45\n # Index 7: N130\n # Index 8: Contractual/Payer Error\n # Index 9: Appeal - Contractual Rate Dispute\n # Index 10: Escalated or Appealed\n # Index 11: Maria Santos\n # Index 12: 04/24/2026\n # Index 13: BLANK\n\n expected_values = [\n None, # placeholder for DEN ID\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"BlueCross TX\",\n \"CO-45\",\n \"N130\",\n \"Contractual/Payer Error\",\n \"Appeal - Contractual Rate Dispute\",\n \"Escalated\", # also accept \"Appealed\"\n \"Maria Santos\",\n \"04/24/2026\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n return s\n\n def norm_date(s):\n \"\"\"Normalize date string to MM/DD/YYYY format.\"\"\"\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\")\n ]\n for pat, fmt in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fmt(m)\n return s.strip()\n\n def cell_to_normalized_string(actual):\n \"\"\"Convert a cell value to a normalized comparable string.\"\"\"\n if actual is None:\n return \"\"\n if isinstance(actual, (datetime.datetime, datetime.date)):\n d = actual if isinstance(actual, datetime.date) else actual.date()\n return f\"{d.month:02d}/{d.day:02d}/{d.year}\"\n return str(actual).strip()\n\n def values_match(actual, expected, col_index):\n a = cell_to_normalized_string(actual).lower()\n e = expected.lower().strip()\n # Special handling for column 10 (Escalated / Appealed)\n if col_index == 10:\n return a in [\"escalated\", \"appealed\"]\n if a == e:\n return True\n # Try date normalization\n if norm_date(a) == norm_date(e):\n return True\n # Loose matching: strip punctuation differences\n a_clean = re.sub(r'[\\s\\-/]+', '', a)\n e_clean = re.sub(r'[\\s\\-/]+', '', e)\n if a_clean == e_clean:\n return True\n return False\n\n def is_blank(val):\n return val is None or str(val).strip() == \"\"\n\n rows_to_check = list(range(2, ws.max_row + 1)) # Check all data rows, but prioritize 9-11\n priority_rows = [9, 10, 11]\n other_rows = [r for r in rows_to_check if r not in priority_rows]\n ordered_rows = priority_rows + other_rows\n\n best_score = 0\n best_details = []\n best_row = None\n\n for row_num in ordered_rows:\n if row_num > ws.max_row:\n continue\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n if len(row_cells) < len(expected_values):\n continue\n\n all_match = True\n match_details = []\n matched_count = 0\n\n # Check column 1 (DEN ID)\n col0_val = cell_to_normalized_string(row_cells[0]).lower()\n if col0_val in valid_den_ids:\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' matches DEN ID pattern\")\n matched_count += 1\n else:\n all_match = False\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' not in valid DEN IDs {valid_den_ids}\")\n\n # Check remaining expected values (index 1 onwards)\n for i in range(1, len(expected_values)):\n exp = expected_values[i]\n if i < len(row_cells):\n if values_match(row_cells[i], exp, i):\n match_details.append(f\"Col {i+1}: '{normalize(row_cells[i])}' matches '{exp}'\")\n matched_count += 1\n else:\n all_match = False\n match_details.append(f\"Col {i+1}: '{cell_to_normalized_string(row_cells[i])}' != '{exp}'\")\n else:\n all_match = False\n match_details.append(f\"Col {i+1}: missing\")\n\n # Check last cell (column index len(expected_values)) is blank\n last_col_idx = len(expected_values)\n if last_col_idx < len(row_cells):\n if not is_blank(row_cells[last_col_idx]):\n all_match = False\n match_details.append(f\"Col {last_col_idx+1} (last): expected BLANK, got '{normalize(row_cells[last_col_idx])}'\")\n else:\n match_details.append(f\"Col {last_col_idx+1} (last): BLANK as expected\")\n matched_count += 1\n else:\n match_details.append(f\"Col {last_col_idx+1} (last): BLANK (beyond data range)\")\n matched_count += 1\n\n total_checks = len(expected_values) + 1 # expected values + blank check\n row_score = matched_count / total_checks\n\n if all_match:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {row_num} matches all expected values. Details: {'; '.join(match_details)}\"}\n\n if row_score > best_score:\n best_score = row_score\n best_details = match_details\n best_row = row_num\n\n # If none matched fully, provide diagnostics for rows 9-11\n diagnostics = []\n for row_num in priority_rows:\n if row_num > ws.max_row:\n diagnostics.append(f\"Row {row_num}: (beyond max_row {ws.max_row})\")\n continue\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n diagnostics.append(f\"Row {row_num}: {[cell_to_normalized_string(c) for c in row_cells]}\")\n\n feedback = (f\"No matching row found. Valid DEN IDs: {valid_den_ids}. \"\n f\"Expected remaining values: {expected_values[1:]} + BLANK (col 10 also accepts 'Appealed'). \"\n f\"Best partial score: {best_score:.2f} in row {best_row}. \"\n f\"Best match details: {'; '.join(best_details)}. \"\n f\"Rows 9-11: {'; '.join(diagnostics)}\")\n return {\"pass\": False, \"score\": best_score * 0.5, \"feedback\": feedback}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705711177",
|
|
"sort_order": 16,
|
|
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be no duplicate rows or Denial IDs.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if f.name.lower() == \"pbc_denial_worklist.xlsx\":\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n rows = []\n header_row = None\n denial_id_col = None\n \n for i, row in enumerate(ws.iter_rows(values_only=True)):\n # Skip completely empty rows\n if all(cell is None or str(cell).strip() == '' for cell in row):\n continue\n if header_row is None:\n header_row = tuple(str(c).strip() if c is not None else '' for c in row)\n # Find the Denial ID column (be flexible with naming)\n for col_idx, h in enumerate(header_row):\n h_lower = h.lower().strip()\n if 'denial' in h_lower and 'id' in h_lower:\n denial_id_col = col_idx\n break\n elif h_lower in ('denialid', 'denial_id', 'denial id'):\n denial_id_col = col_idx\n break\n continue\n # Normalize cell values for comparison\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else ''\n for cell in row\n )\n rows.append((i + 1, normalized, row)) # 1-indexed row number\n \n if not rows:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PBC_Denial_Worklist.xlsx has no data rows (only header or empty), so no duplicates.\"}\n \n issues = []\n \n # Check for duplicate full rows\n seen_rows = {}\n duplicate_rows = []\n for row_num, normalized, raw in rows:\n if normalized in seen_rows:\n duplicate_rows.append((row_num, seen_rows[normalized]))\n else:\n seen_rows[normalized] = row_num\n \n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {dup_row} is a duplicate of row {orig_row}\"\n for dup_row, orig_row in duplicate_rows\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate row(s): {dup_details}\")\n \n # Check for duplicate Denial IDs\n if denial_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for row_num, normalized, raw in rows:\n denial_id = normalized[denial_id_col] if denial_id_col < len(normalized) else ''\n if denial_id and denial_id in seen_ids:\n duplicate_ids.append((row_num, denial_id, seen_ids[denial_id]))\n elif denial_id:\n seen_ids[denial_id] = row_num\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Row {dup_row} has Denial ID '{did}' which duplicates row {orig_row}\"\n for dup_row, did, orig_row in duplicate_ids\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Denial ID(s): {dup_id_details}\")\n else:\n # If we can't find a Denial ID column, note it but don't fail on that alone\n pass\n \n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx has duplicates. \" + \" | \".join(issues)\n }\n \n denial_id_info = f\" Denial ID column found at index {denial_id_col} ('{header_row[denial_id_col]}').\" if denial_id_col is not None else \" No Denial ID column detected (checked row uniqueness only).\"\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx contains {len(rows)} data rows with no duplicate rows or Denial IDs.{denial_id_info}\"\n }\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Denial_Worklist.xlsx: {str(e)}\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1775705797706",
|
|
"sort_order": 17,
|
|
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be a table with 5 total rows (including the header row).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 5 rows (1 header + 4 data)\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows: a row is non-empty if at least one cell has a value\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 5:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 5 total rows (including header) in sheet '{sheet_name}' of PBC_Escalation_Log.xlsx.\"\n }\n \n # If no sheet had exactly 5 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with exactly 5 total rows (including header) in PBC_Escalation_Log.xlsx. Found: {'; '.join(details)}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705829052",
|
|
"sort_order": 18,
|
|
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the following exact values: \"ESC-2026-0008\" (or \"ESC-2026-0009\"), \"04/24/2026\", \"CLM-2026-0234, CLM-2026-0356, CLM-2026-0489, CLM-2026-0612, CLM-2026-0892\" (in any order), \"Operational\", \"Pattern Recurrence\", \"Maria Santos\", \"David Park\", \"Open\", and a BLANK cell.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file with glob to be lenient\n xlsx_files = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if not xlsx_files:\n # Try case-insensitive\n xlsx_files = [f for f in Path(workspace_path).glob(\"**/*.xlsx\") if \"escalation\" in f.name.lower()]\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Escalation_Log.xlsx not found in workspace.\"}\n\n file_path = xlsx_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n expected_claims = {\"CLM-2026-0234\", \"CLM-2026-0356\", \"CLM-2026-0489\", \"CLM-2026-0612\", \"CLM-2026-0892\"}\n # Accept ESC-2026-0008 or ESC-2026-0009 (or nearby numbers to be generous)\n valid_esc_ids = {\"ESC-2026-0008\", \"ESC-2026-0009\", \"ESC-2026-0010\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_row(row_num):\n row_vals = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_vals.append(cell_val)\n\n row_strs = [normalize(v) for v in row_vals]\n feedback_parts = []\n score = 0\n total_checks = 8 # 8 required fields\n\n # Check ESC ID\n found_esc = False\n for s in row_strs:\n for esc_id in valid_esc_ids:\n if esc_id in s:\n found_esc = True\n break\n if found_esc:\n break\n if found_esc:\n score += 1\n feedback_parts.append(\"Found valid ESC ID\")\n else:\n feedback_parts.append(f\"Missing ESC ID (expected one of {valid_esc_ids})\")\n\n # Check date 04/24/2026 - be flexible with date formats\n found_date = False\n for idx, s in enumerate(row_strs):\n if not s:\n continue\n if \"04/24/2026\" in s or \"4/24/2026\" in s or \"2026-04-24\" in s:\n found_date = True\n break\n raw_val = row_vals[idx]\n if hasattr(raw_val, 'strftime'):\n try:\n formatted = raw_val.strftime(\"%m/%d/%Y\")\n if formatted == \"04/24/2026\":\n found_date = True\n break\n except:\n pass\n if hasattr(raw_val, 'year'):\n try:\n if raw_val.year == 2026 and raw_val.month == 4 and raw_val.day == 24:\n found_date = True\n break\n except:\n pass\n if found_date:\n score += 1\n feedback_parts.append(\"Found date 04/24/2026\")\n else:\n feedback_parts.append(\"Missing date 04/24/2026\")\n\n # Check claims - they can be in any order, possibly in one cell or across cells\n found_claims = set()\n for s in row_strs:\n for claim in expected_claims:\n if claim in s:\n found_claims.add(claim)\n if found_claims == expected_claims:\n score += 1\n feedback_parts.append(\"Found all 5 claims\")\n else:\n missing = expected_claims - found_claims\n feedback_parts.append(f\"Missing claims: {missing}\")\n\n # Check Operational\n found_operational = any(\"operational\" in s.lower() for s in row_strs)\n if found_operational:\n score += 1\n feedback_parts.append(\"Found 'Operational'\")\n else:\n feedback_parts.append(\"Missing 'Operational'\")\n\n # Check Pattern Recurrence\n found_pattern = any(\"pattern recurrence\" in s.lower() for s in row_strs)\n if found_pattern:\n score += 1\n feedback_parts.append(\"Found 'Pattern Recurrence'\")\n else:\n feedback_parts.append(\"Missing 'Pattern Recurrence'\")\n\n # Check Maria Santos\n found_maria = any(\"maria santos\" in s.lower() for s in row_strs)\n if found_maria:\n score += 1\n feedback_parts.append(\"Found 'Maria Santos'\")\n else:\n feedback_parts.append(\"Missing 'Maria Santos'\")\n\n # Check David Park\n found_david = any(\"david park\" in s.lower() for s in row_strs)\n if found_david:\n score += 1\n feedback_parts.append(\"Found 'David Park'\")\n else:\n feedback_parts.append(\"Missing 'David Park'\")\n\n # Check Open\n found_open = any(s.lower() == \"open\" for s in row_strs)\n if found_open:\n score += 1\n feedback_parts.append(\"Found 'Open'\")\n else:\n feedback_parts.append(\"Missing 'Open'\")\n\n # Check for at least one BLANK cell in the row (rubric requires a BLANK cell)\n has_blank = any(v is None or str(v).strip() == \"\" for v in row_vals)\n if has_blank:\n feedback_parts.append(\"Found blank cell (as expected)\")\n else:\n feedback_parts.append(\"No blank cell found (minor)\")\n\n return score, total_checks, \"; \".join(feedback_parts)\n\n best_score = 0\n best_total = 8\n best_feedback = \"\"\n best_row = None\n\n # Check rows 4 and 5 as specified, but also be generous and check a wider range\n rows_to_check = set()\n for r in [4, 5]:\n if r <= ws.max_row:\n rows_to_check.add(r)\n # Also check rows 3-10 in case of off-by-one\n for r in range(3, min(ws.max_row + 1, 15)):\n rows_to_check.add(r)\n\n for row_num in sorted(rows_to_check):\n s, t, fb = check_row(row_num)\n if s > best_score:\n best_score = s\n best_total = t\n best_feedback = fb\n best_row = row_num\n\n if best_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No valid rows found in the sheet.\"}\n\n ratio = best_score / best_total\n passed = best_score >= 8\n\n return {\n \"pass\": passed,\n \"score\": ratio,\n \"feedback\": f\"Row {best_row}: {best_score}/{best_total} checks passed. Details: {best_feedback}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705849430",
|
|
"sort_order": 19,
|
|
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the exact values: \"ESC-2026-0009\" (or \"ESC-2026-0008\"), \"04/24/2026\", \"CLM-2026-0967\", \"Financial\", \"Payer Dispute\", \"Maria Santos\", \"David Park\", \"Open\", and BLANK (empty cell).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for leniency\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Try exact path\n fp = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if fp.exists():\n file_path = fp\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # The rubric says ESC-2026-0009 or ESC-2026-0008, but we'll accept a range of ESC IDs\n valid_esc_ids = [\"esc-2026-0008\", \"esc-2026-0009\", \"esc-2026-0010\", \"esc-2026-0011\"]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def check_date(val):\n \"\"\"Check if the value represents 04/24/2026\"\"\"\n if isinstance(val, datetime):\n return val.month == 4 and val.day == 24 and val.year == 2026\n s = normalize(val).lower()\n date_patterns = [\"04/24/2026\", \"4/24/2026\", \"2026-04-24\", \"04-24-2026\", \"april 24, 2026\", \"apr 24, 2026\"]\n return s in date_patterns\n\n def check_blank(val):\n return val is None or str(val).strip() == \"\"\n\n # Search rows 4-10 for leniency (rubric says 4 or 5, but be generous)\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n \n row_strs = [normalize(c).lower() for c in row_cells]\n \n # Check if key identifiers are present in the row\n has_esc = any(any(esc_id in s for esc_id in valid_esc_ids) for s in row_strs)\n has_clm = any(\"clm-2026-0967\" in s for s in row_strs)\n has_financial = any(\"financial\" in s for s in row_strs)\n has_payer = any(\"payer dispute\" in s for s in row_strs)\n has_maria = any(\"maria santos\" in s or \"maria\" in s for s in row_strs)\n has_david = any(\"david park\" in s or \"david\" in s for s in row_strs)\n has_open = any(s == \"open\" for s in row_strs)\n \n # Check date\n has_date = any(check_date(c) for c in row_cells)\n \n checks = {\n \"ESC ID\": has_esc,\n \"CLM-2026-0967\": has_clm,\n \"Financial\": has_financial,\n \"Payer Dispute\": has_payer,\n \"Maria Santos\": has_maria,\n \"David Park\": has_david,\n \"Open\": has_open,\n \"Date 04/24/2026\": has_date\n }\n \n if all(checks.values()):\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row {row_num} in PBC_Escalation_Log.xlsx contains all expected escalation values. Row values: {row_strs}\"\n }\n \n # Gather debug info\n debug = []\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n row_strs = [normalize(c) for c in row_cells]\n if any(s.strip() for s in row_strs): # Only show non-empty rows\n debug.append(f\"Row {row_num}: {row_strs}\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No row (4-10) in PBC_Escalation_Log.xlsx contains the expected escalation entry. Expected ESC-2026-0008/0009, 04/24/2026, CLM-2026-0967, Financial, Payer Dispute, Maria Santos, David Park, Open. Found rows: {'; '.join(debug)}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705853789",
|
|
"sort_order": 20,
|
|
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be no duplicate rows or Escalation IDs",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if \"escalation\" in f.name.lower() and \"log\" in f.name.lower():\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {file_path}: {e}\"}\n\n issues = []\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else \"\"\n for cell in row\n )\n rows.append(normalized)\n\n if not rows:\n continue\n\n header = rows[0]\n data_rows = rows[1:] if len(rows) > 1 else []\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(cell != \"\" for cell in r)]\n\n # Check for duplicate full rows\n seen_rows = set()\n duplicate_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n duplicate_rows.append((i + 2, r))\n else:\n seen_rows.add(r)\n\n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {idx}: {row}\" for idx, row in duplicate_rows[:5]\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate data row(s) in sheet '{sheet_name}'. Examples: {dup_details}\")\n\n # Check for duplicate Escalation IDs\n # Find the column index for Escalation ID (flexible matching)\n esc_id_col = None\n for col_idx, col_name in enumerate(header):\n if col_name and (\"escalation\" in col_name and \"id\" in col_name):\n esc_id_col = col_idx\n break\n \n if esc_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for i, r in enumerate(data_rows):\n if esc_id_col < len(r):\n esc_id = r[esc_id_col]\n if esc_id and esc_id != \"\":\n if esc_id in seen_ids:\n duplicate_ids.append((i + 2, esc_id, seen_ids[esc_id]))\n else:\n seen_ids[esc_id] = i + 2\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Escalation ID '{eid}' in rows {first_row} and {dup_row}\" \n for dup_row, eid, first_row in duplicate_ids[:5]\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Escalation ID(s) in sheet '{sheet_name}'. Examples: {dup_id_details}\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicate issues found in PBC_Escalation_Log.xlsx: \" + \" | \".join(issues)\n }\n\n total_sheets = len(wb.sheetnames)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Escalation IDs found in PBC_Escalation_Log.xlsx across {total_sheets} sheet(s).\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1775705926737",
|
|
"sort_order": 21,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be a table with 11 total rows (including the header row).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for the file with case-insensitive matching\n target = None\n ws_path = Path(workspace_path)\n \n # Try exact name first\n exact = ws_path / \"PBC_RA_Queue.xlsx\"\n if exact.exists():\n target = exact\n else:\n # Try case-insensitive glob\n for f in ws_path.glob(\"*.xlsx\"):\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target = f\n break\n \n if target is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File 'PBC_RA_Queue.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open '{target.name}': {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (a row is non-empty if at least one cell has a value)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 11 total rows (including header) in sheet '{sheet_name}' of '{target.name}'.\"\n }\n \n # If no sheet had exactly 11 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected 11 total rows (including header) in 'PBC_RA_Queue.xlsx', but found: {'; '.join(details)}.\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705946912",
|
|
"sort_order": 22,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0031\" (where 31 can be 32 or 33), \"04/24/2026\", \"CLM-2026-0892\", \"PAT-1042\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the file with case-insensitive search\n file_path = None\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n file_path = c\n break\n if file_path is None:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n for f in xlsx_files:\n if \"ra_queue\" in f.name.lower() or \"raqueue\" in f.name.lower():\n file_path = f\n break\n if file_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in {workspace_path}. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n # The rubric says the RA ID can be RA-2026-0031, 0032, or 0033\n valid_ra_ids = [\"ra-2026-0031\", \"ra-2026-0032\", \"ra-2026-0033\"]\n\n # Core required values that MUST be present:\n # RA ID (flexible from list above)\n # Date: 04/24/2026\n # Claim: CLM-2026-0892\n # Patient: PAT-1042\n # Task: ERA Processing\n # Priority: [Any value is acceptable] - we won't check this\n # Assigned: Maria Santos\n #\n # Status: \"Resolved\" or \"Open\" or BLANK\n # Resolution Date: \"04/24/2026\" or BLANK\n\n core_required = [\n \"04/24/2026\", # date\n \"CLM-2026-0892\", # claim id\n \"PAT-1042\", # patient id\n \"ERA Processing\", # task type\n \"Maria Santos\", # assigned to\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def values_match(cell_val, expected_val):\n n_cell = normalize(cell_val).lower()\n n_exp = expected_val.strip().lower()\n if n_cell == n_exp:\n return True\n # Handle date variations\n if n_exp == \"04/24/2026\":\n if n_cell in [\"4/24/2026\", \"04/24/2026\", \"2026-04-24\", \"04-24-2026\", \"4-24-2026\"]:\n return True\n # Handle minor whitespace/case differences\n if n_cell.replace(\" \", \"\") == n_exp.replace(\" \", \"\"):\n return True\n # Partial matching for things like \"era processing\"\n if n_exp in n_cell or n_cell in n_exp:\n if len(n_exp) > 3 and len(n_cell) > 3:\n return True\n return False\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == \"\" or s.lower() == \"none\"\n\n # Check rows 9, 10, 11 first, then expand search\n priority_rows = [9, 10, 11]\n check_rows = list(range(2, min(ws.max_row + 1, 50)))\n all_rows = priority_rows + [r for r in check_rows if r not in priority_rows]\n\n best_match_count = 0\n best_row_info = \"\"\n\n for row_num in all_rows:\n row_values = []\n for col in range(1, ws.max_column + 1):\n row_values.append(ws.cell(row=row_num, column=col).value)\n\n # Check RA ID\n ra_id_found = False\n for cv in row_values:\n nv = normalize(cv).lower().strip()\n if nv in valid_ra_ids:\n ra_id_found = True\n break\n\n # Check core required values\n core_matched = []\n for exp in core_required:\n found = False\n for cv in row_values:\n if values_match(cv, exp):\n found = True\n break\n core_matched.append(found)\n\n total_core = sum(core_matched) + (1 if ra_id_found else 0)\n if total_core > best_match_count:\n best_match_count = total_core\n best_row_info = f\"Row {row_num}: {[normalize(v) for v in row_values]}\"\n\n # Need RA ID + all core required values (6 total)\n if ra_id_found and all(core_matched):\n # Check status - rubric says \"Resolved\" and \"04/24/2026\" for last two,\n # but also says they could be \"Open\", BLANK instead\n has_resolved = False\n has_open = False\n status_val = \"unknown\"\n\n for cv in row_values:\n nv = normalize(cv).lower()\n if nv == \"resolved\":\n has_resolved = True\n status_val = \"Resolved\"\n if nv == \"open\":\n has_open = True\n status_val = \"Open\"\n\n # Status is acceptable if Resolved, Open, or blank (lenient)\n # Per rubric: last two cells could be \"Open\", BLANK instead of \"Resolved\", \"04/24/2026\"\n # We accept any of these combinations\n status_ok = True # Be lenient per rubric\n\n row_display = [normalize(v) for v in row_values]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row {row_num} contains all expected core values. RA ID found. Status={status_val}. Row contents: {row_display}\"\n }\n\n # If not found, provide debugging info\n debug_info = []\n for row_num in priority_rows:\n row_values = []\n for col in range(1, ws.max_column + 1):\n row_values.append(normalize(ws.cell(row=row_num, column=col).value))\n debug_info.append(f\"Row {row_num}: {row_values}\")\n\n all_debug = []\n for row_num in range(1, min(ws.max_row + 1, 20)):\n row_values = []\n for col in range(1, ws.max_column + 1):\n row_values.append(normalize(ws.cell(row=row_num, column=col).value))\n all_debug.append(f\"Row {row_num}: {row_values}\")\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No row contains all expected core values. Expected RA ID in {valid_ra_ids}, plus {core_required}. Best match had {best_match_count}/6 values in {best_row_info}. Rows 9-11 contents:\\n\" + \"\\n\".join(debug_info) + \"\\n\\nAll rows:\\n\" + \"\\n\".join(all_debug)\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705963295",
|
|
"sort_order": 23,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, row 9, 10, or 11 must contain the exact values: \"RA-2026-0032\" (where 32 can be 31 or 33), \"04/24/2026\", \"CLM-2026-1105\", \"PAT-1058\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\n\ndef normalize(val):\n \"\"\"Normalize a cell value to a comparable string.\"\"\"\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n\ndef values_match(actual, expected):\n \"\"\"Loosely compare two values.\"\"\"\n a = normalize(actual)\n e = expected.strip()\n # Direct match (case-insensitive)\n if a.lower() == e.lower():\n return True\n # Try date parsing: expected is MM/DD/YYYY\n if '/' in e:\n try:\n exp_date = datetime.strptime(e, \"%m/%d/%Y\")\n # Try parsing actual in various formats\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\"]:\n try:\n act_date = datetime.strptime(a, fmt)\n if act_date.date() == exp_date.date():\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n return False\n\n\ndef ra_id_match(actual):\n \"\"\"Check if a cell value matches RA-2026-00{31,32,33}.\"\"\"\n a = normalize(actual).upper()\n return a in [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n\ndef verify(workspace_path, external_services_path=None):\n ws_path = Path(workspace_path)\n # Try case-insensitive search for file\n target_file = None\n for f in ws_path.iterdir():\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target_file = f\n break\n\n if target_file is None:\n # Also try glob\n candidates = list(ws_path.glob(\"*RA_Queue*\")) + list(ws_path.glob(\"*ra_queue*\")) + list(ws_path.glob(\"*Ra_Queue*\")) + list(ws_path.glob(\"*Queue*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n # The core required values (excluding RA ID which has special matching,\n # excluding priority which accepts any value,\n # and excluding status/completion date which are flexible)\n core_expected = [\n \"04/24/2026\",\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"ERA Processing\",\n \"Maria Santos\",\n ]\n\n # Search rows 8-12 to be generous (rubric says 9, 10, or 11)\n search_rows = list(range(8, 13))\n\n for row_num in search_rows:\n row_cells = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_cells.append(cell_val)\n\n # Check if RA ID matches\n ra_found = False\n for cell in row_cells:\n if ra_id_match(cell):\n ra_found = True\n break\n\n if not ra_found:\n continue\n\n # Check if all core expected values can be found in this row\n all_core_found = True\n matched_details = []\n matched_details.append(\" RA ID found (one of RA-2026-0031/32/33)\")\n\n for exp in core_expected:\n found = False\n for i, cell in enumerate(row_cells):\n if values_match(cell, exp):\n found = True\n matched_details.append(f\" Expected '{exp}' -> found '{normalize(cell)}' in column {i+1}\")\n break\n if not found:\n all_core_found = False\n matched_details.append(f\" Expected '{exp}' -> NOT FOUND\")\n break\n\n if all_core_found:\n # Status can be \"Resolved\", \"Open\", or blank - check if present\n status_info = \"\"\n for i, cell in enumerate(row_cells):\n n = normalize(cell).lower()\n if n in [\"resolved\", \"open\"]:\n status_info = f\"Status='{normalize(cell)}' in column {i+1}\"\n break\n if not status_info:\n status_info = \"Status is blank or not found (acceptable)\"\n\n detail = \"\\n\".join(matched_details)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All expected values found in row {row_num} of PBC_RA_Queue.xlsx:\\n{detail}\\n {status_info}\\nPriority: any value accepted.\"\n }\n\n # If not found, provide diagnostic info\n diag = []\n for row_num in search_rows:\n row_vals = []\n for col in range(1, ws.max_column + 1):\n v = ws.cell(row=row_num, column=col).value\n row_vals.append(normalize(v))\n diag.append(f\" Row {row_num}: {row_vals}\")\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected values not found in rows 8-12 of PBC_RA_Queue.xlsx.\\nExpected RA ID: RA-2026-0031/32/33\\nExpected core: {core_expected}\\nStatus: 'Resolved', 'Open', or blank\\nPriority: any value\\nActual rows:\\n\" + \"\\n\".join(diag)\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705979579",
|
|
"sort_order": 24,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0033\" (where 33 can be 32 or 31), \"04/24/2026\", \"CLM-2026-0967\", \"PAT-1063\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Open\", and a BLANK cell (in the appropriate columns).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n if not file_path.exists():\n # Try case-insensitive glob\n candidates = list(Path(workspace_path).glob(\"PBC_RA_Queue*\")) + list(Path(workspace_path).glob(\"PBC_Ra_Queue*\")) + list(Path(workspace_path).glob(\"pbc_ra_queue*\"))\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n acceptable_ra_ids = [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n # Expected fixed values (excluding RA ID, Date, and Priority which are handled separately)\n # The rubric says: RA ID, Date, CLM-2026-0967, PAT-1063, ERA Processing, [Any priority], Maria Santos, Open, BLANK\n expected_fixed = {\n \"CLM\": \"CLM-2026-0967\",\n \"PAT\": \"PAT-1063\",\n \"TYPE\": \"ERA Processing\",\n \"ASSIGNEE\": \"Maria Santos\",\n \"STATUS\": \"Open\"\n }\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def clean(s):\n return s.lower().replace(\" \", \"\").replace(\"-\", \"\").replace(\"/\", \"\").replace(\"_\", \"\")\n\n def match_date(actual):\n a_clean = normalize(actual)\n # Try multiple date formats\n try:\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%B %d, %Y\", \"%m/%d/%y\"]:\n try:\n parsed = datetime.strptime(a_clean, fmt)\n if parsed.month == 4 and parsed.day == 24 and parsed.year == 2026:\n return True\n except:\n pass\n except:\n pass\n # Fallback: clean comparison\n if clean(a_clean) == clean(\"04/24/2026\"):\n return True\n if clean(a_clean) == \"04242026\":\n return True\n return False\n\n def match_ra_id(actual):\n a = normalize(actual)\n if not a:\n return False\n for rid in acceptable_ra_ids:\n if clean(a) == clean(rid):\n return True\n if a.upper().replace(\"-\", \"\").replace(\" \", \"\") == rid.upper().replace(\"-\", \"\"):\n return True\n # Regex: RA-2026-003[1-3] with flexible separators\n pattern = r'(?i)ra[\\-_ ]?2026[\\-_ ]?0{0,2}3[123]'\n if re.search(pattern, a):\n return True\n return False\n\n def fuzzy_match(expected, actual):\n e_norm = normalize(expected)\n a_norm = normalize(actual)\n if not a_norm and not e_norm:\n return True\n if clean(e_norm) == clean(a_norm):\n return True\n if e_norm.lower().strip() == a_norm.lower().strip():\n return True\n return False\n\n def check_row(row_values):\n \"\"\"Check if a row contains all expected values. Returns (True, msg) or (False, msg).\"\"\"\n # Strategy 1: Ordered match\n # Expected column order: A=RA_ID, B=Date, C=CLM, D=PAT, E=Type, F=Priority, G=Assignee, H=Status, I=blank(notes)\n if len(row_values) >= 8:\n ordered_ok = True\n issues = []\n # Column A (index 0): RA ID\n if not match_ra_id(row_values[0]):\n ordered_ok = False\n issues.append(f\"Col A: expected RA-2026-003[1-3], got '{normalize(row_values[0])}'\")\n # Column B (index 1): Date\n if ordered_ok and not match_date(row_values[1]):\n ordered_ok = False\n issues.append(f\"Col B: expected 04/24/2026, got '{normalize(row_values[1])}'\")\n # Column C (index 2): CLM-2026-0967\n if ordered_ok and not fuzzy_match(\"CLM-2026-0967\", row_values[2]):\n ordered_ok = False\n issues.append(f\"Col C: expected CLM-2026-0967, got '{normalize(row_values[2])}'\")\n # Column D (index 3): PAT-1063\n if ordered_ok and not fuzzy_match(\"PAT-1063\", row_values[3]):\n ordered_ok = False\n issues.append(f\"Col D: expected PAT-1063, got '{normalize(row_values[3])}'\")\n # Column E (index 4): ERA Processing\n if ordered_ok and not fuzzy_match(\"ERA Processing\", row_values[4]):\n ordered_ok = False\n issues.append(f\"Col E: expected ERA Processing, got '{normalize(row_values[4])}'\")\n # Column F (index 5): Priority - ANY value is acceptable per rubric\n # No check needed\n # Column G (index 6): Maria Santos\n if ordered_ok and not fuzzy_match(\"Maria Santos\", row_values[6]):\n ordered_ok = False\n issues.append(f\"Col G: expected Maria Santos, got '{normalize(row_values[6])}'\")\n # Column H (index 7): Open\n if ordered_ok and not fuzzy_match(\"Open\", row_values[7]):\n ordered_ok = False\n issues.append(f\"Col H: expected Open, got '{normalize(row_values[7])}'\")\n # Column I (index 8): blank\n if ordered_ok:\n col9_val = row_values[8] if len(row_values) > 8 else None\n if col9_val is None or str(col9_val).strip() == \"\":\n return True, \"ordered match with blank column 9\"\n else:\n ordered_ok = False\n issues.append(f\"Col I: expected BLANK, got '{normalize(col9_val)}'\")\n\n # Strategy 2: Unordered match - find all values somewhere in the row\n found_ra = False\n for rv in row_values:\n if match_ra_id(rv):\n found_ra = True\n break\n\n found_date = False\n for rv in row_values:\n if match_date(rv):\n found_date = True\n break\n\n found_others = True\n missing = []\n for key, exp in expected_fixed.items():\n found = False\n for rv in row_values:\n if fuzzy_match(exp, rv):\n found = True\n break\n if not found:\n found_others = False\n missing.append(exp)\n\n has_blank = any(v is None or str(v).strip() == \"\" for v in row_values)\n\n if found_ra and found_date and found_others and has_blank:\n return True, \"unordered match\"\n\n details = []\n if not found_ra:\n details.append(\"RA ID not found (expected one of RA-2026-0031/0032/0033)\")\n if not found_date:\n details.append(\"Date 04/24/2026 not found\")\n if missing:\n details.append(f\"Missing values: {missing}\")\n if not has_blank:\n details.append(\"No blank cell found\")\n return False, \"; \".join(details)\n\n # Check rows 9, 10, 11\n for row_num in [9, 10, 11]:\n row_values = []\n for col in range(1, max(ws.max_column + 1, 15)):\n cell_val = ws.cell(row=row_num, column=col).value\n row_values.append(cell_val)\n\n ok, msg = check_row(row_values)\n if ok:\n wb.close()\n row_display = [normalize(v) for v in row_values[:min(len(row_values), 12)]]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {row_num} contains all expected values ({msg}): {row_display}\"}\n\n # Diagnostic info\n diag = []\n for row_num in [9, 10, 11]:\n row_values = []\n for col in range(1, min(ws.max_column + 1, 15)):\n cell_val = ws.cell(row=row_num, column=col).value\n row_values.append(normalize(cell_val))\n raw_values = [ws.cell(row=row_num, column=c).value for c in range(1, max(ws.max_column + 1, 15))]\n ok, msg = check_row(raw_values)\n diag.append(f\"Row {row_num}: {row_values} -- Issues: {msg}\")\n\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No matching row found in rows 9-11 of PBC_RA_Queue.xlsx. Expected: RA-2026-003[1-3], 04/24/2026, CLM-2026-0967, PAT-1063, ERA Processing, [Any Priority], Maria Santos, Open, BLANK. Actual rows:\\n\" + \"\\n\".join(diag)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775705982031",
|
|
"sort_order": 25,
|
|
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be no duplicate rows or Queue Entry IDs.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n # Also try case-insensitive search\n if not target.exists():\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n target = c\n break\n\n if not target.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in workspace. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = []\n for cell in row:\n if isinstance(cell, str):\n normalized.append(cell.strip().lower())\n elif cell is None:\n normalized.append(None)\n else:\n normalized.append(cell)\n rows.append(tuple(normalized))\n\n if len(rows) <= 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(rows)} row(s); no duplicates possible.\"}\n\n header = rows[0]\n data_rows = rows[1:]\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(c is not None for c in r)]\n\n issues = []\n\n # Check 1: Duplicate entire rows\n seen_rows = set()\n dup_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n dup_rows.append((i + 2, r)) # +2 for 1-indexed + header\n else:\n seen_rows.add(r)\n\n if dup_rows:\n dup_info = \"; \".join([f\"Row {idx}\" for idx, vals in dup_rows[:5]])\n issues.append(f\"Found {len(dup_rows)} duplicate row(s). Examples: {dup_info}\")\n\n # Check 2: Duplicate Queue Entry IDs\n # Try to find a column that looks like a Queue Entry ID\n qe_col_idx = None\n if header:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue entry id' in h_lower or 'queueentryid' in h_lower or 'queue_entry_id' in h_lower or h_lower == 'id' or 'entry id' in h_lower:\n qe_col_idx = idx\n break\n # If not found, also check for columns starting with 'queue'\n if qe_col_idx is None:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue' in h_lower and 'id' in h_lower:\n qe_col_idx = idx\n break\n\n if qe_col_idx is not None:\n seen_ids = {}\n dup_ids = []\n for i, r in enumerate(data_rows):\n if qe_col_idx < len(r):\n val = r[qe_col_idx]\n if val is not None:\n val_key = str(val).strip().lower()\n if val_key in seen_ids:\n dup_ids.append((i + 2, val))\n else:\n seen_ids[val_key] = i + 2\n if dup_ids:\n dup_id_info = \"; \".join([f\"Row {idx}: ID={val}\" for idx, val in dup_ids[:5]])\n issues.append(f\"Found {len(dup_ids)} duplicate Queue Entry ID(s). Examples: {dup_id_info}\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx has duplicates: \" + \" | \".join(issues)}\n else:\n id_note = f\" Queue Entry ID column found at index {qe_col_idx}, all unique.\" if qe_col_idx is not None else \" No Queue Entry ID column identified; only checked full row uniqueness.\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(data_rows)} data rows with no duplicate rows or Queue Entry IDs.{id_note}\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1775706048803",
|
|
"sort_order": 26,
|
|
"rubric_text": "In `slack_data.json` (in `external_data/final`), there must be exactly 3 total messages in #general and exactly 8 total messages in #rcm-team.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n # Try multiple possible locations for slack_data.json\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n \n # Also check workspace-based paths\n wp = Path(workspace_path)\n candidates.append(wp / \"external_data\" / \"final\" / \"slack_data.json\")\n candidates.append(wp / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find slack_data.json. Checked: {[str(c) for c in candidates]}\"}\n \n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n \n messages = data.get(\"messages\", {})\n \n # Find the general channel and rcm-team channel\n # First, map channel names to IDs from channels info\n channels = data.get(\"channels\", {})\n general_id = None\n rcm_id = None\n for ch_id, ch_info in channels.items():\n name = ch_info.get(\"name\", \"\").lower()\n if name == \"general\":\n general_id = ch_id\n elif name == \"rcm-team\":\n rcm_id = ch_id\n \n # Count messages\n general_msgs = []\n rcm_msgs = []\n \n # Try by channel ID first, then by channel name as key\n for key, msg_list in messages.items():\n if key == general_id or key.lower() == \"general\":\n general_msgs = msg_list\n elif key == rcm_id or key.lower() == \"rcm-team\":\n rcm_msgs = msg_list\n \n general_count = len(general_msgs)\n rcm_count = len(rcm_msgs)\n \n feedback_parts = []\n passed = True\n score = 0.0\n \n if general_count == 3:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). FAIL.\")\n passed = False\n \n if rcm_count == 8:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). FAIL.\")\n passed = False\n \n return {\"pass\": passed, \"score\": score, \"feedback\": \" | \".join(feedback_parts)}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|