DerekSurge
59 lines
30 KiB
JSON
59 lines
30 KiB
JSON
[
|
|
{
|
|
"id": "20e16006-8021-4434-a6c9-3b370efde0f8",
|
|
"sort_order": 0,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: Invoice Number CC-2026-0168 must have Status 'ON HOLD' and Hold Code 'APPR'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in xlsx_files if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n sheet_names = [s.lower() for s in wb.sheetnames]\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n invoice_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if 'invoice' in h and ('number' in h or 'num' in h or 'no' in h or '#' in h):\n invoice_col = c\n elif h == 'invoice number' or h == 'invoice no' or h == 'invoice#':\n invoice_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n \n # Try alternate header matching\n for h, c in headers.items():\n if invoice_col is None and 'invoice' in h:\n invoice_col = c\n if hold_col is None and 'hold' in h:\n hold_col = c\n \n if invoice_col is None or status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find required columns. Headers found: {list(headers.keys())}'}\n \n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=invoice_col).value\n if inv_val and str(inv_val).strip() == 'CC-2026-0168':\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n status_str = str(status_val).strip().upper() if status_val else ''\n hold_str = str(hold_val).strip().upper() if hold_val else ''\n if 'ON HOLD' in status_str or status_str == 'ON HOLD':\n if hold_str == 'APPR':\n return {'pass': True, 'score': 1.0, 'feedback': f'CC-2026-0168 has Status={status_val}, Hold Code={hold_val} as expected'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 has Status={status_val} but Hold Code={hold_val} (expected APPR)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 has Status={status_val} (expected ON HOLD), Hold Code={hold_val}'}\n \n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice CC-2026-0168 not found in Invoice Register'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "600c2927-4972-436d-9c47-c74f620e9e57",
|
|
"sort_order": 1,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: Invoice Number CC-2026-0168 must NOT appear.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n for row in range(1, queue_sheet.max_row + 1):\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=row, column=col).value\n if val and 'CC-2026-0168' in str(val):\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 found in Payment Queue at row {row} - it should not be there'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'CC-2026-0168 correctly absent from Payment Queue'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "a8717ac3-48f0-4ae5-96a0-9d0be7a89e67",
|
|
"sort_order": 2,
|
|
"rubric_text": "In mailbox.json, a sent email to t.okonkwo@meridianpartners.com must exist with subject containing 'Cascade Cleaning' and 'CC-2026-0168', and body containing: 'Tanya' or 'Okonkwo', 'A recurring invoice has been received for a vendor whose agreement has expired. The invoice has been placed on APPR hold and cannot be processed until the agreement is renewed or confirmed.', 'Cascade Cleaning', 'V-000031', 'CC-2026-0168', the date April 1 2026, the amount $2,200, and the date March 31 2026.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n \n target_recipient = 't.okonkwo@meridianpartners.com'\n \n matching_emails = []\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n if target_recipient.lower() in to_addr:\n matching_emails.append(email)\n \n if not matching_emails:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target_recipient}'}\n \n required_subject_terms = ['cascade cleaning', 'cc-2026-0168']\n required_body_terms = [\n 'cascade cleaning',\n 'v-000031',\n 'cc-2026-0168',\n ]\n \n # 'Tanya' or 'Okonkwo' must appear in body\n name_alternatives = ['tanya', 'okonkwo']\n \n # The key sentence from the rubric (check loosely via fragments)\n # \"A recurring invoice has been received for a vendor whose agreement has expired.\n # The invoice has been placed on APPR hold and cannot be processed until the agreement is renewed or confirmed.\"\n key_message_fragments = [\n 'recurring invoice',\n 'agreement',\n 'expired',\n 'hold',\n 'renewed',\n 'confirmed',\n ]\n \n # Amount check - accept $2,200 or 2200 or 2,200 or 2200.00 etc.\n amount_patterns = [r'\\$?2[,.]?200']\n \n # Date terms - accept various formats for April 1 2026 and March 31 2026\n date_patterns_apr1 = [\n r'april\\s*1[,\\s]+2026', r'april\\s*0?1[,\\s]+2026',\n r'apr\\.?\\s*1[,\\s]+2026', r'apr\\.?\\s*0?1[,\\s]+2026',\n r'2026-04-01', r'04/01/2026', r'04-01-2026',\n r'1\\s+april\\s+2026', r'01\\s+april\\s+2026',\n r'april\\s*1\\b', r'apr\\.?\\s*1\\b',\n ]\n date_patterns_mar31 = [\n r'march\\s*31[,\\s]+2026', r'mar\\.?\\s*31[,\\s]+2026',\n r'2026-03-31', r'03/31/2026', r'03-31-2026',\n r'31\\s+march\\s+2026',\n r'march\\s*31\\b', r'mar\\.?\\s*31\\b',\n ]\n \n for email in matching_emails:\n subject = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '') or email.get('body', '')).lower()\n \n subject_ok = all(term in subject for term in required_subject_terms)\n if not subject_ok:\n continue\n \n body_terms_ok = all(term.lower() in body for term in required_body_terms)\n \n # Check name alternatives: 'Tanya' or 'Okonkwo'\n name_ok = any(name in body for name in name_alternatives)\n \n # Check key message fragments (at least 4 of 6 for leniency)\n msg_hits = sum(1 for frag in key_message_fragments if frag in body)\n msg_ok = msg_hits >= 4\n \n amount_ok = any(re.search(p, body) for p in amount_patterns)\n apr1_ok = any(re.search(p, body) for p in date_patterns_apr1)\n mar31_ok = any(re.search(p, body) for p in date_patterns_mar31)\n \n if body_terms_ok and name_ok and msg_ok and amount_ok and apr1_ok and mar31_ok:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found qualifying sent email to {target_recipient} with all required content about CC-2026-0168'}\n \n # Provide detailed failure info\n for email in matching_emails:\n subject = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '') or email.get('body', '')).lower()\n missing = []\n for term in required_subject_terms:\n if term not in subject:\n missing.append(f'subject missing: {term}')\n for term in required_body_terms:\n if term.lower() not in body:\n missing.append(f'body missing: {term}')\n if not any(name in body for name in name_alternatives):\n missing.append('body missing: Tanya or Okonkwo')\n missing_frags = [frag for frag in key_message_fragments if frag not in body]\n if len(missing_frags) > 2:\n missing.append(f'body missing key message fragments: {missing_frags}')\n if not any(re.search(p, body) for p in amount_patterns):\n missing.append('body missing: $2,200 amount')\n if not any(re.search(p, body) for p in date_patterns_apr1):\n missing.append('body missing: April 1, 2026 date')\n if not any(re.search(p, body) for p in date_patterns_mar31):\n missing.append('body missing: March 31, 2026 date')\n return {'pass': False, 'score': 0.0, 'feedback': f'Email found to {target_recipient} but missing required content: {missing}. Body preview: {body[:500]}'}\n \n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email to {target_recipient} matched all requirements (subject must contain Cascade Cleaning and CC-2026-0168)'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "53e3483a-22d8-405c-9af6-64d7faa8b50c",
|
|
"sort_order": 3,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: Invoice IDs INV-018 through INV-026 (all nine) must each have Status 'APPROVED' and no hold code.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if h in ('invoice id', 'inv id', 'id', 'invoice_id') or ('invoice' in h and 'id' in h):\n inv_id_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n elif 'hold' in h and hold_col is None:\n hold_col = c\n \n if inv_id_col is None:\n # Try first column as ID\n inv_id_col = 1\n \n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Status column not found. Headers: {list(headers.keys())}'}\n \n required_ids = {f'INV-0{i}' for i in range(18, 27)}\n found = {}\n \n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in required_ids:\n inv_id = str(inv_val).strip()\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n found[inv_id] = {'status': status_val, 'hold': hold_val}\n \n missing = required_ids - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs not found in register: {missing}'}\n \n failures = []\n for inv_id, vals in found.items():\n status_str = str(vals['status']).strip().upper() if vals['status'] else ''\n hold_str = str(vals['hold']).strip() if vals['hold'] else ''\n if status_str != 'APPROVED':\n failures.append(f'{inv_id} has Status={vals[\"status\"]} (expected APPROVED)')\n if hold_str and hold_str.upper() not in ('', 'NONE', 'N/A', 'NULL'):\n failures.append(f'{inv_id} has Hold Code={vals[\"hold\"]} (expected empty)')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All 9 invoices INV-018 through INV-026 have Status=APPROVED and no hold code'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "860f1841-7c7f-4001-89b3-58b9d47100d0",
|
|
"sort_order": 4,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: INV-018 through INV-026 must each appear with correct Payment Amt values (INV-018: $620.00, INV-019: $8,750.00, INV-020: $2,800.00, INV-021: $4,500.00, INV-022: $14,000.00, INV-023: $3,200.00, INV-024: $1,050.00, INV-025: $5,500.00, INV-026: $5,200.00).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n expected = {\n 'INV-018': 620.00,\n 'INV-019': 8750.00,\n 'INV-020': 2800.00,\n 'INV-021': 4500.00,\n 'INV-022': 14000.00,\n 'INV-023': 3200.00,\n 'INV-024': 1050.00,\n 'INV-025': 5500.00,\n 'INV-026': 5200.00,\n }\n \n headers = {}\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n amt_col = None\n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'invoice id', 'id') and inv_id_col is None:\n inv_id_col = c\n if 'payment' in h and ('amt' in h or 'amount' in h):\n amt_col = c\n elif ('amount' in h or 'amt' in h) and amt_col is None:\n amt_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if amt_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Payment amount column not found. Headers: {list(headers.keys())}'}\n \n def parse_amount(val):\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return float(val)\n s = str(val).replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except:\n return None\n \n found = {}\n for row in range(2, queue_sheet.max_row + 1):\n inv_val = queue_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected:\n inv_id = str(inv_val).strip()\n amt_val = queue_sheet.cell(row=row, column=amt_col).value\n found[inv_id] = parse_amount(amt_val)\n \n missing = set(expected.keys()) - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs missing from Payment Queue: {missing}'}\n \n failures = []\n for inv_id, exp_amt in expected.items():\n actual = found.get(inv_id)\n if actual is None:\n failures.append(f'{inv_id}: could not parse amount')\n else:\n tol = exp_amt * 0.01\n if abs(actual - exp_amt) > tol:\n failures.append(f'{inv_id}: expected {exp_amt}, got {actual}')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Amount mismatches: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All 9 invoices INV-018 through INV-026 found in Payment Queue with correct amounts'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "7b93f221-bd0e-470f-b332-715f3d44329b",
|
|
"sort_order": 5,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: INV-007 must have Status 'ON HOLD' with Hold Code 'MATCH'; INV-008 must have Status 'ON HOLD' with Hold Code 'APPR'; INV-009 must have Status 'ON HOLD' with Hold Code 'MATCH'; INV-011 must have Status 'IC-HOLD'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'id') and inv_id_col is None:\n inv_id_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n elif 'hold' in h and hold_col is None:\n hold_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Status column not found. Headers: {list(headers.keys())}'}\n \n expected = {\n 'INV-007': {'status': 'ON HOLD', 'hold': 'MATCH'},\n 'INV-008': {'status': 'ON HOLD', 'hold': 'APPR'},\n 'INV-009': {'status': 'ON HOLD', 'hold': 'MATCH'},\n 'INV-011': {'status': 'IC-HOLD', 'hold': None},\n }\n \n found = {}\n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected:\n inv_id = str(inv_val).strip()\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n found[inv_id] = {'status': status_val, 'hold': hold_val}\n \n missing = set(expected.keys()) - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs not found: {missing}'}\n \n failures = []\n for inv_id, exp in expected.items():\n actual = found[inv_id]\n actual_status = str(actual['status']).strip().upper() if actual['status'] else ''\n actual_hold = str(actual['hold']).strip().upper() if actual['hold'] else ''\n exp_status = exp['status'].upper()\n \n if actual_status != exp_status:\n failures.append(f'{inv_id}: Status={actual[\"status\"]} (expected {exp[\"status\"]})')\n \n if exp['hold'] is not None:\n if actual_hold != exp['hold'].upper():\n failures.append(f'{inv_id}: Hold Code={actual[\"hold\"]} (expected {exp[\"hold\"]})')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'INV-007, INV-008, INV-009, INV-011 all have correct Status and Hold Code values'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "9519c1fd-5be9-4de0-8f31-3d1006cd104a",
|
|
"sort_order": 6,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: Invoice IDs INV-007, INV-008, INV-009, and INV-011 must NOT appear.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n forbidden = {'INV-007', 'INV-008', 'INV-009', 'INV-011'}\n found_forbidden = []\n \n for row in range(1, queue_sheet.max_row + 1):\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=row, column=col).value\n if val and str(val).strip() in forbidden:\n found_forbidden.append(f'{str(val).strip()} at row {row}')\n \n if found_forbidden:\n return {'pass': False, 'score': 0.0, 'feedback': f'Forbidden Invoice IDs found in Payment Queue: {found_forbidden}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'INV-007, INV-008, INV-009, INV-011 correctly absent from Payment Queue'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "323b236e-507a-4454-8c71-e027177ad6c4",
|
|
"sort_order": 7,
|
|
"rubric_text": "In file `match_log_2026_Q2.xlsx`, Invoice IDs INV-018, INV-021, INV-024, and INV-026 must each appear with Match Date 2026-04-08 and Match Result containing the word \"PASS\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the match log file with flexible matching\n xlsx_files = list(Path(workspace_path).glob('match_log_2026_Q2.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'match' in f.name.lower() and 'log' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'match' in f.name.lower()]\n if not xlsx_files:\n # Also search subdirectories\n xlsx_files = list(Path(workspace_path).rglob('match_log_2026_Q2.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).rglob('*.xlsx') if 'match' in f.name.lower() and 'log' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).rglob('*.xlsx') if 'match' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_log_2026_Q2.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n if len(wb.sheetnames) > 1:\n for name in wb.sheetnames:\n if 'match' in name.lower() or 'log' in name.lower() or '2026' in name:\n ws = wb[name]\n break\n \n # Parse headers flexibly\n headers = {}\n for col in range(1, ws.max_column + 1):\n val = ws.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n date_col = None\n result_col = None\n \n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'id', 'invoice_id', 'inv_id', 'invoiceid') and inv_id_col is None:\n inv_id_col = c\n if 'match' in h and 'date' in h:\n date_col = c\n elif 'date' in h and date_col is None:\n date_col = c\n if 'match' in h and ('result' in h or 'status' in h):\n result_col = c\n elif ('result' in h or 'status' in h) and result_col is None:\n result_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if date_col is None or result_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Required columns not found. Headers found: {list(headers.keys())}'}\n \n # The rubric requires these four invoices with Match Date 2026-04-08 and Match Result containing \"PASS\"\n expected_invoices = {'INV-018', 'INV-021', 'INV-024', 'INV-026'}\n \n found = {}\n for row in range(2, ws.max_row + 1):\n inv_val = ws.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected_invoices:\n inv_id = str(inv_val).strip()\n date_val = ws.cell(row=row, column=date_col).value\n result_val = ws.cell(row=row, column=result_col).value\n found[inv_id] = {'date': date_val, 'result': result_val, 'row': row}\n \n missing = expected_invoices - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs missing from match log: {sorted(missing)}'}\n \n def check_date(val):\n if val is None:\n return False\n if isinstance(val, datetime):\n return val.date() == date(2026, 4, 8)\n if isinstance(val, date):\n return val == date(2026, 4, 8)\n s = str(val).strip()\n return ('2026-04-08' in s or '04/08/2026' in s or '04-08-2026' in s or\n '2026/04/08' in s or 'april 8' in s.lower() or 'apr 8' in s.lower() or\n '8 apr' in s.lower() or '8 april' in s.lower() or\n '4/8/2026' in s or '8/4/2026' in s or '2026.04.08' in s)\n \n def check_result_contains_pass(val):\n \"\"\"Rubric says Match Result must contain the word PASS.\"\"\"\n if val is None:\n return False\n result_str = str(val).strip().upper()\n # Check if the word PASS appears anywhere in the result string\n # This is intentionally loose: PASS, MATCH-PASS, MATCH_PASS, MATCH PASS, 3-Way Pass, etc.\n return 'PASS' in result_str\n \n failures = []\n for inv_id in sorted(expected_invoices):\n actual = found[inv_id]\n if not check_date(actual['date']):\n failures.append(f'{inv_id} (row {actual[\"row\"]}): Match Date={actual[\"date\"]} (expected 2026-04-08)')\n if not check_result_contains_pass(actual['result']):\n failures.append(f'{inv_id} (row {actual[\"row\"]}): Match Result={actual[\"result\"]} (expected to contain the word PASS)')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Match log failures: ' + '; '.join(failures)}\n \n # Collect actual values for detailed feedback\n details = []\n for inv_id in sorted(expected_invoices):\n actual = found[inv_id]\n details.append(f'{inv_id}: date={actual[\"date\"]}, result={actual[\"result\"]}')\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All required invoices (INV-018, INV-021, INV-024, INV-026) found in match log with Match Date 2026-04-08 and Match Result containing PASS. Details: ' + '; '.join(details)}\n",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|