Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,58 @@
|
||||
[
|
||||
{
|
||||
"id": "20e16006-8021-4434-a6c9-3b370efde0f8",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: Invoice Number CC-2026-0168 must have Status 'ON HOLD' and Hold Code 'APPR'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n xlsx_files = [f for f in xlsx_files if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n sheet_names = [s.lower() for s in wb.sheetnames]\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n invoice_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if 'invoice' in h and ('number' in h or 'num' in h or 'no' in h or '#' in h):\n invoice_col = c\n elif h == 'invoice number' or h == 'invoice no' or h == 'invoice#':\n invoice_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n \n # Try alternate header matching\n for h, c in headers.items():\n if invoice_col is None and 'invoice' in h:\n invoice_col = c\n if hold_col is None and 'hold' in h:\n hold_col = c\n \n if invoice_col is None or status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find required columns. Headers found: {list(headers.keys())}'}\n \n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=invoice_col).value\n if inv_val and str(inv_val).strip() == 'CC-2026-0168':\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n status_str = str(status_val).strip().upper() if status_val else ''\n hold_str = str(hold_val).strip().upper() if hold_val else ''\n if 'ON HOLD' in status_str or status_str == 'ON HOLD':\n if hold_str == 'APPR':\n return {'pass': True, 'score': 1.0, 'feedback': f'CC-2026-0168 has Status={status_val}, Hold Code={hold_val} as expected'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 has Status={status_val} but Hold Code={hold_val} (expected APPR)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 has Status={status_val} (expected ON HOLD), Hold Code={hold_val}'}\n \n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice CC-2026-0168 not found in Invoice Register'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "600c2927-4972-436d-9c47-c74f620e9e57",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: Invoice Number CC-2026-0168 must NOT appear.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n for row in range(1, queue_sheet.max_row + 1):\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=row, column=col).value\n if val and 'CC-2026-0168' in str(val):\n return {'pass': False, 'score': 0.0, 'feedback': f'CC-2026-0168 found in Payment Queue at row {row} - it should not be there'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'CC-2026-0168 correctly absent from Payment Queue'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "a8717ac3-48f0-4ae5-96a0-9d0be7a89e67",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In mailbox.json, a sent email to t.okonkwo@meridianpartners.com must exist with subject containing 'Cascade Cleaning' and 'CC-2026-0168', and body containing: 'Tanya' or 'Okonkwo', 'A recurring invoice has been received for a vendor whose agreement has expired. The invoice has been placed on APPR hold and cannot be processed until the agreement is renewed or confirmed.', 'Cascade Cleaning', 'V-000031', 'CC-2026-0168', the date April 1 2026, the amount $2,200, and the date March 31 2026.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n \n target_recipient = 't.okonkwo@meridianpartners.com'\n \n matching_emails = []\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n if target_recipient.lower() in to_addr:\n matching_emails.append(email)\n \n if not matching_emails:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target_recipient}'}\n \n required_subject_terms = ['cascade cleaning', 'cc-2026-0168']\n required_body_terms = [\n 'cascade cleaning',\n 'v-000031',\n 'cc-2026-0168',\n ]\n \n # 'Tanya' or 'Okonkwo' must appear in body\n name_alternatives = ['tanya', 'okonkwo']\n \n # The key sentence from the rubric (check loosely via fragments)\n # \"A recurring invoice has been received for a vendor whose agreement has expired.\n # The invoice has been placed on APPR hold and cannot be processed until the agreement is renewed or confirmed.\"\n key_message_fragments = [\n 'recurring invoice',\n 'agreement',\n 'expired',\n 'hold',\n 'renewed',\n 'confirmed',\n ]\n \n # Amount check - accept $2,200 or 2200 or 2,200 or 2200.00 etc.\n amount_patterns = [r'\\$?2[,.]?200']\n \n # Date terms - accept various formats for April 1 2026 and March 31 2026\n date_patterns_apr1 = [\n r'april\\s*1[,\\s]+2026', r'april\\s*0?1[,\\s]+2026',\n r'apr\\.?\\s*1[,\\s]+2026', r'apr\\.?\\s*0?1[,\\s]+2026',\n r'2026-04-01', r'04/01/2026', r'04-01-2026',\n r'1\\s+april\\s+2026', r'01\\s+april\\s+2026',\n r'april\\s*1\\b', r'apr\\.?\\s*1\\b',\n ]\n date_patterns_mar31 = [\n r'march\\s*31[,\\s]+2026', r'mar\\.?\\s*31[,\\s]+2026',\n r'2026-03-31', r'03/31/2026', r'03-31-2026',\n r'31\\s+march\\s+2026',\n r'march\\s*31\\b', r'mar\\.?\\s*31\\b',\n ]\n \n for email in matching_emails:\n subject = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '') or email.get('body', '')).lower()\n \n subject_ok = all(term in subject for term in required_subject_terms)\n if not subject_ok:\n continue\n \n body_terms_ok = all(term.lower() in body for term in required_body_terms)\n \n # Check name alternatives: 'Tanya' or 'Okonkwo'\n name_ok = any(name in body for name in name_alternatives)\n \n # Check key message fragments (at least 4 of 6 for leniency)\n msg_hits = sum(1 for frag in key_message_fragments if frag in body)\n msg_ok = msg_hits >= 4\n \n amount_ok = any(re.search(p, body) for p in amount_patterns)\n apr1_ok = any(re.search(p, body) for p in date_patterns_apr1)\n mar31_ok = any(re.search(p, body) for p in date_patterns_mar31)\n \n if body_terms_ok and name_ok and msg_ok and amount_ok and apr1_ok and mar31_ok:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found qualifying sent email to {target_recipient} with all required content about CC-2026-0168'}\n \n # Provide detailed failure info\n for email in matching_emails:\n subject = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '') or email.get('body', '')).lower()\n missing = []\n for term in required_subject_terms:\n if term not in subject:\n missing.append(f'subject missing: {term}')\n for term in required_body_terms:\n if term.lower() not in body:\n missing.append(f'body missing: {term}')\n if not any(name in body for name in name_alternatives):\n missing.append('body missing: Tanya or Okonkwo')\n missing_frags = [frag for frag in key_message_fragments if frag not in body]\n if len(missing_frags) > 2:\n missing.append(f'body missing key message fragments: {missing_frags}')\n if not any(re.search(p, body) for p in amount_patterns):\n missing.append('body missing: $2,200 amount')\n if not any(re.search(p, body) for p in date_patterns_apr1):\n missing.append('body missing: April 1, 2026 date')\n if not any(re.search(p, body) for p in date_patterns_mar31):\n miLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "53e3483a-22d8-405c-9af6-64d7faa8b50c",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: Invoice IDs INV-018 through INV-026 (all nine) must each have Status 'APPROVED' and no hold code.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if h in ('invoice id', 'inv id', 'id', 'invoice_id') or ('invoice' in h and 'id' in h):\n inv_id_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n elif 'hold' in h and hold_col is None:\n hold_col = c\n \n if inv_id_col is None:\n # Try first column as ID\n inv_id_col = 1\n \n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Status column not found. Headers: {list(headers.keys())}'}\n \n required_ids = {f'INV-0{i}' for i in range(18, 27)}\n found = {}\n \n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in required_ids:\n inv_id = str(inv_val).strip()\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n found[inv_id] = {'status': status_val, 'hold': hold_val}\n \n missing = required_ids - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs not found in register: {missing}'}\n \n failures = []\n for inv_id, vals in found.items():\n status_str = str(vals['status']).strip().upper() if vals['status'] else ''\n hold_str = str(vals['hold']).strip() if vals['hold'] else ''\n if status_str != 'APPROVED':\n failures.append(f'{inv_id} has Status={vals[\"status\"]} (expected APPROVED)')\n if hold_str and hold_str.upper() not in ('', 'NONE', 'N/A', 'NULL'):\n failures.append(f'{inv_id} has Hold Code={vals[\"hold\"]} (expected empty)')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All 9 invoices INV-018 through INV-026 have Status=APPROVED and no hold code'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "860f1841-7c7f-4001-89b3-58b9d47100d0",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: INV-018 through INV-026 must each appear with correct Payment Amt values (INV-018: $620.00, INV-019: $8,750.00, INV-020: $2,800.00, INV-021: $4,500.00, INV-022: $14,000.00, INV-023: $3,200.00, INV-024: $1,050.00, INV-025: $5,500.00, INV-026: $5,200.00).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n expected = {\n 'INV-018': 620.00,\n 'INV-019': 8750.00,\n 'INV-020': 2800.00,\n 'INV-021': 4500.00,\n 'INV-022': 14000.00,\n 'INV-023': 3200.00,\n 'INV-024': 1050.00,\n 'INV-025': 5500.00,\n 'INV-026': 5200.00,\n }\n \n headers = {}\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n amt_col = None\n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'invoice id', 'id') and inv_id_col is None:\n inv_id_col = c\n if 'payment' in h and ('amt' in h or 'amount' in h):\n amt_col = c\n elif ('amount' in h or 'amt' in h) and amt_col is None:\n amt_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if amt_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Payment amount column not found. Headers: {list(headers.keys())}'}\n \n def parse_amount(val):\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return float(val)\n s = str(val).replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except:\n return None\n \n found = {}\n for row in range(2, queue_sheet.max_row + 1):\n inv_val = queue_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected:\n inv_id = str(inv_val).strip()\n amt_val = queue_sheet.cell(row=row, column=amt_col).value\n found[inv_id] = parse_amount(amt_val)\n \n missing = set(expected.keys()) - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs missing from Payment Queue: {missing}'}\n \n failures = []\n for inv_id, exp_amt in expected.items():\n actual = found.get(inv_id)\n if actual is None:\n failures.append(f'{inv_id}: could not parse amount')\n else:\n tol = exp_amt * 0.01\n if abs(actual - exp_amt) > tol:\n failures.append(f'{inv_id}: expected {exp_amt}, got {actual}')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Amount mismatches: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'All 9 invoices INV-018 through INV-026 found in Payment Queue with correct amounts'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "7b93f221-bd0e-470f-b332-715f3d44329b",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Invoice Register sheet: INV-007 must have Status 'ON HOLD' with Hold Code 'MATCH'; INV-008 must have Status 'ON HOLD' with Hold Code 'APPR'; INV-009 must have Status 'ON HOLD' with Hold Code 'MATCH'; INV-011 must have Status 'IC-HOLD'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n invoice_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n invoice_sheet = wb[name]\n break\n if invoice_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register sheet not found'}\n \n headers = {}\n for col in range(1, invoice_sheet.max_column + 1):\n val = invoice_sheet.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n status_col = None\n hold_col = None\n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'id') and inv_id_col is None:\n inv_id_col = c\n if 'status' in h and 'match' not in h:\n status_col = c\n if 'hold' in h and 'code' in h:\n hold_col = c\n elif 'hold' in h and hold_col is None:\n hold_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Status column not found. Headers: {list(headers.keys())}'}\n \n expected = {\n 'INV-007': {'status': 'ON HOLD', 'hold': 'MATCH'},\n 'INV-008': {'status': 'ON HOLD', 'hold': 'APPR'},\n 'INV-009': {'status': 'ON HOLD', 'hold': 'MATCH'},\n 'INV-011': {'status': 'IC-HOLD', 'hold': None},\n }\n \n found = {}\n for row in range(2, invoice_sheet.max_row + 1):\n inv_val = invoice_sheet.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected:\n inv_id = str(inv_val).strip()\n status_val = invoice_sheet.cell(row=row, column=status_col).value\n hold_val = invoice_sheet.cell(row=row, column=hold_col).value if hold_col else None\n found[inv_id] = {'status': status_val, 'hold': hold_val}\n \n missing = set(expected.keys()) - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs not found: {missing}'}\n \n failures = []\n for inv_id, exp in expected.items():\n actual = found[inv_id]\n actual_status = str(actual['status']).strip().upper() if actual['status'] else ''\n actual_hold = str(actual['hold']).strip().upper() if actual['hold'] else ''\n exp_status = exp['status'].upper()\n \n if actual_status != exp_status:\n failures.append(f'{inv_id}: Status={actual[\"status\"]} (expected {exp[\"status\"]})')\n \n if exp['hold'] is not None:\n if actual_hold != exp['hold'].upper():\n failures.append(f'{inv_id}: Hold Code={actual[\"hold\"]} (expected {exp[\"hold\"]})')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'INV-007, INV-008, INV-009, INV-011 all have correct Status and Hold Code values'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "9519c1fd-5be9-4de0-8f31-3d1006cd104a",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, Payment Queue sheet: Invoice IDs INV-007, INV-008, INV-009, and INV-011 must NOT appear.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('ap_ledger.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'ledger' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n for name in wb.sheetnames:\n if 'queue' in name.lower():\n queue_sheet = wb[name]\n break\n if queue_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Payment Queue sheet not found'}\n \n forbidden = {'INV-007', 'INV-008', 'INV-009', 'INV-011'}\n found_forbidden = []\n \n for row in range(1, queue_sheet.max_row + 1):\n for col in range(1, queue_sheet.max_column + 1):\n val = queue_sheet.cell(row=row, column=col).value\n if val and str(val).strip() in forbidden:\n found_forbidden.append(f'{str(val).strip()} at row {row}')\n \n if found_forbidden:\n return {'pass': False, 'score': 0.0, 'feedback': f'Forbidden Invoice IDs found in Payment Queue: {found_forbidden}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'INV-007, INV-008, INV-009, INV-011 correctly absent from Payment Queue'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "323b236e-507a-4454-8c71-e027177ad6c4",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In file `match_log_2026_Q2.xlsx`, Invoice IDs INV-018, INV-021, INV-024, and INV-026 must each appear with Match Date 2026-04-08 and Match Result containing the word \"PASS\".",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the match log file with flexible matching\n xlsx_files = list(Path(workspace_path).glob('match_log_2026_Q2.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'match' in f.name.lower() and 'log' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob('*.xlsx') if 'match' in f.name.lower()]\n if not xlsx_files:\n # Also search subdirectories\n xlsx_files = list(Path(workspace_path).rglob('match_log_2026_Q2.xlsx'))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).rglob('*.xlsx') if 'match' in f.name.lower() and 'log' in f.name.lower()]\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).rglob('*.xlsx') if 'match' in f.name.lower()]\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_log_2026_Q2.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n if len(wb.sheetnames) > 1:\n for name in wb.sheetnames:\n if 'match' in name.lower() or 'log' in name.lower() or '2026' in name:\n ws = wb[name]\n break\n \n # Parse headers flexibly\n headers = {}\n for col in range(1, ws.max_column + 1):\n val = ws.cell(row=1, column=col).value\n if val:\n headers[str(val).strip().lower()] = col\n \n inv_id_col = None\n date_col = None\n result_col = None\n \n for h, c in headers.items():\n if 'invoice' in h and 'id' in h:\n inv_id_col = c\n elif h in ('inv id', 'id', 'invoice_id', 'inv_id', 'invoiceid') and inv_id_col is None:\n inv_id_col = c\n if 'match' in h and 'date' in h:\n date_col = c\n elif 'date' in h and date_col is None:\n date_col = c\n if 'match' in h and ('result' in h or 'status' in h):\n result_col = c\n elif ('result' in h or 'status' in h) and result_col is None:\n result_col = c\n \n if inv_id_col is None:\n inv_id_col = 1\n \n if date_col is None or result_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Required columns not found. Headers found: {list(headers.keys())}'}\n \n # The rubric requires these four invoices with Match Date 2026-04-08 and Match Result containing \"PASS\"\n expected_invoices = {'INV-018', 'INV-021', 'INV-024', 'INV-026'}\n \n found = {}\n for row in range(2, ws.max_row + 1):\n inv_val = ws.cell(row=row, column=inv_id_col).value\n if inv_val and str(inv_val).strip() in expected_invoices:\n inv_id = str(inv_val).strip()\n date_val = ws.cell(row=row, column=date_col).value\n result_val = ws.cell(row=row, column=result_col).value\n found[inv_id] = {'date': date_val, 'result': result_val, 'row': row}\n \n missing = expected_invoices - set(found.keys())\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Invoice IDs missing from match log: {sorted(missing)}'}\n \n def check_date(val):\n if val is None:\n return False\n if isinstance(val, datetime):\n return val.date() == date(2026, 4, 8)\n if isinstance(val, date):\n return val == date(2026, 4, 8)\n s = str(val).strip()\n return ('2026-04-08' in s or '04/08/2026' in s or '04-08-2026' in s or\n '2026/04/08' in s or 'april 8' in s.lower() or 'apr 8' in s.lower() or\n '8 apr' in s.lower() or '8 april' in s.lower() or\n '4/8/2026' in s or '8/4/2026' in s or '2026.04.08' in s)\n \n def check_result_contains_pass(val):\n \"\"\"Rubric says Match Result must contain the word PASS.\"\"\"\n if val is None:\n return False\n result_str = str(val).strip().upper()\n # Check if the word PASS appears anywhere in the result string\n # This is intentionally loose: PASS, MATCH-PASS, MATCH_PASS, MATCH PASS, 3-Way Pass, etc.\n return 'PASS' in result_str\n \n failures = []\n for inv_id in sorted(expected_invoices):\n actual = found[inv_id]\n if not check_date(actual['date']):\n failures.append(f'{inv_id} (row {actual[\"row\"]}): Match Date={actual[\"date\"]} (expected 2026-04-08)')\n if not check_result_contains_pass(actual['result']):\n failures.append(f'{inv_id} (row {actual[\"row\"]}): Match Result={actual[\"result\"]} (expected to contain the word PASS)')\n \n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Match log failures: ' + '; '.join(failures)}\n \n # Collect actual values for detailLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Today is April 8, 2026. You are an AP Clerk at Meridian Partners. Bring all invoices marked as pending in the Invoice Register current per the SOP and queue for payment, validating each across all files as necessary. Also process any unread emails. Do not directly request approval for any newly queued payments; the relevant people will automatically be contacted as the payment queue is updated.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user