Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,135 @@
|
||||
[
|
||||
{
|
||||
"id": "cb45048d-c17b-4f06-9331-ff74e996e824",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In folder `1099/2025/`, file `W9_Whitfield_Consulting_Group.pdf` must exist.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / '1099' / '2025' / 'W9_Whitfield_Consulting_Group.pdf'\n if target.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'Found W9_Whitfield_Consulting_Group.pdf in 1099/2025/'}\n # Also try case-insensitive search\n folder = Path(workspace_path) / '1099' / '2025'\n if folder.exists():\n matches = [f for f in folder.iterdir() if 'whitfield' in f.name.lower() and f.suffix.lower() == '.pdf']\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield W-9 PDF: {matches[0].name}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'W9_Whitfield_Consulting_Group.pdf not found in 1099/2025/'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "d5570533-815a-4694-b65d-8d62187d7c86",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In folder `1099/2025/`, file `W9_Harrison_Cole_LLP.pdf` must exist.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / '1099' / '2025' / 'W9_Harrison_Cole_LLP.pdf'\n if target.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'Found W9_Harrison_Cole_LLP.pdf in 1099/2025/'}\n folder = Path(workspace_path) / '1099' / '2025'\n if folder.exists():\n matches = [f for f in folder.iterdir() if 'harrison' in f.name.lower() and f.suffix.lower() == '.pdf']\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole W-9 PDF: {matches[0].name}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'W9_Harrison_Cole_LLP.pdf not found in 1099/2025/'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a9da8354-1545-447d-b665-bcd6ac61998d",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `vendor_master.xlsx`, a row for Whitfield Consulting Group, Inc. must exist with: Tax ID 23-7891045, Address containing 450 Independence Mall West, Contact Email d.whitfield@whitfieldcg.com, Payment Terms 2/10 Net 30, Bank Routing 321081669, Bank Account 7720194583, 1099 Reportable Yes, 1099 Type NEC, Status Active, Created Date 09/04/2025",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'vendor_master.xlsx not found in workspace.'}\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n headers = [str(c.value).strip().lower() if c.value else '' for c in next(ws.iter_rows(min_row=1, max_row=1))]\n\n found_row_raw = None\n found_row_idx = None\n for idx, row in enumerate(ws.iter_rows(min_row=2, max_row=ws.max_row), start=2):\n row_values = [cell.value for cell in row]\n row_str = ' '.join(str(v) for v in row_values if v is not None).lower()\n if 'whitfield' in row_str and 'consult' in row_str:\n found_row_raw = row_values\n found_row_idx = idx\n break\n if found_row_raw is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for Whitfield Consulting Group found in vendor_master.xlsx.'}\n\n row_values = []\n for v in found_row_raw:\n if v is None:\n row_values.append('')\n elif isinstance(v, datetime):\n row_values.append(v.strftime('%m/%d/%Y'))\n else:\n row_values.append(str(v).strip())\n row_str = ' '.join(row_values)\n row_lower = row_str.lower()\n\n missing = []\n\n # Tax ID: 23-7891045\n row_str_no_sep = row_str.replace('-', '').replace(' ', '')\n if '23-7891045' not in row_str and '237891045' not in row_str_no_sep:\n missing.append('Tax ID (expected: 23-7891045)')\n\n # Address containing 450 Independence Mall West\n if '450 independence mall west' not in row_lower:\n if not ('450' in row_lower and 'independence' in row_lower and 'mall' in row_lower and 'west' in row_lower):\n missing.append('Address containing 450 Independence Mall West')\n\n # Contact Email\n if 'd.whitfield@whitfieldcg.com' not in row_lower:\n missing.append('Contact Email (expected: d.whitfield@whitfieldcg.com)')\n\n # Payment Terms: 2/10 Net 30\n terms_found = False\n for val in row_values:\n vl = val.lower().replace(' ', '')\n if '2/10net30' in vl or '2/10,net30' in vl or '2/10-net30' in vl:\n terms_found = True\n break\n if '2/10 net 30' in val.lower():\n terms_found = True\n break\n if '2/10' in val and 'net' in val.lower() and '30' in val:\n terms_found = True\n break\n if not terms_found:\n missing.append('Payment Terms (expected: 2/10 Net 30)')\n\n # Bank Routing: 321081669\n if '321081669' not in row_str:\n missing.append('Bank Routing (expected: 321081669)')\n\n # Bank Account: 7720194583\n if '7720194583' not in row_str:\n missing.append('Bank Account (expected: 7720194583)')\n\n # 1099 Reportable: Yes\n reportable_found = False\n for val in row_values:\n if val.strip().lower() in ('yes', 'y', 'true', '1'):\n reportable_found = True\n break\n if not reportable_found:\n missing.append('1099 Reportable (expected: Yes)')\n\n # 1099 Type: NEC\n nec_found = False\n for val in row_values:\n if val.strip().upper() == 'NEC' or 'nec' in val.lower().split():\n nec_found = True\n break\n if not nec_found:\n # Also check as part of a combined value\n for val in row_values:\n if 'nec' in val.lower():\n nec_found = True\n break\n if not nec_found:\n missing.append('1099 Type (expected: NEC)')\n\n # Status: Active\n active_found = False\n for val in row_values:\n if val.strip().lower() == 'active':\n active_found = True\n break\n if not active_found:\n missing.append('Status (expected: Active)')\n\n # Created Date: 09/04/2025\n date_found = False\n for v in found_row_raw:\n nd = normalize_date(v)\n if nd == '09/04/2025':\n date_found = True\n break\n if not date_found:\n # Also check string representations in row_values\n for val in row_values:\n if '09/04/2025' in val or '9/4/2025' in val or '2025-09-04' Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "970d0671-7df2-4425-8f51-254cf30b82b4",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `vendor_master.xlsx`, a row for Harrison & Cole LLP must exist with: Tax ID 23-7654321, Address containing '1200 Market Street', Contact Email s.harrison@harrisoncolellp.com, Payment Terms 2/10 Net 30, Bank Routing 031100209, Bank Account 7742851903, 1099 Reportable Yes, 1099 Type NEC, Status Active, Created Date 09/04/2025",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n # Handle cases like '09/04/25' or '9/4/2025' etc.\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d', '%m/%d/%Y %H:%M:%S',\n '%Y-%m-%d %H:%M:%S', '%m-%d-%y', '%d/%m/%Y'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef verify(workspace_path, external_services_path=None):\n # Search for vendor_master.xlsx broadly\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).rglob('*vendor_master*.xlsx'))\n if not candidates:\n # Try any xlsx that might contain vendor master data\n candidates = list(Path(workspace_path).rglob('*.xlsx'))\n candidates = [c for c in candidates if 'vendor' in c.name.lower()]\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'vendor_master.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n\n # Try to find header row to map columns\n headers = []\n header_row_idx = 1\n for row in ws.iter_rows(min_row=1, max_row=3, values_only=False):\n row_vals = [str(cell.value).strip().lower() if cell.value else '' for cell in row]\n # Check if this looks like a header row\n if any(h in ' '.join(row_vals) for h in ['vendor', 'name', 'tax', 'status']):\n headers = row_vals\n header_row_idx = row[0].row\n break\n\n found_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'harrison' in row_str and 'cole' in row_str:\n found_row = row\n break\n\n if found_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for Harrison & Cole LLP found in vendor_master.xlsx.'}\n\n row_str = ' '.join(str(v) for v in found_row if v is not None)\n row_lower = row_str.lower()\n\n missing = []\n\n # Tax ID - accept with or without dash\n if '23-7654321' not in row_lower and '237654321' not in row_lower:\n missing.append('Tax ID 23-7654321')\n\n # Address containing 1200 Market Street\n if '1200 market street' not in row_lower and '1200 market st' not in row_lower:\n missing.append('Address 1200 Market Street')\n\n # Contact Email\n if 's.harrison@harrisoncolellp.com' not in row_lower:\n missing.append('Contact Email s.harrison@harrisoncolellp.com')\n\n # Payment Terms - be generous with variations\n terms_found = False\n for variant in ['2/10 net 30', '2/10 net30', '2/10net30', '2/10, net 30',\n '2/10-net 30', '2% 10 net 30', '2/10 net-30', '2/10net 30',\n '2/10 net 30', '2%10 net 30', '2% 10, net 30']:\n if variant in row_lower:\n terms_found = True\n break\n if not terms_found:\n missing.append('Payment Terms 2/10 Net 30')\n\n # Bank Routing\n if '031100209' not in row_lower:\n missing.append('Bank Routing 031100209')\n\n # Bank Account\n if '7742851903' not in row_lower:\n missing.append('Bank Account 7742851903')\n\n # 1099 Reportable Yes\n yes_found = False\n for v in found_row:\n if v is not None and str(v).strip().lower() in ('yes', 'y', 'true', '1'):\n yes_found = True\n break\n if not yes_found:\n missing.append('1099 Reportable Yes')\n\n # 1099 Type NEC\n if 'nec' not in row_lower:\n missing.append('1099 Type NEC')\n\n # Status Active\n if 'active' not in row_lower:\n missing.append('Status Active')\n\n # Created Date 09/04/2025 - be generous with date formats\n date_found = False\n for v in found_row:\n nd = normalize_date(v)\n if nd == '09/04/2025':\n date_found = True\n break\n # Also check string representation directly\n if v is not None:\n vs = str(v).strip()\n if vs in ('09/04/2025', '9/4/2025', '2025-09-04', '09-04-2025',\n 'September 4, 2025', 'Sep 4, 2025', '09/04/25', '9/4/25'):\n date_found = True\n break\n if not date_found:\n missing.append('Created Date 09/04/2025')\n\n if missing:\n return {\n 'pass': False,\n 'score': max(0.1, 1.0 - len(missing) * 0.1),\n 'feedbacLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "61f3c9bd-e56d-4d07-937e-7c5ca2857a79",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `vendor_master.xlsx`, NO new row for ClearPath Talent Solutions must be added (duplicate vendor — Tax ID 58-8901234 matches existing vendor Premier Staffing Group V-00008).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n return {'pass': True, 'score': 1.0, 'feedback': 'vendor_master.xlsx not found; no ClearPath row added.'}\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'clearpath' in row_str or ('58-8901234' in row_str and 'premier' not in row_str):\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath Talent Solutions should NOT be added to vendor_master.xlsx (duplicate vendor). Found row: {row_str[:200]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions row found in vendor_master.xlsx — correct, duplicate vendor blocked.'}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "50e0d257-697e-49f9-9cd3-3d7e544f9092",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In ap_ledger.xlsx Invoice Register tab, a row for Whitfield Consulting Group invoice WCG-2025-0087 must exist with: Invoice Date 08/28/2025, PO Number blank/none, Category Non-PO, Amount 4950.00, GL Code 1000-120-6100, Status Entered, Payment Terms 2/10 Net 30, Due Date 09/27/2025, Entered By T. Okonkwo, Entered Date 09/04/2025.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef normalize_amount(value):\n \"\"\"Return a float for any amount-like value, or None if unparseable.\"\"\"\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return float(value)\n s = str(value).strip().replace('$', '').replace(',', '')\n try:\n return float(s)\n except ValueError:\n return None\n\ndef norm_header(h):\n return str(h).strip().lower() if h is not None else ''\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).rglob('*.xlsx'))\n candidates = [c for c in candidates if 'ap' in c.stem.lower() or 'ledger' in c.stem.lower()]\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n\n # Find the Invoice Register sheet\n target_ws = None\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl and 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl or 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab not found in ap_ledger.xlsx.'}\n\n # Read header row\n rows = list(target_ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab is empty.'}\n header = [norm_header(h) for h in rows[0]]\n\n # Required columns (plus Invoice Number column to locate the row)\n required_cols = [\n 'Invoice Number',\n 'Invoice Date', 'PO Number', 'Category', 'Amount', 'GL Code',\n 'Status', 'Payment Terms', 'Due Date', 'Entered By', 'Entered Date',\n ]\n col_idx = {}\n for col in required_cols:\n try:\n col_idx[col] = header.index(col.lower())\n except ValueError:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Column \"{col}\" not found in Invoice Register header. Header row: {rows[0]}'}\n\n # Locate the WCG-2025-0087 row by Invoice Number column\n inv_col = col_idx['Invoice Number']\n found_row = None\n for r in rows[1:]:\n if r is None or inv_col >= len(r):\n continue\n cell = r[inv_col]\n if cell is not None and str(cell).strip().lower() == 'wcg-2025-0087':\n found_row = r\n break\n if found_row is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No row with Invoice Number WCG-2025-0087 found in Invoice Register.'}\n\n def cell(col_name):\n idx = col_idx[col_name]\n return found_row[idx] if idx < len(found_row) else None\n\n missing = []\n\n # Invoice Date: 08/28/2025\n if normalize_date(cell('Invoice Date')) != '08/28/2025':\n missing.append(f'Invoice Date should be 08/28/2025, got {cell(\"Invoice Date\")!r}')\n\n # PO Number: blank/none\n po_val = cell('PO Number')\n if po_val is not None and str(po_val).strip() != '':\n missing.append(f'PO Number should be blank, got {po_val!r}')\n\n # Category: Non-PO\n cat_val = cell('Category')\n cat_norm = str(cat_val).strip().lower().replace(' ', '').replace('_', '') if cat_val is not None else ''\n if cat_norm != 'non-po' and cat_norm != 'nonpo':\n missing.append(f'Category should be Non-PO, got {cat_val!r}')\n\n # Amount: 4950.00\n amt = normalize_amount(cell('Amount'))\n if amt is None or abs(amt - 4950.00) > 0.005:\n missing.append(f'Amount should be 4950.00, got {cell(\"Amount\")!r}')\n\n # GL Code: 1000-120-6100\n gl_val = cell('GL Code')\n if gl_val is None or str(gl_val).strip() != '1000-120-6100':\n missing.append(f'GL Code should be 1000-120-6100, got {gl_val!r}')\n\n # Status: Entered (exact, case-insensitive)\n status_val = cell('Status')\n if status_val is None or str(status_val).strip().lower() != 'entered':\n missing.append(f'Status should be Entered,Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4d825a27-1baf-4c9c-a7ac-e06b45e1a18b",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In ap_ledger.xlsx Invoice Register tab, a row for Harrison & Cole LLP invoice HC-2025-0001 must exist with: Invoice Date 09/04/2025, PO Number blank/none, Category Non-PO, Amount 5000.00, GL Code 1000-120-6100, Status Entered, Payment Terms 2/10 Net 30, Due Date 10/04/2025, Entered By T. Okonkwo, Entered Date 09/04/2025.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef normalize_amount(value):\n \"\"\"Return a float for any number-like value, or None if unparseable.\"\"\"\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return float(value)\n s = str(value).strip().replace('$', '').replace(',', '')\n try:\n return float(s)\n except ValueError:\n return None\n\ndef verify(workspace_path, external_services_path=None):\n # Find ap_ledger.xlsx\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n\n # Find the Invoice Register tab\n target_ws = None\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl and 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab not found in ap_ledger.xlsx.'}\n\n # Build header -> column index map (case-insensitive, trimmed)\n headers = {}\n for i, cell in enumerate(target_ws[1]):\n if cell.value is not None:\n headers[str(cell.value).strip().lower()] = i\n\n # Required column headers\n required_cols = [\n 'invoice date', 'po number', 'category', 'amount', 'gl code',\n 'status', 'payment terms', 'due date', 'entered by', 'entered date'\n ]\n # We also need an Invoice Number column to find the right row\n invoice_num_col = None\n for key in ('invoice number', 'invoice #', 'invoice no', 'invoice no.'):\n if key in headers:\n invoice_num_col = headers[key]\n break\n\n if invoice_num_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find an Invoice Number column in Invoice Register header.'}\n\n missing_cols = [c for c in required_cols if c not in headers]\n if missing_cols:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns in Invoice Register: {missing_cols}'}\n\n # Locate the HC-2025-0001 row\n found_row = None\n for row in target_ws.iter_rows(min_row=2, values_only=True):\n if invoice_num_col < len(row) and row[invoice_num_col] is not None:\n if str(row[invoice_num_col]).strip().lower() == 'hc-2025-0001':\n found_row = row\n break\n\n if found_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for invoice HC-2025-0001 found in Invoice Register.'}\n\n def cell(col_name):\n idx = headers[col_name]\n if idx >= len(found_row):\n return None\n return found_row[idx]\n\n def cell_str(col_name):\n v = cell(col_name)\n return '' if v is None else str(v).strip()\n\n missing = []\n\n # Invoice Date == 09/04/2025\n if normalize_date(cell('invoice date')) != '09/04/2025':\n missing.append(f\"Invoice Date should be 09/04/2025, found: {cell('invoice date')!r}\")\n\n # PO Number blank/none\n po_val = cell('po number')\n if po_val is not None and str(po_val).strip().lower() not in ('', 'none', 'n/a', '-'):\n missing.append(f\"PO Number should be blank/none, found: {po_val!r}\")\n\n # Category == Non-PO\n cat = cell_str('category').lower().replace(' ', '').replace('-', '')\n if cat != 'nonpo':\n missing.append(f\"Category should be Non-PO, found: {cell_str('category')!r}\")\n\n # Amount == 5000.00\n amt = normalize_amount(cell('amount'))\n if amt is None or abs(amt - 5000.00) > 0.001:\n missing.append(f\"Amount should be 5000.00, found: {cell('amount')!r}\")\n\n # GL Code == 1000-120-6100\n if cell_str('gl code') != '1000-120-6100':\n missing.append(f\"GL Code should be 1000-120-6100, found: {cell_str('gl code')!r}\")\n\n # Status == Entered\n if cell_str('status').lower() != 'entered':\n missing.append(f\"Status should be Entered, found: {cell_str('status')!r}\")\n\n # Payment Terms == 2/10 Net 30 (allow minor spacing/comma variation)\n pt = cell_str('payment terms').lower().replace(',', '').replace(' ', '')\n if pt != '2/10net30':\n missing.append(f\"Payment Terms shouLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "b909c1a5-256a-45b2-a355-4bc1554e47b1",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `ap_ledger.xlsx` Invoice Register tab, NO row for ClearPath Talent Solutions or invoice CPT-2025-0412 must exist (duplicate vendor — entire pipeline blocked).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n return {'pass': True, 'score': 1.0, 'feedback': 'ap_ledger.xlsx not found; no ClearPath invoice entered.'}\n wb = openpyxl.load_workbook(candidates[0])\n for ws in wb.worksheets:\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'clearpath' in row_str or 'cpt-2025-0412' in row_str:\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath invoice CPT-2025-0412 should NOT be entered in ap_ledger.xlsx. Found: {row_str[:200]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions / CPT-2025-0412 entry in ap_ledger.xlsx — correct.'}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "e1811786-91b9-483d-b2f4-06dab61c2cef",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In folder `Invoices/2025-09/`, invoice file for Whitfield Consulting Group (WCG-2025-0087) must exist (e.g., INV-V-00038-WCG-2025-0087.pdf or similar).",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n # Try alternate casing\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoices/2025-09/ folder not found in workspace.'}\n matches = [f for f in folder.iterdir() if 'wcg-2025-0087' in f.name.lower() or ('whitfield' in f.name.lower() and f.suffix.lower() == '.pdf')]\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield invoice file: {[f.name for f in matches]}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No invoice file for WCG-2025-0087 found in Invoices/2025-09/. Files present: {[f.name for f in folder.iterdir()]}'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "3f78f653-6bab-4c33-98b4-9bca1b8448fb",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In folder `Invoices/2025-09/`, invoice file for Harrison & Cole LLP (HC-2025-0001) must exist (e.g., INV-V-00039-HC-2025-0001.pdf or similar).",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoices/2025-09/ folder not found in workspace.'}\n matches = [f for f in folder.iterdir() if 'hc-2025-0001' in f.name.lower() or ('harrison' in f.name.lower() and f.suffix.lower() == '.pdf')]\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole invoice file: {[f.name for f in matches]}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No invoice file for HC-2025-0001 found in Invoices/2025-09/. Files present: {[f.name for f in folder.iterdir()]}'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4e0d1eea-4c10-47d0-a7d4-ceb532b6377f",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In folder `Invoices/2025-09/`, NO invoice file for ClearPath Talent Solutions (CPT-2025-0412) must exist.",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': True, 'score': 1.0, 'feedback': 'Invoices/2025-09/ folder not found; no ClearPath invoice filed.'}\n matches = [f for f in folder.iterdir() if 'clearpath' in f.name.lower() or 'cpt-2025-0412' in f.name.lower() or 'cpt' in f.name.lower()]\n if matches:\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath invoice should NOT be filed in Invoices/2025-09/. Found: {[f.name for f in matches]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions invoice found in Invoices/2025-09/ — correct.'}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "251ff8e0-bb26-4e6f-9d77-cbd264f4d756",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In mailbox sent mail (mailbox.json), an approval email for Whitfield Consulting Group invoice WCG-2025-0087 must be sent to jennifer.walsh@meridianpartners.com and maria.santos@meridianpartners.com with CC ap.team@meridianpartners.com, subject containing 'AP APPROVAL REQUIRED' and 'WCG-2025-0087', body containing vendor name, vendor ID (V-00038 or V-00039), invoice number WCG-2025-0087, invoice date 08/28/2025 (or similar format), amount $4,950.00, GL Code 1000-120-6100, and '3 business days'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox for approval email.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.'}\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or email.get('body') or '').lower()\n combined = subject + ' ' + body\n\n # Collect all address fields flexibly\n to_raw = email.get('to_addr') or email.get('to') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join(str(x) for x in to_raw).lower()\n else:\n to_addr = str(to_raw).lower()\n\n cc_raw = email.get('cc_addr') or email.get('cc') or ''\n if isinstance(cc_raw, list):\n cc_addr = ' '.join(str(x) for x in cc_raw).lower()\n else:\n cc_addr = str(cc_raw).lower()\n\n all_addr = to_addr + ' ' + cc_addr\n\n # Must reference WCG-2025-0087 somewhere in subject or body\n if 'wcg-2025-0087' not in combined:\n continue\n\n # Must have AP APPROVAL REQUIRED in subject (lenient with casing/spacing)\n if 'ap approval required' not in subject:\n continue\n\n missing = []\n\n # Check To recipients\n if 'jennifer.walsh@meridianpartners.com' not in all_addr:\n missing.append('jennifer.walsh@meridianpartners.com in To')\n if 'maria.santos@meridianpartners.com' not in all_addr:\n missing.append('maria.santos@meridianpartners.com in To')\n\n # Check CC - be lenient, also accept if it's in To\n if 'ap.team@meridianpartners.com' not in all_addr:\n missing.append('ap.team@meridianpartners.com in CC')\n\n # Body checks\n # Vendor name - accept various forms\n if 'whitfield' not in combined:\n missing.append('vendor name Whitfield')\n\n # Invoice number in body\n if 'wcg-2025-0087' not in body:\n missing.append('invoice number WCG-2025-0087 in body')\n\n # Invoice date - accept multiple date formats for 08/28/2025\n date_found = False\n date_patterns = ['08/28/2025', '8/28/2025', '08-28-2025', '2025-08-28', 'august 28, 2025', 'august 28 2025', '28 august 2025', '08/28/25', '8/28/25']\n for dp in date_patterns:\n if dp in combined:\n date_found = True\n break\n if not date_found:\n missing.append('invoice date 08/28/2025 (or similar)')\n\n # Amount - accept various formats for $4,950.00\n amount_found = False\n amount_patterns = ['4,950', '4950.00', '4950', '$4,950', '4,950.00']\n for ap in amount_patterns:\n if ap in combined:\n amount_found = True\n break\n if not amount_found:\n missing.append('amount $4,950.00')\n\n # GL Code - accept with or without spaces around dashes\n gl_found = False\n gl_patterns = ['1000-120-6100']\n for gp in gl_patterns:\n if gp in combined:\n gl_found = True\n break\n if not gl_found:\n missing.append('GL Code 1000-120-6100')\n\n # 3 business days - be lenient\n if '3 business day' not in combined and 'three business day' not in combined:\n missing.append('3 business days response deadline')\n\n # Vendor ID\n if not ('v-00038' in combined or 'v-00039' in combined):\n missing.append('Vendor ID V-00038 or V-00039')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': 'Whitfield Consulting Group approval email found with all required fields.'}\n\n # Track best match (fewest missing items)\n if best_missing is None or len(missing) < len(best_missing):\n best_missing = missing\n best_match = email\n\n if best_match is not None:\n return {'pass': False, 'score': max(0.1, 0.5 - 0.05 * len(best_missing)), 'feedback': f'Whitfield approval email found but missing: {best_missing}'}\n\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent approval email for WCG-2025-0087 found in mailbox.'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "3e47971f-c177-4411-b691-0f09f3dd9d2f",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In mailbox sent mail (mailbox.json), an approval email for Harrison & Cole LLP invoice HC-2025-0001 must be sent to jennifer.walsh@meridianpartners.com and maria.santos@meridianpartners.com with CC ap.team@meridianpartners.com, subject containing 'AP APPROVAL REQUIRED' and 'HC-2025-0001', body containing vendor name, vendor ID (V-00038 or V-00039), invoice number HC-2025-0001, invoice date 09/04/2025 (or similar format), amount $5,000.00, GL Code 1000-120-6100, and '3 business days'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox for approval email.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.json.'}\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or email.get('body') or '').lower()\n # Combine subject and body for broad searching\n full_text = subject + ' ' + body\n\n # Normalize to/cc fields - could be string or list\n to_raw = email.get('to_addr') or email.get('to') or ''\n cc_raw = email.get('cc_addr') or email.get('cc') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join([str(x) for x in to_raw]).lower()\n else:\n to_addr = str(to_raw).lower()\n if isinstance(cc_raw, list):\n cc_addr = ' '.join([str(x) for x in cc_raw]).lower()\n else:\n cc_addr = str(cc_raw).lower()\n\n # Also check combined to+cc for loose matching\n all_recipients = to_addr + ' ' + cc_addr\n\n # Must reference HC-2025-0001 somewhere\n if 'hc-2025-0001' not in full_text:\n continue\n\n # Must have AP APPROVAL REQUIRED in subject (loose: accept with or without extra words)\n if 'ap approval required' not in subject:\n continue\n\n missing = []\n\n # Check To addresses - be loose, accept in either To or CC\n if 'jennifer.walsh@meridianpartners.com' not in to_addr:\n # Check if it's anywhere in recipients\n if 'jennifer.walsh@meridianpartners.com' not in all_recipients:\n missing.append('jennifer.walsh@meridianpartners.com in To')\n if 'maria.santos@meridianpartners.com' not in to_addr:\n if 'maria.santos@meridianpartners.com' not in all_recipients:\n missing.append('maria.santos@meridianpartners.com in To')\n\n # Check CC\n if 'ap.team@meridianpartners.com' not in cc_addr:\n if 'ap.team@meridianpartners.com' not in all_recipients:\n missing.append('ap.team@meridianpartners.com in CC')\n\n # Check vendor name - be very loose\n if 'harrison' not in full_text and 'cole' not in full_text:\n missing.append('vendor name Harrison & Cole')\n\n # Check invoice number in body\n if 'hc-2025-0001' not in body:\n missing.append('invoice number HC-2025-0001 in body')\n\n # Check invoice date - accept many formats\n date_found = False\n date_patterns = ['09/04/2025', '9/4/2025', '2025-09-04', 'september 4, 2025', 'september 4 2025',\n 'sep 4, 2025', 'sep 04, 2025', 'sep 4 2025', 'sep 04 2025',\n '09-04-2025', '04/09/2025', '09/04/25', '9/4/25', '2025/09/04']\n for dp in date_patterns:\n if dp in body:\n date_found = True\n break\n if not date_found:\n missing.append('invoice date 09/04/2025 (or similar)')\n\n # Check amount - accept many formats: 5,000, 5000, $5,000, $5000, 5,000.00, 5000.00\n amount_found = False\n amount_patterns = ['5,000', '5000', '$5,000', '$5000']\n for ap in amount_patterns:\n if ap in body:\n amount_found = True\n break\n if not amount_found:\n missing.append('amount $5,000.00')\n\n # Check GL Code - accept with or without spaces/dashes\n gl_found = False\n gl_patterns = ['1000-120-6100', '1000 120 6100', '10001206100']\n for gp in gl_patterns:\n if gp in body:\n gl_found = True\n break\n if not gl_found:\n missing.append('GL Code 1000-120-6100')\n\n # Check 3 business days - be loose\n biz_day_found = False\n biz_patterns = ['3 business day', '3 business-day', 'three business day', '3-business-day',\n 'three business-day', '3 working day', 'three working day']\n for bp in biz_patterns:\n if bp in body:\n biz_day_found = True\n break\n if not biz_day_found:\n missing.append('3 business days response deadline')\n\n # Check vendor ID\n if 'v-00038' not inLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "8a358c9a-e182-4438-b0be-9351c0351bb8",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In mailbox sent mail (mailbox.json), NO approval email for ClearPath Talent Solutions or invoice CPT-2025-0412 must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external_services_path provided; skipping mailbox check.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; no ClearPath email sent.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or '').lower()\n if 'clearpath' in subject or 'clearpath' in body or 'cpt-2025-0412' in subject or 'cpt-2025-0412' in body:\n return {'pass': False, 'score': 0.0, 'feedback': f'Approval email for ClearPath should NOT have been sent. Found email with subject: {email.get(\"subject\")}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No approval email for ClearPath Talent Solutions found in sent mail — correct.'}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "e23b68e1-2412-4b5d-9e87-fbe7636fcad8",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In Slack (slack_data.json), channel #approvals must contain a new vendor notification post with exact text: `NEW VENDOR ADDED: Whitfield Consulting Group, Inc., V-00038 or V-00039 (either is acceptable), 1099 Reportable: YES`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n \n # Find the #approvals channel\n approvals_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n if cinfo.get('name', '').lower() == 'approvals':\n approvals_channel_id = cid\n break\n \n if approvals_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#approvals channel not found in slack_data.json.'}\n \n messages = data.get('messages', {}).get(approvals_channel_id, [])\n \n # The rubric says the message should contain:\n # NEW VENDOR ADDED: Whitfield Consulting Group, Inc., V-00038 or V-00039, 1099 Reportable: YES\n # Be loose in matching - check for key fragments case-insensitively\n expected_fragments_base = [\n 'new vendor added',\n 'whitfield consulting group',\n ]\n valid_vendor_ids = ['v-00038', 'v-00039']\n \n # Accept various forms of 1099 reportable yes\n reportable_patterns = [\n '1099 reportable: yes',\n '1099 reportable:yes',\n '1099-reportable: yes',\n '1099 reportable - yes',\n '1099 reportable yes',\n '1099reportable: yes',\n '1099reportable:yes',\n '1099: yes',\n '1099 yes',\n ]\n \n for msg in messages:\n text = (msg.get('text') or '').lower()\n # Check base fragments\n if all(frag in text for frag in expected_fragments_base):\n has_vendor_id = any(vid in text for vid in valid_vendor_ids)\n if has_vendor_id:\n # Split on \"new vendor added\" to isolate each notification\n # This prevents Harrison & Cole's \"YES\" from satisfying Whitfield's check\n segments = re.split(r'new vendor added', text)\n for segment in segments:\n if 'whitfield consulting group' in segment:\n has_reportable = any(rp in segment for rp in reportable_patterns)\n if has_reportable:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield new vendor notification in #approvals: {msg.get(\"text\")}'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found Whitfield notification but 1099 Reportable is not YES. Message: {msg.get(\"text\")}'}\n \n # Collect all messages in the channel for debugging\n all_texts = [msg.get('text', '') for msg in messages]\n return {'pass': False, 'score': 0.0, 'feedback': f'No Whitfield Consulting Group new vendor notification found in #approvals channel. Expected message containing: NEW VENDOR ADDED, Whitfield Consulting Group, Inc., V-00038 or V-00039, 1099 Reportable: YES. Messages found in #approvals: {all_texts}'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "995de855-5315-444e-8765-584fad8962e4",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In Slack (slack_data.json), channel #approvals must contain a new vendor notification post with exact text: `NEW VENDOR ADDED: Harrison & Cole LLP, V-00038 or V-00039 (either is acceptable), 1099 Reportable: YES`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n approvals_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n if cinfo.get('name', '').lower() == 'approvals':\n approvals_channel_id = cid\n break\n\n if approvals_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#approvals channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(approvals_channel_id, [])\n\n all_texts = []\n for msg in messages:\n text = msg.get('text') or ''\n all_texts.append(text)\n lower_text = text.lower()\n\n # Check for key components loosely\n has_new_vendor = 'new vendor' in lower_text\n has_harrison_cole = 'harrison' in lower_text and 'cole' in lower_text\n has_vendor_id = 'v-00038' in lower_text or 'v-00039' in lower_text\n has_1099_yes = ('1099' in lower_text and 'yes' in lower_text)\n\n if has_new_vendor and has_harrison_cole and has_vendor_id and has_1099_yes:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found Harrison & Cole new vendor notification in #approvals: {text}'\n }\n\n # Provide detailed failure feedback\n feedback_parts = ['No Harrison & Cole LLP new vendor notification found in #approvals.']\n feedback_parts.append('Expected text containing: NEW VENDOR ADDED: Harrison & Cole LLP, V-00038 or V-00039, 1099 Reportable: YES')\n if all_texts:\n feedback_parts.append(f'Found {len(all_texts)} message(s) in #approvals. Sample texts: {all_texts[:5]}')\n else:\n feedback_parts.append('No messages found in #approvals channel.')\n\n return {'pass': False, 'score': 0.0, 'feedback': ' | '.join(feedback_parts)}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "1671ba3e-b81e-461b-ac11-e133196369fb",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In Slack (slack_data.json), channel #vendor-management must contain a duplicate vendor notification with exact text: `DUPLICATE VENDOR SUSPECTED: ClearPath Talent Solutions matches Tax ID of existing vendor Premier Staffing Group, V-00008`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n # Find #vendor-management channel (loose matching)\n target_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n cname = cinfo.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if 'vendor' in cname and 'management' in cname:\n target_channel_id = cid\n break\n\n if target_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#vendor-management channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(target_channel_id, [])\n\n # Key fragments to look for (case-insensitive)\n expected_fragments = [\n 'duplicate vendor suspected',\n 'clearpath',\n 'premier staffing group',\n 'v-00008',\n ]\n\n for msg in messages:\n text = (msg.get('text') or '').lower()\n if all(frag.lower() in text for frag in expected_fragments):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found ClearPath duplicate vendor notification in #vendor-management: {msg.get(\"text\")}'}\n\n # Collect all messages for debugging\n all_texts = [msg.get('text', '') for msg in messages]\n return {'pass': False, 'score': 0.0, 'feedback': f'No ClearPath duplicate vendor notification found in #vendor-management. Expected message containing: DUPLICATE VENDOR SUSPECTED: ClearPath Talent Solutions matches Tax ID of existing vendor Premier Staffing Group, V-00008. Found {len(messages)} message(s) in channel: {all_texts[:5]}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "d87b42b7-4674-4037-bea8-407baa5fc939",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In Slack (slack_data.json), channel #accounts-payable must contain an early payment discount notification for Harrison & Cole LLP with exact text: `EARLY PAYMENT DISCOUNT: Harrison & Cole LLP, Invoice HC-2025-0001, Discount $100, Deadline 09/14/2025 (or similar format)`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack for early payment discount notification.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n # Find #accounts-payable channel\n target_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n name = cinfo.get('name', '').lower().replace(' ', '').replace('-', '').replace('_', '')\n if 'accountspayable' in name:\n target_channel_id = cid\n break\n\n if target_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#accounts-payable channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(target_channel_id, [])\n if not isinstance(messages, list):\n return {'pass': False, 'score': 0.0, 'feedback': 'Messages for #accounts-payable channel is not a list.'}\n\n # Be lenient: check for key fragments that indicate the early payment discount notification\n # Required concepts: early payment discount, harrison (& cole), invoice HC-2025-0001, some mention of $100 discount, deadline around 09/14/2025\n for msg in messages:\n text = (msg.get('text') or '').lower()\n \n # Check for core fragments with loose matching\n has_early_discount = ('early payment' in text and 'discount' in text) or 'early payment discount' in text or ('early' in text and 'discount' in text)\n has_harrison = 'harrison' in text\n has_invoice = 'hc-2025-0001' in text or 'hc2025-0001' in text or 'hc 2025 0001' in text or 'hc-2025-001' in text or 'hc20250001' in text\n # Accept $100, 100.00, $100.00, etc.\n has_amount = bool(re.search(r'\\$?100(\\.00)?', text))\n # Accept various date formats for Sep 14, 2025\n has_deadline = bool(re.search(r'09[/\\-]14[/\\-]2025|9[/\\-]14[/\\-]2025|september\\s+14|sep\\.?\\s*14|2025-09-14', text))\n\n if has_early_discount and has_harrison and has_invoice and has_amount and has_deadline:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole early payment discount notification in #accounts-payable: {msg.get(\"text\")}'}\n\n # Provide helpful feedback about what was found\n all_texts = [msg.get('text', '') for msg in messages]\n harrison_msgs = [t for t in all_texts if 'harrison' in t.lower()]\n discount_msgs = [t for t in all_texts if 'discount' in t.lower()]\n \n detail = ''\n if harrison_msgs:\n detail += f' Found {len(harrison_msgs)} message(s) mentioning Harrison but missing other required elements. Texts: {harrison_msgs[:3]}'\n if discount_msgs:\n detail += f' Found {len(discount_msgs)} message(s) mentioning discount. Texts: {discount_msgs[:3]}'\n if not harrison_msgs and not discount_msgs:\n detail = f' Total messages in channel: {len(messages)}. None mention Harrison or discount.'\n if messages:\n sample = [msg.get('text', '')[:100] for msg in messages[:5]]\n detail += f' Sample messages: {sample}'\n\n return {'pass': False, 'score': 0.0, 'feedback': f'No Harrison & Cole early payment discount notification found in #accounts-payable. Expected text containing: EARLY PAYMENT DISCOUNT, Harrison & Cole, Invoice HC-2025-0001, Discount $100, Deadline 09/14/2025.{detail}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "55327dea-2c59-4e4e-a870-8c79e2373a64",
|
||||
"sort_order": 18,
|
||||
"rubric_text": "In Slack (slack_data.json), NO early payment discount notification for Whitfield Consulting Group invoice WCG-2025-0087 must exist in any channel (discount is $99.00, below $100 minimum threshold).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external_services_path provided; skipping Slack check.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; no early payment discount posted.'}\n with open(slack_path) as f:\n data = json.load(f)\n \n for channel_id, msgs in data.get('messages', {}).items():\n for msg in msgs:\n text = (msg.get('text') or '').lower()\n if 'early payment discount' in text and ('whitfield' in text or 'wcg-2025-0087' in text):\n return {'pass': False, 'score': 0.0, 'feedback': f'Early payment discount notification for Whitfield (WCG-2025-0087) should NOT have been posted — $99.00 discount is below $100 minimum threshold. Found: {msg.get(\"text\")}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No early payment discount notification for Whitfield Consulting Group found — correct, discount ($99) is below $100 threshold.'}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
Hey, this is Tara, and my role is AP Specialist. Today is September 4th, 2025 and I could use some help please.
|
||||
|
||||
Can you check the mailbox inbox for new vendor emails with W-9s and invoices. I think there are three that have not been processed yet. I need to get these fully processed ASAP - vendor setup, invoice entry, approvals, to include all steps per the Meridian SOP (AP). All of the company's AP files (vendor master, AP ledger, PO matching logs, recurring agreements, etc.) are on the shared drive.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user