DerekSurge
136 lines
60 KiB
JSON
136 lines
60 KiB
JSON
[
|
|
{
|
|
"id": "cb45048d-c17b-4f06-9331-ff74e996e824",
|
|
"sort_order": 0,
|
|
"rubric_text": "In folder `1099/2025/`, file `W9_Whitfield_Consulting_Group.pdf` must exist.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / '1099' / '2025' / 'W9_Whitfield_Consulting_Group.pdf'\n if target.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'Found W9_Whitfield_Consulting_Group.pdf in 1099/2025/'}\n # Also try case-insensitive search\n folder = Path(workspace_path) / '1099' / '2025'\n if folder.exists():\n matches = [f for f in folder.iterdir() if 'whitfield' in f.name.lower() and f.suffix.lower() == '.pdf']\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield W-9 PDF: {matches[0].name}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'W9_Whitfield_Consulting_Group.pdf not found in 1099/2025/'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "d5570533-815a-4694-b65d-8d62187d7c86",
|
|
"sort_order": 1,
|
|
"rubric_text": "In folder `1099/2025/`, file `W9_Harrison_Cole_LLP.pdf` must exist.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / '1099' / '2025' / 'W9_Harrison_Cole_LLP.pdf'\n if target.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'Found W9_Harrison_Cole_LLP.pdf in 1099/2025/'}\n folder = Path(workspace_path) / '1099' / '2025'\n if folder.exists():\n matches = [f for f in folder.iterdir() if 'harrison' in f.name.lower() and f.suffix.lower() == '.pdf']\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole W-9 PDF: {matches[0].name}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'W9_Harrison_Cole_LLP.pdf not found in 1099/2025/'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "a9da8354-1545-447d-b665-bcd6ac61998d",
|
|
"sort_order": 2,
|
|
"rubric_text": "In `vendor_master.xlsx`, a row for Whitfield Consulting Group, Inc. must exist with: Tax ID 23-7891045, Address containing 450 Independence Mall West, Contact Email d.whitfield@whitfieldcg.com, Payment Terms 2/10 Net 30, Bank Routing 321081669, Bank Account 7720194583, 1099 Reportable Yes, 1099 Type NEC, Status Active, Created Date 09/04/2025",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'vendor_master.xlsx not found in workspace.'}\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n headers = [str(c.value).strip().lower() if c.value else '' for c in next(ws.iter_rows(min_row=1, max_row=1))]\n\n found_row_raw = None\n found_row_idx = None\n for idx, row in enumerate(ws.iter_rows(min_row=2, max_row=ws.max_row), start=2):\n row_values = [cell.value for cell in row]\n row_str = ' '.join(str(v) for v in row_values if v is not None).lower()\n if 'whitfield' in row_str and 'consult' in row_str:\n found_row_raw = row_values\n found_row_idx = idx\n break\n if found_row_raw is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for Whitfield Consulting Group found in vendor_master.xlsx.'}\n\n row_values = []\n for v in found_row_raw:\n if v is None:\n row_values.append('')\n elif isinstance(v, datetime):\n row_values.append(v.strftime('%m/%d/%Y'))\n else:\n row_values.append(str(v).strip())\n row_str = ' '.join(row_values)\n row_lower = row_str.lower()\n\n missing = []\n\n # Tax ID: 23-7891045\n row_str_no_sep = row_str.replace('-', '').replace(' ', '')\n if '23-7891045' not in row_str and '237891045' not in row_str_no_sep:\n missing.append('Tax ID (expected: 23-7891045)')\n\n # Address containing 450 Independence Mall West\n if '450 independence mall west' not in row_lower:\n if not ('450' in row_lower and 'independence' in row_lower and 'mall' in row_lower and 'west' in row_lower):\n missing.append('Address containing 450 Independence Mall West')\n\n # Contact Email\n if 'd.whitfield@whitfieldcg.com' not in row_lower:\n missing.append('Contact Email (expected: d.whitfield@whitfieldcg.com)')\n\n # Payment Terms: 2/10 Net 30\n terms_found = False\n for val in row_values:\n vl = val.lower().replace(' ', '')\n if '2/10net30' in vl or '2/10,net30' in vl or '2/10-net30' in vl:\n terms_found = True\n break\n if '2/10 net 30' in val.lower():\n terms_found = True\n break\n if '2/10' in val and 'net' in val.lower() and '30' in val:\n terms_found = True\n break\n if not terms_found:\n missing.append('Payment Terms (expected: 2/10 Net 30)')\n\n # Bank Routing: 321081669\n if '321081669' not in row_str:\n missing.append('Bank Routing (expected: 321081669)')\n\n # Bank Account: 7720194583\n if '7720194583' not in row_str:\n missing.append('Bank Account (expected: 7720194583)')\n\n # 1099 Reportable: Yes\n reportable_found = False\n for val in row_values:\n if val.strip().lower() in ('yes', 'y', 'true', '1'):\n reportable_found = True\n break\n if not reportable_found:\n missing.append('1099 Reportable (expected: Yes)')\n\n # 1099 Type: NEC\n nec_found = False\n for val in row_values:\n if val.strip().upper() == 'NEC' or 'nec' in val.lower().split():\n nec_found = True\n break\n if not nec_found:\n # Also check as part of a combined value\n for val in row_values:\n if 'nec' in val.lower():\n nec_found = True\n break\n if not nec_found:\n missing.append('1099 Type (expected: NEC)')\n\n # Status: Active\n active_found = False\n for val in row_values:\n if val.strip().lower() == 'active':\n active_found = True\n break\n if not active_found:\n missing.append('Status (expected: Active)')\n\n # Created Date: 09/04/2025\n date_found = False\n for v in found_row_raw:\n nd = normalize_date(v)\n if nd == '09/04/2025':\n date_found = True\n break\n if not date_found:\n # Also check string representations in row_values\n for val in row_values:\n if '09/04/2025' in val or '9/4/2025' in val or '2025-09-04' in val:\n date_found = True\n break\n if not date_found:\n missing.append('Created Date (expected: 09/04/2025)')\n\n if missing:\n return {\n 'pass': False,\n 'score': max(0.0, 1.0 - len(missing) * 0.1),\n 'feedback': f'Whitfield Consulting Group row found at row {found_row_idx} but missing/incorrect fields: {missing}. Row data: {row_str[:500]}'\n }\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Whitfield Consulting Group, Inc. row found at row {found_row_idx} with all required fields: Tax ID (23-7891045), Address (450 Independence Mall West), Email (d.whitfield@whitfieldcg.com), Payment Terms (2/10 Net 30), Bank Routing (321081669), Bank Account (7720194583), 1099 Reportable (Yes), 1099 Type (NEC), Status (Active), Created Date (09/04/2025).'\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "970d0671-7df2-4425-8f51-254cf30b82b4",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `vendor_master.xlsx`, a row for Harrison & Cole LLP must exist with: Tax ID 23-7654321, Address containing '1200 Market Street', Contact Email s.harrison@harrisoncolellp.com, Payment Terms 2/10 Net 30, Bank Routing 031100209, Bank Account 7742851903, 1099 Reportable Yes, 1099 Type NEC, Status Active, Created Date 09/04/2025",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n # Handle cases like '09/04/25' or '9/4/2025' etc.\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d', '%m/%d/%Y %H:%M:%S',\n '%Y-%m-%d %H:%M:%S', '%m-%d-%y', '%d/%m/%Y'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef verify(workspace_path, external_services_path=None):\n # Search for vendor_master.xlsx broadly\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).rglob('*vendor_master*.xlsx'))\n if not candidates:\n # Try any xlsx that might contain vendor master data\n candidates = list(Path(workspace_path).rglob('*.xlsx'))\n candidates = [c for c in candidates if 'vendor' in c.name.lower()]\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'vendor_master.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n\n # Try to find header row to map columns\n headers = []\n header_row_idx = 1\n for row in ws.iter_rows(min_row=1, max_row=3, values_only=False):\n row_vals = [str(cell.value).strip().lower() if cell.value else '' for cell in row]\n # Check if this looks like a header row\n if any(h in ' '.join(row_vals) for h in ['vendor', 'name', 'tax', 'status']):\n headers = row_vals\n header_row_idx = row[0].row\n break\n\n found_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'harrison' in row_str and 'cole' in row_str:\n found_row = row\n break\n\n if found_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for Harrison & Cole LLP found in vendor_master.xlsx.'}\n\n row_str = ' '.join(str(v) for v in found_row if v is not None)\n row_lower = row_str.lower()\n\n missing = []\n\n # Tax ID - accept with or without dash\n if '23-7654321' not in row_lower and '237654321' not in row_lower:\n missing.append('Tax ID 23-7654321')\n\n # Address containing 1200 Market Street\n if '1200 market street' not in row_lower and '1200 market st' not in row_lower:\n missing.append('Address 1200 Market Street')\n\n # Contact Email\n if 's.harrison@harrisoncolellp.com' not in row_lower:\n missing.append('Contact Email s.harrison@harrisoncolellp.com')\n\n # Payment Terms - be generous with variations\n terms_found = False\n for variant in ['2/10 net 30', '2/10 net30', '2/10net30', '2/10, net 30',\n '2/10-net 30', '2% 10 net 30', '2/10 net-30', '2/10net 30',\n '2/10 net 30', '2%10 net 30', '2% 10, net 30']:\n if variant in row_lower:\n terms_found = True\n break\n if not terms_found:\n missing.append('Payment Terms 2/10 Net 30')\n\n # Bank Routing\n if '031100209' not in row_lower:\n missing.append('Bank Routing 031100209')\n\n # Bank Account\n if '7742851903' not in row_lower:\n missing.append('Bank Account 7742851903')\n\n # 1099 Reportable Yes\n yes_found = False\n for v in found_row:\n if v is not None and str(v).strip().lower() in ('yes', 'y', 'true', '1'):\n yes_found = True\n break\n if not yes_found:\n missing.append('1099 Reportable Yes')\n\n # 1099 Type NEC\n if 'nec' not in row_lower:\n missing.append('1099 Type NEC')\n\n # Status Active\n if 'active' not in row_lower:\n missing.append('Status Active')\n\n # Created Date 09/04/2025 - be generous with date formats\n date_found = False\n for v in found_row:\n nd = normalize_date(v)\n if nd == '09/04/2025':\n date_found = True\n break\n # Also check string representation directly\n if v is not None:\n vs = str(v).strip()\n if vs in ('09/04/2025', '9/4/2025', '2025-09-04', '09-04-2025',\n 'September 4, 2025', 'Sep 4, 2025', '09/04/25', '9/4/25'):\n date_found = True\n break\n if not date_found:\n missing.append('Created Date 09/04/2025')\n\n if missing:\n return {\n 'pass': False,\n 'score': max(0.1, 1.0 - len(missing) * 0.1),\n 'feedback': f'Harrison & Cole row found but missing/incorrect fields: {missing}. Row data: {row_str[:500]}'\n }\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Harrison & Cole LLP row found with all required fields in vendor_master.xlsx. Row data: {row_str[:500]}'\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "61f3c9bd-e56d-4d07-937e-7c5ca2857a79",
|
|
"sort_order": 4,
|
|
"rubric_text": "In `vendor_master.xlsx`, NO new row for ClearPath Talent Solutions must be added (duplicate vendor — Tax ID 58-8901234 matches existing vendor Premier Staffing Group V-00008).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('vendor_master.xlsx'))\n if not candidates:\n return {'pass': True, 'score': 1.0, 'feedback': 'vendor_master.xlsx not found; no ClearPath row added.'}\n wb = openpyxl.load_workbook(candidates[0])\n ws = wb.active\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'clearpath' in row_str or ('58-8901234' in row_str and 'premier' not in row_str):\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath Talent Solutions should NOT be added to vendor_master.xlsx (duplicate vendor). Found row: {row_str[:200]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions row found in vendor_master.xlsx — correct, duplicate vendor blocked.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "50e0d257-697e-49f9-9cd3-3d7e544f9092",
|
|
"sort_order": 5,
|
|
"rubric_text": "In ap_ledger.xlsx Invoice Register tab, a row for Whitfield Consulting Group invoice WCG-2025-0087 must exist with: Invoice Date 08/28/2025, PO Number blank/none, Category Non-PO, Amount 4950.00, GL Code 1000-120-6100, Status Entered, Payment Terms 2/10 Net 30, Due Date 09/27/2025, Entered By T. Okonkwo, Entered Date 09/04/2025.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef normalize_amount(value):\n \"\"\"Return a float for any amount-like value, or None if unparseable.\"\"\"\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return float(value)\n s = str(value).strip().replace('$', '').replace(',', '')\n try:\n return float(s)\n except ValueError:\n return None\n\ndef norm_header(h):\n return str(h).strip().lower() if h is not None else ''\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).rglob('*.xlsx'))\n candidates = [c for c in candidates if 'ap' in c.stem.lower() or 'ledger' in c.stem.lower()]\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n\n # Find the Invoice Register sheet\n target_ws = None\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl and 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl or 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab not found in ap_ledger.xlsx.'}\n\n # Read header row\n rows = list(target_ws.iter_rows(values_only=True))\n if not rows:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab is empty.'}\n header = [norm_header(h) for h in rows[0]]\n\n # Required columns (plus Invoice Number column to locate the row)\n required_cols = [\n 'Invoice Number',\n 'Invoice Date', 'PO Number', 'Category', 'Amount', 'GL Code',\n 'Status', 'Payment Terms', 'Due Date', 'Entered By', 'Entered Date',\n ]\n col_idx = {}\n for col in required_cols:\n try:\n col_idx[col] = header.index(col.lower())\n except ValueError:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Column \"{col}\" not found in Invoice Register header. Header row: {rows[0]}'}\n\n # Locate the WCG-2025-0087 row by Invoice Number column\n inv_col = col_idx['Invoice Number']\n found_row = None\n for r in rows[1:]:\n if r is None or inv_col >= len(r):\n continue\n cell = r[inv_col]\n if cell is not None and str(cell).strip().lower() == 'wcg-2025-0087':\n found_row = r\n break\n if found_row is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'No row with Invoice Number WCG-2025-0087 found in Invoice Register.'}\n\n def cell(col_name):\n idx = col_idx[col_name]\n return found_row[idx] if idx < len(found_row) else None\n\n missing = []\n\n # Invoice Date: 08/28/2025\n if normalize_date(cell('Invoice Date')) != '08/28/2025':\n missing.append(f'Invoice Date should be 08/28/2025, got {cell(\"Invoice Date\")!r}')\n\n # PO Number: blank/none\n po_val = cell('PO Number')\n if po_val is not None and str(po_val).strip() != '':\n missing.append(f'PO Number should be blank, got {po_val!r}')\n\n # Category: Non-PO\n cat_val = cell('Category')\n cat_norm = str(cat_val).strip().lower().replace(' ', '').replace('_', '') if cat_val is not None else ''\n if cat_norm != 'non-po' and cat_norm != 'nonpo':\n missing.append(f'Category should be Non-PO, got {cat_val!r}')\n\n # Amount: 4950.00\n amt = normalize_amount(cell('Amount'))\n if amt is None or abs(amt - 4950.00) > 0.005:\n missing.append(f'Amount should be 4950.00, got {cell(\"Amount\")!r}')\n\n # GL Code: 1000-120-6100\n gl_val = cell('GL Code')\n if gl_val is None or str(gl_val).strip() != '1000-120-6100':\n missing.append(f'GL Code should be 1000-120-6100, got {gl_val!r}')\n\n # Status: Entered (exact, case-insensitive)\n status_val = cell('Status')\n if status_val is None or str(status_val).strip().lower() != 'entered':\n missing.append(f'Status should be Entered, got {status_val!r}')\n\n # Payment Terms: 2/10 Net 30\n pt_val = cell('Payment Terms')\n pt_norm = str(pt_val).strip().lower().replace(',', '').replace(' ', ' ') if pt_val is not None else ''\n accepted_terms = {'2/10 net 30', '2/10net30', '2% 10 net 30'}\n if pt_norm not in accepted_terms:\n missing.append(f'Payment Terms should be 2/10 Net 30, got {pt_val!r}')\n\n # Due Date: 09/27/2025\n if normalize_date(cell('Due Date')) != '09/27/2025':\n missing.append(f'Due Date should be 09/27/2025, got {cell(\"Due Date\")!r}')\n\n # Entered By: exactly 'T. Okonkwo'\n eb_val = cell('Entered By')\n if eb_val is None or str(eb_val).strip() != 'T. Okonkwo':\n missing.append(f'Entered By should be exactly \"T. Okonkwo\", got {eb_val!r}')\n\n # Entered Date: 09/04/2025\n if normalize_date(cell('Entered Date')) != '09/04/2025':\n missing.append(f'Entered Date should be 09/04/2025, got {cell(\"Entered Date\")!r}')\n\n if missing:\n return {'pass': False, 'score': 0.5,\n 'feedback': 'WCG-2025-0087 row found but fields incorrect: ' + '; '.join(missing)}\n return {'pass': True, 'score': 1.0,\n 'feedback': 'WCG-2025-0087 row in Invoice Register has all required fields with correct values.'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "4d825a27-1baf-4c9c-a7ac-e06b45e1a18b",
|
|
"sort_order": 6,
|
|
"rubric_text": "In ap_ledger.xlsx Invoice Register tab, a row for Harrison & Cole LLP invoice HC-2025-0001 must exist with: Invoice Date 09/04/2025, PO Number blank/none, Category Non-PO, Amount 5000.00, GL Code 1000-120-6100, Status Entered, Payment Terms 2/10 Net 30, Due Date 10/04/2025, Entered By T. Okonkwo, Entered Date 09/04/2025.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef normalize_date(value):\n \"\"\"Return 'MM/DD/YYYY' for any date-like value, or '' if unparseable.\"\"\"\n if value is None:\n return ''\n if isinstance(value, datetime):\n return value.strftime('%m/%d/%Y')\n s = str(value).strip()\n for fmt in ('%m/%d/%Y', '%m/%d/%y', '%Y-%m-%d', '%m-%d-%Y',\n '%B %d, %Y', '%B %d %Y', '%b %d, %Y', '%b %d %Y',\n '%d-%b-%Y', '%Y/%m/%d'):\n try:\n return datetime.strptime(s, fmt).strftime('%m/%d/%Y')\n except ValueError:\n continue\n return ''\n\ndef normalize_amount(value):\n \"\"\"Return a float for any number-like value, or None if unparseable.\"\"\"\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return float(value)\n s = str(value).strip().replace('$', '').replace(',', '')\n try:\n return float(s)\n except ValueError:\n return None\n\ndef verify(workspace_path, external_services_path=None):\n # Find ap_ledger.xlsx\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace.'}\n\n wb = openpyxl.load_workbook(candidates[0])\n\n # Find the Invoice Register tab\n target_ws = None\n for name in wb.sheetnames:\n nl = name.lower()\n if 'invoice' in nl and 'register' in nl:\n target_ws = wb[name]\n break\n if target_ws is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoice Register tab not found in ap_ledger.xlsx.'}\n\n # Build header -> column index map (case-insensitive, trimmed)\n headers = {}\n for i, cell in enumerate(target_ws[1]):\n if cell.value is not None:\n headers[str(cell.value).strip().lower()] = i\n\n # Required column headers\n required_cols = [\n 'invoice date', 'po number', 'category', 'amount', 'gl code',\n 'status', 'payment terms', 'due date', 'entered by', 'entered date'\n ]\n # We also need an Invoice Number column to find the right row\n invoice_num_col = None\n for key in ('invoice number', 'invoice #', 'invoice no', 'invoice no.'):\n if key in headers:\n invoice_num_col = headers[key]\n break\n\n if invoice_num_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find an Invoice Number column in Invoice Register header.'}\n\n missing_cols = [c for c in required_cols if c not in headers]\n if missing_cols:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns in Invoice Register: {missing_cols}'}\n\n # Locate the HC-2025-0001 row\n found_row = None\n for row in target_ws.iter_rows(min_row=2, values_only=True):\n if invoice_num_col < len(row) and row[invoice_num_col] is not None:\n if str(row[invoice_num_col]).strip().lower() == 'hc-2025-0001':\n found_row = row\n break\n\n if found_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row for invoice HC-2025-0001 found in Invoice Register.'}\n\n def cell(col_name):\n idx = headers[col_name]\n if idx >= len(found_row):\n return None\n return found_row[idx]\n\n def cell_str(col_name):\n v = cell(col_name)\n return '' if v is None else str(v).strip()\n\n missing = []\n\n # Invoice Date == 09/04/2025\n if normalize_date(cell('invoice date')) != '09/04/2025':\n missing.append(f\"Invoice Date should be 09/04/2025, found: {cell('invoice date')!r}\")\n\n # PO Number blank/none\n po_val = cell('po number')\n if po_val is not None and str(po_val).strip().lower() not in ('', 'none', 'n/a', '-'):\n missing.append(f\"PO Number should be blank/none, found: {po_val!r}\")\n\n # Category == Non-PO\n cat = cell_str('category').lower().replace(' ', '').replace('-', '')\n if cat != 'nonpo':\n missing.append(f\"Category should be Non-PO, found: {cell_str('category')!r}\")\n\n # Amount == 5000.00\n amt = normalize_amount(cell('amount'))\n if amt is None or abs(amt - 5000.00) > 0.001:\n missing.append(f\"Amount should be 5000.00, found: {cell('amount')!r}\")\n\n # GL Code == 1000-120-6100\n if cell_str('gl code') != '1000-120-6100':\n missing.append(f\"GL Code should be 1000-120-6100, found: {cell_str('gl code')!r}\")\n\n # Status == Entered\n if cell_str('status').lower() != 'entered':\n missing.append(f\"Status should be Entered, found: {cell_str('status')!r}\")\n\n # Payment Terms == 2/10 Net 30 (allow minor spacing/comma variation)\n pt = cell_str('payment terms').lower().replace(',', '').replace(' ', '')\n if pt != '2/10net30':\n missing.append(f\"Payment Terms should be 2/10 Net 30, found: {cell_str('payment terms')!r}\")\n\n # Due Date == 10/04/2025\n if normalize_date(cell('due date')) != '10/04/2025':\n missing.append(f\"Due Date should be 10/04/2025, found: {cell('due date')!r}\")\n\n # Entered By == exactly 'T. Okonkwo'\n if cell_str('entered by') != 'T. Okonkwo':\n missing.append(f\"Entered By should be exactly 'T. Okonkwo', found: {cell_str('entered by')!r}\")\n\n # Entered Date == 09/04/2025\n if normalize_date(cell('entered date')) != '09/04/2025':\n missing.append(f\"Entered Date should be 09/04/2025, found: {cell('entered date')!r}\")\n\n if missing:\n return {'pass': False, 'score': 0.5, 'feedback': f'HC-2025-0001 row found but has issues: {missing}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'HC-2025-0001 row in Invoice Register has all required field values.'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "b909c1a5-256a-45b2-a355-4bc1554e47b1",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `ap_ledger.xlsx` Invoice Register tab, NO row for ClearPath Talent Solutions or invoice CPT-2025-0412 must exist (duplicate vendor — entire pipeline blocked).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).rglob('ap_ledger.xlsx'))\n if not candidates:\n return {'pass': True, 'score': 1.0, 'feedback': 'ap_ledger.xlsx not found; no ClearPath invoice entered.'}\n wb = openpyxl.load_workbook(candidates[0])\n for ws in wb.worksheets:\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_str = ' '.join(str(v) for v in row if v is not None).lower()\n if 'clearpath' in row_str or 'cpt-2025-0412' in row_str:\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath invoice CPT-2025-0412 should NOT be entered in ap_ledger.xlsx. Found: {row_str[:200]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions / CPT-2025-0412 entry in ap_ledger.xlsx — correct.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "e1811786-91b9-483d-b2f4-06dab61c2cef",
|
|
"sort_order": 8,
|
|
"rubric_text": "In folder `Invoices/2025-09/`, invoice file for Whitfield Consulting Group (WCG-2025-0087) must exist (e.g., INV-V-00038-WCG-2025-0087.pdf or similar).",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n # Try alternate casing\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoices/2025-09/ folder not found in workspace.'}\n matches = [f for f in folder.iterdir() if 'wcg-2025-0087' in f.name.lower() or ('whitfield' in f.name.lower() and f.suffix.lower() == '.pdf')]\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield invoice file: {[f.name for f in matches]}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No invoice file for WCG-2025-0087 found in Invoices/2025-09/. Files present: {[f.name for f in folder.iterdir()]}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "3f78f653-6bab-4c33-98b4-9bca1b8448fb",
|
|
"sort_order": 9,
|
|
"rubric_text": "In folder `Invoices/2025-09/`, invoice file for Harrison & Cole LLP (HC-2025-0001) must exist (e.g., INV-V-00039-HC-2025-0001.pdf or similar).",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'Invoices/2025-09/ folder not found in workspace.'}\n matches = [f for f in folder.iterdir() if 'hc-2025-0001' in f.name.lower() or ('harrison' in f.name.lower() and f.suffix.lower() == '.pdf')]\n if matches:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole invoice file: {[f.name for f in matches]}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No invoice file for HC-2025-0001 found in Invoices/2025-09/. Files present: {[f.name for f in folder.iterdir()]}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "4e0d1eea-4c10-47d0-a7d4-ceb532b6377f",
|
|
"sort_order": 10,
|
|
"rubric_text": "In folder `Invoices/2025-09/`, NO invoice file for ClearPath Talent Solutions (CPT-2025-0412) must exist.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n folder = Path(workspace_path) / 'Invoices' / '2025-09'\n if not folder.exists():\n for alt in Path(workspace_path).rglob('2025-09'):\n if alt.is_dir():\n folder = alt\n break\n else:\n return {'pass': True, 'score': 1.0, 'feedback': 'Invoices/2025-09/ folder not found; no ClearPath invoice filed.'}\n matches = [f for f in folder.iterdir() if 'clearpath' in f.name.lower() or 'cpt-2025-0412' in f.name.lower() or 'cpt' in f.name.lower()]\n if matches:\n return {'pass': False, 'score': 0.0, 'feedback': f'ClearPath invoice should NOT be filed in Invoices/2025-09/. Found: {[f.name for f in matches]}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No ClearPath Talent Solutions invoice found in Invoices/2025-09/ — correct.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "251ff8e0-bb26-4e6f-9d77-cbd264f4d756",
|
|
"sort_order": 11,
|
|
"rubric_text": "In mailbox sent mail (mailbox.json), an approval email for Whitfield Consulting Group invoice WCG-2025-0087 must be sent to jennifer.walsh@meridianpartners.com and maria.santos@meridianpartners.com with CC ap.team@meridianpartners.com, subject containing 'AP APPROVAL REQUIRED' and 'WCG-2025-0087', body containing vendor name, vendor ID (V-00038 or V-00039), invoice number WCG-2025-0087, invoice date 08/28/2025 (or similar format), amount $4,950.00, GL Code 1000-120-6100, and '3 business days'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox for approval email.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.'}\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or email.get('body') or '').lower()\n combined = subject + ' ' + body\n\n # Collect all address fields flexibly\n to_raw = email.get('to_addr') or email.get('to') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join(str(x) for x in to_raw).lower()\n else:\n to_addr = str(to_raw).lower()\n\n cc_raw = email.get('cc_addr') or email.get('cc') or ''\n if isinstance(cc_raw, list):\n cc_addr = ' '.join(str(x) for x in cc_raw).lower()\n else:\n cc_addr = str(cc_raw).lower()\n\n all_addr = to_addr + ' ' + cc_addr\n\n # Must reference WCG-2025-0087 somewhere in subject or body\n if 'wcg-2025-0087' not in combined:\n continue\n\n # Must have AP APPROVAL REQUIRED in subject (lenient with casing/spacing)\n if 'ap approval required' not in subject:\n continue\n\n missing = []\n\n # Check To recipients\n if 'jennifer.walsh@meridianpartners.com' not in all_addr:\n missing.append('jennifer.walsh@meridianpartners.com in To')\n if 'maria.santos@meridianpartners.com' not in all_addr:\n missing.append('maria.santos@meridianpartners.com in To')\n\n # Check CC - be lenient, also accept if it's in To\n if 'ap.team@meridianpartners.com' not in all_addr:\n missing.append('ap.team@meridianpartners.com in CC')\n\n # Body checks\n # Vendor name - accept various forms\n if 'whitfield' not in combined:\n missing.append('vendor name Whitfield')\n\n # Invoice number in body\n if 'wcg-2025-0087' not in body:\n missing.append('invoice number WCG-2025-0087 in body')\n\n # Invoice date - accept multiple date formats for 08/28/2025\n date_found = False\n date_patterns = ['08/28/2025', '8/28/2025', '08-28-2025', '2025-08-28', 'august 28, 2025', 'august 28 2025', '28 august 2025', '08/28/25', '8/28/25']\n for dp in date_patterns:\n if dp in combined:\n date_found = True\n break\n if not date_found:\n missing.append('invoice date 08/28/2025 (or similar)')\n\n # Amount - accept various formats for $4,950.00\n amount_found = False\n amount_patterns = ['4,950', '4950.00', '4950', '$4,950', '4,950.00']\n for ap in amount_patterns:\n if ap in combined:\n amount_found = True\n break\n if not amount_found:\n missing.append('amount $4,950.00')\n\n # GL Code - accept with or without spaces around dashes\n gl_found = False\n gl_patterns = ['1000-120-6100']\n for gp in gl_patterns:\n if gp in combined:\n gl_found = True\n break\n if not gl_found:\n missing.append('GL Code 1000-120-6100')\n\n # 3 business days - be lenient\n if '3 business day' not in combined and 'three business day' not in combined:\n missing.append('3 business days response deadline')\n\n # Vendor ID\n if not ('v-00038' in combined or 'v-00039' in combined):\n missing.append('Vendor ID V-00038 or V-00039')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': 'Whitfield Consulting Group approval email found with all required fields.'}\n\n # Track best match (fewest missing items)\n if best_missing is None or len(missing) < len(best_missing):\n best_missing = missing\n best_match = email\n\n if best_match is not None:\n return {'pass': False, 'score': max(0.1, 0.5 - 0.05 * len(best_missing)), 'feedback': f'Whitfield approval email found but missing: {best_missing}'}\n\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent approval email for WCG-2025-0087 found in mailbox.'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "3e47971f-c177-4411-b691-0f09f3dd9d2f",
|
|
"sort_order": 12,
|
|
"rubric_text": "In mailbox sent mail (mailbox.json), an approval email for Harrison & Cole LLP invoice HC-2025-0001 must be sent to jennifer.walsh@meridianpartners.com and maria.santos@meridianpartners.com with CC ap.team@meridianpartners.com, subject containing 'AP APPROVAL REQUIRED' and 'HC-2025-0001', body containing vendor name, vendor ID (V-00038 or V-00039), invoice number HC-2025-0001, invoice date 09/04/2025 (or similar format), amount $5,000.00, GL Code 1000-120-6100, and '3 business days'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox for approval email.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n if not sent_emails:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails found in mailbox.json.'}\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or email.get('body') or '').lower()\n # Combine subject and body for broad searching\n full_text = subject + ' ' + body\n\n # Normalize to/cc fields - could be string or list\n to_raw = email.get('to_addr') or email.get('to') or ''\n cc_raw = email.get('cc_addr') or email.get('cc') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join([str(x) for x in to_raw]).lower()\n else:\n to_addr = str(to_raw).lower()\n if isinstance(cc_raw, list):\n cc_addr = ' '.join([str(x) for x in cc_raw]).lower()\n else:\n cc_addr = str(cc_raw).lower()\n\n # Also check combined to+cc for loose matching\n all_recipients = to_addr + ' ' + cc_addr\n\n # Must reference HC-2025-0001 somewhere\n if 'hc-2025-0001' not in full_text:\n continue\n\n # Must have AP APPROVAL REQUIRED in subject (loose: accept with or without extra words)\n if 'ap approval required' not in subject:\n continue\n\n missing = []\n\n # Check To addresses - be loose, accept in either To or CC\n if 'jennifer.walsh@meridianpartners.com' not in to_addr:\n # Check if it's anywhere in recipients\n if 'jennifer.walsh@meridianpartners.com' not in all_recipients:\n missing.append('jennifer.walsh@meridianpartners.com in To')\n if 'maria.santos@meridianpartners.com' not in to_addr:\n if 'maria.santos@meridianpartners.com' not in all_recipients:\n missing.append('maria.santos@meridianpartners.com in To')\n\n # Check CC\n if 'ap.team@meridianpartners.com' not in cc_addr:\n if 'ap.team@meridianpartners.com' not in all_recipients:\n missing.append('ap.team@meridianpartners.com in CC')\n\n # Check vendor name - be very loose\n if 'harrison' not in full_text and 'cole' not in full_text:\n missing.append('vendor name Harrison & Cole')\n\n # Check invoice number in body\n if 'hc-2025-0001' not in body:\n missing.append('invoice number HC-2025-0001 in body')\n\n # Check invoice date - accept many formats\n date_found = False\n date_patterns = ['09/04/2025', '9/4/2025', '2025-09-04', 'september 4, 2025', 'september 4 2025',\n 'sep 4, 2025', 'sep 04, 2025', 'sep 4 2025', 'sep 04 2025',\n '09-04-2025', '04/09/2025', '09/04/25', '9/4/25', '2025/09/04']\n for dp in date_patterns:\n if dp in body:\n date_found = True\n break\n if not date_found:\n missing.append('invoice date 09/04/2025 (or similar)')\n\n # Check amount - accept many formats: 5,000, 5000, $5,000, $5000, 5,000.00, 5000.00\n amount_found = False\n amount_patterns = ['5,000', '5000', '$5,000', '$5000']\n for ap in amount_patterns:\n if ap in body:\n amount_found = True\n break\n if not amount_found:\n missing.append('amount $5,000.00')\n\n # Check GL Code - accept with or without spaces/dashes\n gl_found = False\n gl_patterns = ['1000-120-6100', '1000 120 6100', '10001206100']\n for gp in gl_patterns:\n if gp in body:\n gl_found = True\n break\n if not gl_found:\n missing.append('GL Code 1000-120-6100')\n\n # Check 3 business days - be loose\n biz_day_found = False\n biz_patterns = ['3 business day', '3 business-day', 'three business day', '3-business-day',\n 'three business-day', '3 working day', 'three working day']\n for bp in biz_patterns:\n if bp in body:\n biz_day_found = True\n break\n if not biz_day_found:\n missing.append('3 business days response deadline')\n\n # Check vendor ID\n if 'v-00038' not in body and 'v-00039' not in body:\n # Also check full_text in case it's in subject\n if 'v-00038' not in full_text and 'v-00039' not in full_text:\n missing.append('Vendor ID V-00038 or V-00039')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': 'Harrison & Cole LLP approval email found with all required fields: correct To/CC recipients, subject with AP APPROVAL REQUIRED and HC-2025-0001, body with vendor name, vendor ID, invoice number, date, amount $5,000.00, GL Code 1000-120-6100, and 3 business days deadline.'}\n\n # Track best match (fewest missing items)\n if best_missing is None or len(missing) < len(best_missing):\n best_missing = missing\n best_match = email\n\n if best_match is not None:\n score = max(0.1, 1.0 - (len(best_missing) * 0.1))\n return {'pass': False, 'score': score, 'feedback': f'Harrison & Cole approval email found but missing: {best_missing}'}\n\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent approval email for HC-2025-0001 found in mailbox. Looked for email with subject containing AP APPROVAL REQUIRED and HC-2025-0001.'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "8a358c9a-e182-4438-b0be-9351c0351bb8",
|
|
"sort_order": 13,
|
|
"rubric_text": "In mailbox sent mail (mailbox.json), NO approval email for ClearPath Talent Solutions or invoice CPT-2025-0412 must exist.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external_services_path provided; skipping mailbox check.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; no ClearPath email sent.'}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n for email in sent_emails:\n subject = (email.get('subject') or '').lower()\n body = (email.get('body_text') or '').lower()\n if 'clearpath' in subject or 'clearpath' in body or 'cpt-2025-0412' in subject or 'cpt-2025-0412' in body:\n return {'pass': False, 'score': 0.0, 'feedback': f'Approval email for ClearPath should NOT have been sent. Found email with subject: {email.get(\"subject\")}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No approval email for ClearPath Talent Solutions found in sent mail — correct.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "e23b68e1-2412-4b5d-9e87-fbe7636fcad8",
|
|
"sort_order": 14,
|
|
"rubric_text": "In Slack (slack_data.json), channel #approvals must contain a new vendor notification post with exact text: `NEW VENDOR ADDED: Whitfield Consulting Group, Inc., V-00038 or V-00039 (either is acceptable), 1099 Reportable: YES`",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n \n # Find the #approvals channel\n approvals_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n if cinfo.get('name', '').lower() == 'approvals':\n approvals_channel_id = cid\n break\n \n if approvals_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#approvals channel not found in slack_data.json.'}\n \n messages = data.get('messages', {}).get(approvals_channel_id, [])\n \n # The rubric says the message should contain:\n # NEW VENDOR ADDED: Whitfield Consulting Group, Inc., V-00038 or V-00039, 1099 Reportable: YES\n # Be loose in matching - check for key fragments case-insensitively\n expected_fragments_base = [\n 'new vendor added',\n 'whitfield consulting group',\n ]\n valid_vendor_ids = ['v-00038', 'v-00039']\n \n # Accept various forms of 1099 reportable yes\n reportable_patterns = [\n '1099 reportable: yes',\n '1099 reportable:yes',\n '1099-reportable: yes',\n '1099 reportable - yes',\n '1099 reportable yes',\n '1099reportable: yes',\n '1099reportable:yes',\n '1099: yes',\n '1099 yes',\n ]\n \n for msg in messages:\n text = (msg.get('text') or '').lower()\n # Check base fragments\n if all(frag in text for frag in expected_fragments_base):\n has_vendor_id = any(vid in text for vid in valid_vendor_ids)\n if has_vendor_id:\n # Split on \"new vendor added\" to isolate each notification\n # This prevents Harrison & Cole's \"YES\" from satisfying Whitfield's check\n segments = re.split(r'new vendor added', text)\n for segment in segments:\n if 'whitfield consulting group' in segment:\n has_reportable = any(rp in segment for rp in reportable_patterns)\n if has_reportable:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Whitfield new vendor notification in #approvals: {msg.get(\"text\")}'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found Whitfield notification but 1099 Reportable is not YES. Message: {msg.get(\"text\")}'}\n \n # Collect all messages in the channel for debugging\n all_texts = [msg.get('text', '') for msg in messages]\n return {'pass': False, 'score': 0.0, 'feedback': f'No Whitfield Consulting Group new vendor notification found in #approvals channel. Expected message containing: NEW VENDOR ADDED, Whitfield Consulting Group, Inc., V-00038 or V-00039, 1099 Reportable: YES. Messages found in #approvals: {all_texts}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "995de855-5315-444e-8765-584fad8962e4",
|
|
"sort_order": 15,
|
|
"rubric_text": "In Slack (slack_data.json), channel #approvals must contain a new vendor notification post with exact text: `NEW VENDOR ADDED: Harrison & Cole LLP, V-00038 or V-00039 (either is acceptable), 1099 Reportable: YES`",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n approvals_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n if cinfo.get('name', '').lower() == 'approvals':\n approvals_channel_id = cid\n break\n\n if approvals_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#approvals channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(approvals_channel_id, [])\n\n all_texts = []\n for msg in messages:\n text = msg.get('text') or ''\n all_texts.append(text)\n lower_text = text.lower()\n\n # Check for key components loosely\n has_new_vendor = 'new vendor' in lower_text\n has_harrison_cole = 'harrison' in lower_text and 'cole' in lower_text\n has_vendor_id = 'v-00038' in lower_text or 'v-00039' in lower_text\n has_1099_yes = ('1099' in lower_text and 'yes' in lower_text)\n\n if has_new_vendor and has_harrison_cole and has_vendor_id and has_1099_yes:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found Harrison & Cole new vendor notification in #approvals: {text}'\n }\n\n # Provide detailed failure feedback\n feedback_parts = ['No Harrison & Cole LLP new vendor notification found in #approvals.']\n feedback_parts.append('Expected text containing: NEW VENDOR ADDED: Harrison & Cole LLP, V-00038 or V-00039, 1099 Reportable: YES')\n if all_texts:\n feedback_parts.append(f'Found {len(all_texts)} message(s) in #approvals. Sample texts: {all_texts[:5]}')\n else:\n feedback_parts.append('No messages found in #approvals channel.')\n\n return {'pass': False, 'score': 0.0, 'feedback': ' | '.join(feedback_parts)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "1671ba3e-b81e-461b-ac11-e133196369fb",
|
|
"sort_order": 16,
|
|
"rubric_text": "In Slack (slack_data.json), channel #vendor-management must contain a duplicate vendor notification with exact text: `DUPLICATE VENDOR SUSPECTED: ClearPath Talent Solutions matches Tax ID of existing vendor Premier Staffing Group, V-00008`",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n # Find #vendor-management channel (loose matching)\n target_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n cname = cinfo.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if 'vendor' in cname and 'management' in cname:\n target_channel_id = cid\n break\n\n if target_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#vendor-management channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(target_channel_id, [])\n\n # Key fragments to look for (case-insensitive)\n expected_fragments = [\n 'duplicate vendor suspected',\n 'clearpath',\n 'premier staffing group',\n 'v-00008',\n ]\n\n for msg in messages:\n text = (msg.get('text') or '').lower()\n if all(frag.lower() in text for frag in expected_fragments):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found ClearPath duplicate vendor notification in #vendor-management: {msg.get(\"text\")}'}\n\n # Collect all messages for debugging\n all_texts = [msg.get('text', '') for msg in messages]\n return {'pass': False, 'score': 0.0, 'feedback': f'No ClearPath duplicate vendor notification found in #vendor-management. Expected message containing: DUPLICATE VENDOR SUSPECTED: ClearPath Talent Solutions matches Tax ID of existing vendor Premier Staffing Group, V-00008. Found {len(messages)} message(s) in channel: {all_texts[:5]}'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "d87b42b7-4674-4037-bea8-407baa5fc939",
|
|
"sort_order": 17,
|
|
"rubric_text": "In Slack (slack_data.json), channel #accounts-payable must contain an early payment discount notification for Harrison & Cole LLP with exact text: `EARLY PAYMENT DISCOUNT: Harrison & Cole LLP, Invoice HC-2025-0001, Discount $100, Deadline 09/14/2025 (or similar format)`",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack for early payment discount notification.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n with open(slack_path) as f:\n data = json.load(f)\n\n # Find #accounts-payable channel\n target_channel_id = None\n for cid, cinfo in data.get('channels', {}).items():\n name = cinfo.get('name', '').lower().replace(' ', '').replace('-', '').replace('_', '')\n if 'accountspayable' in name:\n target_channel_id = cid\n break\n\n if target_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#accounts-payable channel not found in slack_data.json.'}\n\n messages = data.get('messages', {}).get(target_channel_id, [])\n if not isinstance(messages, list):\n return {'pass': False, 'score': 0.0, 'feedback': 'Messages for #accounts-payable channel is not a list.'}\n\n # Be lenient: check for key fragments that indicate the early payment discount notification\n # Required concepts: early payment discount, harrison (& cole), invoice HC-2025-0001, some mention of $100 discount, deadline around 09/14/2025\n for msg in messages:\n text = (msg.get('text') or '').lower()\n \n # Check for core fragments with loose matching\n has_early_discount = ('early payment' in text and 'discount' in text) or 'early payment discount' in text or ('early' in text and 'discount' in text)\n has_harrison = 'harrison' in text\n has_invoice = 'hc-2025-0001' in text or 'hc2025-0001' in text or 'hc 2025 0001' in text or 'hc-2025-001' in text or 'hc20250001' in text\n # Accept $100, 100.00, $100.00, etc.\n has_amount = bool(re.search(r'\\$?100(\\.00)?', text))\n # Accept various date formats for Sep 14, 2025\n has_deadline = bool(re.search(r'09[/\\-]14[/\\-]2025|9[/\\-]14[/\\-]2025|september\\s+14|sep\\.?\\s*14|2025-09-14', text))\n\n if has_early_discount and has_harrison and has_invoice and has_amount and has_deadline:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found Harrison & Cole early payment discount notification in #accounts-payable: {msg.get(\"text\")}'}\n\n # Provide helpful feedback about what was found\n all_texts = [msg.get('text', '') for msg in messages]\n harrison_msgs = [t for t in all_texts if 'harrison' in t.lower()]\n discount_msgs = [t for t in all_texts if 'discount' in t.lower()]\n \n detail = ''\n if harrison_msgs:\n detail += f' Found {len(harrison_msgs)} message(s) mentioning Harrison but missing other required elements. Texts: {harrison_msgs[:3]}'\n if discount_msgs:\n detail += f' Found {len(discount_msgs)} message(s) mentioning discount. Texts: {discount_msgs[:3]}'\n if not harrison_msgs and not discount_msgs:\n detail = f' Total messages in channel: {len(messages)}. None mention Harrison or discount.'\n if messages:\n sample = [msg.get('text', '')[:100] for msg in messages[:5]]\n detail += f' Sample messages: {sample}'\n\n return {'pass': False, 'score': 0.0, 'feedback': f'No Harrison & Cole early payment discount notification found in #accounts-payable. Expected text containing: EARLY PAYMENT DISCOUNT, Harrison & Cole, Invoice HC-2025-0001, Discount $100, Deadline 09/14/2025.{detail}'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "55327dea-2c59-4e4e-a870-8c79e2373a64",
|
|
"sort_order": 18,
|
|
"rubric_text": "In Slack (slack_data.json), NO early payment discount notification for Whitfield Consulting Group invoice WCG-2025-0087 must exist in any channel (discount is $99.00, below $100 minimum threshold).",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external_services_path provided; skipping Slack check.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'slack_data.json not found; no early payment discount posted.'}\n with open(slack_path) as f:\n data = json.load(f)\n \n for channel_id, msgs in data.get('messages', {}).items():\n for msg in msgs:\n text = (msg.get('text') or '').lower()\n if 'early payment discount' in text and ('whitfield' in text or 'wcg-2025-0087' in text):\n return {'pass': False, 'score': 0.0, 'feedback': f'Early payment discount notification for Whitfield (WCG-2025-0087) should NOT have been posted — $99.00 discount is below $100 minimum threshold. Found: {msg.get(\"text\")}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No early payment discount notification for Whitfield Consulting Group found — correct, discount ($99) is below $100 threshold.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|