DerekSurge
66 lines
46 KiB
JSON
66 lines
46 KiB
JSON
[
|
|
{
|
|
"id": "5fc6c7a8-0f5f-436b-b361-97cedf065876",
|
|
"sort_order": 0,
|
|
"rubric_text": "File `ap_ledger.xlsx` (in workspace) must have a row in the `Invoice Register` sheet with the following values: INV-NGO-4471 for Invoice Number, V-00229 for Vendor ID, Northgate Office Solutions for Vendor Name, 2025-05-19 for Invoice Date, 2025-05-21 for Received Date, PO-2025-00884 for PO Number, 2030.00 (can have dollar signifier such as $ or US$) for Invoice Amount, on hold for Status, MATCH for Hold Code, 6200-010-2200 for GL Code",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/ap_ledger.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n\n # Find Invoice Register sheet (case-insensitive)\n ir_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n ir_sheet = wb[name]\n break\n\n if ir_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'No Invoice Register sheet found. Sheets present: {wb.sheetnames}.'}\n\n target_invoice = 'INV-NGO-4471'\n\n def cell_str(c):\n if c is None:\n return ''\n return str(c).strip()\n\n def normalize_str(s):\n return str(s).strip().lower().replace('$', '').replace(',', '').replace(' ', '').replace('us$', '')\n\n def date_matches(cell_val, target_date_str):\n \"\"\"Check if a cell value matches a target date like '2025-05-19'.\"\"\"\n if cell_val is None:\n return False\n # If it's a datetime/date object\n if isinstance(cell_val, (datetime, date)):\n formatted = cell_val.strftime('%Y-%m-%d')\n if formatted == target_date_str:\n return True\n # String comparison\n s = str(cell_val).strip()\n if target_date_str in s:\n return True\n try:\n target_dt = datetime.strptime(target_date_str, '%Y-%m-%d').date()\n except ValueError:\n return False\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d', '%B %d, %Y', '%b %d, %Y', '%m/%d/%y', '%d-%b-%Y', '%d-%b-%y']:\n try:\n parsed = datetime.strptime(s.split(' ')[0] if ' ' in s and ':' in s else s, fmt).date()\n if parsed == target_dt:\n return True\n except (ValueError, IndexError):\n continue\n try:\n parsed = datetime.strptime(s, '%Y-%m-%d %H:%M:%S').date()\n if parsed == target_dt:\n return True\n except ValueError:\n pass\n return False\n\n def amount_matches(cell_val, target_amount=2030.0):\n if cell_val is None:\n return False\n if isinstance(cell_val, (int, float)):\n if abs(float(cell_val) - target_amount) < 0.015:\n return True\n val = normalize_str(cell_val)\n try:\n num = float(val)\n if abs(num - target_amount) < 0.015:\n return True\n except (ValueError, TypeError):\n pass\n if '2030' in str(cell_val):\n return True\n return False\n\n def status_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).lower().replace('-', ' ').replace('_', ' ')\n return 'on hold' in s or s in ['onhold', 'on hold']\n\n def hold_code_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).upper()\n return s == 'MATCH' or 'MATCH' in s.split()\n\n def gl_matches(cell_val):\n if cell_val is None:\n return False\n return '6200-010-2200' in cell_str(cell_val)\n\n def invoice_id_matches(cell_val):\n if cell_val is None:\n return False\n return target_invoice.lower() in cell_str(cell_val).lower()\n\n def vendor_id_matches(cell_val):\n if cell_val is None:\n return False\n return 'v-00229' in cell_str(cell_val).lower()\n\n def vendor_name_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).lower()\n return 'northgate' in s and 'office' in s\n\n def po_matches(cell_val):\n if cell_val is None:\n return False\n return 'po-2025-00884' in cell_str(cell_val).lower()\n\n # Define matchers for the 10 expected values\n matchers = [\n ('Invoice Number (INV-NGO-4471)', invoice_id_matches),\n ('Vendor ID (V-00229)', vendor_id_matches),\n ('Vendor Name (Northgate Office Solutions)', vendor_name_matches),\n ('Invoice Date (2025-05-19)', lambda c: date_matches(c, '2025-05-19')),\n ('Received Date (2025-05-21)', lambda c: date_matches(c, '2025-05-21')),\n ('PO Number (PO-2025-00884)', po_matches),\n ('Invoice Amount (2030.00)', amount_matches),\n ('Status (on hold)', status_matches),\n ('Hold Code (MATCH)', hold_code_matches),\n ('GL Code (6200-010-2200)', gl_matches),\n ]\n num_matchers = len(matchers)\n\n # Search for rows that contain the target invoice\n candidate_rows = []\n for row in ir_sheet.iter_rows(min_row=2, values_only=True):\n for c in row:\n if c is not None and target_invoice.lower() in str(c).strip().lower():\n candidate_rows.append(row)\n break\n\n if not candidate_rows:\n return {'pass': False, 'score': 0.0, 'feedback': f'No row with invoice number {target_invoice} found in Invoice Register sheet.'}\n\n best_score = 0\n best_checks = {}\n best_row_str = ''\n\n for row in candidate_rows:\n row_vals = list(row)\n row_strs = [cell_str(c) for c in row_vals]\n row_str_combined = ' | '.join(row_strs)\n\n # Try sequential matching: find contiguous subsequence of cells matching matchers in order\n best_seq_score = 0\n best_seq_checks = {}\n\n for start_col in range(len(row_vals)):\n if start_col + num_matchers > len(row_vals):\n break\n checks = {}\n seq_score = 0\n for i, (name, matcher) in enumerate(matchers):\n val = row_vals[start_col + i]\n result = matcher(val)\n checks[name] = result\n if result:\n seq_score += 1\n if seq_score > best_seq_score:\n best_seq_score = seq_score\n best_seq_checks = checks\n\n # Also try loose approach: check each matcher against the whole row (non-sequential)\n loose_checks = {}\n loose_score = 0\n for name, matcher in matchers:\n found = any(matcher(c) for c in row_vals)\n loose_checks[name] = found\n if found:\n loose_score += 1\n\n # Use the better of sequential or loose for scoring\n if best_seq_score >= loose_score:\n row_score = best_seq_score\n row_checks = best_seq_checks\n else:\n row_score = loose_score\n row_checks = loose_checks\n\n if row_score > best_score:\n best_score = row_score\n best_checks = row_checks\n best_row_str = row_str_combined\n\n total_checks = num_matchers\n final_score = best_score / total_checks\n all_pass = all(best_checks.values()) and best_score == total_checks\n\n details = []\n for k, v in best_checks.items():\n details.append(f'{k}: {\"PASS\" if v else \"FAIL\"}')\n\n feedback = (f'Invoice Register row for {target_invoice} found. '\n f'Row contents: [{best_row_str}]. '\n f'Check results ({int(best_score)}/{total_checks}): {\", \".join(details)}.')\n\n return {'pass': all_pass, 'score': final_score, 'feedback': feedback}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "ec32dbcb-7347-4400-a8cd-caf6cc37e87f",
|
|
"sort_order": 1,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, the `Payment Queue` sheet must exist and must NOT contain any row referencing invoice INV-NGO-4471. If the Payment Queue sheet is absent, this criterion fails.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/ap_ledger.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n \n # Look for a Payment Queue sheet with loose matching\n pq_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n pq_sheet = wb[name]\n break\n \n if pq_sheet is None:\n # Rubric says: \"If the Payment Queue sheet is absent, this criterion fails.\"\n return {'pass': False, 'score': 0.0, 'feedback': f'No Payment Queue sheet found in ap_ledger.xlsx. Available sheets: {wb.sheetnames}. The rubric requires this sheet to exist.'}\n \n target = 'INV-NGO-4471'\n for row_idx, row in enumerate(pq_sheet.iter_rows(values_only=True), start=1):\n for cell in row:\n if cell is not None and target.lower() in str(cell).strip().lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 found in Payment Queue sheet (row {row_idx}). The rubric requires that INV-NGO-4471 must NOT appear in the Payment Queue.'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Payment Queue sheet (\"{pq_sheet.title}\") exists in ap_ledger.xlsx and INV-NGO-4471 is correctly absent from it.'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "c054354f-bd75-4b58-b8a2-a04d8fc978ef",
|
|
"sort_order": 2,
|
|
"rubric_text": "In file `match_log_2025_Q2.xlsx`, there must be a row with the following values: 2025-05-21 for Match Date, PO-2025-00884 for PO Number, V-00229 for Vendor ID, Northgate Office Solutions for Vendor Name, INV-NGO-4471 for Invoice Number, 1950.00 (must be exact value, can have dollar signifiers) for PO Amount, 2030.00 (must be exact value, can have dollar signifiers) for Invoice Amount, 97.50 (must be exact value, can have dollar signifiers) for Unit Price (PO), 101.50 (must be exact value, can have dollar signifiers) for Unit Price (Invoice), 4.10 (±0.5% is allowed, can have % sign) for Price Variance %, 4.00 (must be exact value, can have dollar signifiers) for Price Variance $, 20 (must be exact value) for Qty (PO), 20 (must be exact value) for Qty (Invoice), 80.00 (must be exact value, can have dollar signifiers) for Total Variance $, 4.10 (±0.5% is allowed, can have % sign) for Total Variance %, \"Pass\" for Unit Price Result, \"Fail\" for Total Amount Result, \"MATCH Hold\" for Overall Result, \"MATCH for Hold Code\"",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/match_log_2025_Q2.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_log_2025_Q2.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n\n target_invoice = 'INV-NGO-4471'\n found_rows = []\n found_sheet = None\n\n for ws in wb.worksheets:\n for row in ws.iter_rows(values_only=True):\n row_strs = [str(c).strip() if c is not None else '' for c in row]\n if target_invoice in row_strs:\n found_rows.append(row)\n found_sheet = ws.title\n break\n if found_rows:\n break\n\n if not found_rows:\n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 not found in any sheet of match_log_2025_Q2.xlsx'}\n\n raw_row = found_rows[0]\n\n def parse_num(val):\n \"\"\"Strip $, commas, %, whitespace and convert to float.\"\"\"\n s = str(val).replace('$', '').replace(',', '').replace('%', '').strip()\n return float(s)\n\n def normalize(val):\n \"\"\"Return a cleaned string representation.\"\"\"\n if val is None:\n return ''\n return str(val).strip()\n\n def check_date(val, expected_str):\n \"\"\"Check if val represents the date 2025-05-21.\"\"\"\n if isinstance(val, (datetime, date)):\n if isinstance(val, datetime):\n d = val.date()\n else:\n d = val\n return d == date(2025, 5, 21)\n s = normalize(val)\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d', '%m/%d/%y']:\n try:\n parsed = datetime.strptime(s.split(' ')[0], fmt).date()\n if parsed == date(2025, 5, 21):\n return True\n except:\n pass\n if '2025-05-21' in s or '05/21/2025' in s or '5/21/2025' in s or '21/05/2025' in s or '2025/05/21' in s:\n return True\n return False\n\n row_vals = [normalize(v) for v in raw_row]\n\n def find_in_row_str(expected):\n for v in row_vals:\n if v.lower() == expected.lower():\n return True\n return False\n\n def find_in_row_str_contains(expected):\n for v in row_vals:\n if expected.lower() in v.lower():\n return True\n return False\n\n # Collect all numeric values from the row\n numeric_cells = []\n for i, v in enumerate(raw_row):\n try:\n num = parse_num(v)\n numeric_cells.append((i, num, v))\n except:\n pass\n\n def find_exact_num_in_row(expected, tolerance=0.011, exclude_indices=None):\n if exclude_indices is None:\n exclude_indices = set()\n for i, num, raw in numeric_cells:\n if i not in exclude_indices and abs(num - expected) <= tolerance:\n return i, num, raw\n return None\n\n def find_pct_in_row(expected_pct, tolerance=0.5, exclude_indices=None):\n if exclude_indices is None:\n exclude_indices = set()\n for i, num, raw in numeric_cells:\n if i in exclude_indices:\n continue\n if abs(num - expected_pct) <= tolerance:\n return i, num, raw\n if abs(num * 100 - expected_pct) <= tolerance:\n return i, num, raw\n return None\n\n issues = []\n details = []\n used_indices = set()\n\n # 1. Date: 2025-05-21\n date_found = False\n for v in raw_row:\n if check_date(v, '2025-05-21'):\n date_found = True\n details.append(f'Date 2025-05-21: found \\u2713')\n break\n if not date_found:\n issues.append(f'Date 2025-05-21 not found in row. Row values: {row_vals}')\n\n # 2. PO-2025-00884\n if find_in_row_str('PO-2025-00884'):\n details.append('PO Number PO-2025-00884: found \\u2713')\n else:\n issues.append(f'PO-2025-00884 not found in row')\n\n # 3. V-00229\n if find_in_row_str('V-00229'):\n details.append('Vendor ID V-00229: found \\u2713')\n else:\n issues.append(f'V-00229 not found in row')\n\n # 4. Northgate Office Solutions\n if find_in_row_str_contains('Northgate Office Solutions'):\n details.append('Vendor Name Northgate Office Solutions: found \\u2713')\n else:\n issues.append(f'Northgate Office Solutions not found in row')\n\n # 5. INV-NGO-4471 (already confirmed)\n details.append('Invoice INV-NGO-4471: found \\u2713')\n\n # 6. 1950.00 (PO Amount)\n result = find_exact_num_in_row(1950.0, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'PO Amount 1950.00: found {result[2]} \\u2713')\n else:\n issues.append('Value 1950.00 (PO Amount) not found in row')\n\n # 7. 2030.00 (Invoice Amount)\n result = find_exact_num_in_row(2030.0, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Invoice Amount 2030.00: found {result[2]} \\u2713')\n else:\n issues.append('Value 2030.00 (Invoice Amount) not found in row')\n\n # 8. 97.50 (PO Unit Price)\n result = find_exact_num_in_row(97.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'PO Unit Price 97.50: found {result[2]} \\u2713')\n else:\n issues.append('Value 97.50 (PO Unit Price) not found in row')\n\n # 9. 101.50 (Invoice Unit Price)\n result = find_exact_num_in_row(101.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Invoice Unit Price 101.50: found {result[2]} \\u2713')\n else:\n issues.append('Value 101.50 (Invoice Unit Price) not found in row')\n\n # 10. 4.10 (\\u00b10.5%) - Price Variance %\n result = find_pct_in_row(4.1, tolerance=0.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Price Variance % ~4.10: found {result[2]} \\u2713')\n else:\n issues.append('Value ~4.10% (Price Variance %) not found in row')\n\n # 11. 4.00 (Price Variance $)\n result = find_exact_num_in_row(4.0, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Price Variance $ 4.00: found {result[2]} \\u2713')\n else:\n issues.append('Value 4.00 (Price Variance $) not found in row')\n\n # 12. 20 (PO Qty)\n result = find_exact_num_in_row(20.0, tolerance=0.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'PO Qty 20: found {result[2]} \\u2713')\n else:\n issues.append('Value 20 (PO Qty) not found in row')\n\n # 13. 20 (Invoice Qty)\n result = find_exact_num_in_row(20.0, tolerance=0.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Invoice Qty 20: found {result[2]} \\u2713')\n else:\n issues.append('Second value of 20 (Invoice Qty) not found in row')\n\n # 14. 80.00 (Total Variance $)\n result = find_exact_num_in_row(80.0, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Total Variance $ 80.00: found {result[2]} \\u2713')\n else:\n issues.append('Value 80.00 (Total Variance $) not found in row')\n\n # 15. 4.10 (\\u00b10.5%) - Total Variance %\n result = find_pct_in_row(4.1, tolerance=0.5, exclude_indices=used_indices)\n if result:\n used_indices.add(result[0])\n details.append(f'Total Variance % ~4.10: found {result[2]} \\u2713')\n else:\n issues.append('Second value ~4.10% (Total Variance %) not found in row')\n\n # 16. \"Pass\" (Unit Price Result)\n if find_in_row_str('Pass'):\n details.append('Unit Price Result \"Pass\": found \\u2713')\n else:\n issues.append('\"Pass\" not found in row')\n\n # 17. \"Fail\" (Total Amount Result)\n if find_in_row_str('Fail'):\n details.append('Total Amount Result \"Fail\": found \\u2713')\n else:\n issues.append('\"Fail\" not found in row')\n\n # 18. \"MATCH Hold\" (Overall Result) - look for cell containing both \"match\" and \"hold\"\n match_hold_found = False\n for v in row_vals:\n vl = v.lower()\n if 'match' in vl and 'hold' in vl:\n match_hold_found = True\n details.append(f'Overall Result \"MATCH Hold\": found \"{v}\" \\u2713')\n break\n if not match_hold_found:\n issues.append('\"MATCH Hold\" not found in row')\n\n # 19. \"MATCH\" for Hold Code - accept \"MATCH\" as a standalone cell or within hold code context\n match_code_found = False\n for v in row_vals:\n vl = v.lower().strip()\n # Accept exact \"match\" or common variants\n if vl == 'match':\n match_code_found = True\n details.append(f'Hold Code \"MATCH\": found \"{v}\" \\u2713')\n break\n if not match_code_found:\n # Also accept '3-way match', 'three-way match', 'three way match', '3 way match'\n for v in row_vals:\n vl = v.lower().strip()\n if vl in ['3-way match', 'three-way match', 'three way match', '3 way match']:\n match_code_found = True\n details.append(f'Hold Code \"MATCH\": found \"{v}\" \\u2713')\n break\n if not match_code_found:\n # Be even more lenient: accept if \"match\" appears in a cell that is NOT the \"MATCH Hold\" cell\n for v in row_vals:\n vl = v.lower().strip()\n if 'match' in vl and 'hold' not in vl:\n match_code_found = True\n details.append(f'Hold Code \"MATCH\": found \"{v}\" \\u2713')\n break\n if not match_code_found:\n issues.append('\"MATCH\" (as Hold Code) not found in row')\n\n if issues:\n feedback = 'Match log row issues: ' + '; '.join(issues)\n if details:\n feedback += ' | Passed checks: ' + '; '.join(details)\n return {'pass': False, 'score': 0.0, 'feedback': feedback}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'Match log row for INV-NGO-4471 verified: ' + '; '.join(details)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "43f55ad7-21c3-4486-914d-c86bf87238d4",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `match_result_INV-NGO-4471.txt` (in workspace), the file must reference: the \"MATCH hold\" outcome, the $80.00 total amount variance and/or the 4.1% figure, the ±$50 and/or ±2% thresholds, the unit price check passing, and the next action of vendor email/price clarification.",
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file in workspace\n candidates = list(Path(workspace_path).rglob('match_result_INV-NGO-4471.txt'))\n \n if not candidates:\n # Try glob with case variations\n candidates = list(Path(workspace_path).rglob('match_result*4471*.txt'))\n \n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_result_INV-NGO-4471.txt not found in workspace'}\n \n content = candidates[0].read_text(encoding='utf-8', errors='replace')\n content_lower = content.lower()\n issues = []\n found_items = []\n \n # 1. Check MATCH hold outcome\n match_hold_found = False\n if 'match hold' in content_lower or 'match_hold' in content_lower:\n match_hold_found = True\n elif 'match' in content_lower and 'hold' in content_lower:\n match_hold_found = True\n elif 'hold' in content_lower and ('three-way' in content_lower or '3-way' in content_lower or 'three way' in content_lower or '3 way' in content_lower):\n match_hold_found = True\n elif 'exception' in content_lower and ('hold' in content_lower or 'flag' in content_lower or 'fail' in content_lower):\n match_hold_found = True\n elif 'fail' in content_lower and 'match' in content_lower:\n match_hold_found = True\n elif 'mismatch' in content_lower and ('hold' in content_lower or 'exception' in content_lower or 'flag' in content_lower):\n match_hold_found = True\n elif 'on hold' in content_lower:\n match_hold_found = True\n elif 'placed' in content_lower and 'hold' in content_lower:\n match_hold_found = True\n \n if match_hold_found:\n found_items.append('MATCH hold outcome')\n else:\n issues.append('MATCH hold outcome not mentioned')\n \n # 2. Check $80 variance or ~4.1%\n has_80 = bool(re.search(r'\\$?80(\\.00)?', content))\n has_4_1 = bool(re.search(r'4\\.0[0-9]|4\\.1[0-9]?', content))\n if has_80 or has_4_1:\n parts = []\n if has_80:\n parts.append('$80 variance')\n if has_4_1:\n parts.append('~4.1% figure')\n found_items.append(' and '.join(parts))\n else:\n issues.append('Neither $80.00 variance nor ~4.1% figure found in file')\n \n # 3. Check threshold references (±$50 and/or ±2%)\n has_50 = bool(re.search(r'\\$?50', content))\n has_2pct = bool(re.search(r'2\\s*%|±\\s*2|\\+/?-\\s*2|2\\s*percent', content_lower))\n if has_50 or has_2pct:\n parts = []\n if has_50:\n parts.append('$50 threshold')\n if has_2pct:\n parts.append('2% threshold')\n found_items.append(' and '.join(parts))\n else:\n issues.append('Neither $50 threshold nor 2% threshold referenced in file')\n \n # 4. Check unit price check passing\n unit_price_found = False\n if 'unit price' in content_lower or 'unit_price' in content_lower:\n unit_price_found = True\n elif 'per-unit' in content_lower or 'per unit' in content_lower:\n unit_price_found = True\n elif 'price' in content_lower and any(w in content_lower for w in ['pass', 'passed', 'match', 'matched', 'correct', 'ok', 'good', 'confirmed', 'agree', 'consistent', 'verified', 'check']):\n unit_price_found = True\n elif 'line item' in content_lower and any(w in content_lower for w in ['pass', 'passed', 'match', 'matched', 'correct', 'verified', 'check']):\n unit_price_found = True\n \n if unit_price_found:\n found_items.append('unit price check')\n else:\n issues.append('Unit price check pass not mentioned')\n \n # 5. Check next action: vendor email / price clarification\n has_vendor_action = False\n # Multiple ways this could be expressed\n if ('email' in content_lower or 'contact' in content_lower or 'notify' in content_lower or 'reach out' in content_lower or 'send' in content_lower or 'communicate' in content_lower) and ('vendor' in content_lower or 'supplier' in content_lower or 'northgate' in content_lower):\n has_vendor_action = True\n if 'clarif' in content_lower or 'discrepanc' in content_lower or 'dispute' in content_lower or 'reconcil' in content_lower:\n has_vendor_action = True\n if 'vendor' in content_lower and ('price' in content_lower or 'amount' in content_lower or 'invoice' in content_lower or 'review' in content_lower or 'resolution' in content_lower or 'resolve' in content_lower):\n has_vendor_action = True\n if 'northgate' in content_lower and ('contact' in content_lower or 'email' in content_lower or 'notify' in content_lower or 'clarif' in content_lower or 'reach' in content_lower):\n has_vendor_action = True\n # Check for SOP step references as alternative evidence\n if re.search(r'6\\.1|step\\s*2\\s*b|2b', content_lower):\n has_vendor_action = True\n # Accept general next steps mentioning investigation/follow-up with vendor\n if ('next step' in content_lower or 'action' in content_lower or 'follow' in content_lower) and ('vendor' in content_lower or 'northgate' in content_lower or 'supplier' in content_lower):\n has_vendor_action = True\n \n if has_vendor_action:\n found_items.append('vendor email/price clarification next step')\n else:\n issues.append('Vendor email/price clarification next step not mentioned')\n \n # Scoring: partial credit\n total_checks = 5\n passed_checks = total_checks - len(issues)\n score = passed_checks / total_checks\n \n if not issues:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'match_result_INV-NGO-4471.txt correctly references: {\", \".join(found_items)}'\n }\n else:\n return {\n 'pass': False,\n 'score': score,\n 'feedback': f'match_result_INV-NGO-4471.txt issues: {\"; \".join(issues)}. Found: {\", \".join(found_items) if found_items else \"nothing matching\"}'\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "fb6593a0-7e58-4dac-9515-c6979c7a9783",
|
|
"sort_order": 4,
|
|
"rubric_text": "In `slack_data.json`, there must be a message to the #ap-exceptions channel with: \"text\": \"PRICE VARIANCE: Northgate Office Solutions, V-00229, Invoice INV-NGO-4471, PO Price $97.50, Invoice Price $101.50, Variance 4.10%\" (the 4.10% can be ±0.5%)",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data'}\n \n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n \n with open(slack_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n \n # Find #ap-exceptions channel\n channels = data.get('channels', {})\n exceptions_channel_id = None\n for cid, cdata in channels.items():\n if cdata.get('name', '').lower() == 'ap-exceptions':\n exceptions_channel_id = cid\n break\n \n if exceptions_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#ap-exceptions channel not found in Slack data'}\n \n messages = data.get('messages', {}).get(exceptions_channel_id, [])\n \n target = 'INV-NGO-4471'\n relevant_msgs = [m for m in messages if isinstance(m, dict) and target in m.get('text', '')]\n \n if not relevant_msgs:\n return {'pass': False, 'score': 0.0, 'feedback': f'No message referencing {target} found in #ap-exceptions'}\n \n issues_per_msg = []\n for msg in relevant_msgs:\n text = msg.get('text', '')\n text_lower = text.lower()\n msg_issues = []\n \n # Check for PRICE VARIANCE mention (case insensitive)\n if 'price variance' not in text_lower:\n msg_issues.append('\"PRICE VARIANCE\" label missing')\n \n # Check vendor name (loosely)\n if 'northgate' not in text_lower:\n msg_issues.append('vendor name (Northgate Office Solutions) missing')\n \n # Check vendor ID (case insensitive)\n if 'v-00229' not in text_lower:\n msg_issues.append('vendor ID V-00229 missing')\n \n # PO price: accept $97.50 or unit-level equivalents, or total $1,950.00\n has_po_price = any(x in text for x in ['97.50', '97.5', '1950', '1,950'])\n if not has_po_price:\n msg_issues.append('PO price ($97.50 or $1,950.00) missing')\n \n # Invoice price: accept $101.50 or total $2,030.00\n has_inv_price = any(x in text for x in ['101.50', '101.5', '2030', '2,030'])\n if not has_inv_price:\n msg_issues.append('Invoice price ($101.50 or $2,030.00) missing')\n \n # Variance ~4.10% (±0.5% so accept 3.6% to 4.6%)\n # Look for percentage patterns in text\n pct_matches = re.findall(r'(\\d+\\.\\d+)\\s*%', text)\n has_variance = False\n for pct_str in pct_matches:\n try:\n pct_val = float(pct_str)\n if 3.6 <= pct_val <= 4.6:\n has_variance = True\n break\n except ValueError:\n pass\n # Also accept integer-like \"4%\" loosely\n if not has_variance:\n int_pct_matches = re.findall(r'(\\d+)\\s*%', text)\n for pct_str in int_pct_matches:\n try:\n pct_val = float(pct_str)\n if 3.6 <= pct_val <= 4.6:\n has_variance = True\n break\n except ValueError:\n pass\n if not has_variance:\n msg_issues.append('Variance percentage (~4.10% \\u00b10.5%) missing')\n \n if not msg_issues:\n return {'pass': True, 'score': 1.0, 'feedback': f'#ap-exceptions contains a valid PRICE VARIANCE message for INV-NGO-4471 with all required fields. Message text: {text[:300]}'}\n issues_per_msg.append(f'Message missing: {\"; \".join(msg_issues)}')\n \n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 messages found in #ap-exceptions but have issues: ' + ' | '.join(issues_per_msg)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "4ad6ef7b-6e46-4e91-8e37-a897671ff3de",
|
|
"sort_order": 5,
|
|
"rubric_text": "In `mailbox.json`, there must be an email with the following:\n\"folder\": \"Sent\",\n\"subject\": \"Meridian Partners - Price Discrepancy on Invoice INV-NGO-4471\",\n\"to_addr\" must contain \"@northgateoffice.com\",\nand the body text must contain the PO price ($1950.00 or $97.50), invoiced price ($2030.00 or $101.50) and the variance percentage (4.10%, can be ±0.5%)",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox.json for sent email'}\n\n mailbox_file = Path(external_services_path) / 'mailbox.json'\n if not mailbox_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n with open(mailbox_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n target_invoice = 'INV-NGO-4471'\n\n # Find emails sent to @northgateoffice.com with the invoice number in the subject\n matching = []\n for e in sent_emails:\n to_addr = e.get('to_addr', '').lower()\n subject = e.get('subject', '')\n if '@northgateoffice.com' in to_addr and target_invoice.lower() in subject.lower():\n matching.append(e)\n\n if not matching:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to @northgateoffice.com with {target_invoice} in subject. Sent emails: {[{\"to\": e.get(\"to_addr\"), \"subject\": e.get(\"subject\")} for e in sent_emails]}'}\n\n issues_all = []\n for email in matching:\n subject = email.get('subject', '')\n body = email.get('body_text', '') or email.get('body', '') or ''\n body_lower = body.lower()\n subject_lower = subject.lower()\n email_issues = []\n\n # (1) Subject must contain \"Price Discrepancy\"\n if 'price discrepancy' not in subject_lower:\n email_issues.append(f'Subject does not contain \"Price Discrepancy\". Subject was: \"{subject}\"')\n\n # (2) Body must reference PO price: $1950.00 or $97.50 (total or unit)\n po_price_found = False\n for token in ['1950.00', '1950', '1,950.00', '1,950', '97.50', '97.5']:\n if token in body:\n po_price_found = True\n break\n if not po_price_found:\n email_issues.append('PO price ($1950.00 or $97.50) not found in body')\n\n # (3) Body must reference invoiced price: $2030.00 or $101.50 (total or unit)\n inv_price_found = False\n for token in ['2030.00', '2030', '2,030.00', '2,030', '101.50', '101.5']:\n if token in body:\n inv_price_found = True\n break\n if not inv_price_found:\n email_issues.append('Invoiced price ($2030.00 or $101.50) not found in body')\n\n # (4) Body must reference variance percentage ~4.10% (accept 3.60% to 4.60%, i.e. 4.10 +/- 0.5)\n variance_found = False\n # Search for any percentage-like number in the body and check range\n pct_pattern = re.compile(r'(\\d+\\.\\d+)\\s*%')\n for m in pct_pattern.finditer(body):\n val = float(m.group(1))\n if 3.60 <= val <= 4.60:\n variance_found = True\n break\n # Also accept bare mentions like \"4.1\" or \"4.10\" without % sign near context words\n if not variance_found:\n bare_pattern = re.compile(r'\\b(\\d+\\.\\d+)\\b')\n for m in bare_pattern.finditer(body):\n val = float(m.group(1))\n if 3.60 <= val <= 4.60:\n variance_found = True\n break\n if not variance_found:\n email_issues.append('Variance percentage (~4.10%, +/-0.5%) not found in body')\n\n if not email_issues:\n return {'pass': True, 'score': 1.0, 'feedback': f'Sent email to @northgateoffice.com found with correct subject containing \"Price Discrepancy\" and \"{target_invoice}\", and body references PO price, invoice price, and variance ~4.1%.'}\n issues_all.append(f'Email (subject=\"{subject}\"): ' + '; '.join(email_issues))\n\n return {'pass': False, 'score': 0.0, 'feedback': 'Matching sent emails have issues: ' + ' | '.join(issues_all)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1774379677398",
|
|
"sort_order": 6,
|
|
"rubric_text": "In `slack_data.json` (in external_data/final) the channels #ap-escalations, #ap-payments, #ap-approvals must not contain any messages referencing INV-NGO-4471.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n # Locate the final slack_data.json\n final_candidates = []\n if external_services_path:\n final_candidates.append(Path(external_services_path) / \"slack_data.json\")\n final_candidates.append(workspace_path / \"external_data\" / \"final\" / \"slack_data.json\")\n\n final_slack_path = None\n for p in final_candidates:\n if p.exists():\n final_slack_path = p\n break\n\n if final_slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No final slack_data.json found; cannot confirm any messages were added, so passing by default (generous interpretation).\"}\n\n # Load final data\n try:\n with open(final_slack_path, 'r', encoding='utf-8') as f:\n final_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse final slack_data.json ({e}); passing generously.\"}\n\n # Helper to unwrap possible nested structure\n def unwrap(data):\n if isinstance(data, dict) and \"slack_data\" in data:\n return data[\"slack_data\"]\n return data\n\n final_data = unwrap(final_data)\n\n # The three channels we care about\n target_channel_names = {\"ap-escalations\", \"ap-payments\", \"ap-approvals\"}\n\n # Build a map from channel name -> channel id\n channels_info = final_data.get(\"channels\", {}) if isinstance(final_data, dict) else {}\n if not isinstance(channels_info, dict):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No channels dict found in slack_data.json; pass by default.\"}\n\n channel_name_to_id = {}\n for channel_id, channel_info in channels_info.items():\n if isinstance(channel_info, dict):\n name = (channel_info.get(\"name\", \"\") or \"\").strip().lstrip(\"#\").lower()\n if name in target_channel_names:\n channel_name_to_id[name] = channel_id\n\n if not channel_name_to_id:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"None of the target channels (#ap-escalations, #ap-payments, #ap-approvals) found in final slack_data.json; pass by default.\"}\n\n messages_data = final_data.get(\"messages\", {}) if isinstance(final_data, dict) else {}\n if not isinstance(messages_data, dict):\n messages_data = {}\n\n invoice_ref = \"INV-NGO-4471\"\n # Use a case-insensitive pattern that matches the invoice number with or without dashes\n pattern = re.compile(re.escape(invoice_ref), re.IGNORECASE)\n\n violations = []\n\n for chan_name, chan_id in channel_name_to_id.items():\n chan_msgs = messages_data.get(chan_id, [])\n if not isinstance(chan_msgs, list):\n continue\n for msg in chan_msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\") or \"\"\n if pattern.search(text):\n user = msg.get(\"user\", \"unknown\")\n preview = text[:150]\n violations.append(f\" - #{chan_name} (channel {chan_id}): User {user}: {preview}\")\n\n if violations:\n detail = \"\\n\".join(violations)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violations)} message(s) referencing {invoice_ref} in the restricted channels:\\n{detail}\"\n }\n\n checked = \", \".join([f\"#{n} ({channel_name_to_id[n]})\" for n in sorted(channel_name_to_id.keys())])\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No messages referencing {invoice_ref} found in checked channels: {checked}.\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1774605866883",
|
|
"sort_order": 7,
|
|
"rubric_text": "In file `ap_ledger.xlsx`, the `PO Register` sheet must contain the following cell values: G5=97.50 (can have dollar signifier), I5=1950.00 (can have dollar signifier)",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{file_path}' does not exist.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open '{file_path}': {e}\"}\n \n # Check that 'PO Register' sheet exists\n sheet_names = wb.sheetnames\n po_register_sheet = None\n for name in sheet_names:\n if name.strip().lower() == \"po register\":\n po_register_sheet = wb[name]\n break\n \n if po_register_sheet is None:\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'PO Register' not found in ap_ledger.xlsx. Sheets found: {sheet_names}\"}\n \n def parse_numeric(val):\n \"\"\"Parse a cell value to a float, stripping dollar signs, commas, whitespace.\"\"\"\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return float(val)\n s = str(val).strip()\n # Remove dollar sign, commas, whitespace\n s = s.replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except (ValueError, TypeError):\n return None\n \n g5_val = po_register_sheet['G5'].value\n i5_val = po_register_sheet['I5'].value\n \n g5_num = parse_numeric(g5_val)\n i5_num = parse_numeric(i5_val)\n \n issues = []\n passed_checks = []\n \n # Check G5 = 97.50 with ±1% tolerance\n expected_g5 = 97.50\n if g5_num is None:\n issues.append(f\"G5: expected {expected_g5}, but got '{g5_val}' (could not parse as number)\")\n elif abs(g5_num - expected_g5) > expected_g5 * 0.01:\n issues.append(f\"G5: expected {expected_g5}, but got {g5_num} (raw: '{g5_val}')\")\n else:\n passed_checks.append(f\"G5={g5_num} (expected {expected_g5})\")\n \n # Check I5 = 1950.00 with ±1% tolerance\n expected_i5 = 1950.00\n if i5_num is None:\n issues.append(f\"I5: expected {expected_i5}, but got '{i5_val}' (could not parse as number)\")\n elif abs(i5_num - expected_i5) > expected_i5 * 0.01:\n issues.append(f\"I5: expected {expected_i5}, but got {i5_num} (raw: '{i5_val}')\")\n else:\n passed_checks.append(f\"I5={i5_num} (expected {expected_i5})\")\n \n wb.close()\n \n if issues:\n score = len(passed_checks) / (len(passed_checks) + len(issues))\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"PO Register cell value mismatches: {'; '.join(issues)}. Passed: {'; '.join(passed_checks) if passed_checks else 'none'}.\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PO Register sheet verified successfully. {'; '.join(passed_checks)}.\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1774610562264",
|
|
"sort_order": 8,
|
|
"rubric_text": "File `match_result_INV-NGO-4471.txt` must exist in the workspace.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"match_result_INV-NGO-4471.txt\"\n if target_file.exists() and target_file.is_file():\n # Additionally check that the file is not empty (i.e., it has some content)\n content = target_file.read_text(encoding='utf-8', errors='replace').strip()\n if len(content) > 0:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"File 'match_result_INV-NGO-4471.txt' exists and contains content ({len(content)} characters).\"\n }\n else:\n # Be generous: even an empty file technically \"exists\", so still pass\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"File 'match_result_INV-NGO-4471.txt' exists but is empty. Passing since the rubric only requires existence.\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'match_result_INV-NGO-4471.txt' was not found in the workspace at '{workspace_path}'.\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|