66 lines
46 KiB
JSON
66 lines
46 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "5fc6c7a8-0f5f-436b-b361-97cedf065876",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "File `ap_ledger.xlsx` (in workspace) must have a row in the `Invoice Register` sheet with the following values: INV-NGO-4471 for Invoice Number, V-00229 for Vendor ID, Northgate Office Solutions for Vendor Name, 2025-05-19 for Invoice Date, 2025-05-21 for Received Date, PO-2025-00884 for PO Number, 2030.00 (can have dollar signifier such as $ or US$) for Invoice Amount, on hold for Status, MATCH for Hold Code, 6200-010-2200 for GL Code",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/ap_ledger.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n\n # Find Invoice Register sheet (case-insensitive)\n ir_sheet = None\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n ir_sheet = wb[name]\n break\n\n if ir_sheet is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'No Invoice Register sheet found. Sheets present: {wb.sheetnames}.'}\n\n target_invoice = 'INV-NGO-4471'\n\n def cell_str(c):\n if c is None:\n return ''\n return str(c).strip()\n\n def normalize_str(s):\n return str(s).strip().lower().replace('$', '').replace(',', '').replace(' ', '').replace('us$', '')\n\n def date_matches(cell_val, target_date_str):\n \"\"\"Check if a cell value matches a target date like '2025-05-19'.\"\"\"\n if cell_val is None:\n return False\n # If it's a datetime/date object\n if isinstance(cell_val, (datetime, date)):\n formatted = cell_val.strftime('%Y-%m-%d')\n if formatted == target_date_str:\n return True\n # String comparison\n s = str(cell_val).strip()\n if target_date_str in s:\n return True\n try:\n target_dt = datetime.strptime(target_date_str, '%Y-%m-%d').date()\n except ValueError:\n return False\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d', '%B %d, %Y', '%b %d, %Y', '%m/%d/%y', '%d-%b-%Y', '%d-%b-%y']:\n try:\n parsed = datetime.strptime(s.split(' ')[0] if ' ' in s and ':' in s else s, fmt).date()\n if parsed == target_dt:\n return True\n except (ValueError, IndexError):\n continue\n try:\n parsed = datetime.strptime(s, '%Y-%m-%d %H:%M:%S').date()\n if parsed == target_dt:\n return True\n except ValueError:\n pass\n return False\n\n def amount_matches(cell_val, target_amount=2030.0):\n if cell_val is None:\n return False\n if isinstance(cell_val, (int, float)):\n if abs(float(cell_val) - target_amount) < 0.015:\n return True\n val = normalize_str(cell_val)\n try:\n num = float(val)\n if abs(num - target_amount) < 0.015:\n return True\n except (ValueError, TypeError):\n pass\n if '2030' in str(cell_val):\n return True\n return False\n\n def status_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).lower().replace('-', ' ').replace('_', ' ')\n return 'on hold' in s or s in ['onhold', 'on hold']\n\n def hold_code_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).upper()\n return s == 'MATCH' or 'MATCH' in s.split()\n\n def gl_matches(cell_val):\n if cell_val is None:\n return False\n return '6200-010-2200' in cell_str(cell_val)\n\n def invoice_id_matches(cell_val):\n if cell_val is None:\n return False\n return target_invoice.lower() in cell_str(cell_val).lower()\n\n def vendor_id_matches(cell_val):\n if cell_val is None:\n return False\n return 'v-00229' in cell_str(cell_val).lower()\n\n def vendor_name_matches(cell_val):\n if cell_val is None:\n return False\n s = cell_str(cell_val).lower()\n return 'northgate' in s and 'office' in s\n\n def po_matches(cell_val):\n if cell_va
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ec32dbcb-7347-4400-a8cd-caf6cc37e87f",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `ap_ledger.xlsx`, the `Payment Queue` sheet must exist and must NOT contain any row referencing invoice INV-NGO-4471. If the Payment Queue sheet is absent, this criterion fails.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/ap_ledger.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ap_ledger.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n \n # Look for a Payment Queue sheet with loose matching\n pq_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n pq_sheet = wb[name]\n break\n \n if pq_sheet is None:\n # Rubric says: \"If the Payment Queue sheet is absent, this criterion fails.\"\n return {'pass': False, 'score': 0.0, 'feedback': f'No Payment Queue sheet found in ap_ledger.xlsx. Available sheets: {wb.sheetnames}. The rubric requires this sheet to exist.'}\n \n target = 'INV-NGO-4471'\n for row_idx, row in enumerate(pq_sheet.iter_rows(values_only=True), start=1):\n for cell in row:\n if cell is not None and target.lower() in str(cell).strip().lower():\n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 found in Payment Queue sheet (row {row_idx}). The rubric requires that INV-NGO-4471 must NOT appear in the Payment Queue.'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Payment Queue sheet (\"{pq_sheet.title}\") exists in ap_ledger.xlsx and INV-NGO-4471 is correctly absent from it.'}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "c054354f-bd75-4b58-b8a2-a04d8fc978ef",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In file `match_log_2025_Q2.xlsx`, there must be a row with the following values: 2025-05-21 for Match Date, PO-2025-00884 for PO Number, V-00229 for Vendor ID, Northgate Office Solutions for Vendor Name, INV-NGO-4471 for Invoice Number, 1950.00 (must be exact value, can have dollar signifiers) for PO Amount, 2030.00 (must be exact value, can have dollar signifiers) for Invoice Amount, 97.50 (must be exact value, can have dollar signifiers) for Unit Price (PO), 101.50 (must be exact value, can have dollar signifiers) for Unit Price (Invoice), 4.10 (±0.5% is allowed, can have % sign) for Price Variance %, 4.00 (must be exact value, can have dollar signifiers) for Price Variance $, 20 (must be exact value) for Qty (PO), 20 (must be exact value) for Qty (Invoice), 80.00 (must be exact value, can have dollar signifiers) for Total Variance $, 4.10 (±0.5% is allowed, can have % sign) for Total Variance %, \"Pass\" for Unit Price Result, \"Fail\" for Total Amount Result, \"MATCH Hold\" for Overall Result, \"MATCH for Hold Code\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/match_log_2025_Q2.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_log_2025_Q2.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n\n target_invoice = 'INV-NGO-4471'\n found_rows = []\n found_sheet = None\n\n for ws in wb.worksheets:\n for row in ws.iter_rows(values_only=True):\n row_strs = [str(c).strip() if c is not None else '' for c in row]\n if target_invoice in row_strs:\n found_rows.append(row)\n found_sheet = ws.title\n break\n if found_rows:\n break\n\n if not found_rows:\n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 not found in any sheet of match_log_2025_Q2.xlsx'}\n\n raw_row = found_rows[0]\n\n def parse_num(val):\n \"\"\"Strip $, commas, %, whitespace and convert to float.\"\"\"\n s = str(val).replace('$', '').replace(',', '').replace('%', '').strip()\n return float(s)\n\n def normalize(val):\n \"\"\"Return a cleaned string representation.\"\"\"\n if val is None:\n return ''\n return str(val).strip()\n\n def check_date(val, expected_str):\n \"\"\"Check if val represents the date 2025-05-21.\"\"\"\n if isinstance(val, (datetime, date)):\n if isinstance(val, datetime):\n d = val.date()\n else:\n d = val\n return d == date(2025, 5, 21)\n s = normalize(val)\n for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d', '%m/%d/%y']:\n try:\n parsed = datetime.strptime(s.split(' ')[0], fmt).date()\n if parsed == date(2025, 5, 21):\n return True\n except:\n pass\n if '2025-05-21' in s or '05/21/2025' in s or '5/21/2025' in s or '21/05/2025' in s or '2025/05/21' in s:\n return True\n return False\n\n row_vals = [normalize(v) for v in raw_row]\n\n def find_in_row_str(expected):\n for v in row_vals:\n if v.lower() == expected.lower():\n return True\n return False\n\n def find_in_row_str_contains(expected):\n for v in row_vals:\n if expected.lower() in v.lower():\n return True\n return False\n\n # Collect all numeric values from the row\n numeric_cells = []\n for i, v in enumerate(raw_row):\n try:\n num = parse_num(v)\n numeric_cells.append((i, num, v))\n except:\n pass\n\n def find_exact_num_in_row(expected, tolerance=0.011, exclude_indices=None):\n if exclude_indices is None:\n exclude_indices = set()\n for i, num, raw in numeric_cells:\n if i not in exclude_indices and abs(num - expected) <= tolerance:\n return i, num, raw\n return None\n\n def find_pct_in_row(expected_pct, tolerance=0.5, exclude_indices=None):\n if exclude_indices is None:\n exclude_indices = set()\n for i, num, raw in numeric_cells:\n if i in exclude_indices:\n continue\n if abs(num - expected_pct) <= tolerance:\n return i, num, raw\n if abs(num * 100 - expected_pct) <= tolerance:\n return i, num, raw\n return None\n\n issues = []\n details = []\n used_indices = set()\n\n # 1. Date: 2025-05-21\n date_found = False\n for v in raw_row:\n if check_date(v, '2025-05-21'):\n date_found = True\n details.append(f'Date 2025-05-21: found \\u2713')\n break\n if not date_found:\n issues.append(f'Date 2025-05-21 not found in row. Row values: {row_vals}')\n\n #
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "43f55ad7-21c3-4486-914d-c86bf87238d4",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `match_result_INV-NGO-4471.txt` (in workspace), the file must reference: the \"MATCH hold\" outcome, the $80.00 total amount variance and/or the 4.1% figure, the ±$50 and/or ±2% thresholds, the unit price check passing, and the next action of vendor email/price clarification.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file in workspace\n candidates = list(Path(workspace_path).rglob('match_result_INV-NGO-4471.txt'))\n \n if not candidates:\n # Try glob with case variations\n candidates = list(Path(workspace_path).rglob('match_result*4471*.txt'))\n \n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'match_result_INV-NGO-4471.txt not found in workspace'}\n \n content = candidates[0].read_text(encoding='utf-8', errors='replace')\n content_lower = content.lower()\n issues = []\n found_items = []\n \n # 1. Check MATCH hold outcome\n match_hold_found = False\n if 'match hold' in content_lower or 'match_hold' in content_lower:\n match_hold_found = True\n elif 'match' in content_lower and 'hold' in content_lower:\n match_hold_found = True\n elif 'hold' in content_lower and ('three-way' in content_lower or '3-way' in content_lower or 'three way' in content_lower or '3 way' in content_lower):\n match_hold_found = True\n elif 'exception' in content_lower and ('hold' in content_lower or 'flag' in content_lower or 'fail' in content_lower):\n match_hold_found = True\n elif 'fail' in content_lower and 'match' in content_lower:\n match_hold_found = True\n elif 'mismatch' in content_lower and ('hold' in content_lower or 'exception' in content_lower or 'flag' in content_lower):\n match_hold_found = True\n elif 'on hold' in content_lower:\n match_hold_found = True\n elif 'placed' in content_lower and 'hold' in content_lower:\n match_hold_found = True\n \n if match_hold_found:\n found_items.append('MATCH hold outcome')\n else:\n issues.append('MATCH hold outcome not mentioned')\n \n # 2. Check $80 variance or ~4.1%\n has_80 = bool(re.search(r'\\$?80(\\.00)?', content))\n has_4_1 = bool(re.search(r'4\\.0[0-9]|4\\.1[0-9]?', content))\n if has_80 or has_4_1:\n parts = []\n if has_80:\n parts.append('$80 variance')\n if has_4_1:\n parts.append('~4.1% figure')\n found_items.append(' and '.join(parts))\n else:\n issues.append('Neither $80.00 variance nor ~4.1% figure found in file')\n \n # 3. Check threshold references (±$50 and/or ±2%)\n has_50 = bool(re.search(r'\\$?50', content))\n has_2pct = bool(re.search(r'2\\s*%|±\\s*2|\\+/?-\\s*2|2\\s*percent', content_lower))\n if has_50 or has_2pct:\n parts = []\n if has_50:\n parts.append('$50 threshold')\n if has_2pct:\n parts.append('2% threshold')\n found_items.append(' and '.join(parts))\n else:\n issues.append('Neither $50 threshold nor 2% threshold referenced in file')\n \n # 4. Check unit price check passing\n unit_price_found = False\n if 'unit price' in content_lower or 'unit_price' in content_lower:\n unit_price_found = True\n elif 'per-unit' in content_lower or 'per unit' in content_lower:\n unit_price_found = True\n elif 'price' in content_lower and any(w in content_lower for w in ['pass', 'passed', 'match', 'matched', 'correct', 'ok', 'good', 'confirmed', 'agree', 'consistent', 'verified', 'check']):\n unit_price_found = True\n elif 'line item' in content_lower and any(w in content_lower for w in ['pass', 'passed', 'match', 'matched', 'correct', 'verified', 'check']):\n unit_price_found = True\n \n if unit_price_found:\n found_items.append('unit price check')\n else:\n issues.append('Unit price check pass not mentioned')\n \n # 5. Check next action: vendor email / price clarification\n has_vendor_action = False\n # Multiple ways this could be expressed\n if ('email' in content_lower or 'contact' in content_lower or 'notify' in content_lower or 'reach out' in content_lower or 'send' in content_lower or 'communicate' in content_lower) and ('vendor' in
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "fb6593a0-7e58-4dac-9515-c6979c7a9783",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `slack_data.json`, there must be a message to the #ap-exceptions channel with: \"text\": \"PRICE VARIANCE: Northgate Office Solutions, V-00229, Invoice INV-NGO-4471, PO Price $97.50, Invoice Price $101.50, Variance 4.10%\" (the 4.10% can be ±0.5%)",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data'}\n \n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n \n with open(slack_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n \n # Find #ap-exceptions channel\n channels = data.get('channels', {})\n exceptions_channel_id = None\n for cid, cdata in channels.items():\n if cdata.get('name', '').lower() == 'ap-exceptions':\n exceptions_channel_id = cid\n break\n \n if exceptions_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#ap-exceptions channel not found in Slack data'}\n \n messages = data.get('messages', {}).get(exceptions_channel_id, [])\n \n target = 'INV-NGO-4471'\n relevant_msgs = [m for m in messages if isinstance(m, dict) and target in m.get('text', '')]\n \n if not relevant_msgs:\n return {'pass': False, 'score': 0.0, 'feedback': f'No message referencing {target} found in #ap-exceptions'}\n \n issues_per_msg = []\n for msg in relevant_msgs:\n text = msg.get('text', '')\n text_lower = text.lower()\n msg_issues = []\n \n # Check for PRICE VARIANCE mention (case insensitive)\n if 'price variance' not in text_lower:\n msg_issues.append('\"PRICE VARIANCE\" label missing')\n \n # Check vendor name (loosely)\n if 'northgate' not in text_lower:\n msg_issues.append('vendor name (Northgate Office Solutions) missing')\n \n # Check vendor ID (case insensitive)\n if 'v-00229' not in text_lower:\n msg_issues.append('vendor ID V-00229 missing')\n \n # PO price: accept $97.50 or unit-level equivalents, or total $1,950.00\n has_po_price = any(x in text for x in ['97.50', '97.5', '1950', '1,950'])\n if not has_po_price:\n msg_issues.append('PO price ($97.50 or $1,950.00) missing')\n \n # Invoice price: accept $101.50 or total $2,030.00\n has_inv_price = any(x in text for x in ['101.50', '101.5', '2030', '2,030'])\n if not has_inv_price:\n msg_issues.append('Invoice price ($101.50 or $2,030.00) missing')\n \n # Variance ~4.10% (±0.5% so accept 3.6% to 4.6%)\n # Look for percentage patterns in text\n pct_matches = re.findall(r'(\\d+\\.\\d+)\\s*%', text)\n has_variance = False\n for pct_str in pct_matches:\n try:\n pct_val = float(pct_str)\n if 3.6 <= pct_val <= 4.6:\n has_variance = True\n break\n except ValueError:\n pass\n # Also accept integer-like \"4%\" loosely\n if not has_variance:\n int_pct_matches = re.findall(r'(\\d+)\\s*%', text)\n for pct_str in int_pct_matches:\n try:\n pct_val = float(pct_str)\n if 3.6 <= pct_val <= 4.6:\n has_variance = True\n break\n except ValueError:\n pass\n if not has_variance:\n msg_issues.append('Variance percentage (~4.10% \\u00b10.5%) missing')\n \n if not msg_issues:\n return {'pass': True, 'score': 1.0, 'feedback': f'#ap-exceptions contains a valid PRICE VARIANCE message for INV-NGO-4471 with all required fields. Message text: {text[:300]}'}\n issues_per_msg.append(f'Message missing: {\"; \".join(msg_issues)}')\n \n return {'pass': False, 'score': 0.0, 'feedback': f'INV-NGO-4471 messages found in #ap-exceptions but have issues: ' + ' | '.join(issues_per_msg)}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "4ad6ef7b-6e46-4e91-8e37-a897671ff3de",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In `mailbox.json`, there must be an email with the following:\n\"folder\": \"Sent\",\n\"subject\": \"Meridian Partners - Price Discrepancy on Invoice INV-NGO-4471\",\n\"to_addr\" must contain \"@northgateoffice.com\",\nand the body text must contain the PO price ($1950.00 or $97.50), invoiced price ($2030.00 or $101.50) and the variance percentage (4.10%, can be ±0.5%)",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox.json for sent email'}\n\n mailbox_file = Path(external_services_path) / 'mailbox.json'\n if not mailbox_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n with open(mailbox_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n\n target_invoice = 'INV-NGO-4471'\n\n # Find emails sent to @northgateoffice.com with the invoice number in the subject\n matching = []\n for e in sent_emails:\n to_addr = e.get('to_addr', '').lower()\n subject = e.get('subject', '')\n if '@northgateoffice.com' in to_addr and target_invoice.lower() in subject.lower():\n matching.append(e)\n\n if not matching:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to @northgateoffice.com with {target_invoice} in subject. Sent emails: {[{\"to\": e.get(\"to_addr\"), \"subject\": e.get(\"subject\")} for e in sent_emails]}'}\n\n issues_all = []\n for email in matching:\n subject = email.get('subject', '')\n body = email.get('body_text', '') or email.get('body', '') or ''\n body_lower = body.lower()\n subject_lower = subject.lower()\n email_issues = []\n\n # (1) Subject must contain \"Price Discrepancy\"\n if 'price discrepancy' not in subject_lower:\n email_issues.append(f'Subject does not contain \"Price Discrepancy\". Subject was: \"{subject}\"')\n\n # (2) Body must reference PO price: $1950.00 or $97.50 (total or unit)\n po_price_found = False\n for token in ['1950.00', '1950', '1,950.00', '1,950', '97.50', '97.5']:\n if token in body:\n po_price_found = True\n break\n if not po_price_found:\n email_issues.append('PO price ($1950.00 or $97.50) not found in body')\n\n # (3) Body must reference invoiced price: $2030.00 or $101.50 (total or unit)\n inv_price_found = False\n for token in ['2030.00', '2030', '2,030.00', '2,030', '101.50', '101.5']:\n if token in body:\n inv_price_found = True\n break\n if not inv_price_found:\n email_issues.append('Invoiced price ($2030.00 or $101.50) not found in body')\n\n # (4) Body must reference variance percentage ~4.10% (accept 3.60% to 4.60%, i.e. 4.10 +/- 0.5)\n variance_found = False\n # Search for any percentage-like number in the body and check range\n pct_pattern = re.compile(r'(\\d+\\.\\d+)\\s*%')\n for m in pct_pattern.finditer(body):\n val = float(m.group(1))\n if 3.60 <= val <= 4.60:\n variance_found = True\n break\n # Also accept bare mentions like \"4.1\" or \"4.10\" without % sign near context words\n if not variance_found:\n bare_pattern = re.compile(r'\\b(\\d+\\.\\d+)\\b')\n for m in bare_pattern.finditer(body):\n val = float(m.group(1))\n if 3.60 <= val <= 4.60:\n variance_found = True\n break\n if not variance_found:\n email_issues.append('Variance percentage (~4.10%, +/-0.5%) not found in body')\n\n if not email_issues:\n return {'pass': True, 'score': 1.0, 'feedback': f'Sent email to @northgateoffice.com found with correct subject containing \"Price Discrepancy\" and \"{target_invoice}\", and body references PO price, invoice price, and variance ~4.1%.'}\n issues_all.append(f'Email (subject=\"{subject}\"): ' + '; '.join(email_issues))\n\n return {'pass': False, 'score': 0.0, 'feedback': 'Matching sent
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774379677398",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `slack_data.json` (in external_data/final) the channels #ap-escalations, #ap-payments, #ap-approvals must not contain any messages referencing INV-NGO-4471.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n # Locate the final slack_data.json\n final_candidates = []\n if external_services_path:\n final_candidates.append(Path(external_services_path) / \"slack_data.json\")\n final_candidates.append(workspace_path / \"external_data\" / \"final\" / \"slack_data.json\")\n\n final_slack_path = None\n for p in final_candidates:\n if p.exists():\n final_slack_path = p\n break\n\n if final_slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No final slack_data.json found; cannot confirm any messages were added, so passing by default (generous interpretation).\"}\n\n # Load final data\n try:\n with open(final_slack_path, 'r', encoding='utf-8') as f:\n final_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse final slack_data.json ({e}); passing generously.\"}\n\n # Helper to unwrap possible nested structure\n def unwrap(data):\n if isinstance(data, dict) and \"slack_data\" in data:\n return data[\"slack_data\"]\n return data\n\n final_data = unwrap(final_data)\n\n # The three channels we care about\n target_channel_names = {\"ap-escalations\", \"ap-payments\", \"ap-approvals\"}\n\n # Build a map from channel name -> channel id\n channels_info = final_data.get(\"channels\", {}) if isinstance(final_data, dict) else {}\n if not isinstance(channels_info, dict):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No channels dict found in slack_data.json; pass by default.\"}\n\n channel_name_to_id = {}\n for channel_id, channel_info in channels_info.items():\n if isinstance(channel_info, dict):\n name = (channel_info.get(\"name\", \"\") or \"\").strip().lstrip(\"#\").lower()\n if name in target_channel_names:\n channel_name_to_id[name] = channel_id\n\n if not channel_name_to_id:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"None of the target channels (#ap-escalations, #ap-payments, #ap-approvals) found in final slack_data.json; pass by default.\"}\n\n messages_data = final_data.get(\"messages\", {}) if isinstance(final_data, dict) else {}\n if not isinstance(messages_data, dict):\n messages_data = {}\n\n invoice_ref = \"INV-NGO-4471\"\n # Use a case-insensitive pattern that matches the invoice number with or without dashes\n pattern = re.compile(re.escape(invoice_ref), re.IGNORECASE)\n\n violations = []\n\n for chan_name, chan_id in channel_name_to_id.items():\n chan_msgs = messages_data.get(chan_id, [])\n if not isinstance(chan_msgs, list):\n continue\n for msg in chan_msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\") or \"\"\n if pattern.search(text):\n user = msg.get(\"user\", \"unknown\")\n preview = text[:150]\n violations.append(f\" - #{chan_name} (channel {chan_id}): User {user}: {preview}\")\n\n if violations:\n detail = \"\\n\".join(violations)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violations)} message(s) referencing {invoice_ref} in the restricted channels:\\n{detail}\"\n }\n\n checked = \", \".join([f\"#{n} ({channel_name_to_id[n]})\" for n in sorted(channel_name_to_id.keys())])\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No messages referencing {invoice_ref} found in checked channels: {checked}.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774605866883",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In file `ap_ledger.xlsx`, the `PO Register` sheet must contain the following cell values: G5=97.50 (can have dollar signifier), I5=1950.00 (can have dollar signifier)",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{file_path}' does not exist.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open '{file_path}': {e}\"}\n \n # Check that 'PO Register' sheet exists\n sheet_names = wb.sheetnames\n po_register_sheet = None\n for name in sheet_names:\n if name.strip().lower() == \"po register\":\n po_register_sheet = wb[name]\n break\n \n if po_register_sheet is None:\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'PO Register' not found in ap_ledger.xlsx. Sheets found: {sheet_names}\"}\n \n def parse_numeric(val):\n \"\"\"Parse a cell value to a float, stripping dollar signs, commas, whitespace.\"\"\"\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return float(val)\n s = str(val).strip()\n # Remove dollar sign, commas, whitespace\n s = s.replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except (ValueError, TypeError):\n return None\n \n g5_val = po_register_sheet['G5'].value\n i5_val = po_register_sheet['I5'].value\n \n g5_num = parse_numeric(g5_val)\n i5_num = parse_numeric(i5_val)\n \n issues = []\n passed_checks = []\n \n # Check G5 = 97.50 with ±1% tolerance\n expected_g5 = 97.50\n if g5_num is None:\n issues.append(f\"G5: expected {expected_g5}, but got '{g5_val}' (could not parse as number)\")\n elif abs(g5_num - expected_g5) > expected_g5 * 0.01:\n issues.append(f\"G5: expected {expected_g5}, but got {g5_num} (raw: '{g5_val}')\")\n else:\n passed_checks.append(f\"G5={g5_num} (expected {expected_g5})\")\n \n # Check I5 = 1950.00 with ±1% tolerance\n expected_i5 = 1950.00\n if i5_num is None:\n issues.append(f\"I5: expected {expected_i5}, but got '{i5_val}' (could not parse as number)\")\n elif abs(i5_num - expected_i5) > expected_i5 * 0.01:\n issues.append(f\"I5: expected {expected_i5}, but got {i5_num} (raw: '{i5_val}')\")\n else:\n passed_checks.append(f\"I5={i5_num} (expected {expected_i5})\")\n \n wb.close()\n \n if issues:\n score = len(passed_checks) / (len(passed_checks) + len(issues))\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"PO Register cell value mismatches: {'; '.join(issues)}. Passed: {'; '.join(passed_checks) if passed_checks else 'none'}.\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PO Register sheet verified successfully. {'; '.join(passed_checks)}.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1774610562264",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "File `match_result_INV-NGO-4471.txt` must exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"match_result_INV-NGO-4471.txt\"\n if target_file.exists() and target_file.is_file():\n # Additionally check that the file is not empty (i.e., it has some content)\n content = target_file.read_text(encoding='utf-8', errors='replace').strip()\n if len(content) > 0:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"File 'match_result_INV-NGO-4471.txt' exists and contains content ({len(content)} characters).\"\n }\n else:\n # Be generous: even an empty file technically \"exists\", so still pass\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"File 'match_result_INV-NGO-4471.txt' exists but is empty. Passing since the rubric only requires existence.\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'match_result_INV-NGO-4471.txt' was not found in the workspace at '{workspace_path}'.\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|