Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,114 @@
|
||||
[
|
||||
{
|
||||
"id": "852af8c1-546e-4f60-b5e7-cf9f9a98b8ce",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to lindon@luminarylogistics.com must exist with subject containing 'INV-82419' and body containing the incomplete invoice notice about missing address fields (City, State, Zip Code) and instructions to resubmit to ap@meridianpartners.com.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_emails = [\n e for e in sent_emails\n if \"lindon@luminarylogistics.com\" in (e.get(\"to_addr\", \"\") or \"\").lower()\n ]\n \n if not target_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to lindon@luminarylogistics.com.\"}\n \n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n combined = subject + \" \" + body\n \n # Check subject contains INV-82419\n has_inv_subject = \"inv-82419\" in subject\n \n # Check body contains INV-82419 reference\n has_inv_body = \"inv-82419\" in body\n \n # Check body mentions missing address fields: City, State, Zip Code\n has_city = \"city\" in body\n has_state = \"state\" in body\n has_zip = any(term in body for term in [\"zip\", \"postal\"])\n # At least mention of incomplete/missing and address-related fields\n has_missing_notice = any(term in body for term in [\"missing\", \"incomplete\", \"not included\", \"lacking\", \"absent\", \"required\"])\n \n # Check resubmit instructions\n has_resubmit = any(term in body for term in [\"resubmit\", \"re-submit\", \"corrected\", \"resend\", \"re-send\", \"send\", \"updated\", \"revise\", \"provide\"])\n \n # Check ap email\n has_ap_email = \"ap@meridianpartners.com\" in body\n \n # Require: subject has INV-82419, body mentions missing address fields (city, state, zip), resubmit instructions, and ap email\n if has_inv_subject and has_missing_notice and has_city and has_state and has_zip and has_resubmit and has_ap_email:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found valid email to lindon@luminarylogistics.com about INV-82419 with incomplete invoice notice about missing address fields (City, State, Zip Code) and instructions to resubmit to ap@meridianpartners.com. Subject: {email.get('subject', '')}\"}\n \n # Report what we found for debugging\n details = []\n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n detail = {\n \"subject\": email.get(\"subject\", \"\"),\n \"has_inv_subject\": \"inv-82419\" in subject,\n \"has_city\": \"city\" in body,\n \"has_state\": \"state\" in body,\n \"has_zip\": any(t in body for t in [\"zip\", \"postal\"]),\n \"has_missing_notice\": any(t in body for t in [\"missing\", \"incomplete\", \"not included\", \"lacking\", \"absent\", \"required\"]),\n \"has_resubmit\": any(t in body for t in [\"resubmit\", \"re-submit\", \"corrected\", \"resend\", \"re-send\", \"send\", \"updated\", \"revise\", \"provide\"]),\n \"has_ap_email\": \"ap@meridianpartners.com\" in body\n }\n details.append(detail)\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(target_emails)} email(s) to lindon@luminarylogistics.com but none matched all required content. Details: {json.dumps(details, indent=2)}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "9f1cd0b2-b083-4e8d-9308-dae522acc509",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `Ap_ledger.xlsx`, the Invoice Register sheet must NOT contain a row for invoice INV-82419.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception as e:\n continue\n \n # Find invoice register sheet\n register_sheet = None\n for sheet_name in wb.sheetnames:\n if \"register\" in sheet_name.lower() or \"invoice\" in sheet_name.lower():\n register_sheet = wb[sheet_name]\n break\n \n if register_sheet is None:\n # Try first sheet\n if wb.sheetnames:\n register_sheet = wb[wb.sheetnames[0]]\n \n if register_sheet is None:\n continue\n \n for row in register_sheet.iter_rows(values_only=True):\n for cell in row:\n if cell is not None and \"inv-82419\" in str(cell).lower():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"INV-82419 found in Invoice Register sheet in {xlsx_path.name} - it should NOT be listed.\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-82419 is not present in the Invoice Register sheet in {xlsx_path.name}, as required.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read a suitable xlsx file with an Invoice Register sheet.\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "6e8e36a8-e868-4af4-bc06-8cb2578ea253",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to greg.kendrick@kunluncreativestudio.com must exist with subject containing 'INV-KC-20260305' and body containing: amount $6,019.44, pending internal approval, and 5 additional business days.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_emails = [\n e for e in sent_emails\n if \"greg.kendrick@kunluncreativestudio.com\" in (e.get(\"to_addr\", \"\") or \"\")\n ]\n \n if not target_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to greg.kendrick@kunluncreativestudio.com.\"}\n \n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n \n if \"inv-kc-20260305\" not in subject and \"inv-kc-20260305\" not in body:\n continue\n \n has_inv = \"inv-kc-20260305\" in body\n has_amount = \"6,019.44\" in body or \"6019.44\" in body\n has_pending = any(term in body for term in [\"pending\", \"approval\", \"approve\"])\n has_days = \"5\" in body and \"business day\" in body\n \n if has_inv and has_amount and has_pending and has_days:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found valid status update email to greg.kendrick@kunluncreativestudio.com about INV-KC-20260305 with all required content.\"}\n \n found_subjects = [e.get(\"subject\", \"\") for e in target_emails]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found emails to greg.kendrick@kunluncreativestudio.com but none matched all required content (INV-KC-20260305, $6,019.44, pending approval, 5 business days). Subjects: {found_subjects}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "76697ce7-241c-48a7-87d2-57eb944b78aa",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `Ap_ledger.xlsx`, the Payment Queue sheet must list INV-4839201-24 with Vendor 'Sentinel Cybersecurity Solutions', invoice date 3/16/2026, due date 4/15/2026, amount $13,837, GL Account 6400-010-1100, no PO number, and Status 'Approved'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n # Find Payment Queue sheet\n payment_sheet = None\n for sheet_name in wb.sheetnames:\n if \"payment\" in sheet_name.lower() and \"queue\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n if payment_sheet is None:\n for sheet_name in wb.sheetnames:\n if \"queue\" in sheet_name.lower() or \"payment\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n \n if payment_sheet is None:\n continue\n \n rows = list(payment_sheet.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-4839201-24\" not in combined:\n continue\n \n # Found the row - check all required fields\n errors = []\n \n # Vendor check\n if \"sentinel cybersecurity\" not in combined:\n errors.append(\"Vendor 'Sentinel Cybersecurity Solutions' not found in row\")\n \n # Amount check - $13,837\n has_amount = \"13,837\" in combined or \"13837\" in combined\n if not has_amount:\n # Check with tolerance\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 13837) <= 138.37: # 1% tolerance\n has_amount = True\n break\n except:\n pass\n if not has_amount:\n errors.append(\"Amount $13,837 not found in row\")\n \n # GL Account check\n if \"6400-010-1100\" not in combined:\n errors.append(\"GL Account 6400-010-1100 not found in row\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found in row\")\n \n # Invoice date check (3/16/2026)\n has_inv_date = any(d in combined for d in [\"3/16/2026\", \"03/16/2026\", \"2026-03-16\", \"march 16\"])\n if not has_inv_date:\n errors.append(\"Invoice date 3/16/2026 not found in row\")\n \n # Due date check (4/15/2026)\n has_due_date = any(d in combined for d in [\"4/15/2026\", \"04/15/2026\", \"2026-04-15\", \"april 15\"])\n if not has_due_date:\n errors.append(\"Due date 4/15/2026 not found in row\")\n \n if errors:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found INV-4839201-24 in Payment Queue but issues: {'; '.join(errors)}. Row: {row_vals}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-4839201-24 found in Payment Queue with all required fields verified. Row: {row_vals}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-4839201-24 not found in any Payment Queue sheet in workspace xlsx files.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "6929234d-ab72-4732-90bb-865f606bf268",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `match_log_2026_Q1.xlsx` (or similar match log file), a row for INV-004322 must exist with: Match Date 3/16/2026, Vendor 'Main Street Office Products', PO PO-2026-00532, PO Amount $1,276.14, GR Number GR-20260311-004, GR Date 3/11/2026, Invoice Date 3/16/2026, Invoice Amount $1,356.14, GL Account 6200-000-5100, PO Variance $80.00, Status 'Approved'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find match log file\n match_files = list(Path(workspace_path).glob(\"*match*log*.xlsx\")) + list(Path(workspace_path).glob(\"*match*.xlsx\")) + list(Path(workspace_path).glob(\"match_log*.xlsx\"))\n if not match_files:\n match_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not match_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found for match log.\"}\n \n for xlsx_path in match_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-004322\" not in combined:\n continue\n \n errors = []\n \n # Vendor check\n if \"main street office products\" not in combined:\n errors.append(\"Vendor 'Main Street Office Products' not found\")\n \n # PO check\n if \"po-2026-00532\" not in combined:\n errors.append(\"PO number PO-2026-00532 not found\")\n \n # PO Amount check $1,276.14\n has_po_amt = \"1,276.14\" in combined or \"1276.14\" in combined\n if not has_po_amt:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1276.14) <= 12.76:\n has_po_amt = True\n break\n except:\n pass\n if not has_po_amt:\n errors.append(\"PO Amount $1,276.14 not found\")\n \n # GR Number check\n if \"gr-20260311-004\" not in combined:\n errors.append(\"GR Number GR-20260311-004 not found\")\n \n # GR Date check 3/11/2026\n has_gr_date = any(d in combined for d in [\"3/11/2026\", \"03/11/2026\", \"2026-03-11\", \"march 11\"])\n if not has_gr_date:\n errors.append(\"GR Date 3/11/2026 not found\")\n \n # Invoice Amount $1,356.14\n has_inv_amt = \"1,356.14\" in combined or \"1356.14\" in combined\n if not has_inv_amt:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1356.14) <= 13.56:\n has_inv_amt = True\n break\n except:\n pass\n if not has_inv_amt:\n errors.append(\"Invoice Amount $1,356.14 not found\")\n \n # GL Account check\n if \"6200-000-5100\" not in combined:\n errors.append(\"GL Account 6200-000-5100 not found\")\n \n # PO Variance $80.00\n has_variance = \"80\" in combined\n if not has_variance:\n errors.append(\"PO Variance $80.00 not found\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found\")\n \n # Match date 3/16/2026\n has_match_date = any(d in combined for d in [\"3/16/2026\", \"03/16/2026\", \"2026-03-16\", \"march 16\"])\n if not has_match_date:\n errors.append(\"Match Date 3/16/2026 not found\")\n \n if errors:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found INV-004322 in {xlsx_path.name} sheet '{sheet_name}' but issues: {'; '.join(errors)}. Row: {row_vals}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-004322 match log row verified in {xlsx_path.name} sheet '{sheet_name}' with all required fields.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-004322 not found in any match log xlsx file in workspace.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "9e5460f7-30b2-45c0-9f50-04b43f5b8b6e",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `Ap_ledger.xlsx`, the Payment Queue sheet must list INV-004322 with Vendor 'Main Street Office Products', invoice date 3/16/2026, due date 4/15/2026, amount $1,356.14, GL Account 6200-000-5100, PO number PO-2026-00532, and Status 'Approved'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n # Find Payment Queue sheet\n payment_sheet = None\n for sheet_name in wb.sheetnames:\n if \"payment\" in sheet_name.lower() and \"queue\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n if payment_sheet is None:\n for sheet_name in wb.sheetnames:\n if \"queue\" in sheet_name.lower() or \"payment\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n \n if payment_sheet is None:\n continue\n \n rows = list(payment_sheet.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-004322\" not in combined:\n continue\n \n errors = []\n \n # Vendor check\n if \"main street office products\" not in combined:\n errors.append(\"Vendor 'Main Street Office Products' not found\")\n \n # Amount check $1,356.14\n has_amount = \"1,356.14\" in combined or \"1356.14\" in combined\n if not has_amount:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1356.14) <= 13.56:\n has_amount = True\n break\n except:\n pass\n if not has_amount:\n errors.append(\"Amount $1,356.14 not found\")\n \n # GL Account check\n if \"6200-000-5100\" not in combined:\n errors.append(\"GL Account 6200-000-5100 not found\")\n \n # PO Number check\n if \"po-2026-00532\" not in combined:\n errors.append(\"PO number PO-2026-00532 not found\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found\")\n \n # Invoice date 3/16/2026\n has_inv_date = any(d in combined for d in [\"3/16/2026\", \"03/16/2026\", \"2026-03-16\", \"march 16\"])\n if not has_inv_date:\n errors.append(\"Invoice date 3/16/2026 not found\")\n \n # Due date 4/15/2026\n has_due_date = any(d in combined for d in [\"4/15/2026\", \"04/15/2026\", \"2026-04-15\", \"april 15\"])\n if not has_due_date:\n errors.append(\"Due date 4/15/2026 not found\")\n \n if errors:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found INV-004322 in Payment Queue but issues: {'; '.join(errors)}. Row: {row_vals}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-004322 found in Payment Queue with all required fields verified. Row: {row_vals}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-004322 not found in any Payment Queue sheet in workspace xlsx files.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "813da462-d30a-4309-98cb-be45d71e3de3",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `slack_data.json`, the #ap-exceptions channel must contain following message: \"PRICE VARIANCE: Main Street Office Products, 432319, Invoice INV-004322, PO Price $59.99, Invoice Price $79.99, Variance 33.34%\"",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check Slack.\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find #ap-exceptions channel ID\n ap_exceptions_channel_id = None\n for ch_id, ch_info in channels.items():\n if \"ap-exceptions\" in ch_info.get(\"name\", \"\").lower():\n ap_exceptions_channel_id = ch_id\n break\n \n if ap_exceptions_channel_id is None:\n # Try searching messages in channels with related names\n all_channel_messages = []\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get(\"name\", \"\")\n if \"ap\" in ch_name.lower() or \"exception\" in ch_name.lower():\n all_channel_messages.extend(messages.get(ch_id, []))\n if not all_channel_messages:\n # Search all messages as fallback\n for ch_id, msgs in messages.items():\n all_channel_messages.extend(msgs)\n else:\n all_channel_messages = messages.get(ap_exceptions_channel_id, [])\n \n # The rubric requires a message like:\n # \"PRICE VARIANCE: Main Street Office Products, 432319, Invoice INV-004322, PO Price $59.99, Invoice Price $79.99, Variance 33.34%\"\n # We check for the key components loosely\n \n for msg in all_channel_messages:\n text = (msg.get(\"text\", \"\") or \"\")\n text_lower = text.lower()\n \n has_price_variance = \"price variance\" in text_lower\n has_vendor = \"main street office products\" in text_lower\n has_po_num = \"432319\" in text\n has_inv = \"inv-004322\" in text_lower\n has_po_price = \"59.99\" in text\n has_inv_price = \"79.99\" in text\n # Accept variance around 33.34% loosely\n has_variance_pct = \"33.3\" in text or \"33.4\" in text\n \n if has_price_variance and has_vendor and has_po_num and has_inv and has_po_price and has_inv_price and has_variance_pct:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found ap-exceptions Slack message with all required price variance details: {text[:300]}\"}\n \n # Provide detailed failure info\n sample_msgs = [msg.get(\"text\", \"\")[:200] for msg in all_channel_messages[:10]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No Slack message found in #ap-exceptions with all required price variance details (PRICE VARIANCE, Main Street Office Products, 432319, INV-004322, $59.99, $79.99, ~33.34%). Sample messages found: {sample_msgs}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775621743582",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In Ap_ledger.xlsx, on the Invoice Register sheet, INV-KC-20260305's status must be 'APPR' or 'APPR Hold' (case-insensitive).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n\n # Find the AP ledger file with flexible naming\n candidates = []\n for pattern in ['**/Ap_ledger.xlsx', '**/AP_Ledger.xlsx', '**/ap_ledger.xlsx',\n '**/AP_ledger.xlsx', '**/Ap_Ledger.xlsx']:\n candidates.extend(workspace.glob(pattern))\n\n if not candidates:\n for f in workspace.glob('**/*.xlsx'):\n if 'ledger' in f.name.lower() and 'ap' in f.name.lower():\n candidates.append(f)\n\n direct = workspace / \"Ap_ledger.xlsx\"\n if direct.exists() and direct not in candidates:\n candidates.insert(0, direct)\n\n if not candidates:\n all_xlsx = list(workspace.glob('**/*.xlsx'))\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"AP Ledger file not found. Available xlsx files: {[str(f.relative_to(workspace)) for f in all_xlsx]}\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Failed to open workbook {file_path.name}: {e}\"}\n\n # Find the Invoice Register sheet (case-insensitive, flexible)\n sheet = None\n for name in wb.sheetnames:\n normalized = name.strip().lower().replace(' ', '').replace('_', '')\n if normalized in ('invoiceregister', 'invoicereg'):\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'reg' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Sheet 'Invoice Register' not found. Available sheets: {wb.sheetnames}\"}\n\n # Headers are in row 2. Match exactly (case-insensitive, trimmed).\n HEADER_ROW = 2\n invoice_col = None\n status_col = None\n for col_idx in range(1, sheet.max_column + 1):\n val = sheet.cell(row=HEADER_ROW, column=col_idx).value\n if val is None:\n continue\n header = str(val).strip().lower()\n if header == 'invoice #':\n invoice_col = col_idx\n elif header == 'status':\n status_col = col_idx\n\n if invoice_col is None or status_col is None:\n row2_values = [sheet.cell(row=HEADER_ROW, column=c).value\n for c in range(1, sheet.max_column + 1)]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Could not locate 'Invoice #' and/or 'Status' headers in row {HEADER_ROW}. Row {HEADER_ROW} values: {row2_values}\"}\n\n # Find the target invoice row\n target_invoice = 'INV-KC-20260305'\n found_status = None\n for row_idx in range(HEADER_ROW + 1, sheet.max_row + 1):\n cell_val = sheet.cell(row=row_idx, column=invoice_col).value\n if cell_val is None:\n continue\n if str(cell_val).strip().upper() == target_invoice.upper():\n status_val = sheet.cell(row=row_idx, column=status_col).value\n found_status = str(status_val).strip() if status_val is not None else None\n break\n\n if found_status is None:\n # Distinguish \"not found\" vs \"found but empty status\"\n for row_idx in range(HEADER_ROW + 1, sheet.max_row + 1):\n cell_val = sheet.cell(row=row_idx, column=invoice_col).value\n if cell_val is not None and str(cell_val).strip().upper() == target_invoice.upper():\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Invoice '{target_invoice}' found but its Status cell is empty.\"}\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Invoice '{target_invoice}' not found in the Invoice Register sheet.\"}\n\n # Case-insensitive status check\n status_lower = found_status.lower().strip()\n if status_lower in ('appr', 'appr hold'):\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Invoice '{target_invoice}' has status '{found_status}', which matches an acceptable value.\"}\n\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Invoice '{target_invoice}' has status '{found_status}', expected 'APPR' or 'APPR Hold' (case-insensitive).\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775677892670",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "The file `match_log_2026_Q1.xlsx` must not contain a row with invoice number INV-82419.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2026_Q1.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open file: {e}\"}\n \n target = \"INV-82419\"\n target_lower = target.lower().strip()\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).strip().lower()\n if cell_str == target_lower or target_lower in cell_str:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found INV-82419 in sheet '{sheet_name}', row {row_idx}. The file must NOT contain this invoice number.\"\n }\n \n wb.close()\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"Confirmed: match_log_2026_Q1.xlsx does not contain a row with invoice number INV-82419.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775677952539",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `Ap_ledger.xlsx`, the Invoice Register sheet must NOT contain a row for invoice INV-004322",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find Ap_ledger.xlsx with flexible matching\n fp = Path(workspace_path) / \"Ap_ledger.xlsx\"\n if not fp.exists():\n # Try case-insensitive glob\n candidates = list(Path(workspace_path).glob(\"**/[Aa][Pp]_[Ll]edger*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ledger*.xlsx\"))\n if candidates:\n fp = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: Ap_ledger.xlsx in {workspace_path}\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n\n # Find the Invoice Register sheet (flexible on case/spacing)\n sheet = None\n for name in wb.sheetnames:\n normalized = name.strip().lower().replace(' ', '').replace('_', '')\n if normalized in ['invoiceregister', 'invoicesregister']:\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if name.strip() == 'Invoice Register':\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"'Invoice Register' sheet not found. Available sheets: {wb.sheetnames}\"}\n\n target = 'INV-004322'\n found = False\n found_row = None\n for row_idx, row in enumerate(sheet.iter_rows(min_row=1, values_only=True), start=1):\n for cell in row:\n if cell is not None:\n cell_str = str(cell).strip().upper()\n if target.upper() in cell_str:\n found = True\n found_row = row_idx\n break\n if found:\n break\n\n if found:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Invoice {target} was found in the Invoice Register sheet at row {found_row}. It should NOT be present.\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Confirmed: Invoice {target} is not present in the Invoice Register sheet of {fp.name}.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775677983569",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `slack_data.json`, the channel C002 must not contain a message with \"Luminary Logistics\" in it.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for slack_data.json\n slack_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n # Also check workspace_path\n if not slack_path:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if not slack_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in either external_services_path or workspace_path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n \n # The rubric says channel C002 must not contain a message with \"Luminary Logistics\".\n # slack_data.json can have different formats. Handle both:\n # Format A (dict-based): messages is a dict keyed by channel ID -> list of message objects\n # Format B (list-based): channels is a list of channel objects with messages inside\n \n messages_to_check = []\n \n # Try Format A: data[\"messages\"][\"C002\"] -> list of messages\n if \"messages\" in slack_data and isinstance(slack_data[\"messages\"], dict):\n channel_messages = slack_data[\"messages\"].get(\"C002\", [])\n if isinstance(channel_messages, list):\n messages_to_check.extend(channel_messages)\n \n # Try Format B: data[\"channels\"] is a list, find channel with id C002\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for channel in channels:\n ch_id = channel.get(\"id\", \"\") or channel.get(\"channel_id\", \"\")\n if ch_id == \"C002\":\n msgs = channel.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_to_check.extend(msgs)\n elif isinstance(channels, dict):\n # Format: channels is a dict keyed by channel ID\n ch = channels.get(\"C002\")\n if ch and isinstance(ch, dict):\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_to_check.extend(msgs)\n \n if not messages_to_check:\n # No messages found for C002 at all - that means no offending message exists, so pass\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel C002 has no messages (or was not found), so no 'Luminary Logistics' message exists. Condition satisfied.\"}\n \n for msg in messages_to_check:\n # Check all text-like fields\n msg_text = \"\"\n for key in [\"message\", \"text\", \"body\", \"content\"]:\n val = msg.get(key, \"\")\n if val:\n msg_text += \" \" + str(val)\n if \"luminary logistics\" in msg_text.lower():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a message containing 'Luminary Logistics' in channel C002: '{msg_text.strip()[:200]}'\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No message containing 'Luminary Logistics' was found in channel C002. Checked {len(messages_to_check)} messages.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775678010037",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `slack_data.json`, channel C002 must not contain a message with \"Kunlun Creative Studio\" in it.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Find slack_data.json - check external_services_path first, then workspace\n slack_path = None\n \n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # The rubric says: channel C002 must not contain a message with \"Kunlun Creative Studio\"\n # Per the Slack data format, messages are keyed by channel ID under \"messages\"\n messages_dict = slack_data.get(\"messages\", {})\n \n # Get messages for channel C002\n c002_messages = messages_dict.get(\"C002\", [])\n \n if not isinstance(c002_messages, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected messages for C002 to be a list, got {type(c002_messages).__name__}\"}\n \n # Check each message for \"Kunlun Creative Studio\"\n found_messages = []\n for msg in c002_messages:\n msg_text = \"\"\n if isinstance(msg, dict):\n # Collect all text-like fields\n for field in [\"text\", \"message\", \"content\", \"body\"]:\n val = msg.get(field, \"\")\n if val:\n msg_text += \" \" + str(val)\n # Also check attachments if any\n attachments = msg.get(\"attachments\", [])\n if isinstance(attachments, list):\n for att in attachments:\n if isinstance(att, dict):\n for field in [\"text\", \"fallback\", \"pretext\", \"title\"]:\n val = att.get(field, \"\")\n if val:\n msg_text += \" \" + str(val)\n elif isinstance(msg, str):\n msg_text = msg\n \n if \"kunlun creative studio\" in msg_text.lower():\n found_messages.append(msg_text.strip()[:200])\n \n if found_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(found_messages)} message(s) in channel C002 containing 'Kunlun Creative Studio'. First match: {found_messages[0]}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No messages containing 'Kunlun Creative Studio' found in channel C002. Checked {len(c002_messages)} messages.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775678035580",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `slack_data.json`, channel C002 must contain a message that reads: \"PAYMENT AUTHORIZATION REQUIRED: Sentinel Cybersecurity Solutions | Invoice INV-4839201-24 | Amount $13,837.00 | Recurring Invoice | Tanya Okonkwo and Rachel Whitfield\"",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # The slack_data.json could be in workspace or external_services\n slack_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Required fragments to look for (case-insensitive, flexible)\n required_fragments = [\n \"PAYMENT AUTHORIZATION REQUIRED\",\n \"Sentinel Cybersecurity Solutions\",\n \"INV-4839201-24\",\n \"13,837.00\",\n \"Recurring Invoice\",\n \"Tanya Okonkwo\",\n \"Rachel Whitfield\"\n ]\n \n # Try to find messages in channel C002 using the new slack data format\n # The new format has: messages as dict keyed by channel ID -> list of message objects\n # Also try the old format: channels as list with messages inside\n \n all_messages = []\n \n # New format: messages dict keyed by channel ID\n messages_dict = slack_data.get(\"messages\", {})\n if isinstance(messages_dict, dict):\n # Try C002 directly\n c002_msgs = messages_dict.get(\"C002\", [])\n if isinstance(c002_msgs, list):\n all_messages.extend(c002_msgs)\n \n # Also search all channels in case C002 is referenced differently\n # But prioritize C002\n \n # Old format: channels as list\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for ch in channels:\n ch_id = ch.get(\"channel_id\", \"\") or ch.get(\"id\", \"\")\n ch_name = ch.get(\"channel_name\", \"\") or ch.get(\"name\", \"\")\n if ch_id == \"C002\" or \"ap-approvals\" in ch_name.lower().replace(\" \", \"-\"):\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n all_messages.extend(msgs)\n elif isinstance(channels, dict):\n # channels could be a dict keyed by channel ID\n c002_ch = channels.get(\"C002\", {})\n if c002_ch:\n # Channel info found, messages are in the messages dict\n pass\n # Also check by name\n for ch_id, ch_info in channels.items():\n if ch_id == \"C002\" or (isinstance(ch_info, dict) and \"ap-approvals\" in (ch_info.get(\"name\", \"\") or \"\").lower()):\n # Messages for this channel are in the messages_dict\n ch_msgs = messages_dict.get(ch_id, []) if isinstance(messages_dict, dict) else []\n if isinstance(ch_msgs, list):\n all_messages.extend(ch_msgs)\n \n if not all_messages:\n # Provide debug info\n available_keys = list(slack_data.keys())\n msg_keys = list(messages_dict.keys()) if isinstance(messages_dict, dict) else \"not a dict\"\n ch_info = \"\"\n if isinstance(channels, list):\n ch_info = str([{\"id\": c.get(\"channel_id\", c.get(\"id\", \"\")), \"name\": c.get(\"channel_name\", c.get(\"name\", \"\"))} for c in channels])\n elif isinstance(channels, dict):\n ch_info = str({k: v.get(\"name\", \"\") if isinstance(v, dict) else v for k, v in channels.items()})\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found for channel C002. Top-level keys: {available_keys}. Messages keys: {msg_keys}. Channels info: {ch_info}\"}\n \n # Search messages for required content\n for msg in all_messages:\n # Handle different message formats\n msg_text = \"\"\n if isinstance(msg, dict):\n msg_text = msg.get(\"message\", \"\") or msg.get(\"text\", \"\") or msg.get(\"content\", \"\") or \"\"\n elif isinstance(msg, str):\n msg_text = msg\n \n if not msg_text:\n continue\n \n msg_upper = msg_text.upper()\n \n all_found = True\n missing = []\n for frag in required_fragments:\n frag_upper = frag.upper()\n if frag_upper in msg_upper:\n continue\n # Try alternate for the amount: maybe without comma or with dollar sign variations\n if frag == \"13,837.00\":\n if \"13837.00\" in msg_text or \"13,837\" in msg_text or \"13837\" in msg_text:\n continue\nLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775678101946",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `slack_data.json`, channel C002 must contain a message that reads: \"SELF-APPROVED: Main Street Office Products | Invoice INV-004322 | Amount $1,356.14 | 3-way match\"",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # The slack_data.json could be in the workspace or in external_services\n slack_file = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_file = c\n break\n \n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external_services path.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n # The rubric says channel C002 must contain the message.\n # slack_data.json can have two formats:\n # Format A (new): { \"channels\": { \"C002\": {...} }, \"messages\": { \"C002\": [...] } }\n # Format B (old): { \"channels\": [ { \"channel_id\": \"C002\", \"messages\": [...] } ] }\n \n channel_messages = []\n \n # Try Format A: messages dict keyed by channel ID\n messages_dict = slack_data.get(\"messages\", {})\n if isinstance(messages_dict, dict):\n if \"C002\" in messages_dict:\n msgs = messages_dict[\"C002\"]\n if isinstance(msgs, list):\n channel_messages.extend(msgs)\n \n # Try Format B: channels as a list with embedded messages\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for ch in channels:\n ch_id = ch.get(\"channel_id\", \"\") or ch.get(\"id\", \"\")\n ch_name = ch.get(\"channel_name\", \"\") or ch.get(\"name\", \"\")\n if ch_id == \"C002\" or \"ap-approvals\" in ch_name.lower().replace(\" \", \"\").replace(\"-\", \"\"):\n ch_msgs = ch.get(\"messages\", [])\n if isinstance(ch_msgs, list):\n channel_messages.extend(ch_msgs)\n elif isinstance(channels, dict):\n # channels could also be a dict keyed by channel ID\n if \"C002\" in channels:\n ch = channels[\"C002\"]\n ch_msgs = ch.get(\"messages\", [])\n if isinstance(ch_msgs, list):\n channel_messages.extend(ch_msgs)\n \n if not channel_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found for channel C002 in slack_data.json. Top-level keys: {list(slack_data.keys())}\"}\n \n # Target fragments we need to find in a single message (case-insensitive)\n required_fragments = [\n \"self-approved\",\n \"main street office products\",\n \"inv-004322\",\n \"1,356.14\",\n \"3-way match\"\n ]\n \n all_message_texts = []\n for msg in channel_messages:\n text = \"\"\n if isinstance(msg, dict):\n text = msg.get(\"message\", \"\") or msg.get(\"text\", \"\") or msg.get(\"content\", \"\") or \"\"\n elif isinstance(msg, str):\n text = msg\n all_message_texts.append(text)\n text_lower = text.lower()\n \n # Check if all required fragments are present\n found_all = True\n missing = []\n for frag in required_fragments:\n frag_lower = frag.lower()\n if frag_lower not in text_lower:\n # Try alternate for amount: without comma\n if frag == \"1,356.14\":\n if \"1356.14\" in text_lower:\n continue\n # Try alternate for self-approved: with or without hyphen\n if frag == \"self-approved\":\n if \"self approved\" in text_lower or \"selfapproved\" in text_lower:\n continue\n # Try alternate for 3-way match\n if frag == \"3-way match\":\n if \"3 way match\" in text_lower or \"3way match\" in text_lower or \"three-way match\" in text_lower or \"three way match\" in text_lower:\n continue\n found_all = False\n missing.append(frag)\n \n if found_all:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching message in channel C002: '{text}'\"}\n \n # If we didn't find a perfect match, report what we found\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No message in channel C002 matched. Expected a message containing all of: {required_fragments}. Messages found in channel: {all_message_texts}\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775678154191",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `scott@mainstreetofficeproducts.com` with subject line \"Meridian Partners - Price Discrepancy on Invoice INV-004322\" must NOT exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox), so we need external_services_path\n # But also check workspace_path for mailbox.json as a fallback\n mailbox_path = None\n \n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found at either external_services_path or workspace_path. Since no emails exist, the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not read mailbox.json: {e}. Since we cannot confirm the email exists, PASS by default.\"}\n \n emails = mailbox_data.get(\"emails\", [])\n \n target_to = \"scott@mainstreetofficeproducts.com\"\n target_subject = \"Meridian Partners - Price Discrepancy on Invoice INV-004322\"\n \n for email in emails:\n folder = str(email.get(\"folder\", \"\")).strip().lower()\n to_addr = str(email.get(\"to_addr\", \"\")).strip().lower()\n subject = str(email.get(\"subject\", \"\")).strip()\n \n # Check if it's a sent email\n if folder != \"sent\":\n continue\n \n # Check recipient (case-insensitive)\n if to_addr != target_to.lower():\n continue\n \n # Check subject (case-insensitive comparison)\n if subject.lower() == target_subject.lower():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found a sent email to '{email.get('to_addr')}' with subject '{subject}'. This email must NOT exist per the rubric.\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No sent email to '{target_to}' with subject '{target_subject}' was found in mailbox.json. The prohibited email does not exist.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775678196872",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, the Payment Queue must contain rows for exactly these vendors and no others: 'Deloitte Consulting LLP', 'Jones Day LLP', 'Workday Inc', '225 Wacker LLC', 'Aramark Workplace Services', 'Iron Mountain Inc', 'Sentinel Cybersecurity Solutions', 'Main Street Office Products'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'ap_ledger.xlsx' not found in workspace.\"}\n \n expected_vendors = {\n 'deloitte consulting llp',\n 'jones day llp',\n 'workday inc',\n '225 wacker llc',\n 'aramark workplace services',\n 'iron mountain inc',\n 'sentinel cybersecurity solutions',\n 'main street office products'\n }\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'ap_ledger.xlsx': {e}\"}\n \n # Find the Payment Queue sheet\n payment_queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n payment_queue_sheet = wb[name]\n break\n \n if payment_queue_sheet is None:\n # Maybe it's a section within a sheet; look for \"Payment Queue\" as a header in all sheets\n for name in wb.sheetnames:\n ws = wb[name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row, max_col=ws.max_column, values_only=False):\n for cell in row:\n if cell.value and isinstance(cell.value, str) and 'payment queue' in cell.value.lower():\n payment_queue_sheet = ws\n break\n if payment_queue_sheet:\n break\n if payment_queue_sheet:\n break\n \n if payment_queue_sheet is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a 'Payment Queue' sheet or section in ap_ledger.xlsx. Sheets found: {wb.sheetnames}\"}\n \n ws = payment_queue_sheet\n \n # Find the vendor column - look for header row\n vendor_col = None\n header_row = None\n data_start_row = None\n \n # Strategy: find a row that contains a header like 'vendor' or 'vendor name' or 'company'\n # Also look for the \"Payment Queue\" label to know where data starts\n payment_queue_label_row = None\n \n for row_idx in range(1, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n if 'payment queue' in val.lower():\n payment_queue_label_row = row_idx\n \n # Search for header row (possibly after Payment Queue label)\n search_start = payment_queue_label_row if payment_queue_label_row else 1\n \n for row_idx in range(search_start, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n val_lower = val.strip().lower()\n if val_lower in ('vendor', 'vendor name', 'vendor_name', 'company', 'company name', 'payee'):\n vendor_col = col_idx\n header_row = row_idx\n data_start_row = row_idx + 1\n break\n if vendor_col:\n break\n \n if vendor_col is None:\n # Fallback: try to find vendor names in any column\n # Scan all cells for known vendor names\n found_vendors = set()\n all_text_values = []\n for row_idx in range(1, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n all_text_values.append(val.strip())\n if val.strip().lower() in expected_vendors:\n found_vendors.add(val.strip().lower())\n \n if found_vendors:\n missing = expected_vendors - found_vendors\n extra_candidates = set()\n # Can't reliably determine extra without knowing column structure\n if missing:\n return {\"pass\": False, \"score\": len(found_vendors) / len(expected_vendors), \"feedback\": f\"Found vendors (no header detected): {found_vendors}. Missing: {missing}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found all expected vendors in Payment Queue (no explicit header detected): {found_vendors}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find vendor column or any expected vendor names in Payment Queue sheet. All text values found: {all_text_values[:50]}\"}\n \n # Read vendor names from the data rows\n found_vendors_raw = []\n for row_idx in range(data_start_row, ws.max_row + 1):\n val = ws.cell(row=row_idx, cLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
It's March 16, 2026. Please process any invoices that have come in today, plus the Kunlun Creative one from earlier this month, all the way through payment posting. Of course, everything should be done in accordance with the SOP. Files attached to inbound emails have already been downloaded, so you can already access them.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user