115 lines
64 KiB
JSON
115 lines
64 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "852af8c1-546e-4f60-b5e7-cf9f9a98b8ce",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In `mailbox.json`, a sent email to lindon@luminarylogistics.com must exist with subject containing 'INV-82419' and body containing the incomplete invoice notice about missing address fields (City, State, Zip Code) and instructions to resubmit to ap@meridianpartners.com.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_emails = [\n e for e in sent_emails\n if \"lindon@luminarylogistics.com\" in (e.get(\"to_addr\", \"\") or \"\").lower()\n ]\n \n if not target_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to lindon@luminarylogistics.com.\"}\n \n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n combined = subject + \" \" + body\n \n # Check subject contains INV-82419\n has_inv_subject = \"inv-82419\" in subject\n \n # Check body contains INV-82419 reference\n has_inv_body = \"inv-82419\" in body\n \n # Check body mentions missing address fields: City, State, Zip Code\n has_city = \"city\" in body\n has_state = \"state\" in body\n has_zip = any(term in body for term in [\"zip\", \"postal\"])\n # At least mention of incomplete/missing and address-related fields\n has_missing_notice = any(term in body for term in [\"missing\", \"incomplete\", \"not included\", \"lacking\", \"absent\", \"required\"])\n \n # Check resubmit instructions\n has_resubmit = any(term in body for term in [\"resubmit\", \"re-submit\", \"corrected\", \"resend\", \"re-send\", \"send\", \"updated\", \"revise\", \"provide\"])\n \n # Check ap email\n has_ap_email = \"ap@meridianpartners.com\" in body\n \n # Require: subject has INV-82419, body mentions missing address fields (city, state, zip), resubmit instructions, and ap email\n if has_inv_subject and has_missing_notice and has_city and has_state and has_zip and has_resubmit and has_ap_email:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found valid email to lindon@luminarylogistics.com about INV-82419 with incomplete invoice notice about missing address fields (City, State, Zip Code) and instructions to resubmit to ap@meridianpartners.com. Subject: {email.get('subject', '')}\"}\n \n # Report what we found for debugging\n details = []\n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n detail = {\n \"subject\": email.get(\"subject\", \"\"),\n \"has_inv_subject\": \"inv-82419\" in subject,\n \"has_city\": \"city\" in body,\n \"has_state\": \"state\" in body,\n \"has_zip\": any(t in body for t in [\"zip\", \"postal\"]),\n \"has_missing_notice\": any(t in body for t in [\"missing\", \"incomplete\", \"not included\", \"lacking\", \"absent\", \"required\"]),\n \"has_resubmit\": any(t in body for t in [\"resubmit\", \"re-submit\", \"corrected\", \"resend\", \"re-send\", \"send\", \"updated\", \"revise\", \"provide\"]),\n \"has_ap_email\": \"ap@meridianpartners.com\" in body\n }\n details.append(detail)\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(target_emails)} email(s) to lindon@luminarylogistics.com but none matched all required content. Details: {json.dumps(details, indent=2)}\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "9f1cd0b2-b083-4e8d-9308-dae522acc509",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In `Ap_ledger.xlsx`, the Invoice Register sheet must NOT contain a row for invoice INV-82419.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception as e:\n continue\n \n # Find invoice register sheet\n register_sheet = None\n for sheet_name in wb.sheetnames:\n if \"register\" in sheet_name.lower() or \"invoice\" in sheet_name.lower():\n register_sheet = wb[sheet_name]\n break\n \n if register_sheet is None:\n # Try first sheet\n if wb.sheetnames:\n register_sheet = wb[wb.sheetnames[0]]\n \n if register_sheet is None:\n continue\n \n for row in register_sheet.iter_rows(values_only=True):\n for cell in row:\n if cell is not None and \"inv-82419\" in str(cell).lower():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"INV-82419 found in Invoice Register sheet in {xlsx_path.name} - it should NOT be listed.\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-82419 is not present in the Invoice Register sheet in {xlsx_path.name}, as required.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read a suitable xlsx file with an Invoice Register sheet.\"}",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "6e8e36a8-e868-4af4-bc06-8cb2578ea253",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In `mailbox.json`, a sent email to greg.kendrick@kunluncreativestudio.com must exist with subject containing 'INV-KC-20260305' and body containing: amount $6,019.44, pending internal approval, and 5 additional business days.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_emails = [\n e for e in sent_emails\n if \"greg.kendrick@kunluncreativestudio.com\" in (e.get(\"to_addr\", \"\") or \"\")\n ]\n \n if not target_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to greg.kendrick@kunluncreativestudio.com.\"}\n \n for email in target_emails:\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n \n if \"inv-kc-20260305\" not in subject and \"inv-kc-20260305\" not in body:\n continue\n \n has_inv = \"inv-kc-20260305\" in body\n has_amount = \"6,019.44\" in body or \"6019.44\" in body\n has_pending = any(term in body for term in [\"pending\", \"approval\", \"approve\"])\n has_days = \"5\" in body and \"business day\" in body\n \n if has_inv and has_amount and has_pending and has_days:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found valid status update email to greg.kendrick@kunluncreativestudio.com about INV-KC-20260305 with all required content.\"}\n \n found_subjects = [e.get(\"subject\", \"\") for e in target_emails]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found emails to greg.kendrick@kunluncreativestudio.com but none matched all required content (INV-KC-20260305, $6,019.44, pending approval, 5 business days). Subjects: {found_subjects}\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "76697ce7-241c-48a7-87d2-57eb944b78aa",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `Ap_ledger.xlsx`, the Payment Queue sheet must list INV-4839201-24 with Vendor 'Sentinel Cybersecurity Solutions', invoice date 3/16/2026, due date 4/15/2026, amount $13,837, GL Account 6400-010-1100, no PO number, and Status 'Approved'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n # Find Payment Queue sheet\n payment_sheet = None\n for sheet_name in wb.sheetnames:\n if \"payment\" in sheet_name.lower() and \"queue\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n if payment_sheet is None:\n for sheet_name in wb.sheetnames:\n if \"queue\" in sheet_name.lower() or \"payment\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n \n if payment_sheet is None:\n continue\n \n rows = list(payment_sheet.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-4839201-24\" not in combined:\n continue\n \n # Found the row - check all required fields\n errors = []\n \n # Vendor check\n if \"sentinel cybersecurity\" not in combined:\n errors.append(\"Vendor 'Sentinel Cybersecurity Solutions' not found in row\")\n \n # Amount check - $13,837\n has_amount = \"13,837\" in combined or \"13837\" in combined\n if not has_amount:\n # Check with tolerance\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 13837) <= 138.37: # 1% tolerance\n has_amount = True\n break\n except:\n pass\n if not has_amount:\n errors.append(\"Amount $13,837 not found in row\")\n \n # GL Account check\n if \"6400-010-1100\" not in combined:\n errors.append(\"GL Account 6400-010-1100 not found in row\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found in row\")\n \n # Invoice date check (3/16/2026)\n has_inv_date = any(d in combined for d in [\"3/16/2026\", \"03/16/2026\", \"2026-03-16\", \"march 16\"])\n if not has_inv_date:\n errors.append(\"Invoice date 3/16/2026 not found in row\")\n \n # Due date check (4/15/2026)\n has_due_date = any(d in combined for d in [\"4/15/2026\", \"04/15/2026\", \"2026-04-15\", \"april 15\"])\n if not has_due_date:\n errors.append(\"Due date 4/15/2026 not found in row\")\n \n if errors:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found INV-4839201-24 in Payment Queue but issues: {'; '.join(errors)}. Row: {row_vals}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-4839201-24 found in Payment Queue with all required fields verified. Row: {row_vals}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-4839201-24 not found in any Payment Queue sheet in workspace xlsx files.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "6929234d-ab72-4732-90bb-865f606bf268",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `match_log_2026_Q1.xlsx` (or similar match log file), a row for INV-004322 must exist with: Match Date 3/16/2026, Vendor 'Main Street Office Products', PO PO-2026-00532, PO Amount $1,276.14, GR Number GR-20260311-004, GR Date 3/11/2026, Invoice Date 3/16/2026, Invoice Amount $1,356.14, GL Account 6200-000-5100, PO Variance $80.00, Status 'Approved'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find match log file\n match_files = list(Path(workspace_path).glob(\"*match*log*.xlsx\")) + list(Path(workspace_path).glob(\"*match*.xlsx\")) + list(Path(workspace_path).glob(\"match_log*.xlsx\"))\n if not match_files:\n match_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not match_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found for match log.\"}\n \n for xlsx_path in match_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-004322\" not in combined:\n continue\n \n errors = []\n \n # Vendor check\n if \"main street office products\" not in combined:\n errors.append(\"Vendor 'Main Street Office Products' not found\")\n \n # PO check\n if \"po-2026-00532\" not in combined:\n errors.append(\"PO number PO-2026-00532 not found\")\n \n # PO Amount check $1,276.14\n has_po_amt = \"1,276.14\" in combined or \"1276.14\" in combined\n if not has_po_amt:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1276.14) <= 12.76:\n has_po_amt = True\n break\n except:\n pass\n if not has_po_amt:\n errors.append(\"PO Amount $1,276.14 not found\")\n \n # GR Number check\n if \"gr-20260311-004\" not in combined:\n errors.append(\"GR Number GR-20260311-004 not found\")\n \n # GR Date check 3/11/2026\n has_gr_date = any(d in combined for d in [\"3/11/2026\", \"03/11/2026\", \"2026-03-11\", \"march 11\"])\n if not has_gr_date:\n errors.append(\"GR Date 3/11/2026 not found\")\n \n # Invoice Amount $1,356.14\n has_inv_amt = \"1,356.14\" in combined or \"1356.14\" in combined\n if not has_inv_amt:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1356.14) <= 13.56:\n has_inv_amt = True\n break\n except:\n pass\n if not has_inv_amt:\n errors.append(\"Invoice Amount $1,356.14 not found\")\n \n # GL Account check\n if \"6200-000-5100\" not in combined:\n errors.append(\"GL Account 6200-000-5100 not found\")\n \n # PO Variance $80.00\n has_variance = \"80\" in combined\n if not has_variance:\n errors.append(\"PO Variance $80.00 not found\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found\")\n \n # Match date 3/16/2026\n has_
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "9e5460f7-30b2-45c0-9f50-04b43f5b8b6e",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In `Ap_ledger.xlsx`, the Payment Queue sheet must list INV-004322 with Vendor 'Main Street Office Products', invoice date 3/16/2026, due date 4/15/2026, amount $1,356.14, GL Account 6200-000-5100, PO number PO-2026-00532, and Status 'Approved'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"*ledger*.xlsx\")) + list(Path(workspace_path).glob(\"*Ledger*.xlsx\")) + list(Path(workspace_path).glob(\"Ap_ledger.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace.\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n except Exception:\n continue\n \n # Find Payment Queue sheet\n payment_sheet = None\n for sheet_name in wb.sheetnames:\n if \"payment\" in sheet_name.lower() and \"queue\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n if payment_sheet is None:\n for sheet_name in wb.sheetnames:\n if \"queue\" in sheet_name.lower() or \"payment\" in sheet_name.lower():\n payment_sheet = wb[sheet_name]\n break\n \n if payment_sheet is None:\n continue\n \n rows = list(payment_sheet.iter_rows(values_only=True))\n \n for row in rows:\n row_vals = [str(c) if c is not None else \"\" for c in row]\n row_lower = [v.lower() for v in row_vals]\n combined = \" \".join(row_lower)\n \n if \"inv-004322\" not in combined:\n continue\n \n errors = []\n \n # Vendor check\n if \"main street office products\" not in combined:\n errors.append(\"Vendor 'Main Street Office Products' not found\")\n \n # Amount check $1,356.14\n has_amount = \"1,356.14\" in combined or \"1356.14\" in combined\n if not has_amount:\n for val in row_vals:\n try:\n num = float(val.replace(\",\", \"\").replace(\"$\", \"\").strip())\n if abs(num - 1356.14) <= 13.56:\n has_amount = True\n break\n except:\n pass\n if not has_amount:\n errors.append(\"Amount $1,356.14 not found\")\n \n # GL Account check\n if \"6200-000-5100\" not in combined:\n errors.append(\"GL Account 6200-000-5100 not found\")\n \n # PO Number check\n if \"po-2026-00532\" not in combined:\n errors.append(\"PO number PO-2026-00532 not found\")\n \n # Status check\n if \"approved\" not in combined:\n errors.append(\"Status 'Approved' not found\")\n \n # Invoice date 3/16/2026\n has_inv_date = any(d in combined for d in [\"3/16/2026\", \"03/16/2026\", \"2026-03-16\", \"march 16\"])\n if not has_inv_date:\n errors.append(\"Invoice date 3/16/2026 not found\")\n \n # Due date 4/15/2026\n has_due_date = any(d in combined for d in [\"4/15/2026\", \"04/15/2026\", \"2026-04-15\", \"april 15\"])\n if not has_due_date:\n errors.append(\"Due date 4/15/2026 not found\")\n \n if errors:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found INV-004322 in Payment Queue but issues: {'; '.join(errors)}. Row: {row_vals}\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"INV-004322 found in Payment Queue with all required fields verified. Row: {row_vals}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-004322 not found in any Payment Queue sheet in workspace xlsx files.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "813da462-d30a-4309-98cb-be45d71e3de3",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `slack_data.json`, the #ap-exceptions channel must contain following message: \"PRICE VARIANCE: Main Street Office Products, 432319, Invoice INV-004322, PO Price $59.99, Invoice Price $79.99, Variance 33.34%\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check Slack.\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find #ap-exceptions channel ID\n ap_exceptions_channel_id = None\n for ch_id, ch_info in channels.items():\n if \"ap-exceptions\" in ch_info.get(\"name\", \"\").lower():\n ap_exceptions_channel_id = ch_id\n break\n \n if ap_exceptions_channel_id is None:\n # Try searching messages in channels with related names\n all_channel_messages = []\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get(\"name\", \"\")\n if \"ap\" in ch_name.lower() or \"exception\" in ch_name.lower():\n all_channel_messages.extend(messages.get(ch_id, []))\n if not all_channel_messages:\n # Search all messages as fallback\n for ch_id, msgs in messages.items():\n all_channel_messages.extend(msgs)\n else:\n all_channel_messages = messages.get(ap_exceptions_channel_id, [])\n \n # The rubric requires a message like:\n # \"PRICE VARIANCE: Main Street Office Products, 432319, Invoice INV-004322, PO Price $59.99, Invoice Price $79.99, Variance 33.34%\"\n # We check for the key components loosely\n \n for msg in all_channel_messages:\n text = (msg.get(\"text\", \"\") or \"\")\n text_lower = text.lower()\n \n has_price_variance = \"price variance\" in text_lower\n has_vendor = \"main street office products\" in text_lower\n has_po_num = \"432319\" in text\n has_inv = \"inv-004322\" in text_lower\n has_po_price = \"59.99\" in text\n has_inv_price = \"79.99\" in text\n # Accept variance around 33.34% loosely\n has_variance_pct = \"33.3\" in text or \"33.4\" in text\n \n if has_price_variance and has_vendor and has_po_num and has_inv and has_po_price and has_inv_price and has_variance_pct:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found ap-exceptions Slack message with all required price variance details: {text[:300]}\"}\n \n # Provide detailed failure info\n sample_msgs = [msg.get(\"text\", \"\")[:200] for msg in all_channel_messages[:10]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No Slack message found in #ap-exceptions with all required price variance details (PRICE VARIANCE, Main Street Office Products, 432319, INV-004322, $59.99, $79.99, ~33.34%). Sample messages found: {sample_msgs}\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775621743582",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In Ap_ledger.xlsx, on the Invoice Register sheet, INV-KC-20260305's status must be 'APPR' or 'APPR Hold' (case-insensitive).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n\n # Find the AP ledger file with flexible naming\n candidates = []\n for pattern in ['**/Ap_ledger.xlsx', '**/AP_Ledger.xlsx', '**/ap_ledger.xlsx',\n '**/AP_ledger.xlsx', '**/Ap_Ledger.xlsx']:\n candidates.extend(workspace.glob(pattern))\n\n if not candidates:\n for f in workspace.glob('**/*.xlsx'):\n if 'ledger' in f.name.lower() and 'ap' in f.name.lower():\n candidates.append(f)\n\n direct = workspace / \"Ap_ledger.xlsx\"\n if direct.exists() and direct not in candidates:\n candidates.insert(0, direct)\n\n if not candidates:\n all_xlsx = list(workspace.glob('**/*.xlsx'))\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"AP Ledger file not found. Available xlsx files: {[str(f.relative_to(workspace)) for f in all_xlsx]}\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Failed to open workbook {file_path.name}: {e}\"}\n\n # Find the Invoice Register sheet (case-insensitive, flexible)\n sheet = None\n for name in wb.sheetnames:\n normalized = name.strip().lower().replace(' ', '').replace('_', '')\n if normalized in ('invoiceregister', 'invoicereg'):\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'reg' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Sheet 'Invoice Register' not found. Available sheets: {wb.sheetnames}\"}\n\n # Headers are in row 2. Match exactly (case-insensitive, trimmed).\n HEADER_ROW = 2\n invoice_col = None\n status_col = None\n for col_idx in range(1, sheet.max_column + 1):\n val = sheet.cell(row=HEADER_ROW, column=col_idx).value\n if val is None:\n continue\n header = str(val).strip().lower()\n if header == 'invoice #':\n invoice_col = col_idx\n elif header == 'status':\n status_col = col_idx\n\n if invoice_col is None or status_col is None:\n row2_values = [sheet.cell(row=HEADER_ROW, column=c).value\n for c in range(1, sheet.max_column + 1)]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Could not locate 'Invoice #' and/or 'Status' headers in row {HEADER_ROW}. Row {HEADER_ROW} values: {row2_values}\"}\n\n # Find the target invoice row\n target_invoice = 'INV-KC-20260305'\n found_status = None\n for row_idx in range(HEADER_ROW + 1, sheet.max_row + 1):\n cell_val = sheet.cell(row=row_idx, column=invoice_col).value\n if cell_val is None:\n continue\n if str(cell_val).strip().upper() == target_invoice.upper():\n status_val = sheet.cell(row=row_idx, column=status_col).value\n found_status = str(status_val).strip() if status_val is not None else None\n break\n\n if found_status is None:\n # Distinguish \"not found\" vs \"found but empty status\"\n for row_idx in range(HEADER_ROW + 1, sheet.max_row + 1):\n cell_val = sheet.cell(row=row_idx, column=invoice_col).value\n if cell_val is not None and str(cell_val).strip().upper() == target_invoice.upper():\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Invoice '{target_invoice}' found but its Status cell is empty.\"}\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Invoice '{target_invoice}' not found in the Invoice Register sheet.\"}\n\n # Case-insensitive status check\n status_lower =
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775677892670",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "The file `match_log_2026_Q1.xlsx` must not contain a row with invoice number INV-82419.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2026_Q1.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open file: {e}\"}\n \n target = \"INV-82419\"\n target_lower = target.lower().strip()\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(values_only=True), start=1):\n for cell_val in row:\n if cell_val is not None:\n cell_str = str(cell_val).strip().lower()\n if cell_str == target_lower or target_lower in cell_str:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found INV-82419 in sheet '{sheet_name}', row {row_idx}. The file must NOT contain this invoice number.\"\n }\n \n wb.close()\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"Confirmed: match_log_2026_Q1.xlsx does not contain a row with invoice number INV-82419.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775677952539",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "In `Ap_ledger.xlsx`, the Invoice Register sheet must NOT contain a row for invoice INV-004322",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find Ap_ledger.xlsx with flexible matching\n fp = Path(workspace_path) / \"Ap_ledger.xlsx\"\n if not fp.exists():\n # Try case-insensitive glob\n candidates = list(Path(workspace_path).glob(\"**/[Aa][Pp]_[Ll]edger*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ledger*.xlsx\"))\n if candidates:\n fp = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: Ap_ledger.xlsx in {workspace_path}\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n\n # Find the Invoice Register sheet (flexible on case/spacing)\n sheet = None\n for name in wb.sheetnames:\n normalized = name.strip().lower().replace(' ', '').replace('_', '')\n if normalized in ['invoiceregister', 'invoicesregister']:\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower() and 'register' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if name.strip() == 'Invoice Register':\n sheet = wb[name]\n break\n if sheet is None:\n for name in wb.sheetnames:\n if 'invoice' in name.lower():\n sheet = wb[name]\n break\n if sheet is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"'Invoice Register' sheet not found. Available sheets: {wb.sheetnames}\"}\n\n target = 'INV-004322'\n found = False\n found_row = None\n for row_idx, row in enumerate(sheet.iter_rows(min_row=1, values_only=True), start=1):\n for cell in row:\n if cell is not None:\n cell_str = str(cell).strip().upper()\n if target.upper() in cell_str:\n found = True\n found_row = row_idx\n break\n if found:\n break\n\n if found:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Invoice {target} was found in the Invoice Register sheet at row {found_row}. It should NOT be present.\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Confirmed: Invoice {target} is not present in the Invoice Register sheet of {fp.name}.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775677983569",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "In `slack_data.json`, the channel C002 must not contain a message with \"Luminary Logistics\" in it.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for slack_data.json\n slack_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n # Also check workspace_path\n if not slack_path:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if not slack_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in either external_services_path or workspace_path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n \n # The rubric says channel C002 must not contain a message with \"Luminary Logistics\".\n # slack_data.json can have different formats. Handle both:\n # Format A (dict-based): messages is a dict keyed by channel ID -> list of message objects\n # Format B (list-based): channels is a list of channel objects with messages inside\n \n messages_to_check = []\n \n # Try Format A: data[\"messages\"][\"C002\"] -> list of messages\n if \"messages\" in slack_data and isinstance(slack_data[\"messages\"], dict):\n channel_messages = slack_data[\"messages\"].get(\"C002\", [])\n if isinstance(channel_messages, list):\n messages_to_check.extend(channel_messages)\n \n # Try Format B: data[\"channels\"] is a list, find channel with id C002\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for channel in channels:\n ch_id = channel.get(\"id\", \"\") or channel.get(\"channel_id\", \"\")\n if ch_id == \"C002\":\n msgs = channel.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_to_check.extend(msgs)\n elif isinstance(channels, dict):\n # Format: channels is a dict keyed by channel ID\n ch = channels.get(\"C002\")\n if ch and isinstance(ch, dict):\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_to_check.extend(msgs)\n \n if not messages_to_check:\n # No messages found for C002 at all - that means no offending message exists, so pass\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel C002 has no messages (or was not found), so no 'Luminary Logistics' message exists. Condition satisfied.\"}\n \n for msg in messages_to_check:\n # Check all text-like fields\n msg_text = \"\"\n for key in [\"message\", \"text\", \"body\", \"content\"]:\n val = msg.get(key, \"\")\n if val:\n msg_text += \" \" + str(val)\n if \"luminary logistics\" in msg_text.lower():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a message containing 'Luminary Logistics' in channel C002: '{msg_text.strip()[:200]}'\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No message containing 'Luminary Logistics' was found in channel C002. Checked {len(messages_to_check)} messages.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775678010037",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "In `slack_data.json`, channel C002 must not contain a message with \"Kunlun Creative Studio\" in it.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Find slack_data.json - check external_services_path first, then workspace\n slack_path = None\n \n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # The rubric says: channel C002 must not contain a message with \"Kunlun Creative Studio\"\n # Per the Slack data format, messages are keyed by channel ID under \"messages\"\n messages_dict = slack_data.get(\"messages\", {})\n \n # Get messages for channel C002\n c002_messages = messages_dict.get(\"C002\", [])\n \n if not isinstance(c002_messages, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected messages for C002 to be a list, got {type(c002_messages).__name__}\"}\n \n # Check each message for \"Kunlun Creative Studio\"\n found_messages = []\n for msg in c002_messages:\n msg_text = \"\"\n if isinstance(msg, dict):\n # Collect all text-like fields\n for field in [\"text\", \"message\", \"content\", \"body\"]:\n val = msg.get(field, \"\")\n if val:\n msg_text += \" \" + str(val)\n # Also check attachments if any\n attachments = msg.get(\"attachments\", [])\n if isinstance(attachments, list):\n for att in attachments:\n if isinstance(att, dict):\n for field in [\"text\", \"fallback\", \"pretext\", \"title\"]:\n val = att.get(field, \"\")\n if val:\n msg_text += \" \" + str(val)\n elif isinstance(msg, str):\n msg_text = msg\n \n if \"kunlun creative studio\" in msg_text.lower():\n found_messages.append(msg_text.strip()[:200])\n \n if found_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(found_messages)} message(s) in channel C002 containing 'Kunlun Creative Studio'. First match: {found_messages[0]}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No messages containing 'Kunlun Creative Studio' found in channel C002. Checked {len(c002_messages)} messages.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775678035580",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "In `slack_data.json`, channel C002 must contain a message that reads: \"PAYMENT AUTHORIZATION REQUIRED: Sentinel Cybersecurity Solutions | Invoice INV-4839201-24 | Amount $13,837.00 | Recurring Invoice | Tanya Okonkwo and Rachel Whitfield\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # The slack_data.json could be in workspace or external_services\n slack_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Required fragments to look for (case-insensitive, flexible)\n required_fragments = [\n \"PAYMENT AUTHORIZATION REQUIRED\",\n \"Sentinel Cybersecurity Solutions\",\n \"INV-4839201-24\",\n \"13,837.00\",\n \"Recurring Invoice\",\n \"Tanya Okonkwo\",\n \"Rachel Whitfield\"\n ]\n \n # Try to find messages in channel C002 using the new slack data format\n # The new format has: messages as dict keyed by channel ID -> list of message objects\n # Also try the old format: channels as list with messages inside\n \n all_messages = []\n \n # New format: messages dict keyed by channel ID\n messages_dict = slack_data.get(\"messages\", {})\n if isinstance(messages_dict, dict):\n # Try C002 directly\n c002_msgs = messages_dict.get(\"C002\", [])\n if isinstance(c002_msgs, list):\n all_messages.extend(c002_msgs)\n \n # Also search all channels in case C002 is referenced differently\n # But prioritize C002\n \n # Old format: channels as list\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for ch in channels:\n ch_id = ch.get(\"channel_id\", \"\") or ch.get(\"id\", \"\")\n ch_name = ch.get(\"channel_name\", \"\") or ch.get(\"name\", \"\")\n if ch_id == \"C002\" or \"ap-approvals\" in ch_name.lower().replace(\" \", \"-\"):\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n all_messages.extend(msgs)\n elif isinstance(channels, dict):\n # channels could be a dict keyed by channel ID\n c002_ch = channels.get(\"C002\", {})\n if c002_ch:\n # Channel info found, messages are in the messages dict\n pass\n # Also check by name\n for ch_id, ch_info in channels.items():\n if ch_id == \"C002\" or (isinstance(ch_info, dict) and \"ap-approvals\" in (ch_info.get(\"name\", \"\") or \"\").lower()):\n # Messages for this channel are in the messages_dict\n ch_msgs = messages_dict.get(ch_id, []) if isinstance(messages_dict, dict) else []\n if isinstance(ch_msgs, list):\n all_messages.extend(ch_msgs)\n \n if not all_messages:\n # Provide debug info\n available_keys = list(slack_data.keys())\n msg_keys = list(messages_dict.keys()) if isinstance(messages_dict, dict) else \"not a dict\"\n ch_info = \"\"\n if isinstance(channels, list):\n ch_info = str([{\"id\": c.get(\"channel_id\", c.get(\"id\", \"\")), \"name\": c.get(\"channel_name\", c.get(\"name\", \"\"))} for c in channels])\n elif isinstance(channels, dict):\n ch_info = str({k: v.get(\"name\", \"\") if isinstance(v, dict) else v for k, v in channels.items()})\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found for channel C002. Top-level keys: {available_keys}. Messages keys: {msg_keys}. Channels info: {ch_info}\"}\n \n # Search messages for r
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775678101946",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "In `slack_data.json`, channel C002 must contain a message that reads: \"SELF-APPROVED: Main Street Office Products | Invoice INV-004322 | Amount $1,356.14 | 3-way match\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # The slack_data.json could be in the workspace or in external_services\n slack_file = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_file = c\n break\n \n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external_services path.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n # The rubric says channel C002 must contain the message.\n # slack_data.json can have two formats:\n # Format A (new): { \"channels\": { \"C002\": {...} }, \"messages\": { \"C002\": [...] } }\n # Format B (old): { \"channels\": [ { \"channel_id\": \"C002\", \"messages\": [...] } ] }\n \n channel_messages = []\n \n # Try Format A: messages dict keyed by channel ID\n messages_dict = slack_data.get(\"messages\", {})\n if isinstance(messages_dict, dict):\n if \"C002\" in messages_dict:\n msgs = messages_dict[\"C002\"]\n if isinstance(msgs, list):\n channel_messages.extend(msgs)\n \n # Try Format B: channels as a list with embedded messages\n channels = slack_data.get(\"channels\", [])\n if isinstance(channels, list):\n for ch in channels:\n ch_id = ch.get(\"channel_id\", \"\") or ch.get(\"id\", \"\")\n ch_name = ch.get(\"channel_name\", \"\") or ch.get(\"name\", \"\")\n if ch_id == \"C002\" or \"ap-approvals\" in ch_name.lower().replace(\" \", \"\").replace(\"-\", \"\"):\n ch_msgs = ch.get(\"messages\", [])\n if isinstance(ch_msgs, list):\n channel_messages.extend(ch_msgs)\n elif isinstance(channels, dict):\n # channels could also be a dict keyed by channel ID\n if \"C002\" in channels:\n ch = channels[\"C002\"]\n ch_msgs = ch.get(\"messages\", [])\n if isinstance(ch_msgs, list):\n channel_messages.extend(ch_msgs)\n \n if not channel_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found for channel C002 in slack_data.json. Top-level keys: {list(slack_data.keys())}\"}\n \n # Target fragments we need to find in a single message (case-insensitive)\n required_fragments = [\n \"self-approved\",\n \"main street office products\",\n \"inv-004322\",\n \"1,356.14\",\n \"3-way match\"\n ]\n \n all_message_texts = []\n for msg in channel_messages:\n text = \"\"\n if isinstance(msg, dict):\n text = msg.get(\"message\", \"\") or msg.get(\"text\", \"\") or msg.get(\"content\", \"\") or \"\"\n elif isinstance(msg, str):\n text = msg\n all_message_texts.append(text)\n text_lower = text.lower()\n \n # Check if all required fragments are present\n found_all = True\n missing = []\n for frag in required_fragments:\n frag_lower = frag.lower()\n if frag_lower not in text_lower:\n # Try alternate for amount: without comma\n if frag == \"1,356.14\":\n if \"1356.14\" in text_lower:\n continue\n # Try alternate for self-approved: with or without hyphen\n if frag == \"self-approved\":\n if \"self approved\" in text_lower or \"selfapproved\" in text_lower:\n continue\n # Try alternate for 3
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775678154191",
|
||
|
|
"sort_order": 14,
|
||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `scott@mainstreetofficeproducts.com` with subject line \"Meridian Partners - Price Discrepancy on Invoice INV-004322\" must NOT exist.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox), so we need external_services_path\n # But also check workspace_path for mailbox.json as a fallback\n mailbox_path = None\n \n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if not mailbox_path:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found at either external_services_path or workspace_path. Since no emails exist, the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not read mailbox.json: {e}. Since we cannot confirm the email exists, PASS by default.\"}\n \n emails = mailbox_data.get(\"emails\", [])\n \n target_to = \"scott@mainstreetofficeproducts.com\"\n target_subject = \"Meridian Partners - Price Discrepancy on Invoice INV-004322\"\n \n for email in emails:\n folder = str(email.get(\"folder\", \"\")).strip().lower()\n to_addr = str(email.get(\"to_addr\", \"\")).strip().lower()\n subject = str(email.get(\"subject\", \"\")).strip()\n \n # Check if it's a sent email\n if folder != \"sent\":\n continue\n \n # Check recipient (case-insensitive)\n if to_addr != target_to.lower():\n continue\n \n # Check subject (case-insensitive comparison)\n if subject.lower() == target_subject.lower():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found a sent email to '{email.get('to_addr')}' with subject '{subject}'. This email must NOT exist per the rubric.\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No sent email to '{target_to}' with subject '{target_subject}' was found in mailbox.json. The prohibited email does not exist.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775678196872",
|
||
|
|
"sort_order": 15,
|
||
|
|
"rubric_text": "In `ap_ledger.xlsx`, the Payment Queue must contain rows for exactly these vendors and no others: 'Deloitte Consulting LLP', 'Jones Day LLP', 'Workday Inc', '225 Wacker LLC', 'Aramark Workplace Services', 'Iron Mountain Inc', 'Sentinel Cybersecurity Solutions', 'Main Street Office Products'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'ap_ledger.xlsx' not found in workspace.\"}\n \n expected_vendors = {\n 'deloitte consulting llp',\n 'jones day llp',\n 'workday inc',\n '225 wacker llc',\n 'aramark workplace services',\n 'iron mountain inc',\n 'sentinel cybersecurity solutions',\n 'main street office products'\n }\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'ap_ledger.xlsx': {e}\"}\n \n # Find the Payment Queue sheet\n payment_queue_sheet = None\n for name in wb.sheetnames:\n if 'payment' in name.lower() and 'queue' in name.lower():\n payment_queue_sheet = wb[name]\n break\n \n if payment_queue_sheet is None:\n # Maybe it's a section within a sheet; look for \"Payment Queue\" as a header in all sheets\n for name in wb.sheetnames:\n ws = wb[name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row, max_col=ws.max_column, values_only=False):\n for cell in row:\n if cell.value and isinstance(cell.value, str) and 'payment queue' in cell.value.lower():\n payment_queue_sheet = ws\n break\n if payment_queue_sheet:\n break\n if payment_queue_sheet:\n break\n \n if payment_queue_sheet is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a 'Payment Queue' sheet or section in ap_ledger.xlsx. Sheets found: {wb.sheetnames}\"}\n \n ws = payment_queue_sheet\n \n # Find the vendor column - look for header row\n vendor_col = None\n header_row = None\n data_start_row = None\n \n # Strategy: find a row that contains a header like 'vendor' or 'vendor name' or 'company'\n # Also look for the \"Payment Queue\" label to know where data starts\n payment_queue_label_row = None\n \n for row_idx in range(1, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n if 'payment queue' in val.lower():\n payment_queue_label_row = row_idx\n \n # Search for header row (possibly after Payment Queue label)\n search_start = payment_queue_label_row if payment_queue_label_row else 1\n \n for row_idx in range(search_start, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n val_lower = val.strip().lower()\n if val_lower in ('vendor', 'vendor name', 'vendor_name', 'company', 'company name', 'payee'):\n vendor_col = col_idx\n header_row = row_idx\n data_start_row = row_idx + 1\n break\n if vendor_col:\n break\n \n if vendor_col is None:\n # Fallback: try to find vendor names in any column\n # Scan all cells for known vendor names\n found_vendors = set()\n all_text_values = []\n for row_idx in range(1, ws.max_row + 1):\n for col_idx in range(1, ws.max_column + 1):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val and isinstance(val, str):\n all_text_values.append(val.strip())\n if val.strip().lower() in expected_vendors:\n found_vendors.add(val.strip().lower())\n \n if found_vendors:\n missing = expected_vendor
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|