Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,177 @@
|
||||
[
|
||||
{
|
||||
"id": "a309c711-6d28-46ca-8b80-b0eda8c78b0c",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `slack_data.json`, there must be NO message containing 'AUDIT HOLD APPLIED' in channel C006 (#vendor-management) after timestamp 1757687400.000042, confirming no audit hold was incorrectly applied to SecureNet Compliance (V-00011) after that time.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n threshold_ts = 1757687400.000042\n found = []\n for channel_id, msgs in messages.items():\n for msg in msgs:\n try:\n ts = float(msg.get(\"ts\", 0))\n except (ValueError, TypeError):\n continue\n if ts > threshold_ts and \"AUDIT HOLD APPLIED\" in msg.get(\"text\", \"\"):\n found.append({\"channel\": channel_id, \"ts\": msg.get(\"ts\"), \"text\": msg.get(\"text\", \"\")[:100]})\n if found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found)} message(s) containing 'AUDIT HOLD APPLIED' after timestamp {threshold_ts}: {found}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No 'AUDIT HOLD APPLIED' messages found after timestamp 1757687400.000042. Correct — no improper audit hold was applied to SecureNet Compliance.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "d51b8a8e-eae7-4d3f-abab-978637870b80",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `mailbox.json`, there must be NO sent email to billing@securenetcompliance.com or raj.patel@meridianpartners.com, confirming no improper payment or response was sent regarding SecureNet Compliance (V-00011, Invoice SNC-2025-001).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n from datetime import datetime, timezone\n\n # Any sent email after the last pre-existing one (2025-09-12T09:00:00-04:00 = 13:00 UTC)\n # is considered model-generated during the task.\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n forbidden_addresses = [\n \"billing@securenetcompliance.com\",\n \"raj.patel@meridianpartners.com\",\n ]\n emails = data.get(\"emails\", [])\n violations = []\n for email in emails:\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue # skip pre-existing sent emails\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addresses:\n if addr in to_addr:\n violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:80]}'\")\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to forbidden SecureNet Compliance addresses: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to billing@securenetcompliance.com or raj.patel@meridianpartners.com. Correct — legal threat was not actioned.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "9fc28e2d-fe54-4b39-b7c9-345dae70b338",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `mailbox.json`, there must be NO sent email to bmatthews@pinnacleprintsvc.com, bmatthews@pinnacleprint-services.com, or bmatthews@pinnacleprint.com, and NO sent email to maria.santos@meridianpartners.com, marcus.webb@meridianpartners.com, or raj.patel@meridianpartners.com regarding Pinnacle Print Services (V-00003) bank change request — confirming no bank change was processed for a suspected BEC fraud vendor.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n # Any sent email after the last pre-existing one (end of 2025-09-15)\n # is considered model-generated during the task.\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n forbidden_addresses = [\n \"bmatthews@pinnacleprintsvc.com\",\n \"bmatthews@pinnacleprint-services.com\",\n \"bmatthews@pinnacleprint.com\",\n \"maria.santos@meridianpartners.com\",\n \"marcus.webb@meridianpartners.com\",\n \"raj.patel@meridianpartners.com\",\n ]\n\n emails = data.get(\"emails\", [])\n violations = []\n for email in emails:\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue # skip pre-existing sent emails\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addresses:\n if addr in to_addr:\n violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:80]}'\")\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to forbidden addresses related to Pinnacle Print Services BEC bank change request: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to Pinnacle vendor addresses or internal team members. Correct — bank change request was not processed.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "78ef4e7c-fe51-4573-b827-3ce3baea02ea",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email with 'in_reply_to' referencing 'cm-38720@cascadeit.com' sent to ar@cascadeit.com, with body text referencing credit memo CM-38720, the amount $2,000.00, and original invoice INV-38720 totaling $1,950.00, requesting a corrected credit memo or explanation.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n candidate = None\n for email in sent_emails:\n if \"ar@cascadeit.com\" in (email.get(\"to_addr\") or \"\").lower():\n candidate = email\n break\n\n if candidate is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to ar@cascadeit.com. Expected a dispute email regarding credit memo CM-38720.\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n s = s.replace(',', '')\n return s.strip()\n\n body = normalize(candidate.get(\"body_text\") or \"\")\n\n # Expected full body (from rubric draft), broken into complete sentences for verification:\n # \"Dear Brian Choi, We received credit memo CM-38720 dated 2025 for $2,000.00.\n # This amount exceeds the original invoice INV-38720 totaling $1,950.00.\n # We are unable to apply this credit until clarification is received.\n # Please provide a corrected credit memo or written explanation.\n # Meridian Partners Accounts Payable\"\n checks = {\n \"greeting (Dear Brian Choi)\":\n \"dear brian choi\" in body,\n \"cm receipt sentence (CM-38720 dated [2025-09-11 or 09/11/2025] for $2,000.00)\":\n (\"we received credit memo cm-38720 dated 2025-09-11 for $2000.00\" in body or\n \"we received credit memo cm-38720 dated 09/11/2025 for $2000.00\" in body),\n \"exceeds invoice sentence (INV-38720 totaling $1,950.00)\":\n \"this amount exceeds the original invoice inv-38720 totaling $1950.00\" in body,\n \"unable to apply sentence\":\n \"we are unable to apply this credit until clarification is received\" in body,\n \"corrected cm or explanation sentence\":\n \"please provide a corrected credit memo or written explanation\" in body,\n \"sign-off (Meridian Partners Accounts Payable)\":\n \"meridian partners accounts payable\" in body\n }\n\n\n failed = [k for k, v in checks.items() if not v]\n passed_count = len(checks) - len(failed)\n score = round(passed_count / len(checks), 2)\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Sent email to ar@cascadeit.com passes all {len(checks)} checks. Correct dispute email sent for CM-38720.\"}\n else:\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Sent email to ar@cascadeit.com found but {len(failed)}/{len(checks)} checks failed. Missing: {failed}. Body: {(candidate.get('body_text') or '')[:500]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "14c8a33e-8008-453d-99f2-be9f347219fe",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `slack_data.json`, there must be NO message containing 'VENDOR OVERPAYMENT' in any channel, confirming no overpayment was incorrectly recorded for Cascade IT Hardware (V-00020).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n found = []\n for channel_id, msgs in messages.items():\n for msg in msgs:\n if \"VENDOR OVERPAYMENT\" in msg.get(\"text\", \"\"):\n found.append({\"channel\": channel_id, \"ts\": msg.get(\"ts\"), \"text\": msg.get(\"text\", \"\")[:100]})\n if found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found)} message(s) containing 'VENDOR OVERPAYMENT' — this should not exist for Cascade IT Hardware: {found}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No 'VENDOR OVERPAYMENT' messages found in Slack. Correct — no overpayment was incorrectly recorded.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "9fbe2123-3ea0-401b-bbd3-7f0d36867544",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `mailbox.json`, there must be NO sent email to invoices@apexoffice.com or lchen@apexoffice.com, and in `slack_data.json` there must be NO message containing 'DUPLICATE DETECTED' for Apex Office Supply (V-00001, INV-2025-09745).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping checks.\"}\n\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n email_violations = []\n slack_violations = []\n\n # Check mailbox — no sent email to Apex addresses after task start\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n mail_data = json.load(f)\n forbidden_addrs = [\"invoices@apexoffice.com\", \"lchen@apexoffice.com\"]\n for email in mail_data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addrs:\n if addr in to_addr:\n email_violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:60]}'\")\n break\n else:\n email_violations.append(\"mailbox.json not found\")\n\n # Check Slack — no DUPLICATE DETECTED message in any channel\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n for channel_id, msgs in slack_data.get(\"messages\", {}).items():\n for msg in msgs:\n if \"DUPLICATE DETECTED\" in msg.get(\"text\", \"\"):\n slack_violations.append(f\"'DUPLICATE DETECTED' in channel {channel_id}: {msg.get('text', '')[:80]}\")\n else:\n slack_violations.append(\"slack_data.json not found\")\n\n all_violations = email_violations + slack_violations\n if not all_violations:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to Apex forbidden addresses and no 'DUPLICATE DETECTED' Slack messages. Correct.\"}\n\n failed_checks = (1 if email_violations else 0) + (1 if slack_violations else 0)\n score = round(1.0 - failed_checks * 0.5, 2)\n feedback_parts = []\n if email_violations:\n feedback_parts.append(f\"Email violations: {email_violations}\")\n if slack_violations:\n feedback_parts.append(f\"Slack violations: {slack_violations}\")\n return {\"pass\": False, \"score\": score, \"feedback\": \" | \".join(feedback_parts)}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "7903560e-f58b-4c0f-8880-7f0c858541f6",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `slack_data.json`, channel C004 (#approvals) must contain a message with 'INVOICE APPROVED' for Apex Office Supply V-00001, Invoice INV-2025-09745, Amount $742.50, PO-2025-00123, AND a message with 'INVOICE APPROVED' for Redwood Furniture Co V-00021, Invoice RF-8125, Amount $1,900.00, PO-2025-00124.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n c004_msgs = data.get(\"messages\", {}).get(\"C004\", [])\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n def check_invoice_approved(msgs, vendor_name, vendor_id, invoice_num, amount, po_num):\n # SOP format: \"INVOICE APPROVED: [VENDOR NAME], [VENDOR ID], Invoice [INVOICE NUMBER], Amount $[AMOUNT], [PO NUMBER]\"\n full_expected = normalize(\n f\"INVOICE APPROVED: {vendor_name}, {vendor_id}, Invoice {invoice_num}, Amount ${amount}, {po_num}\"\n )\n\n for msg in msgs:\n if full_expected in normalize(msg.get(\"text\", \"\")):\n return True, 1.0, None\n\n # Full match failed — find candidate and score by field\n parts = {\n \"INVOICE APPROVED keyword\": \"invoice approved\" ,\n f\"vendor name ({vendor_name})\": normalize(vendor_name),\n f\"vendor id ({vendor_id})\": normalize(vendor_id),\n f\"invoice number ({invoice_num})\": f\"invoice {normalize(invoice_num)}\",\n f\"amount (${amount})\": normalize(f\"${amount}\"),\n f\"po number ({po_num})\": normalize(po_num),\n }\n\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in msgs:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"invoice approved\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n return False, 0.0, f\"No INVOICE APPROVED message found in C004 for {invoice_num}\"\n\n score = round(best_passed / len(parts), 2)\n return False, score, f\"INVOICE APPROVED for {invoice_num} found but missing fields: {best_missing}. Message: {best_text[:200]}\"\n\n apex_pass, apex_score, apex_err = check_invoice_approved(\n c004_msgs, \"Apex Office Supply\", \"V-00001\", \"INV-2025-09745\", \"742.50\", \"PO-2025-00123\"\n )\n redwood_pass, redwood_score, redwood_err = check_invoice_approved(\n c004_msgs, \"Redwood Furniture Co\", \"V-00021\", \"RF-8125\", \"1,900.00\", \"PO-2025-00124\"\n )\n\n if apex_pass and redwood_pass:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found correct INVOICE APPROVED messages in C004 for Apex Office Supply (INV-2025-09745, $742.50, PO-2025-00123) and Redwood Furniture Co (RF-8125, $1,900.00, PO-2025-00124).\"}\n\n errors = []\n if not apex_pass:\n errors.append(f\"Apex: {apex_err}\")\n if not redwood_pass:\n errors.append(f\"Redwood: {redwood_err}\")\n\n overall_score = round((apex_score + redwood_score) / 2, 2)\n return {\"pass\": False, \"score\": overall_score, \"feedback\": f\"{2 - len(errors)}/2 INVOICE APPROVED checks passed. \" + \" | \".join(errors)}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "71ba8859-2da6-421d-9d88-8a986f5fd8bc",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `mailbox.json`, there must be NO sent email to billing@redwoodfurniture.com, confirming no unnecessary outbound communication was sent to Redwood Furniture Co.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n violations = []\n for email in data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue\n if \"billing@redwoodfurniture.com\" in (email.get(\"to_addr\") or \"\").lower():\n violations.append(f\"Sent email to billing@redwoodfurniture.com — subject: '{email.get('subject', '')[:80]}'\")\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to billing@redwoodfurniture.com — should not exist for a routine partial invoice: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to billing@redwoodfurniture.com. Correct.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876401233",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `vendor_master.xlsx`, row 4 must contain exactly the following values: V-00003 | Pinnacle Print Services | 58-3456789 | 45 Press Ln Marietta GA 30060 | Brian Matthews | bmatthews@pinnacleprint.com | (336) 555-0291 | Net 30 | Truliant Federal Credit Union | 253175494 | 0082-1147-3355 | Yes | NEC | Active | 2023-04-01 | 0",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"vendor_master.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File vendor_master.xlsx not found at {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n sheet_name = next((n for n in wb.sheetnames if \"vendor\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No vendor sheet found in vendor_master.xlsx. Sheets: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open vendor_master.xlsx: {e}\"}\n\n\n expected_values = [\n \"V-00003\",\n \"Pinnacle Print Services\",\n \"58-3456789\",\n \"45 Press Ln Marietta GA 30060\",\n \"Brian Matthews\",\n \"bmatthews@pinnacleprint.com\",\n \"(336) 555-0291\",\n \"Net 30\",\n \"Truliant Federal Credit Union\",\n \"253175494\",\n \"0082-1147-3355\",\n \"Yes\",\n \"NEC\",\n \"Active\",\n \"2023-04-01\",\n \"0\"\n ]\n\n row_num = 4\n row_data = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_data.append(cell_val)\n\n if not row_data or all(v is None for v in row_data):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row {row_num} is empty in vendor_master.xlsx\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n # Handle dates that might be datetime objects\n if hasattr(val, 'strftime'):\n s = val.strftime('%Y-%m-%d')\n # Remove trailing .0 for integers stored as floats\n if re.match(r'^\\d+\\.0$', s):\n s = s[:-2]\n return s\n\n def values_match(actual, expected):\n a = normalize(actual).lower()\n e = expected.strip().lower()\n if a == e:\n return True\n # Try numeric comparison with tolerance\n try:\n a_num = float(a)\n e_num = float(e)\n if e_num == 0:\n return abs(a_num) < 0.01\n return abs(a_num - e_num) / max(abs(e_num), 1e-9) < 0.05\n except (ValueError, TypeError):\n pass\n # Try removing all whitespace and comparing\n if re.sub(r'\\s+', '', a) == re.sub(r'\\s+', '', e):\n return True\n return False\n\n mismatches = []\n matched = 0\n for i, expected in enumerate(expected_values):\n if i < len(row_data):\n actual = row_data[i]\n if values_match(actual, expected):\n matched += 1\n else:\n mismatches.append(f\"Column {i+1}: expected '{expected}', got '{normalize(actual)}'\")\n else:\n mismatches.append(f\"Column {i+1}: expected '{expected}', but column does not exist\")\n\n total = len(expected_values)\n score = matched / total if total > 0 else 0.0\n\n if len(mismatches) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 4 in vendor_master.xlsx matches all {total} expected values: {[normalize(v) for v in row_data[:total]]}\"}\n else:\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Row 4 in vendor_master.xlsx has {len(mismatches)} mismatches out of {total} columns. Mismatches: {'; '.join(mismatches)}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876416854",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `slack_data.json`: In the #vendor-management channel (\"id\": \"C006\"), a message must be posted containing: BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com\"\n full_expected = normalize(\n \"BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com\"\n )\n\n parts = {\n \"BANK CHANGE REQUEST keyword\": \"bank change request\",\n \"vendor name (Pinnacle Print Services)\": \"pinnacle print services\",\n \"vendor id (V-00003)\": \"v-00003\",\n \"contact name (Brian Matthews)\": \"brian matthews\",\n \"request email (bmatthews@pinnacleprintsvc.com)\": \"bmatthews@pinnacleprintsvc.com\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct BANK CHANGE REQUEST message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"bank change request\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No BANK CHANGE REQUEST message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"BANK CHANGE REQUEST message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876423985",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `slack_data.json`: In #vendor-management (`\"id\": \"C006\"`), a message must be posted indicating POTENTIAL BEC FRAUD for Pinnacle Print Services, V-00003, with sender email `bmatthews@pinnacleprintsvc.com` and on-file email `bmatthews@pinnacleprint.com`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"POTENTIAL BEC FRAUD: Pinnacle Print Services, V-00003, Sender email bmatthews@pinnacleprintsvc.com, on-file email bmatthews@pinnacleprint.com\"\n full_expected = normalize(\n \"POTENTIAL BEC FRAUD: Pinnacle Print Services, V-00003, Sender email bmatthews@pinnacleprintsvc.com, on-file email bmatthews@pinnacleprint.com\"\n )\n\n parts = {\n \"POTENTIAL BEC FRAUD keyword\": \"potential bec fraud\",\n \"vendor name (Pinnacle Print Services)\": \"pinnacle print services\",\n \"vendor id (V-00003)\": \"v-00003\",\n \"sender email (bmatthews@pinnacleprintsvc.com)\": \"bmatthews@pinnacleprintsvc.com\",\n \"on-file email (bmatthews@pinnacleprint.com)\": \"bmatthews@pinnacleprint.com\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct POTENTIAL BEC FRAUD message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"bec fraud\" not in text_norm and \"potential bec\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No POTENTIAL BEC FRAUD message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"POTENTIAL BEC FRAUD message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876445763",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `slack_data.json`: In #vendor-management channel (\"id\": \"C006\"), a message must be posted containing: LEGAL THREAT, SecureNet Compliance, V-00011, and Invoice SNC-2025-001.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"LEGAL THREAT: SecureNet Compliance, V-00011, Invoice SNC-2025-001\"\n full_expected = normalize(\n \"LEGAL THREAT: SecureNet Compliance, V-00011, Invoice SNC-2025-001\"\n )\n\n parts = {\n \"LEGAL THREAT keyword\": \"legal threat\",\n \"vendor name (SecureNet Compliance)\": \"securenet compliance\",\n \"vendor id (V-00011)\": \"v-00011\",\n \"invoice number (SNC-2025-001)\": \"snc-2025-001\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct LEGAL THREAT message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"legal threat\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No LEGAL THREAT message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"LEGAL THREAT message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876458214",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `vendor_master.xlsx`, cell P21 (Outstanding Credit for Cascade IT Hardware) is 0.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"vendor_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'vendor_master.xlsx' not found in workspace at {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n sheet_name = next((n for n in wb.sheetnames if \"vendor\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No vendor sheet found in vendor_master.xlsx. Sheets: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n \n cell_value = ws['P21'].value\n \n # Try to convert to a number for comparison\n if cell_value is None:\n # None could be interpreted as 0 (empty cell) - be generous\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P21 is empty/None, which is treated as 0. PASS.\"}\n \n try:\n numeric_value = float(cell_value)\n except (ValueError, TypeError):\n # If it's a string like \"0\" or \"0.00\" or similar\n stripped = str(cell_value).strip().replace(',', '').replace('$', '')\n try:\n numeric_value = float(stripped)\n except (ValueError, TypeError):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P21 contains '{cell_value}' which cannot be interpreted as a number.\"}\n \n # Check if value is 0 (with tolerance: absolute tolerance of 0.01 since expected is 0)\n if abs(numeric_value) <= 0.01:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P21 in vendor_master.xlsx is {cell_value}, which is effectively 0. PASS.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P21 in vendor_master.xlsx is {cell_value} (numeric: {numeric_value}), expected 0.\"}\n \n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading vendor_master.xlsx: {str(e)}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876463672",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 31 must contain the following values: 1030 | V-00020 | Cascade IT Hardware | INV-38720 | 2025-09-02 | PO-2025-00117 | PO | 1,950.00 | 1000-100-6400 | DISP-HOLD | Net 30 | 2025-10-02 | T. Okonkwo | 2025-09-03 | | | 0 | Placed on DISP-HOLD on 09/16/2025 — CM-38720.pdf",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n filepath = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {filepath}\"}\n\n try:\n wb = openpyxl.load_workbook(str(filepath), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1030 (INV-38720) is at sheet row 31. Search by invoice number as fallback.\n row = 31\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\") != \"INV-38720\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if ws.cell(row=r, column=4).value == \"INV-38720\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-38720 not found in Invoice Register. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match(actual, expected):\n if expected == \"\":\n return actual is None or normalize(actual) == \"\"\n a = normalize(actual)\n e = normalize(expected)\n if a == e:\n return True\n # Numeric tolerance\n try:\n return abs(float(a.replace('$','')) - float(e.replace('$',''))) / max(abs(float(e.replace('$',''))), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n # Expected row: 1030 | V-00020 | Cascade IT Hardware | INV-38720 | 2025-09-02 | PO-2025-00117 |\n # PO | 1,950.00 | 1000-100-6400 | DISP-HOLD | Net 30 | 2025-10-02 |\n # T. Okonkwo | 2025-09-03 | | | 0 | Placed on DISP-HOLD on 09/16/2025 — CM-38720.pdf\n expected = [\n \"1030\", \"V-00020\", \"Cascade IT Hardware\", \"INV-38720\", \"2025-09-02\",\n \"PO-2025-00117\", \"PO\", \"1950.00\", \"1000-100-6400\", \"DISP-HOLD\",\n \"Net 30\", \"2025-10-02\", \"T. Okonkwo\", \"2025-09-03\", \"\", \"\", \"0\",\n \"Placed on DISP-HOLD on 09/16/2025 - CM-38720.pdf\",\n ]\n\n failures = []\n matched = 0\n for i, exp in enumerate(expected):\n actual = row_values[i] if i < len(row_values) else None\n if match(actual, exp):\n matched += 1\n else:\n failures.append(f\"Col {i+1}: expected '{exp}', got '{normalize(actual)}'\")\n\n total = len(expected)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Invoice Register row {row} matches all {total} expected values for INV-38720 (DISP-HOLD).\"}\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Invoice Register row {row} (INV-38720): {len(failures)}/{total} mismatches — {'; '.join(failures)}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876497121",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `match_log_2025_Q3.xlsx`, row 25 must contain the following values: MTH-Q3-024 | 2025-09-16 | PO-2025-00123 | V-00001 | Apex Office Supply | INV-2025-09745 | 4.95 | 4.95 | 0 | 300 | 150 | 300 | 1,485.00 | 742.50 | 0 | 0 | Pass | Partial invoice – 300 units received, 150 units invoiced. Remaining units: 0.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2025_Q3.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # MTH-Q3-024 is at sheet row 25. Search by match ID as fallback.\n row = 25\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip().lower() != \"mth-q3-024\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip().lower() == \"mth-q3-024\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"MTH-Q3-024 not found in match log. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_notes(actual, expected):\n # Allow trailing punctuation/whitespace on the actual value.\n a = normalize(actual)\n a = re.sub(r'[\\s.,;:!?]+$', '', a)\n e = normalize(expected)\n e = re.sub(r'[\\s.,;:!?]+$', '', e)\n return a == e\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n # Expected row (from rubric draft):\n # MTH-Q3-024 | 2025-09-16 | PO-2025-00123 | V-00001 | Apex Office Supply | INV-2025-09745 |\n # 4.95 | 4.95 | 0 | 300 | 150 | 300 | 1,485.00 | 742.50 | 0 | 0 | Pass |\n # Partial invoice – 300 units received, 150 units invoiced. Remaining units: 0\n checks = [\n (\"Col 1 (Match ID)\", lambda v: match_str(v, \"MTH-Q3-024\")),\n (\"Col 2 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 3 (PO)\", lambda v: match_str(v, \"PO-2025-00123\")),\n (\"Col 4 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 5 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 6 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 7 (Unit Price PO)\", lambda v: match_num(v, 4.95)),\n (\"Col 8 (Unit Price Inv)\", lambda v: match_num(v, 4.95)),\n (\"Col 9 (Variance %)\", lambda v: match_num(v, 0)),\n (\"Col 10 (GR Qty)\", lambda v: match_num(v, 300)),\n (\"Col 11 (Inv Qty)\", lambda v: match_num(v, 150)),\n (\"Col 12 (PO Qty)\", lambda v: match_num(v, 300)),\n (\"Col 13 (PO Amount)\", lambda v: match_num(v, 1485.00)),\n (\"Col 14 (Inv Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 15 (Variance $)\", lambda v: match_num(v, 0)),\n (\"Col 16 (Flag)\", lambda v: match_num(v, 0)),\n (\"Col 17 (Result)\", lambda v: match_str(v, \"Pass\")),\n (\"Col 18 (Notes)\", lambda v: match_notes(v, \"Partial invoice - 300 units received, 150 units invoiced. Remaining units: 0\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Match log row {row} matches all {total} expected values for MTH-Q3-024.\"}\n return {\"pass\": False, \"score\": score, \"Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876500258",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, on the \"Payment Queue\" sheet, row 30 contains the following values: PAY-0029 | V-00001 | Apex Office Supply | INV-2025-09745 | 742.50 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Payment Queue' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # PAY-0029 is at sheet row 30. Search by payment ID as fallback.\n row = 30\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip() != \"PAY-0029\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip() == \"PAY-0029\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PAY-0029 not found in Payment Queue.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: PAY-0029 | V-00001 | Apex Office Supply | INV-2025-09745 | 742.50 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending\n checks = [\n (\"Col 1 (Payment ID)\", lambda v: match_str(v, \"PAY-0029\")),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 5 (Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 6 (Method)\", lambda v: match_str(v, \"ACH\")),\n (\"Col 7 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 8 (Batch ID)\", lambda v: match_str(v, \"BAT-2025-0916\")),\n (\"Col 9 (empty)\", lambda v: match_empty(v)),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"Pending\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Payment Queue row {row} matches all {total} expected values for PAY-0029.\"}\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Payment Queue row {row} (PAY-0029): {len(failures)}/{total} mismatches — {'; '.join(failures)}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876509007",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 47 must contain the following values: 1046 | V-00001 | Apex Office Supply | INV-2025-09745 | 2025-09-12 | PO-2025-00123 | PO | 742.50 | 1000-100-6200 | APPROVED | Net 30 | 2025-10-12 | T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1046 is at sheet row 47. Search by invoice number as fallback.\n row = 47\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\").strip() != \"INV-2025-09745\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=4).value or \"\").strip() == \"INV-2025-09745\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-2025-09745 not found in Invoice Register.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: 1046 | V-00001 | Apex Office Supply | INV-2025-09745 | 2025-09-12 | PO-2025-00123 |\n # PO | 742.50 | 1000-100-6200 | APPROVED | Net 30 | 2025-10-12 |\n # T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]\n checks = [\n (\"Col 1 (Inv Reg ID)\", lambda v: match_num(v, 1046)),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 5 (Invoice Date)\", lambda v: match_date(v, \"2025-09-12\")),\n (\"Col 6 (PO)\", lambda v: match_str(v, \"PO-2025-00123\")),\n (\"Col 7 (Type)\", lambda v: match_str(v, \"PO\")),\n (\"Col 8 (Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 9 (GL Code)\", lambda v: match_str(v, \"1000-100-6200\")),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"APPROVED\")),\n (\"Col 11 (Terms)\", lambda v: match_str(v, \"Net 30\")),\n (\"Col 12 (Due Date)\", lambda v: match_date(v, \"2025-10-12\")),\n (\"Col 13 (Entered By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 14 (Entered Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 15 (Approved By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 16 (Approved Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 17 (On Hold)\", lambda v: match_num(v, 0)),\n (\"Col 18 (Notes)\", lambda v: match_empty(v)),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Invoice Register row {row} matches all {total} expected values for INV-2025-09745 (ID 1046, APPROVED).\"}\n return {\"pass\": False, \"score\": score, \"feedbLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876514496",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 48 must contain the following values: 1047 | V-00021 | Redwood Furniture Co | RF-8125 | 2025-09-11 | PO-2025-00124 | PO | 1,900.00 | 1000-100-6500 | APPROVED | Net 45 | 2025-10-26 | T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1047 is at sheet row 48. Search by invoice number as fallback.\n row = 48\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\").strip() != \"RF-8125\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=4).value or \"\").strip() == \"RF-8125\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"RF-8125 not found in Invoice Register.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: 1047 | V-00021 | Redwood Furniture Co | RF-8125 | 2025-09-11 | PO-2025-00124 |\n # PO | 1,900.00 | 1000-100-6500 | APPROVED | Net 45 | 2025-10-26 |\n # T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]\n checks = [\n (\"Col 1 (Inv Reg ID)\", lambda v: match_num(v, 1047)),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 5 (Invoice Date)\", lambda v: match_date(v, \"2025-09-11\")),\n (\"Col 6 (PO)\", lambda v: match_str(v, \"PO-2025-00124\")),\n (\"Col 7 (Type)\", lambda v: match_str(v, \"PO\")),\n (\"Col 8 (Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 9 (GL Code)\", lambda v: match_str(v, \"1000-100-6500\")),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"APPROVED\")),\n (\"Col 11 (Terms)\", lambda v: match_str(v, \"Net 45\")),\n (\"Col 12 (Due Date)\", lambda v: match_date(v, \"2025-10-26\")),\n (\"Col 13 (Entered By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 14 (Entered Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 15 (Approved By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 16 (Approved Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 17 (On Hold)\", lambda v: match_num(v, 0)),\n (\"Col 18 (Notes)\", lambda v: match_empty(v)),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Invoice Register row {row} matches all {total} expected values for RF-8125 (ID 1047, APPROVED).\"}\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Invoice Register row {rowLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876517591",
|
||||
"sort_order": 18,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, on the \"Payment Queue\" sheet, row 31 contains: PAY-0030 | V-00021 | Redwood Furniture Co | RF-8125 | 1,900.00 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Payment Queue' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # PAY-0030 is at sheet row 31. Search by payment ID as fallback.\n row = 31\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip() != \"PAY-0030\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip() == \"PAY-0030\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PAY-0030 not found in Payment Queue.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: PAY-0030 | V-00021 | Redwood Furniture Co | RF-8125 | 1,900.00 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending\n checks = [\n (\"Col 1 (Payment ID)\", lambda v: match_str(v, \"PAY-0030\")),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 5 (Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 6 (Method)\", lambda v: match_str(v, \"ACH\")),\n (\"Col 7 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 8 (Batch ID)\", lambda v: match_str(v, \"BAT-2025-0916\")),\n (\"Col 9 (empty)\", lambda v: match_empty(v)),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"Pending\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Payment Queue row {row} matches all {total} expected values for PAY-0030.\"}\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Payment Queue row {row} (PAY-0030): {len(failures)}/{total} mismatches — {'; '.join(failures)}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775876522265",
|
||||
"sort_order": 19,
|
||||
"rubric_text": "In `match_log_2025_Q3.xlsx`, row 26 must contain the following values: MTH-Q3-025 | 2025-09-16 | PO-2025-00124 | V-00021 | Redwood Furniture Co | RF-8125 | 380.00 | 380.00 | 0 | 10 | 5 | 10 | 3,800.00 | 1,900.00 | 0 | 0 | Pass | Partial invoice – 10 units received, 5 units invoiced. Remaining units: 0.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2025_Q3.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # MTH-Q3-025 is at sheet row 26. Search by match ID as fallback.\n row = 26\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip().lower() != \"mth-q3-025\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip().lower() == \"mth-q3-025\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"MTH-Q3-025 not found in match log. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_notes(actual, expected):\n # Allow trailing punctuation/whitespace on the actual value.\n a = normalize(actual)\n a = re.sub(r'[\\s.,;:!?]+$', '', a)\n e = normalize(expected)\n e = re.sub(r'[\\s.,;:!?]+$', '', e)\n return a == e\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n # Expected row (from rubric draft):\n # MTH-Q3-025 | 2025-09-16 | PO-2025-00124 | V-00021 | Redwood Furniture Co | RF-8125 |\n # 380.00 | 380.00 | 0 | 10 | 5 | 10 | 3,800.00 | 1,900.00 | 0 | 0 | Pass |\n # Partial invoice – 10 units received, 5 units invoiced. Remaining units: 0\n checks = [\n (\"Col 1 (Match ID)\", lambda v: match_str(v, \"MTH-Q3-025\")),\n (\"Col 2 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 3 (PO)\", lambda v: match_str(v, \"PO-2025-00124\")),\n (\"Col 4 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 5 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 6 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 7 (Unit Price PO)\", lambda v: match_num(v, 380.00)),\n (\"Col 8 (Unit Price Inv)\", lambda v: match_num(v, 380.00)),\n (\"Col 9 (Variance %)\", lambda v: match_num(v, 0)),\n (\"Col 10 (GR Qty)\", lambda v: match_num(v, 10)),\n (\"Col 11 (Inv Qty)\", lambda v: match_num(v, 5)),\n (\"Col 12 (PO Qty)\", lambda v: match_num(v, 10)),\n (\"Col 13 (PO Amount)\", lambda v: match_num(v, 3800.00)),\n (\"Col 14 (Inv Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 15 (Variance $)\", lambda v: match_num(v, 0)),\n (\"Col 16 (Flag)\", lambda v: match_num(v, 0)),\n (\"Col 17 (Result)\", lambda v: match_str(v, \"Pass\")),\n (\"Col 18 (Notes)\", lambda v: match_notes(v, \"Partial invoice - 10 units received, 5 units invoiced. Remaining units: 0\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Match log row {row} matches all {total} expected values for MTH-Q3-025.\"}\n return {\"pass\": False, \"score\": score, \"feedback\": fLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775877705459",
|
||||
"sort_order": 20,
|
||||
"rubric_text": "In `mailbox.json`, there are exactly 8 emails in the Sent folder.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n mailbox_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"mailbox.json\" if external_services_path else None,\n workspace_path / \"mailbox.json\",\n ]):\n if candidate.exists():\n mailbox_path = candidate\n break\n\n if mailbox_path:\n try:\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n mailbox_data = json.load(f)\n sent = [e for e in mailbox_data.get(\"emails\", []) if str(e.get(\"folder\", \"\")).lower() == \"sent\"]\n count = len(sent)\n if count == 8:\n feedback = \"PASS: Sent folder has exactly 8 emails.\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": feedback}\n else:\n feedback = f\"FAIL: Sent folder expected 8 emails, found {count}.\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading mailbox.json: {e}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: mailbox.json not found.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "cf326947-0f8d-4e79-9f2a-dfe2f78b8fa7",
|
||||
"sort_order": 21,
|
||||
"rubric_text": "In `slack_data.json`, there are exactly 5 messages in #general (C001), 11 in #finance (C002), 15 in #accounts-payable (C003), 9 in #approvals (C004), and 4 in #vendor-management (C006).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n workspace_path / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n expected_counts = {\n \"C001\": (\"#general\", 5),\n \"C002\": (\"#finance\", 11),\n \"C003\": (\"#accounts-payable\", 15),\n \"C004\": (\"#approvals\", 9),\n \"C006\": (\"#vendor-management\", 4),\n }\n\n if slack_path:\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n messages = slack_data.get(\"messages\", {})\n channel_results = []\n all_pass = True\n for cid, (cname, expected) in expected_counts.items():\n actual = len(messages.get(cid, []))\n if actual == expected:\n channel_results.append(f\" {cname} ({cid}): PASS ({actual})\")\n else:\n channel_results.append(f\" {cname} ({cid}): FAIL - expected {expected}, found {actual}\")\n all_pass = False\n if all_pass:\n feedback = \"PASS: Slack message counts correct.\\n\" + \"\\n\".join(channel_results)\n return {\"pass\": True, \"score\": 1.0, \"feedback\": feedback}\n else:\n feedback = \"FAIL: Slack message counts mismatch.\\n\" + \"\\n\".join(channel_results)\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading slack_data.json: {e}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: slack_data.json not found.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a2b92c1c-4245-4fce-9e04-b3033efcff46",
|
||||
"sort_order": 22,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, the \"Invoice Register\" sheet has exactly 47 data rows.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ap_path = workspace_path / \"ap_ledger.xlsx\"\n\n if not ap_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: ap_ledger.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ap_path, read_only=True, data_only=True)\n ir_sheet = next((wb[n] for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if ir_sheet:\n data_rows = sum(1 for r in ir_sheet.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 47:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Invoice Register has exactly 47 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Invoice Register expected 47 data rows, found {data_rows}.\"}\n else:\n sheets = wb.sheetnames\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Invoice Register sheet not found. Sheets: {sheets}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading ap_ledger.xlsx: {e}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "d7475f15-9b80-4809-8f4d-83cc0556f241",
|
||||
"sort_order": 23,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, the \"Payment Queue\" sheet has exactly 30 data rows.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ap_path = workspace_path / \"ap_ledger.xlsx\"\n\n if not ap_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: ap_ledger.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ap_path, read_only=True, data_only=True)\n pq_sheet = next((wb[n] for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if pq_sheet:\n data_rows = sum(1 for r in pq_sheet.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 30:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Payment Queue has exactly 30 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Payment Queue expected 30 data rows, found {data_rows}.\"}\n else:\n sheets = wb.sheetnames\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Payment Queue sheet not found. Sheets: {sheets}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading ap_ledger.xlsx: {e}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "08400c56-c184-4958-b2fb-eec6f2de75dc",
|
||||
"sort_order": 24,
|
||||
"rubric_text": "In `match_log_2025_Q3.xlsx`, there are exactly 25 data rows.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ml_path = workspace_path / \"match_log_2025_Q3.xlsx\"\n\n if not ml_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: match_log_2025_Q3.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ml_path, read_only=True, data_only=True)\n ws = wb.active\n data_rows = sum(1 for r in ws.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 25:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: match_log_2025_Q3.xlsx has exactly 25 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: match_log_2025_Q3.xlsx expected 25 data rows, found {data_rows}.\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading match_log_2025_Q3.xlsx: {e}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Hello, today is 9/16/2025 9:00AM, and I could use some help with new emails that have come in. I am working on a special project, but my manager wants the emails processed ASAP. Can you check the inbox for any unread emails and follow the steps in the SOP-FIN-AP-004 to complete all the required steps?
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user