178 lines
83 KiB
JSON
178 lines
83 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "a309c711-6d28-46ca-8b80-b0eda8c78b0c",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In `slack_data.json`, there must be NO message containing 'AUDIT HOLD APPLIED' in channel C006 (#vendor-management) after timestamp 1757687400.000042, confirming no audit hold was incorrectly applied to SecureNet Compliance (V-00011) after that time.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n threshold_ts = 1757687400.000042\n found = []\n for channel_id, msgs in messages.items():\n for msg in msgs:\n try:\n ts = float(msg.get(\"ts\", 0))\n except (ValueError, TypeError):\n continue\n if ts > threshold_ts and \"AUDIT HOLD APPLIED\" in msg.get(\"text\", \"\"):\n found.append({\"channel\": channel_id, \"ts\": msg.get(\"ts\"), \"text\": msg.get(\"text\", \"\")[:100]})\n if found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found)} message(s) containing 'AUDIT HOLD APPLIED' after timestamp {threshold_ts}: {found}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No 'AUDIT HOLD APPLIED' messages found after timestamp 1757687400.000042. Correct — no improper audit hold was applied to SecureNet Compliance.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d51b8a8e-eae7-4d3f-abab-978637870b80",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be NO sent email to billing@securenetcompliance.com or raj.patel@meridianpartners.com, confirming no improper payment or response was sent regarding SecureNet Compliance (V-00011, Invoice SNC-2025-001).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n from datetime import datetime, timezone\n\n # Any sent email after the last pre-existing one (2025-09-12T09:00:00-04:00 = 13:00 UTC)\n # is considered model-generated during the task.\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n forbidden_addresses = [\n \"billing@securenetcompliance.com\",\n \"raj.patel@meridianpartners.com\",\n ]\n emails = data.get(\"emails\", [])\n violations = []\n for email in emails:\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue # skip pre-existing sent emails\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addresses:\n if addr in to_addr:\n violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:80]}'\")\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to forbidden SecureNet Compliance addresses: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to billing@securenetcompliance.com or raj.patel@meridianpartners.com. Correct — legal threat was not actioned.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9fc28e2d-fe54-4b39-b7c9-345dae70b338",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be NO sent email to bmatthews@pinnacleprintsvc.com, bmatthews@pinnacleprint-services.com, or bmatthews@pinnacleprint.com, and NO sent email to maria.santos@meridianpartners.com, marcus.webb@meridianpartners.com, or raj.patel@meridianpartners.com regarding Pinnacle Print Services (V-00003) bank change request — confirming no bank change was processed for a suspected BEC fraud vendor.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n # Any sent email after the last pre-existing one (end of 2025-09-15)\n # is considered model-generated during the task.\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n forbidden_addresses = [\n \"bmatthews@pinnacleprintsvc.com\",\n \"bmatthews@pinnacleprint-services.com\",\n \"bmatthews@pinnacleprint.com\",\n \"maria.santos@meridianpartners.com\",\n \"marcus.webb@meridianpartners.com\",\n \"raj.patel@meridianpartners.com\",\n ]\n\n emails = data.get(\"emails\", [])\n violations = []\n for email in emails:\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue # skip pre-existing sent emails\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addresses:\n if addr in to_addr:\n violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:80]}'\")\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to forbidden addresses related to Pinnacle Print Services BEC bank change request: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to Pinnacle vendor addresses or internal team members. Correct — bank change request was not processed.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "78ef4e7c-fe51-4573-b827-3ce3baea02ea",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be a sent email with 'in_reply_to' referencing 'cm-38720@cascadeit.com' sent to ar@cascadeit.com, with body text referencing credit memo CM-38720, the amount $2,000.00, and original invoice INV-38720 totaling $1,950.00, requesting a corrected credit memo or explanation.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n candidate = None\n for email in sent_emails:\n if \"ar@cascadeit.com\" in (email.get(\"to_addr\") or \"\").lower():\n candidate = email\n break\n\n if candidate is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to ar@cascadeit.com. Expected a dispute email regarding credit memo CM-38720.\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n s = s.replace(',', '')\n return s.strip()\n\n body = normalize(candidate.get(\"body_text\") or \"\")\n\n # Expected full body (from rubric draft), broken into complete sentences for verification:\n # \"Dear Brian Choi, We received credit memo CM-38720 dated 2025 for $2,000.00.\n # This amount exceeds the original invoice INV-38720 totaling $1,950.00.\n # We are unable to apply this credit until clarification is received.\n # Please provide a corrected credit memo or written explanation.\n # Meridian Partners Accounts Payable\"\n checks = {\n \"greeting (Dear Brian Choi)\":\n \"dear brian choi\" in body,\n \"cm receipt sentence (CM-38720 dated [2025-09-11 or 09/11/2025] for $2,000.00)\":\n (\"we received credit memo cm-38720 dated 2025-09-11 for $2000.00\" in body or\n \"we received credit memo cm-38720 dated 09/11/2025 for $2000.00\" in body),\n \"exceeds invoice sentence (INV-38720 totaling $1,950.00)\":\n \"this amount exceeds the original invoice inv-38720 totaling $1950.00\" in body,\n \"unable to apply sentence\":\n \"we are unable to apply this credit until clarification is received\" in body,\n \"corrected cm or explanation sentence\":\n \"please provide a corrected credit memo or written explanation\" in body,\n \"sign-off (Meridian Partners Accounts Payable)\":\n \"meridian partners accounts payable\" in body\n }\n\n\n failed = [k for k, v in checks.items() if not v]\n passed_count = len(checks) - len(failed)\n score = round(passed_count / len(checks), 2)\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Sent email to ar@cascadeit.com passes all {len(checks)} checks. Correct dispute email sent for CM-38720.\"}\n else:\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Sent email to ar@cascadeit.com found but {len(failed)}/{len(checks)} checks failed. Missing: {failed}. Body: {(candidate.get('body_text') or '')[:500]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "14c8a33e-8008-453d-99f2-be9f347219fe",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In `slack_data.json`, there must be NO message containing 'VENDOR OVERPAYMENT' in any channel, confirming no overpayment was incorrectly recorded for Cascade IT Hardware (V-00020).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n found = []\n for channel_id, msgs in messages.items():\n for msg in msgs:\n if \"VENDOR OVERPAYMENT\" in msg.get(\"text\", \"\"):\n found.append({\"channel\": channel_id, \"ts\": msg.get(\"ts\"), \"text\": msg.get(\"text\", \"\")[:100]})\n if found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found)} message(s) containing 'VENDOR OVERPAYMENT' — this should not exist for Cascade IT Hardware: {found}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No 'VENDOR OVERPAYMENT' messages found in Slack. Correct — no overpayment was incorrectly recorded.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9fbe2123-3ea0-401b-bbd3-7f0d36867544",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be NO sent email to invoices@apexoffice.com or lchen@apexoffice.com, and in `slack_data.json` there must be NO message containing 'DUPLICATE DETECTED' for Apex Office Supply (V-00001, INV-2025-09745).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping checks.\"}\n\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n email_violations = []\n slack_violations = []\n\n # Check mailbox — no sent email to Apex addresses after task start\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n mail_data = json.load(f)\n forbidden_addrs = [\"invoices@apexoffice.com\", \"lchen@apexoffice.com\"]\n for email in mail_data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n for addr in forbidden_addrs:\n if addr in to_addr:\n email_violations.append(f\"Sent email to {addr} — subject: '{email.get('subject', '')[:60]}'\")\n break\n else:\n email_violations.append(\"mailbox.json not found\")\n\n # Check Slack — no DUPLICATE DETECTED message in any channel\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n for channel_id, msgs in slack_data.get(\"messages\", {}).items():\n for msg in msgs:\n if \"DUPLICATE DETECTED\" in msg.get(\"text\", \"\"):\n slack_violations.append(f\"'DUPLICATE DETECTED' in channel {channel_id}: {msg.get('text', '')[:80]}\")\n else:\n slack_violations.append(\"slack_data.json not found\")\n\n all_violations = email_violations + slack_violations\n if not all_violations:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to Apex forbidden addresses and no 'DUPLICATE DETECTED' Slack messages. Correct.\"}\n\n failed_checks = (1 if email_violations else 0) + (1 if slack_violations else 0)\n score = round(1.0 - failed_checks * 0.5, 2)\n feedback_parts = []\n if email_violations:\n feedback_parts.append(f\"Email violations: {email_violations}\")\n if slack_violations:\n feedback_parts.append(f\"Slack violations: {slack_violations}\")\n return {\"pass\": False, \"score\": score, \"feedback\": \" | \".join(feedback_parts)}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "7903560e-f58b-4c0f-8880-7f0c858541f6",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel C004 (#approvals) must contain a message with 'INVOICE APPROVED' for Apex Office Supply V-00001, Invoice INV-2025-09745, Amount $742.50, PO-2025-00123, AND a message with 'INVOICE APPROVED' for Redwood Furniture Co V-00021, Invoice RF-8125, Amount $1,900.00, PO-2025-00124.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping Slack check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n c004_msgs = data.get(\"messages\", {}).get(\"C004\", [])\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n def check_invoice_approved(msgs, vendor_name, vendor_id, invoice_num, amount, po_num):\n # SOP format: \"INVOICE APPROVED: [VENDOR NAME], [VENDOR ID], Invoice [INVOICE NUMBER], Amount $[AMOUNT], [PO NUMBER]\"\n full_expected = normalize(\n f\"INVOICE APPROVED: {vendor_name}, {vendor_id}, Invoice {invoice_num}, Amount ${amount}, {po_num}\"\n )\n\n for msg in msgs:\n if full_expected in normalize(msg.get(\"text\", \"\")):\n return True, 1.0, None\n\n # Full match failed — find candidate and score by field\n parts = {\n \"INVOICE APPROVED keyword\": \"invoice approved\" ,\n f\"vendor name ({vendor_name})\": normalize(vendor_name),\n f\"vendor id ({vendor_id})\": normalize(vendor_id),\n f\"invoice number ({invoice_num})\": f\"invoice {normalize(invoice_num)}\",\n f\"amount (${amount})\": normalize(f\"${amount}\"),\n f\"po number ({po_num})\": normalize(po_num),\n }\n\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in msgs:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"invoice approved\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n return False, 0.0, f\"No INVOICE APPROVED message found in C004 for {invoice_num}\"\n\n score = round(best_passed / len(parts), 2)\n return False, score, f\"INVOICE APPROVED for {invoice_num} found but missing fields: {best_missing}. Message: {best_text[:200]}\"\n\n apex_pass, apex_score, apex_err = check_invoice_approved(\n c004_msgs, \"Apex Office Supply\", \"V-00001\", \"INV-2025-09745\", \"742.50\", \"PO-2025-00123\"\n )\n redwood_pass, redwood_score, redwood_err = check_invoice_approved(\n c004_msgs, \"Redwood Furniture Co\", \"V-00021\", \"RF-8125\", \"1,900.00\", \"PO-2025-00124\"\n )\n\n if apex_pass and redwood_pass:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found correct INVOICE APPROVED messages in C004 for Apex Office Supply (INV-2025-09745, $742.50, PO-2025-00123) and Redwood Furniture Co (RF-8125, $1,900.00, PO-2025-00124).\"}\n\n errors = []\n if not apex_pass:\n errors.append(f\"Apex: {apex_err}\")\n if not redwood_pass:\n errors.append(f\"Redwood: {redwood_err}\")\n\n overall_score = round((apex_score + redwood_score) / 2, 2)\n return {\"pass\": False, \"score\": overall_score, \"feedback\": f\"{2 - len(errors)}/2 INVOICE APPROVED checks passed. \" + \" | \".join(errors)}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "71ba8859-2da6-421d-9d88-8a986f5fd8bc",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be NO sent email to billing@redwoodfurniture.com, confirming no unnecessary outbound communication was sent to Redwood Furniture Co.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime, timezone\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping mailbox check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path.\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n THRESHOLD_UTC = datetime(2025, 9, 15, 23, 59, 59, tzinfo=timezone.utc)\n\n def parse_sent_ts(email):\n for field in (\"sent_at\", \"date\", \"timestamp\"):\n val = email.get(field)\n if val:\n try:\n ts = datetime.fromisoformat(str(val))\n if ts.tzinfo is None:\n ts = ts.replace(tzinfo=timezone.utc)\n return ts.astimezone(timezone.utc)\n except Exception:\n pass\n return None\n\n violations = []\n for email in data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n ts = parse_sent_ts(email)\n if ts is None or ts <= THRESHOLD_UTC:\n continue\n if \"billing@redwoodfurniture.com\" in (email.get(\"to_addr\") or \"\").lower():\n violations.append(f\"Sent email to billing@redwoodfurniture.com — subject: '{email.get('subject', '')[:80]}'\")\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} model-sent email(s) to billing@redwoodfurniture.com — should not exist for a routine partial invoice: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No model-sent emails to billing@redwoodfurniture.com. Correct.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876401233",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In `vendor_master.xlsx`, row 4 must contain exactly the following values: V-00003 | Pinnacle Print Services | 58-3456789 | 45 Press Ln Marietta GA 30060 | Brian Matthews | bmatthews@pinnacleprint.com | (336) 555-0291 | Net 30 | Truliant Federal Credit Union | 253175494 | 0082-1147-3355 | Yes | NEC | Active | 2023-04-01 | 0",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"vendor_master.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File vendor_master.xlsx not found at {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n sheet_name = next((n for n in wb.sheetnames if \"vendor\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No vendor sheet found in vendor_master.xlsx. Sheets: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open vendor_master.xlsx: {e}\"}\n\n\n expected_values = [\n \"V-00003\",\n \"Pinnacle Print Services\",\n \"58-3456789\",\n \"45 Press Ln Marietta GA 30060\",\n \"Brian Matthews\",\n \"bmatthews@pinnacleprint.com\",\n \"(336) 555-0291\",\n \"Net 30\",\n \"Truliant Federal Credit Union\",\n \"253175494\",\n \"0082-1147-3355\",\n \"Yes\",\n \"NEC\",\n \"Active\",\n \"2023-04-01\",\n \"0\"\n ]\n\n row_num = 4\n row_data = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_data.append(cell_val)\n\n if not row_data or all(v is None for v in row_data):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row {row_num} is empty in vendor_master.xlsx\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n # Handle dates that might be datetime objects\n if hasattr(val, 'strftime'):\n s = val.strftime('%Y-%m-%d')\n # Remove trailing .0 for integers stored as floats\n if re.match(r'^\\d+\\.0$', s):\n s = s[:-2]\n return s\n\n def values_match(actual, expected):\n a = normalize(actual).lower()\n e = expected.strip().lower()\n if a == e:\n return True\n # Try numeric comparison with tolerance\n try:\n a_num = float(a)\n e_num = float(e)\n if e_num == 0:\n return abs(a_num) < 0.01\n return abs(a_num - e_num) / max(abs(e_num), 1e-9) < 0.05\n except (ValueError, TypeError):\n pass\n # Try removing all whitespace and comparing\n if re.sub(r'\\s+', '', a) == re.sub(r'\\s+', '', e):\n return True\n return False\n\n mismatches = []\n matched = 0\n for i, expected in enumerate(expected_values):\n if i < len(row_data):\n actual = row_data[i]\n if values_match(actual, expected):\n matched += 1\n else:\n mismatches.append(f\"Column {i+1}: expected '{expected}', got '{normalize(actual)}'\")\n else:\n mismatches.append(f\"Column {i+1}: expected '{expected}', but column does not exist\")\n\n total = len(expected_values)\n score = matched / total if total > 0 else 0.0\n\n if len(mismatches) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 4 in vendor_master.xlsx matches all {total} expected values: {[normalize(v) for v in row_data[:total]]}\"}\n else:\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Row 4 in vendor_master.xlsx has {len(mismatches)} mismatches out of {total} columns. Mismatches: {'; '.join(mismatches)}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876416854",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In `slack_data.json`: In the #vendor-management channel (\"id\": \"C006\"), a message must be posted containing: BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com\"\n full_expected = normalize(\n \"BANK CHANGE REQUEST: Pinnacle Print Services, V-00003, Requested by Brian Matthews, Email bmatthews@pinnacleprintsvc.com\"\n )\n\n parts = {\n \"BANK CHANGE REQUEST keyword\": \"bank change request\",\n \"vendor name (Pinnacle Print Services)\": \"pinnacle print services\",\n \"vendor id (V-00003)\": \"v-00003\",\n \"contact name (Brian Matthews)\": \"brian matthews\",\n \"request email (bmatthews@pinnacleprintsvc.com)\": \"bmatthews@pinnacleprintsvc.com\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct BANK CHANGE REQUEST message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"bank change request\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No BANK CHANGE REQUEST message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"BANK CHANGE REQUEST message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876423985",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In `slack_data.json`: In #vendor-management (`\"id\": \"C006\"`), a message must be posted indicating POTENTIAL BEC FRAUD for Pinnacle Print Services, V-00003, with sender email `bmatthews@pinnacleprintsvc.com` and on-file email `bmatthews@pinnacleprint.com`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"POTENTIAL BEC FRAUD: Pinnacle Print Services, V-00003, Sender email bmatthews@pinnacleprintsvc.com, on-file email bmatthews@pinnacleprint.com\"\n full_expected = normalize(\n \"POTENTIAL BEC FRAUD: Pinnacle Print Services, V-00003, Sender email bmatthews@pinnacleprintsvc.com, on-file email bmatthews@pinnacleprint.com\"\n )\n\n parts = {\n \"POTENTIAL BEC FRAUD keyword\": \"potential bec fraud\",\n \"vendor name (Pinnacle Print Services)\": \"pinnacle print services\",\n \"vendor id (V-00003)\": \"v-00003\",\n \"sender email (bmatthews@pinnacleprintsvc.com)\": \"bmatthews@pinnacleprintsvc.com\",\n \"on-file email (bmatthews@pinnacleprint.com)\": \"bmatthews@pinnacleprint.com\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct POTENTIAL BEC FRAUD message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"bec fraud\" not in text_norm and \"potential bec\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No POTENTIAL BEC FRAUD message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"POTENTIAL BEC FRAUD message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876445763",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In `slack_data.json`: In #vendor-management channel (\"id\": \"C006\"), a message must be posted containing: LEGAL THREAT, SecureNet Compliance, V-00011, and Invoice SNC-2025-001.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n Path(workspace_path) / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n messages = slack_data.get(\"messages\", {}).get(\"C006\", [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #vendor-management (C006).\"}\n\n def normalize(s):\n s = str(s).lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s.strip()\n\n # SOP format: \"LEGAL THREAT: SecureNet Compliance, V-00011, Invoice SNC-2025-001\"\n full_expected = normalize(\n \"LEGAL THREAT: SecureNet Compliance, V-00011, Invoice SNC-2025-001\"\n )\n\n parts = {\n \"LEGAL THREAT keyword\": \"legal threat\",\n \"vendor name (SecureNet Compliance)\": \"securenet compliance\",\n \"vendor id (V-00011)\": \"v-00011\",\n \"invoice number (SNC-2025-001)\": \"snc-2025-001\",\n }\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if full_expected in text_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct LEGAL THREAT message in C006. Message: {msg.get('text', '')[:300]}\"}\n\n # No exact match — find best candidate and score by field\n best_passed = 0\n best_missing = list(parts.keys())\n best_text = \"\"\n\n for msg in messages:\n text_norm = normalize(msg.get(\"text\", \"\"))\n if \"legal threat\" not in text_norm:\n continue\n missing = [label for label, term in parts.items() if term not in text_norm]\n passed = len(parts) - len(missing)\n if passed > best_passed:\n best_passed = passed\n best_missing = missing\n best_text = msg.get(\"text\", \"\")\n\n if best_passed == 0:\n sample = [m.get(\"text\", \"\")[:80] for m in messages[:3]]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No LEGAL THREAT message found in C006. C006 has {len(messages)} message(s). Sample: {sample}\"}\n\n score = round(best_passed / len(parts), 2)\n return {\"pass\": False, \"score\": score, \"feedback\": f\"LEGAL THREAT message found in C006 but missing fields: {best_missing}. Message: {best_text[:300]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876458214",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In `vendor_master.xlsx`, cell P21 (Outstanding Credit for Cascade IT Hardware) is 0.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"vendor_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'vendor_master.xlsx' not found in workspace at {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n sheet_name = next((n for n in wb.sheetnames if \"vendor\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No vendor sheet found in vendor_master.xlsx. Sheets: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n \n cell_value = ws['P21'].value\n \n # Try to convert to a number for comparison\n if cell_value is None:\n # None could be interpreted as 0 (empty cell) - be generous\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P21 is empty/None, which is treated as 0. PASS.\"}\n \n try:\n numeric_value = float(cell_value)\n except (ValueError, TypeError):\n # If it's a string like \"0\" or \"0.00\" or similar\n stripped = str(cell_value).strip().replace(',', '').replace('$', '')\n try:\n numeric_value = float(stripped)\n except (ValueError, TypeError):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P21 contains '{cell_value}' which cannot be interpreted as a number.\"}\n \n # Check if value is 0 (with tolerance: absolute tolerance of 0.01 since expected is 0)\n if abs(numeric_value) <= 0.01:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P21 in vendor_master.xlsx is {cell_value}, which is effectively 0. PASS.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P21 in vendor_master.xlsx is {cell_value} (numeric: {numeric_value}), expected 0.\"}\n \n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading vendor_master.xlsx: {str(e)}\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876463672",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 31 must contain the following values: 1030 | V-00020 | Cascade IT Hardware | INV-38720 | 2025-09-02 | PO-2025-00117 | PO | 1,950.00 | 1000-100-6400 | DISP-HOLD | Net 30 | 2025-10-02 | T. Okonkwo | 2025-09-03 | | | 0 | Placed on DISP-HOLD on 09/16/2025 — CM-38720.pdf",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n filepath = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {filepath}\"}\n\n try:\n wb = openpyxl.load_workbook(str(filepath), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1030 (INV-38720) is at sheet row 31. Search by invoice number as fallback.\n row = 31\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\") != \"INV-38720\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if ws.cell(row=r, column=4).value == \"INV-38720\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-38720 not found in Invoice Register. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match(actual, expected):\n if expected == \"\":\n return actual is None or normalize(actual) == \"\"\n a = normalize(actual)\n e = normalize(expected)\n if a == e:\n return True\n # Numeric tolerance\n try:\n return abs(float(a.replace('$','')) - float(e.replace('$',''))) / max(abs(float(e.replace('$',''))), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n # Expected row: 1030 | V-00020 | Cascade IT Hardware | INV-38720 | 2025-09-02 | PO-2025-00117 |\n # PO | 1,950.00 | 1000-100-6400 | DISP-HOLD | Net 30 | 2025-10-02 |\n # T. Okonkwo | 2025-09-03 | | | 0 | Placed on DISP-HOLD on 09/16/2025 — CM-38720.pdf\n expected = [\n \"1030\", \"V-00020\", \"Cascade IT Hardware\", \"INV-38720\", \"2025-09-02\",\n \"PO-2025-00117\", \"PO\", \"1950.00\", \"1000-100-6400\", \"DISP-HOLD\",\n \"Net 30\", \"2025-10-02\", \"T. Okonkwo\", \"2025-09-03\", \"\", \"\", \"0\",\n \"Placed on DISP-HOLD on 09/16/2025 - CM-38720.pdf\",\n ]\n\n failures = []\n matched = 0\n for i, exp in enumerate(expected):\n actual = row_values[i] if i < len(row_values) else None\n if match(actual, exp):\n matched += 1\n else:\n failures.append(f\"Col {i+1}: expected '{exp}', got '{normalize(actual)}'\")\n\n total = len(expected)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Invoice Register row {row} matches all {total} expected values for INV-38720 (DISP-HOLD).\"}\n return {\"pass\": False, \"score\": score, \"feedback\": f\"Invoice Register row {row} (INV-38720): {len(failures)}/{total} mismatches — {'; '.join(failures)}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876497121",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "In `match_log_2025_Q3.xlsx`, row 25 must contain the following values: MTH-Q3-024 | 2025-09-16 | PO-2025-00123 | V-00001 | Apex Office Supply | INV-2025-09745 | 4.95 | 4.95 | 0 | 300 | 150 | 300 | 1,485.00 | 742.50 | 0 | 0 | Pass | Partial invoice – 300 units received, 150 units invoiced. Remaining units: 0.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2025_Q3.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # MTH-Q3-024 is at sheet row 25. Search by match ID as fallback.\n row = 25\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip().lower() != \"mth-q3-024\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip().lower() == \"mth-q3-024\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"MTH-Q3-024 not found in match log. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_notes(actual, expected):\n # Allow trailing punctuation/whitespace on the actual value.\n a = normalize(actual)\n a = re.sub(r'[\\s.,;:!?]+$', '', a)\n e = normalize(expected)\n e = re.sub(r'[\\s.,;:!?]+$', '', e)\n return a == e\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n # Expected row (from rubric draft):\n # MTH-Q3-024 | 2025-09-16 | PO-2025-00123 | V-00001 | Apex Office Supply | INV-2025-09745 |\n # 4.95 | 4.95 | 0 | 300 | 150 | 300 | 1,485.00 | 742.50 | 0 | 0 | Pass |\n # Partial invoice – 300 units received, 150 units invoiced. Remaining units: 0\n checks = [\n (\"Col 1 (Match ID)\", lambda v: match_str(v, \"MTH-Q3-024\")),\n (\"Col 2 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 3 (PO)\", lambda v: match_str(v, \"PO-2025-00123\")),\n (\"Col 4 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 5 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 6 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 7 (Unit Price PO)\", lambda v: match_num(v, 4.95)),\n (\"Col 8 (Unit Price Inv)\", lambda v: match_num(v, 4.95)),\n (\"Col 9 (Variance %)\", lambda v: match_num(v, 0)),\n (\"Col 10 (GR Qty)\", lambda v: match_num(v, 300)),\n (\"Col 11 (Inv Qty)\", lambda v: match_num(v, 150)),\n (\"Col 12 (PO Qty)\", lambda v: match_num(v, 300)),\n (\"Col 13 (PO Amount)\", lambda v: match_num(v, 1485.00)),\n (\"Col 14 (Inv Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 15 (Variance $)\",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876500258",
|
|||
|
|
"sort_order": 15,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, on the \"Payment Queue\" sheet, row 30 contains the following values: PAY-0029 | V-00001 | Apex Office Supply | INV-2025-09745 | 742.50 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Payment Queue' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # PAY-0029 is at sheet row 30. Search by payment ID as fallback.\n row = 30\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip() != \"PAY-0029\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip() == \"PAY-0029\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PAY-0029 not found in Payment Queue.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: PAY-0029 | V-00001 | Apex Office Supply | INV-2025-09745 | 742.50 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending\n checks = [\n (\"Col 1 (Payment ID)\", lambda v: match_str(v, \"PAY-0029\")),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 5 (Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 6 (Method)\", lambda v: match_str(v, \"ACH\")),\n (\"Col 7 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 8 (Batch ID)\", lambda v: match_str(v, \"BAT-2025-0916\")),\n (\"Col 9 (empty)\", lambda v: match_empty(v)),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"Pending\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Payment Queue row {row} matches all {total} expected values for PAY-0029.\"}\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876509007",
|
|||
|
|
"sort_order": 16,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 47 must contain the following values: 1046 | V-00001 | Apex Office Supply | INV-2025-09745 | 2025-09-12 | PO-2025-00123 | PO | 742.50 | 1000-100-6200 | APPROVED | Net 30 | 2025-10-12 | T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1046 is at sheet row 47. Search by invoice number as fallback.\n row = 47\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\").strip() != \"INV-2025-09745\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=4).value or \"\").strip() == \"INV-2025-09745\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"INV-2025-09745 not found in Invoice Register.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: 1046 | V-00001 | Apex Office Supply | INV-2025-09745 | 2025-09-12 | PO-2025-00123 |\n # PO | 742.50 | 1000-100-6200 | APPROVED | Net 30 | 2025-10-12 |\n # T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]\n checks = [\n (\"Col 1 (Inv Reg ID)\", lambda v: match_num(v, 1046)),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00001\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Apex Office Supply\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"INV-2025-09745\")),\n (\"Col 5 (Invoice Date)\", lambda v: match_date(v, \"2025-09-12\")),\n (\"Col 6 (PO)\", lambda v: match_str(v, \"PO-2025-00123\")),\n (\"Col 7 (Type)\", lambda v: match_str(v, \"PO\")),\n (\"Col 8 (Amount)\", lambda v: match_num(v, 742.50)),\n (\"Col 9 (GL Code)\", lambda v: match_str(v, \"1000-100-6200\")),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"APPROVED\")),\n (\"Col 11 (Terms)\", lambda v: match_str(v, \"Net 30\")),\n (\"Col 12 (Due Date)\", lambda v: match_date(v, \"2025-10-12\")),\n (\"Col 13 (Entered By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 14 (Entered Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876514496",
|
|||
|
|
"sort_order": 17,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, on the `Invoice Register` sheet, row 48 must contain the following values: 1047 | V-00021 | Redwood Furniture Co | RF-8125 | 2025-09-11 | PO-2025-00124 | PO | 1,900.00 | 1000-100-6500 | APPROVED | Net 45 | 2025-10-26 | T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Invoice Register' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # ID 1047 is at sheet row 48. Search by invoice number as fallback.\n row = 48\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[3] if len(row_values) > 3 else \"\").strip() != \"RF-8125\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=4).value or \"\").strip() == \"RF-8125\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"RF-8125 not found in Invoice Register.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: 1047 | V-00021 | Redwood Furniture Co | RF-8125 | 2025-09-11 | PO-2025-00124 |\n # PO | 1,900.00 | 1000-100-6500 | APPROVED | Net 45 | 2025-10-26 |\n # T. Okonkwo | 2025-09-16 | T. Okonkwo | 2025-09-16 | 0 | [empty]\n checks = [\n (\"Col 1 (Inv Reg ID)\", lambda v: match_num(v, 1047)),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 5 (Invoice Date)\", lambda v: match_date(v, \"2025-09-11\")),\n (\"Col 6 (PO)\", lambda v: match_str(v, \"PO-2025-00124\")),\n (\"Col 7 (Type)\", lambda v: match_str(v, \"PO\")),\n (\"Col 8 (Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 9 (GL Code)\", lambda v: match_str(v, \"1000-100-6500\")),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"APPROVED\")),\n (\"Col 11 (Terms)\", lambda v: match_str(v, \"Net 45\")),\n (\"Col 12 (Due Date)\", lambda v: match_date(v, \"2025-10-26\")),\n (\"Col 13 (Entered By)\", lambda v: match_str(v, \"T. Okonkwo\")),\n (\"Col 14 (Entered Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 15 (Approved By)\",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876517591",
|
|||
|
|
"sort_order": 18,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, on the \"Payment Queue\" sheet, row 31 contains: PAY-0030 | V-00021 | Redwood Furniture Co | RF-8125 | 1,900.00 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"ap_ledger.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open ap_ledger.xlsx: {e}\"}\n\n sheet_name = next((n for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if sheet_name is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sheet 'Payment Queue' not found. Available: {wb.sheetnames}\"}\n ws = wb[sheet_name]\n\n # PAY-0030 is at sheet row 31. Search by payment ID as fallback.\n row = 31\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip() != \"PAY-0030\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip() == \"PAY-0030\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PAY-0030 not found in Payment Queue.\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n def match_empty(actual):\n return actual is None or str(actual).strip() == \"\"\n\n # Expected: PAY-0030 | V-00021 | Redwood Furniture Co | RF-8125 | 1,900.00 | ACH | 2025-09-16 | BAT-2025-0916 | | Pending\n checks = [\n (\"Col 1 (Payment ID)\", lambda v: match_str(v, \"PAY-0030\")),\n (\"Col 2 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 3 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 4 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 5 (Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 6 (Method)\", lambda v: match_str(v, \"ACH\")),\n (\"Col 7 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 8 (Batch ID)\", lambda v: match_str(v, \"BAT-2025-0916\")),\n (\"Col 9 (empty)\", lambda v: match_empty(v)),\n (\"Col 10 (Status)\", lambda v: match_str(v, \"Pending\")),\n ]\n\n failures = []\n matched = 0\n for i, (label, check_fn) in enumerate(checks):\n actual = row_values[i] if i < len(row_values) else None\n if check_fn(actual):\n matched += 1\n else:\n failures.append(f\"{label}: got '{normalize(actual)}'\")\n\n total = len(checks)\n score = round(matched / total, 2)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Payment Queue row {row} matches all {total} expected values for PAY-0030.\"}\n retu
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775876522265",
|
|||
|
|
"sort_order": 19,
|
|||
|
|
"rubric_text": "In `match_log_2025_Q3.xlsx`, row 26 must contain the following values: MTH-Q3-025 | 2025-09-16 | PO-2025-00124 | V-00021 | Redwood Furniture Co | RF-8125 | 380.00 | 380.00 | 0 | 10 | 5 | 10 | 3,800.00 | 1,900.00 | 0 | 0 | Pass | Partial invoice – 10 units received, 5 units invoiced. Remaining units: 0.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"match_log_2025_Q3.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # MTH-Q3-025 is at sheet row 26. Search by match ID as fallback.\n row = 26\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n if str(row_values[0] if row_values else \"\").strip().lower() != \"mth-q3-025\":\n found_row = None\n for r in range(2, ws.max_row + 1):\n if str(ws.cell(row=r, column=1).value or \"\").strip().lower() == \"mth-q3-025\":\n found_row = r\n break\n if found_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"MTH-Q3-025 not found in match log. Has the row been added?\"}\n row = found_row\n row_values = [ws.cell(row=row, column=c).value for c in range(1, ws.max_column + 1)]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n s = str(val).strip().lower()\n s = re.sub(r'\\s+', ' ', s)\n s = s.replace(',', '')\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n return s\n\n def match_str(actual, expected):\n return normalize(actual) == normalize(expected)\n\n def match_notes(actual, expected):\n # Allow trailing punctuation/whitespace on the actual value.\n a = normalize(actual)\n a = re.sub(r'[\\s.,;:!?]+$', '', a)\n e = normalize(expected)\n e = re.sub(r'[\\s.,;:!?]+$', '', e)\n return a == e\n\n def match_num(actual, expected):\n try:\n a = float(str(actual).replace(',', '').strip())\n e = float(str(expected).replace(',', '').strip())\n if e == 0:\n return abs(a) < 0.01\n return abs(a - e) / max(abs(e), 0.001) <= 0.05\n except (ValueError, TypeError):\n return False\n\n def match_date(actual, expected_str):\n if isinstance(actual, (datetime, date)):\n return actual.strftime(\"%Y-%m-%d\") == expected_str\n return normalize(actual) == expected_str\n\n # Expected row (from rubric draft):\n # MTH-Q3-025 | 2025-09-16 | PO-2025-00124 | V-00021 | Redwood Furniture Co | RF-8125 |\n # 380.00 | 380.00 | 0 | 10 | 5 | 10 | 3,800.00 | 1,900.00 | 0 | 0 | Pass |\n # Partial invoice – 10 units received, 5 units invoiced. Remaining units: 0\n checks = [\n (\"Col 1 (Match ID)\", lambda v: match_str(v, \"MTH-Q3-025\")),\n (\"Col 2 (Date)\", lambda v: match_date(v, \"2025-09-16\")),\n (\"Col 3 (PO)\", lambda v: match_str(v, \"PO-2025-00124\")),\n (\"Col 4 (Vendor ID)\", lambda v: match_str(v, \"V-00021\")),\n (\"Col 5 (Vendor Name)\", lambda v: match_str(v, \"Redwood Furniture Co\")),\n (\"Col 6 (Invoice)\", lambda v: match_str(v, \"RF-8125\")),\n (\"Col 7 (Unit Price PO)\", lambda v: match_num(v, 380.00)),\n (\"Col 8 (Unit Price Inv)\", lambda v: match_num(v, 380.00)),\n (\"Col 9 (Variance %)\", lambda v: match_num(v, 0)),\n (\"Col 10 (GR Qty)\", lambda v: match_num(v, 10)),\n (\"Col 11 (Inv Qty)\", lambda v: match_num(v, 5)),\n (\"Col 12 (PO Qty)\", lambda v: match_num(v, 10)),\n (\"Col 13 (PO Amount)\", lambda v: match_num(v, 3800.00)),\n (\"Col 14 (Inv Amount)\", lambda v: match_num(v, 1900.00)),\n (\"Col 15 (Variance $)\", lambda
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1775877705459",
|
|||
|
|
"sort_order": 20,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there are exactly 8 emails in the Sent folder.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n mailbox_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"mailbox.json\" if external_services_path else None,\n workspace_path / \"mailbox.json\",\n ]):\n if candidate.exists():\n mailbox_path = candidate\n break\n\n if mailbox_path:\n try:\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n mailbox_data = json.load(f)\n sent = [e for e in mailbox_data.get(\"emails\", []) if str(e.get(\"folder\", \"\")).lower() == \"sent\"]\n count = len(sent)\n if count == 8:\n feedback = \"PASS: Sent folder has exactly 8 emails.\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": feedback}\n else:\n feedback = f\"FAIL: Sent folder expected 8 emails, found {count}.\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading mailbox.json: {e}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: mailbox.json not found.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "cf326947-0f8d-4e79-9f2a-dfe2f78b8fa7",
|
|||
|
|
"sort_order": 21,
|
|||
|
|
"rubric_text": "In `slack_data.json`, there are exactly 5 messages in #general (C001), 11 in #finance (C002), 15 in #accounts-payable (C003), 9 in #approvals (C004), and 4 in #vendor-management (C006).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n\n slack_path = None\n for candidate in filter(None, [\n Path(external_services_path) / \"slack_data.json\" if external_services_path else None,\n workspace_path / \"slack_data.json\",\n ]):\n if candidate.exists():\n slack_path = candidate\n break\n\n expected_counts = {\n \"C001\": (\"#general\", 5),\n \"C002\": (\"#finance\", 11),\n \"C003\": (\"#accounts-payable\", 15),\n \"C004\": (\"#approvals\", 9),\n \"C006\": (\"#vendor-management\", 4),\n }\n\n if slack_path:\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n messages = slack_data.get(\"messages\", {})\n channel_results = []\n all_pass = True\n for cid, (cname, expected) in expected_counts.items():\n actual = len(messages.get(cid, []))\n if actual == expected:\n channel_results.append(f\" {cname} ({cid}): PASS ({actual})\")\n else:\n channel_results.append(f\" {cname} ({cid}): FAIL - expected {expected}, found {actual}\")\n all_pass = False\n if all_pass:\n feedback = \"PASS: Slack message counts correct.\\n\" + \"\\n\".join(channel_results)\n return {\"pass\": True, \"score\": 1.0, \"feedback\": feedback}\n else:\n feedback = \"FAIL: Slack message counts mismatch.\\n\" + \"\\n\".join(channel_results)\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading slack_data.json: {e}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: slack_data.json not found.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "a2b92c1c-4245-4fce-9e04-b3033efcff46",
|
|||
|
|
"sort_order": 22,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, the \"Invoice Register\" sheet has exactly 47 data rows.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ap_path = workspace_path / \"ap_ledger.xlsx\"\n\n if not ap_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: ap_ledger.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ap_path, read_only=True, data_only=True)\n ir_sheet = next((wb[n] for n in wb.sheetnames if \"invoice\" in n.lower() and \"register\" in n.lower()), None)\n if ir_sheet:\n data_rows = sum(1 for r in ir_sheet.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 47:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Invoice Register has exactly 47 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Invoice Register expected 47 data rows, found {data_rows}.\"}\n else:\n sheets = wb.sheetnames\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Invoice Register sheet not found. Sheets: {sheets}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading ap_ledger.xlsx: {e}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d7475f15-9b80-4809-8f4d-83cc0556f241",
|
|||
|
|
"sort_order": 23,
|
|||
|
|
"rubric_text": "In `ap_ledger.xlsx`, the \"Payment Queue\" sheet has exactly 30 data rows.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ap_path = workspace_path / \"ap_ledger.xlsx\"\n\n if not ap_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: ap_ledger.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ap_path, read_only=True, data_only=True)\n pq_sheet = next((wb[n] for n in wb.sheetnames if \"payment\" in n.lower() and \"queue\" in n.lower()), None)\n if pq_sheet:\n data_rows = sum(1 for r in pq_sheet.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 30:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Payment Queue has exactly 30 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Payment Queue expected 30 data rows, found {data_rows}.\"}\n else:\n sheets = wb.sheetnames\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Payment Queue sheet not found. Sheets: {sheets}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading ap_ledger.xlsx: {e}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "08400c56-c184-4958-b2fb-eec6f2de75dc",
|
|||
|
|
"sort_order": 24,
|
|||
|
|
"rubric_text": "In `match_log_2025_Q3.xlsx`, there are exactly 25 data rows.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n ml_path = workspace_path / \"match_log_2025_Q3.xlsx\"\n\n if not ml_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: match_log_2025_Q3.xlsx not found.\"}\n\n try:\n wb = openpyxl.load_workbook(ml_path, read_only=True, data_only=True)\n ws = wb.active\n data_rows = sum(1 for r in ws.iter_rows(min_row=2, values_only=True) if any(c is not None for c in r))\n wb.close()\n if data_rows == 25:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: match_log_2025_Q3.xlsx has exactly 25 data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: match_log_2025_Q3.xlsx expected 25 data rows, found {data_rows}.\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Error reading match_log_2025_Q3.xlsx: {e}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
}
|
|||
|
|
]
|