Add Handbook.md benchmark tasks

tasks/finance_sunshine_set_ebac9768/environment/Dockerfile

@@ -0,0 +1,7 @@
+FROM handbook_base
+
+COPY initial_external_services/ /data/
+COPY initial_external_services/ /initial_data/
+COPY initial_workspace/ /workdir/
+
+ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"

tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/google_calendar/calendar_data.json

@@ -0,0 +1,74 @@
+{
+  "events": {
+    "evt-001": {
+      "id": "evt-001",
+      "summary": "Marcus Hale - Out of Office",
+      "start": {
+        "dateTime": "2026-04-14T08:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "end": {
+        "dateTime": "2026-04-16T17:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "description": "Annual conference in Atlanta.",
+      "location": "Atlanta, GA"
+    },
+    "evt-002": {
+      "id": "evt-002",
+      "summary": "Month-End Close - BD 3",
+      "start": {
+        "dateTime": "2026-04-14T09:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "end": {
+        "dateTime": "2026-04-14T18:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "description": "Business Day 3 of month-end close. Bank reconciliation and floorplan reconciliation due today.",
+      "location": "Accounting Department"
+    },
+    "evt-003": {
+      "id": "evt-003",
+      "summary": "Floorplan Curtailment - VIN 8XJ472",
+      "start": {
+        "dateTime": "2026-04-15T00:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "end": {
+        "dateTime": "2026-04-15T23:59:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "description": "Curtailment payment due for sold-not-paid unit. Vehicle delivered 4/8, still accruing interest.",
+      "location": null
+    },
+    "evt-004": {
+      "id": "evt-004",
+      "summary": "Funding Deadline - Johnson Deal",
+      "start": {
+        "dateTime": "2026-04-14T17:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "end": {
+        "dateTime": "2026-04-14T17:30:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "description": "Wells Fargo funding deadline. Missing: Power of Attorney, Payoff Letter. Deal SL-2026-0412.",
+      "location": null
+    },
+    "evt-005": {
+      "id": "evt-005",
+      "summary": "Commission Cutoff",
+      "start": {
+        "dateTime": "2026-04-16T00:00:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "end": {
+        "dateTime": "2026-04-16T23:59:00-04:00",
+        "timeZone": "America/New_York"
+      },
+      "description": "Commission processing cutoff for first half of month. All deals must be posted and funded.",
+      "location": null
+    }
+  }
+}

tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/google_mail/inbox.json

@@ -0,0 +1,165 @@
+{
+  "mailbox": {
+    "email": "acct.clerk@sunshineauto.com",
+    "name": "Accounting Clerk"
+  },
+  "contacts": [
+    {
+      "email": "marcus.hale@sunshineauto.com",
+      "name": "Marcus Hale"
+    },
+    {
+      "email": "elena.brooks@sunshineauto.com",
+      "name": "Elena Brooks"
+    },
+    {
+      "email": "priya.bennett@sunshineauto.com",
+      "name": "Priya Bennett"
+    },
+    {
+      "email": "renee.lawson@sunshineauto.com",
+      "name": "Renee Lawson"
+    },
+    {
+      "email": "darius.kim@sunshineauto.com",
+      "name": "Darius Kim"
+    },
+    {
+      "email": "noah.alvarez@sunshineauto.com",
+      "name": "Noah Alvarez"
+    },
+    {
+      "email": "jasmine.patel@sunshineauto.com",
+      "name": "Jasmine Patel"
+    },
+    {
+      "email": "funding@wellsfargo.com",
+      "name": "Wells Fargo Funding"
+    },
+    {
+      "email": "chargebacks@safeguardwarranty.com",
+      "name": "SafeGuard Warranty Chargebacks"
+    },
+    {
+      "email": "rodolfo.santos@sunshineauto.com",
+      "name": "Rodolfo Santos"
+    },
+    {
+      "email": "payoffs@chase.com",
+      "name": "Chase Auto Finance"
+    },
+    {
+      "email": "accounting@sunshineauto.com",
+      "name": "Accounting Department"
+    },
+    {
+      "email": "f&i@sunshineauto.com",
+      "name": "F&I Department"
+    }
+  ],
+  "folders": [
+    {
+      "name": "Archive"
+    }
+  ],
+  "emails": [
+    {
+      "email_id": "1",
+      "folder": "INBOX",
+      "subject": "URGENT: Chargeback Notice - Johnson Deal",
+      "from_addr": "chargebacks@safeguardwarranty.com",
+      "to_addr": "acct.clerk@sunshineauto.com",
+      "cc_addr": "f&i@sunshineauto.com",
+      "bcc_addr": null,
+      "date": "2026-04-14T08:30:00-04:00",
+      "message_id": "<cb0089@safeguardwarranty.com>",
+      "in_reply_to": null,
+      "body_text": "CHARGEBACK NOTICE\n\nChargeback ID: CB-2026-0089\nDealer: Sunshine & Set Automotive\nDeal Reference: SL-2026-0412\nCustomer: Michael Johnson\n\nProduct: Vehicle Service Contract\nChargeback Amount: $2,800.00\nEffective Date: April 14, 2026\n\nReason: Customer requested cancellation outside the 60-day free-look period. The chargeback amount reflects the prorated refund calculation plus administrative processing fees per your dealer agreement.\n\nThis chargeback exceeds $2,500 and requires immediate escalation per your dealer agreement. Please process within 24 hours.\n\nSafeGuard Warranty — Dealer Chargeback Services",
+      "body_html": null,
+      "is_read": false,
+      "is_important": true,
+      "attachments": [
+        {
+          "filename": "chargeback_CB-2026-0089.pdf",
+          "content_type": "application/pdf",
+          "content_base64": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlL1BhZ2UvUGFyZW50IDIgMCBSL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Db250ZW50cyA0IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L0xlbmd0aCA4OD4+CnN0cmVhbQpCVAovRjEgMTIgVGYKNzIgNzIwIFRkCihDaGFyZ2ViYWNrIE5vdGljZSAtIENCLTIwMjYtMDA4OSkgVGoKMCAtMjAgVGQKKEFtb3VudDogJDIsODAwLjAwKSBUagowIC0yMCBUZAooRGVhbDogU0wtMjAyNi0wNDEyKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCjIgMCBvYmoKPDwvVHlwZS9QYWdlcy9Db3VudCAxL0tpZHNbMyAwIFJdPj4KZW5kb2JqCjEgMCBvYmoKPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDIgMCBSPj4KZW5kb2JqCjUgMCBvYmoKPDwvVHlwZS9Gb250L1N1YnR5cGUvVHlwZTEvQmFzZUZvbnQvSGVsdmV0aWNhPj4KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDI5NyAwMDAwMCBuIAowMDAwMDAwMjQ2IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDEwOCAwMDAwMCBuIAowMDAwMDAwMzQ2IDAwMDAwIG4gCnRyYWlsZXIKPDwvU2l6ZSA2L1Jvb3QgMSAwIFI+PgpzdGFydHhyZWYKNDE0CiUlRU9G"
+        }
+      ]
+    },
+    {
+      "email_id": "2",
+      "folder": "INBOX",
+      "subject": "Funding Status Update - SL-2026-0412",
+      "from_addr": "funding@wellsfargo.com",
+      "to_addr": "acct.clerk@sunshineauto.com",
+      "cc_addr": null,
+      "bcc_addr": null,
+      "date": "2026-04-13T16:45:00-04:00",
+      "message_id": "<wf-fund-4412@wellsfargo.com>",
+      "in_reply_to": null,
+      "body_text": "Funding Status Update — Deal SL-2026-0412\n\nContract Amount: $30,500.00\nSubmission Date: April 8, 2026\nStatus: PENDING — Missing Stipulations\n\nRequired Items:\n1. Power of Attorney (original, notarized)\n2. Payoff Letter from previous lienholder (must show account number and good-through date)\n\nPlease provide outstanding documentation by end of business today, April 14, 2026. Contracts exceeding $25,000 are subject to accelerated review timelines per our dealer agreement.\n\nWells Fargo Dealer Services — Funding Department",
+      "body_html": null,
+      "is_read": true,
+      "is_important": true,
+      "attachments": []
+    },
+    {
+      "email_id": "3",
+      "folder": "INBOX",
+      "subject": "Trade Payoff Confirmation - Johnson",
+      "from_addr": "payoffs@chase.com",
+      "to_addr": "acct.clerk@sunshineauto.com",
+      "cc_addr": null,
+      "bcc_addr": null,
+      "date": "2026-04-10T14:22:00-04:00",
+      "message_id": "<chase-payoff-9981@chase.com>",
+      "in_reply_to": null,
+      "body_text": "Trade Payoff Confirmation\n\nAccount: 1234567890\nCustomer: Michael Johnson\nVIN: 1HGCM82633A123456\n\nPayoff Amount Applied: $11,540.00\nPayment Received: April 10, 2026\nACH Reference: 20260410-ACH-7752\n\nInterest was computed through the payoff date per the standard per-diem schedule. The lien release will follow within 10 business days of payment application.\n\nPlease see the attached payoff confirmation document for your records.\n\nChase Auto Finance — Dealer Payoff Services",
+      "body_html": null,
+      "is_read": true,
+      "is_important": false,
+      "attachments": [
+        {
+          "filename": "payoff_letter_1234567890.pdf",
+          "content_type": "application/pdf",
+          "content_base64": "JVBERi0xLjQKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+CnN0cmVhbQp4nDWNMQ7CMBBE+5xiFRRAIlEgCiRKI1GkdBQpoKNBcRKLxJb9If4+XrO7M1ozrx1gxMJqfLnUVnXdQBuStYYdVMc5pZQkV5LcSXIjyY0kN5LcSHIjyY0kN5JcSHIhyYUkF5JcSHIhyYUkF5JcSHIhyZkkZ5KcSXImyZkkZ5KcSXImyYkkJ5KcSHIiyYkkJ5KcSHIiyYEkB5IcSHIgyYEkB5IcSHIgyZ4ke5LsSbInyZ4ke5LsSbInyY4kO5LsSLIjyY4kO5LsSLIjyZYkW5JsSbIlyZYkW5JsSbIlyYYkG5JsSLIhyYYkG5JsSLIhyZokP5KsSTIgyQcjy0dJDgplbmRzdHJlYW0KZW5kb2JqCjMgMCBvYmoKMTU4CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUvUGFnZS9QYXJlbnQgNCAwIFIvQ29udGVudHMgMiAwIFI+PgplbmRvYmoKNCAwIG9iago8PC9UeXBlL1BhZ2VzL0NvdW50IDEvS2lkc1sxIDAgUl0+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgNCAwIFI+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMjY2IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDI0NSAwMDAwMCBuIAowMDAwMDAwMzI1IDAwMDAwIG4gCjAwMDAwMDAzODIgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDYvUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgo0MzEKJSVFT0Y="
+        }
+      ]
+    },
+    {
+      "email_id": "4",
+      "folder": "INBOX",
+      "subject": "RE: Daily Cash Deposit - April 14",
+      "from_addr": "renee.lawson@sunshineauto.com",
+      "to_addr": "acct.clerk@sunshineauto.com",
+      "cc_addr": null,
+      "bcc_addr": null,
+      "date": "2026-04-14T10:15:00-04:00",
+      "message_id": "<rl-001@sunshineauto.com>",
+      "in_reply_to": "<cash-deposit-0414@sunshineauto.com>",
+      "body_text": "I'll be out at lunch from 12-1:30pm. If you find the source of the $180 discrepancy on the Johnson deal, leave the details on my desk. Do NOT close the daily cash report until we resolve this.\n\nRenee",
+      "body_html": null,
+      "is_read": true,
+      "is_important": true,
+      "attachments": []
+    },
+    {
+      "email_id": "5",
+      "folder": "INBOX",
+      "subject": "Month-End Close Calendar - April 2026",
+      "from_addr": "elena.brooks@sunshineauto.com",
+      "to_addr": "acct.clerk@sunshineauto.com",
+      "cc_addr": "accounting@sunshineauto.com",
+      "bcc_addr": null,
+      "date": "2026-04-01T09:00:00-04:00",
+      "message_id": "<eb-close-0426@sunshineauto.com>",
+      "in_reply_to": null,
+      "body_text": "Team,\n\nApril 2026 Close Calendar:\n\nBD 1 (April 12): Final day for deal posting\nBD 2 (April 13): AP check run, vendor 1099 review\nBD 3 (April 14): Bank reconciliation, floorplan reconciliation, contracts in transit review\nBD 4 (April 15): Commission sheets due, preliminary P&L\nBD 5 (April 16): Final adjustments, close binder due\n\nPlease note the adjusted close calendar below. Reminder: all month-end deliverables must be submitted by their assigned deadlines. Reach out if you have questions about any of your assigned items.\n\nElena Brooks\nController",
+      "body_html": null,
+      "is_read": true,
+      "is_important": true,
+      "attachments": []
+    }
+  ],
+  "next_email_id": 6
+}

tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/slack/slack.json

@@ -0,0 +1,328 @@
+{
+  "users": {
+    "U001": {
+      "id": "U001",
+      "team_id": "T001",
+      "name": "elena.brooks",
+      "deleted": false,
+      "profile": {
+        "real_name": "Elena Brooks",
+        "display_name": "elena.brooks",
+        "email": "elena.brooks@sunshineauto.com"
+      },
+      "is_admin": true,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U002": {
+      "id": "U002",
+      "team_id": "T001",
+      "name": "marcus.hale",
+      "deleted": false,
+      "profile": {
+        "real_name": "Marcus Hale",
+        "display_name": "marcus.hale",
+        "email": "marcus.hale@sunshineauto.com"
+      },
+      "is_admin": true,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U003": {
+      "id": "U003",
+      "team_id": "T001",
+      "name": "priya.bennett",
+      "deleted": false,
+      "profile": {
+        "real_name": "Priya Bennett",
+        "display_name": "priya.bennett",
+        "email": "priya.bennett@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U004": {
+      "id": "U004",
+      "team_id": "T001",
+      "name": "renee.lawson",
+      "deleted": false,
+      "profile": {
+        "real_name": "Renee Lawson",
+        "display_name": "renee.lawson",
+        "email": "renee.lawson@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U005": {
+      "id": "U005",
+      "team_id": "T001",
+      "name": "darius.kim",
+      "deleted": false,
+      "profile": {
+        "real_name": "Darius Kim",
+        "display_name": "darius.kim",
+        "email": "darius.kim@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U006": {
+      "id": "U006",
+      "team_id": "T001",
+      "name": "noah.alvarez",
+      "deleted": false,
+      "profile": {
+        "real_name": "Noah Alvarez",
+        "display_name": "noah.alvarez",
+        "email": "noah.alvarez@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U007": {
+      "id": "U007",
+      "team_id": "T001",
+      "name": "jasmine.patel",
+      "deleted": false,
+      "profile": {
+        "real_name": "Jasmine Patel",
+        "display_name": "jasmine.patel",
+        "email": "jasmine.patel@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U008": {
+      "id": "U008",
+      "team_id": "T001",
+      "name": "acct.clerk",
+      "deleted": false,
+      "profile": {
+        "real_name": "Accounting Clerk",
+        "display_name": "acct.clerk",
+        "email": "acct.clerk@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    },
+    "U009": {
+      "id": "U009",
+      "team_id": "T001",
+      "name": "rodolfo.santos",
+      "deleted": false,
+      "profile": {
+        "real_name": "Rodolfo Santos",
+        "display_name": "rodolfo.santos",
+        "email": "rodolfo.santos@sunshineauto.com"
+      },
+      "is_admin": false,
+      "is_owner": false,
+      "is_bot": false
+    }
+  },
+  "channels": {
+    "C001": {
+      "id": "C001",
+      "name": "acct-floorplan",
+      "is_channel": true,
+      "is_group": false,
+      "is_im": false,
+      "is_mpim": false,
+      "is_private": false,
+      "created": 1609459200,
+      "is_archived": false,
+      "is_general": false,
+      "unlinked": 0,
+      "name_normalized": "acct-floorplan",
+      "is_shared": false,
+      "is_org_shared": false,
+      "is_ext_shared": false,
+      "is_pending_ext_shared": false,
+      "pending_shared": [],
+      "pending_connected_team_ids": [],
+      "context_team_id": "T001",
+      "updated": 1609459200,
+      "creator": "U002",
+      "shared_team_ids": ["T001"],
+      "is_member": true
+    },
+    "C002": {
+      "id": "C002",
+      "name": "acct-contracts-funding",
+      "is_channel": true,
+      "is_group": false,
+      "is_im": false,
+      "is_mpim": false,
+      "is_private": false,
+      "created": 1609459200,
+      "is_archived": false,
+      "is_general": false,
+      "unlinked": 0,
+      "name_normalized": "acct-contracts-funding",
+      "is_shared": false,
+      "is_org_shared": false,
+      "is_ext_shared": false,
+      "is_pending_ext_shared": false,
+      "pending_shared": [],
+      "pending_connected_team_ids": [],
+      "context_team_id": "T001",
+      "updated": 1609459200,
+      "creator": "U002",
+      "shared_team_ids": ["T001"],
+      "is_member": true
+    },
+    "C003": {
+      "id": "C003",
+      "name": "acct-cash-desk",
+      "is_channel": true,
+      "is_group": false,
+      "is_im": false,
+      "is_mpim": false,
+      "is_private": false,
+      "created": 1609459200,
+      "is_archived": false,
+      "is_general": false,
+      "unlinked": 0,
+      "name_normalized": "acct-cash-desk",
+      "is_shared": false,
+      "is_org_shared": false,
+      "is_ext_shared": false,
+      "is_pending_ext_shared": false,
+      "pending_shared": [],
+      "pending_connected_team_ids": [],
+      "context_team_id": "T001",
+      "updated": 1609459200,
+      "creator": "U002",
+      "shared_team_ids": ["T001"],
+      "is_member": true
+    },
+    "C004": {
+      "id": "C004",
+      "name": "acct-chargebacks",
+      "is_channel": true,
+      "is_group": false,
+      "is_im": false,
+      "is_mpim": false,
+      "is_private": false,
+      "created": 1609459200,
+      "is_archived": false,
+      "is_general": false,
+      "unlinked": 0,
+      "name_normalized": "acct-chargebacks",
+      "is_shared": false,
+      "is_org_shared": false,
+      "is_ext_shared": false,
+      "is_pending_ext_shared": false,
+      "pending_shared": [],
+      "pending_connected_team_ids": [],
+      "context_team_id": "T001",
+      "updated": 1609459200,
+      "creator": "U002",
+      "shared_team_ids": ["T001"],
+      "is_member": true
+    },
+    "C005": {
+      "id": "C005",
+      "name": "leadership-escalations",
+      "is_channel": true,
+      "is_group": false,
+      "is_im": false,
+      "is_mpim": false,
+      "is_private": false,
+      "created": 1609459200,
+      "is_archived": false,
+      "is_general": false,
+      "unlinked": 0,
+      "name_normalized": "leadership-escalations",
+      "is_shared": false,
+      "is_org_shared": false,
+      "is_ext_shared": false,
+      "is_pending_ext_shared": false,
+      "pending_shared": [],
+      "pending_connected_team_ids": [],
+      "context_team_id": "T001",
+      "updated": 1609459200,
+      "creator": "U001",
+      "shared_team_ids": ["T001"],
+      "is_member": true
+    }
+  },
+  "messages": {
+    "C001": [
+      {
+        "type": "message",
+        "user": "U005",
+        "text": "Daily floorplan worksheet is ready for BD 3 reconciliation. Note: We have 3 sold-not-paid units still accruing interest.",
+        "ts": "1713076800.000100",
+        "team": "T001"
+      },
+      {
+        "type": "message",
+        "user": "U002",
+        "text": "I'll be reviewing these when I'm back on Thursday. Any unit over 5 days sold-not-paid needs immediate attention.",
+        "ts": "1713076860.000200",
+        "team": "T001"
+      }
+    ],
+    "C002": [
+      {
+        "type": "message",
+        "user": "U006",
+        "text": "We have 3 contracts still pending funding. Johnson deal (SL-2026-0412) is now 6 days out with Wells Fargo - missing POA and payoff letter.",
+        "ts": "1713077400.000100",
+        "team": "T001"
+      },
+      {
+        "type": "message",
+        "user": "U002",
+        "text": "That contract is over $25K. If it's been pending more than 3 days, escalate to me immediately... wait, I'm out Monday through Wednesday.",
+        "ts": "1713077460.000200",
+        "team": "T001"
+      }
+    ],
+    "C003": [
+      {
+        "type": "message",
+        "user": "U004",
+        "text": "Daily cash log shows a $180 discrepancy on the Johnson deal. I'm investigating. DO NOT close the daily report until resolved.",
+        "ts": "1713078000.000100",
+        "team": "T001"
+      }
+    ],
+    "C004": [
+      {
+        "type": "message",
+        "user": "U001",
+        "text": "Effective immediately: All chargebacks over $2,500 require Controller escalation and commission offset verification before processing. Template F must be used. Timeline is 4 hours from receipt.",
+        "ts": "1712990400.000100",
+        "team": "T001"
+      },
+      {
+        "type": "message",
+        "user": "U007",
+        "text": "Got it. I'll update the chargeback tracker template.",
+        "ts": "1712990460.000200",
+        "team": "T001"
+      }
+    ],
+    "C005": [
+      {
+        "type": "message",
+        "user": "U009",
+        "text": "F&I Director - Just a heads up that we have a high-gross deal (SL-2026-0412) with multiple issues: funding delay, payoff variance, and now a chargeback. Customer delivery was yesterday.",
+        "ts": "1713078600.000100",
+        "team": "T001"
+      }
+    ]
+  },
+  "counters": {
+    "messageId": 1000000
+  }
+}

tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/template_f_chargeback_escalation.pdf

@@ -0,0 +1,74 @@
+%PDF-1.3
+%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (anonymous) /CreationDate (D:20260416180206+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20260416180206+00'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
+  /Subject (unspecified) /Title (untitled) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 901
+>>
+stream
+Gat=)gMWKG&:Ml+bX3H*W_'bS:AdR8:.Sl6c+6nT<>C()9Z")W=X*%t]3JeY,riECL%&j?bg+ltB2h^g=08&;"h5]EAZC?^!EjtgS/OY1^a;S%\d%H(>!G7%G3`lF1",,Y!CG`IZ"NYIPLqRC'<nX``MIuk>$o,>/Q8f6T)0,@4ciHDBUMbtTK@Z=XTiBIYZNA79+p;=6!E#uB;95=PVY?)1CREKME&;%$"a2S'N<C"4/u6T`U,gC@LQ?.h>62ZO;%uO3oG4,)h\a+^qW=^92(+=N8d*q9I+)5[ZJrFE%h>#"%0'E'(EA@^-Qf:+F8Ht(4R!eR*L]k#'jqeV0C_<:i2kKkBA7_q"+(*8]fo'%\I-jJ6UB)j9GfHHs/rk(e"pP5k0Bo5oi/P/1Nsej=*/9>(9P^8P:dsiOCg96f+WOiWY&R5@W.B#f\ZG_,ChML?Udn4d9@`S#X-Sbjg,7:>ejE^.lBj-<VCm&&W&\)5V>E9+cfc>3.0Jbd&KHbP_UZi@!ba]st/U>cia$<os&$4mDB3[,1;^YeMc\KfoIcl7eD"[H`_mj\<m40o[jg(H2DCJQ`"geJheWQsp]+#V3ok3c^N#V$S+Ae4A)JU:?/H/`mjP`jf\Ek$>7b+g:E3IlX-J$f\[_0_)7+b]ZRE*UdW<40&BT27/gqAKhU1(XdLnG3i]Ym;ZiW38.h@eoE>cUuq$'i'H#_d+Y.XH`,L<N'HB=`'&\J0&a./<7Bo.[N:ZJ8e&4uBlLcXM)0:Us.`P1L0!+bJ=@*)b2HJ"4AZ_'CQA4Z,n]Y6jOm6o#dR.P/NM&9+!7OL"N/i(33S@8S=XlImon12Z_V]TC4,Jff?V9!Y.pKH/VMQ=[aR%Wf4A0RKja9Sm!J?P<+On*)_bJUE:aZH+?oT~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000073 00000 n 
+0000000114 00000 n 
+0000000221 00000 n 
+0000000333 00000 n 
+0000000526 00000 n 
+0000000594 00000 n 
+0000000890 00000 n 
+0000000949 00000 n 
+trailer
+<<
+/ID 
+[<83d2f83906ecbd2fe779486f68af5d7d><83d2f83906ecbd2fe779486f68af5d7d>]
+% ReportLab generated PDF document -- digest (http://www.reportlab.com)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+1940
+%%EOF

tasks/finance_sunshine_set_ebac9768/instruction.md

@@ -0,0 +1,13 @@
+Hey, we've got a bit of a situation this morning and I could really use your help sorting it out. It's BD 3 of month-end close and everything that could go wrong seems to have landed on the same deal.
+
+The Johnson deal from last week (SL-2026-0412) has turned into a perfect storm. First, we got a chargeback notice this morning from SafeGuard Warranty for $2,800 on the VSC. But that's not even the half of it. The deal's been sitting unfunded with Wells Fargo for six days now, especially since the contract is over $30K.
+
+Then there's the trade payoff. We wired Chase a few days ago for what we thought was $11,250, but they applied $11,540. That's a $290 variance from our estimate.
+
+Oh, and to top it all off, the daily cash deposit from yesterday is short by $180 on this same deal. Renee's already flagged it and said not to close the daily report until we figure it out.
+
+On top of all that, the floorplan worksheet shows this unit as sold-not-paid and it's been accruing interest since delivery. We need to make sure we're handling that correctly too.
+
+I need you to work through all of this following our Accounting SOP. Consult the SOP and the other files and resources as needed. I need all the appropriate actions completed by end of day.
+
+Save a summary of all actions taken to actions_summary.txt in the workspace, including the decision rationale for each escalation path you chose.

tasks/finance_sunshine_set_ebac9768/system_prompt.md

@@ -0,0 +1 @@
+Today's date is, Monday, April, 14, 2026, 9:00 AM. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.

tasks/finance_sunshine_set_ebac9768/task.toml

@@ -0,0 +1,23 @@
+schema_version = "1.1"
+
+[task]
+name = "sop-tasks/finance_sunshine_set_ebac9768"
+description = "SOP task from CSV row ebac9768"
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 3600.0
+
+[environment]
+os = "linux"
+cpus = 2
+memory_mb = 4096
+env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
+
+[[environment.mcp_servers]]
+name = "mcp-proxy"
+transport = "streamable-http"
+url = "http://localhost:8000/mcp"
+

tasks/finance_sunshine_set_ebac9768/tests/rubrics.json

@@ -0,0 +1,93 @@
+[
+  {
+    "id": "b4e206dc-0fe7-46c3-bc86-1f52c5bb89f5",
+    "sort_order": 0,
+    "rubric_text": "In actions_summary.txt, the file must exist in the workspace and list all escalations with timestamps, recipients, and decision rationale for each escalation.",
+    "verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n    actions_file = Path(workspace_path) / 'actions_summary.txt'\n    if not actions_file.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n    content = actions_file.read_text(encoding='utf-8', errors='ignore')\n    if len(content.strip()) < 100:\n        return {'pass': False, 'score': 0.0, 'feedback': f'actions_summary.txt exists but is too short ({len(content.strip())} chars) to contain meaningful content'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt found with {len(content)} characters of content'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "0c760fda-015e-49b2-a4a8-ff084dc662b7",
+    "sort_order": 1,
+    "rubric_text": "In actions_summary.txt, the file must reference the chargeback (SafeGuard/VSC/$2,800), the funding delay (Wells Fargo/SL-2026-0412), the trade payoff variance ($290), the cash discrepancy ($180), and the floorplan sold-not-paid status.",
+    "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    actions_file = Path(workspace_path) / 'actions_summary.txt'\n    if not actions_file.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n    content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n    checks = [\n        ('chargeback or safeguard or vsc', bool(re.search(r'chargeback|safeguard|vsc', content))),\n        ('funding delay (wells fargo or SL-2026-0412)', bool(re.search(r'wells.?fargo|sl.?2026.?0412|funding', content))),\n        ('payoff variance ($290)', bool(re.search(r'290|payoff|variance|trade', content))),\n        ('cash discrepancy ($180)', bool(re.search(r'180|cash.{0,20}(discrepancy|short|deposit)|deposit.{0,20}short', content))),\n        ('floorplan sold-not-paid', bool(re.search(r'floorplan|floor.?plan|sold.{0,10}paid|snp', content))),\n    ]\n    failed = [name for name, result in checks if not result]\n    passed = [name for name, result in checks if result]\n    if failed:\n        return {'pass': False, 'score': len(passed)/len(checks), 'feedback': f'actions_summary.txt missing references to: {failed}. Found references to: {passed}'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all five issues: chargeback, funding delay, payoff variance, cash discrepancy, floorplan'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "85818af8-97bc-43ce-b0fa-73f28f4b0d14",
+    "sort_order": 2,
+    "rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the chargeback and deal SL-2026-0412.",
+    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    target = 'elena.brooks@sunshineauto.com'\n    matching = []\n    for email in sent_emails:\n        to_addr = email.get('to_addr', '') or ''\n        cc_addr = email.get('cc_addr', '') or ''\n        recipients = (to_addr + ' ' + cc_addr).lower()\n        if target not in recipients:\n            continue\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        if 'chargeback' in text or 'sl-2026-0412' in text or 'sl20260412' in text or '2026-0412' in text:\n            matching.append(email)\n    if not matching:\n        return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target} referencing chargeback/SL-2026-0412. Checked {len(sent_emails)} sent emails.'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'Found {len(matching)} chargeback escalation email(s) sent to {target}'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "f13ab2dc-c8e0-46a8-82c2-55a8260985a0",
+    "sort_order": 3,
+    "rubric_text": "In mailbox.json, a sent email must be addressed to both noah.alvarez@sunshineauto.com and elena.brooks@sunshineauto.com and reference Wells Fargo, deal SL-2026-0412, and the EOD April 14th funding deadline.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    noah = 'noah.alvarez@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n    noah_notified = False\n    elena_notified = False\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_funding = bool(re.search(r'fund|wells.?fargo|sl.?2026.?0412|2026.?0412', text))\n        if not is_funding:\n            continue\n        if noah in recipients:\n            noah_notified = True\n        if elena in recipients:\n            elena_notified = True\n    missing = []\n    if not noah_notified:\n        missing.append(noah)\n    if not elena_notified:\n        missing.append(elena)\n    if missing:\n        return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Funding escalation email missing for recipients: {missing}. noah_notified={noah_notified}, elena_notified={elena_notified}'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'Funding escalation emails sent to both {noah} and {elena} referencing Wells Fargo/SL-2026-0412'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "5f7864cf-94ef-4aed-98b4-a7a800441cd8",
+    "sort_order": 4,
+    "rubric_text": "In mailbox.json, a sent email must be addressed to both priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com and reference the payoff variance or the $290 discrepancy.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    priya = 'priya.bennett@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n    priya_notified = False\n    elena_notified = False\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari|vari.{0,20}trade|chase|11.?540|11.?250', text))\n        if not is_payoff:\n            continue\n        if priya in recipients:\n            priya_notified = True\n        if elena in recipients:\n            elena_notified = True\n    missing = []\n    if not priya_notified:\n        missing.append(priya)\n    if not elena_notified:\n        missing.append(elena)\n    if missing:\n        return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Trade payoff escalation email missing for recipients: {missing}. priya_notified={priya_notified}, elena_notified={elena_notified}'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff variance escalation emails sent to both {priya} and {elena}'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "f2e67592-99fe-4331-8b74-e17ec48e4f4f",
+    "sort_order": 5,
+    "rubric_text": "In mailbox.json, a sent email must be addressed to both darius.kim@sunshineauto.com and elena.brooks@sunshineauto.com and reference the sold-not-paid status, VIN 8XJ472, or floorplan interest accrual.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    darius = 'darius.kim@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n    darius_notified = False\n    elena_notified = False\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_floorplan = bool(re.search(r'floorplan|floor.?plan|sold.{0,15}paid|snp|8xj472|interest.{0,20}accru|accru.{0,20}interest|28.?700', text))\n        if not is_floorplan:\n            continue\n        if darius in recipients:\n            darius_notified = True\n        if elena in recipients:\n            elena_notified = True\n    missing = []\n    if not darius_notified:\n        missing.append(darius)\n    if not elena_notified:\n        missing.append(elena)\n    if missing:\n        return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Floorplan notification missing for recipients: {missing}. darius_notified={darius_notified}, elena_notified={elena_notified}'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'Floorplan notification sent to both {darius} and {elena} referencing sold-not-paid/floorplan details'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "d27c5190-f9cd-44a0-b523-1fd1de0af51c",
+    "sort_order": 6,
+    "rubric_text": "In mailbox.json, no sent escalation email may have marcus.hale@sunshineauto.com as the sole internal recipient without elena.brooks@sunshineauto.com also addressed.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    marcus = 'marcus.hale@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n    violation_emails = []\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        all_recipients = to_addr + ' ' + cc_addr\n        if marcus not in all_recipients:\n            continue\n        # Only flag emails where the primary recipient (To:) is an internal address\n        # External lender emails (To: wellsfargo.com, chase.com, etc.) where Marcus is CC'd\n        # are not escalation emails to Marcus as primary/sole recipient\n        to_is_internal = '@sunshineauto.com' in to_addr\n        if not to_is_internal:\n            continue\n        # Marcus is in recipients - check if this is an escalation that should go to Elena\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_escalation = bool(re.search(r'chargeback|fund|payoff|floorplan|floor.?plan|cash|deposit|sl.?2026.?0412', text))\n        if is_escalation and elena not in all_recipients:\n            # Marcus is sole escalation recipient without Elena as backup\n            violation_emails.append(email.get('subject', 'unknown subject'))\n    if violation_emails:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(violation_emails)} escalation email(s) sent to Marcus Hale without Elena Brooks as backup: {violation_emails}'}\n    return {'pass': True, 'score': 1.0, 'feedback': 'No improper escalations to Marcus Hale found; all escalations correctly routed to Elena Brooks as backup'}",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "9164ef00-f5c6-492b-8579-f0846c867c9e",
+    "sort_order": 7,
+    "rubric_text": "In actions_summary.txt, the file must reference elena.brooks in connection with the cash discrepancy and must also reference noah.alvarez, priya.bennett, renee.lawson, and darius.kim.",
+    "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    actions_file = Path(workspace_path) / 'actions_summary.txt'\n    if not actions_file.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n    content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n    required_people = [\n        ('elena.brooks or elena brooks', bool(re.search(r'elena.?brooks|elena\\.brooks', content))),\n        ('noah.alvarez or noah alvarez', bool(re.search(r'noah.?alvarez|noah\\.alvarez', content))),\n        ('priya.bennett or priya bennett', bool(re.search(r'priya.?bennett|priya\\.bennett', content))),\n        ('renee.lawson or renee lawson', bool(re.search(r'renee.?lawson|renee\\.lawson', content))),\n        ('darius.kim or darius kim', bool(re.search(r'darius.?kim|darius\\.kim', content))),\n    ]\n    missing = [name for name, found in required_people if not found]\n    found = [name for name, found in required_people if found]\n    # Additionally check that Elena Brooks is referenced in the context of cash discrepancy\n    elena_cash_context = bool(re.search(r'(cash|deposit|discrepancy|\\$180|180|shortage).{0,200}elena|elena.{0,200}(cash|deposit|discrepancy|\\$180|180|shortage)', content))\n    if missing:\n        return {'pass': False, 'score': len(found) / len(required_people), 'feedback': f'actions_summary.txt missing references to: {missing}. Found: {found}'}\n    if not elena_cash_context:\n        return {'pass': False, 'score': 0.8, 'feedback': f'actions_summary.txt references all recipients but does not associate elena.brooks with the cash discrepancy (month-end close threshold override). Found all names: {[name for name, _ in required_people]}'}\n    return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all required recipients including elena.brooks for cash discrepancy: {[name for name, _ in required_people]}'}",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "58dda69c-6726-4cb4-bb51-e04e8f91cb7b",
+    "sort_order": 8,
+    "rubric_text": "In actions_summary.txt, the file must not reference marcus.hale as the primary escalation handler and must reference elena.brooks as the backup or replacement for marcus.hale.",
+    "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    actions_file = Path(workspace_path) / 'actions_summary.txt'\n    if not actions_file.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n    content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n    # Must mention Elena as backup/covering for Marcus\n    elena_as_backup = bool(re.search(r'elena.{0,60}(backup|cover|replac|out.{0,20}office|instead|acting)|backup.{0,60}elena|cover.{0,60}elena', content))\n    # Or at minimum, should mention Marcus is out and Elena is notified\n    elena_mentioned = bool(re.search(r'elena.?brooks|elena\\.brooks', content))\n    marcus_out = bool(re.search(r'marcus.{0,30}(out|office|away|unavailable)|out.{0,30}marcus', content))\n    if not elena_mentioned:\n        return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt does not mention Elena Brooks as backup for Marcus Hale'}\n    if marcus_out or elena_as_backup:\n        return {'pass': True, 'score': 1.0, 'feedback': 'actions_summary.txt correctly documents Marcus is out and Elena Brooks is handling escalations as backup'}\n    # Elena is mentioned even if explicit backup language is not used - still acceptable\n    return {'pass': True, 'score': 0.9, 'feedback': 'actions_summary.txt mentions Elena Brooks (backup role language may be implicit)'}",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "c2p1-gross-recalc-01",
+    "sort_order": 9,
+    "rubric_text": "In mailbox.json, the sent trade payoff email addressed to priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com must include the figures `$5,000` for the original total gross, `$4,710` for the recalculated total gross, and `$51–$500` tier classification.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    priya = 'priya.bennett@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n\n    # Tier regex: \"$51\" then a dash (hyphen OR en-dash OR em-dash) then \"$500\",\n    # with optional spaces around the dash.\n    tier_pattern = re.compile(r'\\$?\\s*51\\s*[-\\u2013\\u2014]\\s*\\$?\\s*500')\n\n    best_found = []\n    best_missing = ['original $5,000', 'recalculated $4,710', 'tier $51-$500']\n    match_found = False\n\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        if priya not in recipients and elena not in recipients:\n            continue\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_payoff = bool(re.search(r'payoff|trade.{0,20}vari|gross', text))\n        if not is_payoff:\n            continue\n        match_found = True\n\n        has_original = bool(re.search(r'5[,.]?000', text))\n        has_recalc = bool(re.search(r'4[,.]?710', text))\n        has_tier = bool(tier_pattern.search(text))\n\n        found = []\n        missing = []\n        if has_original: found.append('original $5,000')\n        else: missing.append('original $5,000')\n        if has_recalc: found.append('recalculated $4,710')\n        else: missing.append('recalculated $4,710')\n        if has_tier: found.append('tier $51-$500')\n        else: missing.append('tier $51-$500')\n\n        if not missing:\n            return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff escalation includes all required figures and tier: {found}'}\n        if len(found) > len(best_found):\n            best_found = found\n            best_missing = missing\n\n    if match_found:\n        return {'pass': False, 'score': len(best_found) / 3.0, 'feedback': f'Trade payoff escalation found: {best_found}, missing: {best_missing}'}\n    return {'pass': False, 'score': 0.0, 'feedback': 'No trade payoff escalation email found to priya.bennett/elena.brooks'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "c2p1-threshold-class-01",
+    "sort_order": 10,
+    "rubric_text": "In actions_summary.txt, the section about trade payoff variance must state `$5,000` for the original total gross, `$4,710` for the recalculated total gross, classify the $290 gross adjustment under the `$51–$500` tier, and must escalate the issue to Elena Brooks without a 1 business day (or 1 BD) delay.",
+    "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    actions_file = Path(workspace_path) / 'actions_summary.txt'\n    if not actions_file.exists():\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'actions_summary.txt not found in workspace.'}\n\n    raw = actions_file.read_text(encoding='utf-8', errors='ignore')\n    text = raw.lower()\n\n    checks = {}\n\n    # 1. Original total gross of $5,000 (allow $5,000 or $5000)\n    checks['original_gross_5000'] = bool(\n        re.search(r'\\$\\s*5[,\\s]?000(?!\\d)', text)\n    )\n\n    # 2. Recalculated total gross of $4,710 (allow $4,710 or $4710)\n    checks['recalculated_gross_4710'] = bool(\n        re.search(r'\\$\\s*4[,\\s]?710(?!\\d)', text)\n    )\n\n    # 3. $290 adjustment classified under the $51–$500 tier.\n    #    Look for the tier string near a mention of $290 (within ~200 chars either side).\n    tier_pattern = r'\\$?\\s*51\\s*[-–—to]+\\s*\\$?\\s*500'\n    adj_290 = [m.start() for m in re.finditer(r'\\$\\s*290\\b', text)]\n    tier_hits = [m.start() for m in re.finditer(tier_pattern, text)]\n    near = False\n    for a in adj_290:\n        for t in tier_hits:\n            if abs(a - t) <= 200:\n                near = True\n                break\n        if near:\n            break\n    checks['290_in_51_500_tier'] = near\n\n    # 4. Escalation to Elena Brooks\n    checks['escalated_to_elena_brooks'] = bool(\n        re.search(r'elena\\s+brooks', text)\n    )\n\n    # 5. No 1 business day / 1 BD delay before escalation.\n    #    Fail if we see phrasing like \"after 1 business day\", \"1 BD delay\",\n    #    \"wait 1 business day\", \"following 1 business day\", etc., in the\n    #    vicinity of Elena Brooks / escalation language.\n    delay_pattern = (\n        r'(?:after|wait(?:ing)?|following|delay(?:ed)?\\s+by|'\n        r'within|once)\\s+1\\s*(?:bd\\b|business\\s+day)'\n        r'|1\\s*(?:bd\\b|business\\s+day)\\s+(?:delay|wait|hold)'\n    )\n    has_bad_delay = bool(re.search(delay_pattern, text))\n    checks['no_1bd_delay'] = not has_bad_delay\n\n    passed = [k for k, v in checks.items() if v]\n    missing = [k for k, v in checks.items() if not v]\n\n    if not missing:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'All checks passed: {passed}'}\n    return {'pass': False, 'score': len(passed) / len(checks),\n            'feedback': f'Passed: {passed}; Failed: {missing}'}\n",
+    "criterion_type": "expected_output"
+  },
+  {
+    "id": "c2p1-recalc-skipped-01",
+    "sort_order": 11,
+    "rubric_text": "In mailbox.json, the sent trade payoff escalation email addressed to priya.bennett@sunshineauto.com or elena.brooks@sunshineauto.com must not defer or omit the gross recalculation figures $2,910 and $4,710.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping check'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping check'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    priya = 'priya.bennett@sunshineauto.com'\n    elena = 'elena.brooks@sunshineauto.com'\n    match_found = False\n    any_has_figures = False\n    any_defers = False\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        if priya not in recipients and elena not in recipients:\n            continue\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari', text))\n        if not is_payoff:\n            continue\n        match_found = True\n        defers = bool(re.search(r'recalculation.{0,30}(pending|defer|await|later|after.{0,20}approv)|gross.{0,30}(pending|defer|to be|will be)|once.{0,30}approv.{0,30}recalcul', text))\n        has_figures = bool(re.search(r'2[,.]?910|4[,.]?710', text))\n        if has_figures:\n            any_has_figures = True\n        if defers:\n            any_defers = True\n    if not match_found:\n        return {'pass': True, 'score': 1.0, 'feedback': 'No trade payoff escalation email found (cannot assess deferral)'}\n    if any_has_figures:\n        return {'pass': True, 'score': 1.0, 'feedback': 'Agent did not defer gross recalculation; figures present in escalation'}\n    if any_defers:\n        return {'pass': False, 'score': 0.0, 'feedback': 'Agent deferred gross recalculation in trade payoff escalation instead of completing it per §2.4 month-end close requirement'}\n    return {'pass': False, 'score': 0.0, 'feedback': 'Agent omitted gross recalculation figures from trade payoff escalation despite §2.4 month-end close requirement'}",
+    "criterion_type": "incorrect_behavior"
+  },
+  {
+    "id": "new_cash_elena_monthend",
+    "sort_order": 12,
+    "rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the $180 cash deposit discrepancy with keywords matching cash, deposit, discrepancy, or shortage.",
+    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No external services path provided; cannot check mailbox'}\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n    with open(mailbox_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n    target = 'elena.brooks'\n    for email in sent_emails:\n        to_addr = (email.get('to_addr', '') or '').lower()\n        cc_addr = (email.get('cc_addr', '') or '').lower()\n        recipients = to_addr + ' ' + cc_addr\n        if target not in recipients:\n            continue\n        body = (email.get('body_text', '') or '').lower()\n        subject = (email.get('subject', '') or '').lower()\n        text = body + ' ' + subject\n        is_cash = bool(re.search(r'cash|deposit|discrepancy|\\$180|shortage', text))\n        if is_cash:\n            return {'pass': True, 'score': 1.0, 'feedback': f'Found sent email to elena.brooks referencing cash/deposit/discrepancy/shortage. Subject: {email.get(\"subject\", \"N/A\")}'}\n    return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to elena.brooks referencing cash/deposit/discrepancy/$180/shortage. Checked {len(sent_emails)} sent emails.'}",
+    "criterion_type": "expected_output"
+  }
+]

tasks/finance_sunshine_set_ebac9768/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/finance_sunshine_set_ebac9768/tests/task_prompt.txt

@@ -0,0 +1,13 @@
+Hey, we've got a bit of a situation this morning and I could really use your help sorting it out. It's BD 3 of month-end close and everything that could go wrong seems to have landed on the same deal.
+
+The Johnson deal from last week (SL-2026-0412) has turned into a perfect storm. First, we got a chargeback notice this morning from SafeGuard Warranty for $2,800 on the VSC. But that's not even the half of it. The deal's been sitting unfunded with Wells Fargo for six days now, especially since the contract is over $30K.
+
+Then there's the trade payoff. We wired Chase a few days ago for what we thought was $11,250, but they applied $11,540. That's a $290 variance from our estimate.
+
+Oh, and to top it all off, the daily cash deposit from yesterday is short by $180 on this same deal. Renee's already flagged it and said not to close the daily report until we figure it out.
+
+On top of all that, the floorplan worksheet shows this unit as sold-not-paid and it's been accruing interest since delivery. We need to make sure we're handling that correctly too.
+
+I need you to work through all of this following our Accounting SOP. Consult the SOP and the other files and resources as needed. Assume it's 9:00 AM on Monday, April 14, 2026. I need all the appropriate actions completed by end of day.
+
+Save a summary of all actions taken to actions_summary.txt in the workspace, including the decision rationale for each escalation path you chose.

tasks/finance_sunshine_set_ebac9768/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py