surge-ai/handbook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Handbook.md benchmark tasks

Browse Source
This commit is contained in:
DerekSurge
2026-06-24 12:44:34 -07:00
commit 72f9a65917
1800 changed files with 323589 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/finance_sunshine_set_ebac9768/environment/Dockerfile
+7
View File
@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/google_calendar/calendar_data.json
+74
View File
@@ -0,0 +1,74 @@
{
"events": {
"evt-001": {
"id": "evt-001",
"summary": "Marcus Hale - Out of Office",
"start": {
"dateTime": "2026-04-14T08:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-04-16T17:00:00-04:00",
"timeZone": "America/New_York"
},
"description": "Annual conference in Atlanta.",
"location": "Atlanta, GA"
},
"evt-002": {
"id": "evt-002",
"summary": "Month-End Close - BD 3",
"start": {
"dateTime": "2026-04-14T09:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-04-14T18:00:00-04:00",
"timeZone": "America/New_York"
},
"description": "Business Day 3 of month-end close. Bank reconciliation and floorplan reconciliation due today.",
"location": "Accounting Department"
},
"evt-003": {
"id": "evt-003",
"summary": "Floorplan Curtailment - VIN 8XJ472",
"start": {
"dateTime": "2026-04-15T00:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-04-15T23:59:00-04:00",
"timeZone": "America/New_York"
},
"description": "Curtailment payment due for sold-not-paid unit. Vehicle delivered 4/8, still accruing interest.",
"location": null
},
"evt-004": {
"id": "evt-004",
"summary": "Funding Deadline - Johnson Deal",
"start": {
"dateTime": "2026-04-14T17:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-04-14T17:30:00-04:00",
"timeZone": "America/New_York"
},
"description": "Wells Fargo funding deadline. Missing: Power of Attorney, Payoff Letter. Deal SL-2026-0412.",
"location": null
},
"evt-005": {
"id": "evt-005",
"summary": "Commission Cutoff",
"start": {
"dateTime": "2026-04-16T00:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-04-16T23:59:00-04:00",
"timeZone": "America/New_York"
},
"description": "Commission processing cutoff for first half of month. All deals must be posted and funded.",
"location": null
}
}
}
tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/google_mail/inbox.json
+165
View File
@@ -0,0 +1,165 @@
{
"mailbox": {
"email": "acct.clerk@sunshineauto.com",
"name": "Accounting Clerk"
},
"contacts": [
{
"email": "marcus.hale@sunshineauto.com",
"name": "Marcus Hale"
},
{
"email": "elena.brooks@sunshineauto.com",
"name": "Elena Brooks"
},
{
"email": "priya.bennett@sunshineauto.com",
"name": "Priya Bennett"
},
{
"email": "renee.lawson@sunshineauto.com",
"name": "Renee Lawson"
},
{
"email": "darius.kim@sunshineauto.com",
"name": "Darius Kim"
},
{
"email": "noah.alvarez@sunshineauto.com",
"name": "Noah Alvarez"
},
{
"email": "jasmine.patel@sunshineauto.com",
"name": "Jasmine Patel"
},
{
"email": "funding@wellsfargo.com",
"name": "Wells Fargo Funding"
},
{
"email": "chargebacks@safeguardwarranty.com",
"name": "SafeGuard Warranty Chargebacks"
},
{
"email": "rodolfo.santos@sunshineauto.com",
"name": "Rodolfo Santos"
},
{
"email": "payoffs@chase.com",
"name": "Chase Auto Finance"
},
{
"email": "accounting@sunshineauto.com",
"name": "Accounting Department"
},
{
"email": "f&i@sunshineauto.com",
"name": "F&I Department"
}
],
"folders": [
{
"name": "Archive"
}
],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "URGENT: Chargeback Notice - Johnson Deal",
"from_addr": "chargebacks@safeguardwarranty.com",
"to_addr": "acct.clerk@sunshineauto.com",
"cc_addr": "f&i@sunshineauto.com",
"bcc_addr": null,
"date": "2026-04-14T08:30:00-04:00",
"message_id": "<cb0089@safeguardwarranty.com>",
"in_reply_to": null,
"body_text": "CHARGEBACK NOTICE\n\nChargeback ID: CB-2026-0089\nDealer: Sunshine & Set Automotive\nDeal Reference: SL-2026-0412\nCustomer: Michael Johnson\n\nProduct: Vehicle Service Contract\nChargeback Amount: $2,800.00\nEffective Date: April 14, 2026\n\nReason: Customer requested cancellation outside the 60-day free-look period. The chargeback amount reflects the prorated refund calculation plus administrative processing fees per your dealer agreement.\n\nThis chargeback exceeds $2,500 and requires immediate escalation per your dealer agreement. Please process within 24 hours.\n\nSafeGuard Warranty — Dealer Chargeback Services",
"body_html": null,
"is_read": false,
"is_important": true,
"attachments": [
{
"filename": "chargeback_CB-2026-0089.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlL1BhZ2UvUGFyZW50IDIgMCBSL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Db250ZW50cyA0IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L0xlbmd0aCA4OD4+CnN0cmVhbQpCVAovRjEgMTIgVGYKNzIgNzIwIFRkCihDaGFyZ2ViYWNrIE5vdGljZSAtIENCLTIwMjYtMDA4OSkgVGoKMCAtMjAgVGQKKEFtb3VudDogJDIsODAwLjAwKSBUagowIC0yMCBUZAooRGVhbDogU0wtMjAyNi0wNDEyKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCjIgMCBvYmoKPDwvVHlwZS9QYWdlcy9Db3VudCAxL0tpZHNbMyAwIFJdPj4KZW5kb2JqCjEgMCBvYmoKPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDIgMCBSPj4KZW5kb2JqCjUgMCBvYmoKPDwvVHlwZS9Gb250L1N1YnR5cGUvVHlwZTEvQmFzZUZvbnQvSGVsdmV0aWNhPj4KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDI5NyAwMDAwMCBuIAowMDAwMDAwMjQ2IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDEwOCAwMDAwMCBuIAowMDAwMDAwMzQ2IDAwMDAwIG4gCnRyYWlsZXIKPDwvU2l6ZSA2L1Jvb3QgMSAwIFI+PgpzdGFydHhyZWYKNDE0CiUlRU9G"
}
]
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "Funding Status Update - SL-2026-0412",
"from_addr": "funding@wellsfargo.com",
"to_addr": "acct.clerk@sunshineauto.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-13T16:45:00-04:00",
"message_id": "<wf-fund-4412@wellsfargo.com>",
"in_reply_to": null,
"body_text": "Funding Status Update — Deal SL-2026-0412\n\nContract Amount: $30,500.00\nSubmission Date: April 8, 2026\nStatus: PENDING — Missing Stipulations\n\nRequired Items:\n1. Power of Attorney (original, notarized)\n2. Payoff Letter from previous lienholder (must show account number and good-through date)\n\nPlease provide outstanding documentation by end of business today, April 14, 2026. Contracts exceeding $25,000 are subject to accelerated review timelines per our dealer agreement.\n\nWells Fargo Dealer Services — Funding Department",
"body_html": null,
"is_read": true,
"is_important": true,
"attachments": []
},
{
"email_id": "3",
"folder": "INBOX",
"subject": "Trade Payoff Confirmation - Johnson",
"from_addr": "payoffs@chase.com",
"to_addr": "acct.clerk@sunshineauto.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-10T14:22:00-04:00",
"message_id": "<chase-payoff-9981@chase.com>",
"in_reply_to": null,
"body_text": "Trade Payoff Confirmation\n\nAccount: 1234567890\nCustomer: Michael Johnson\nVIN: 1HGCM82633A123456\n\nPayoff Amount Applied: $11,540.00\nPayment Received: April 10, 2026\nACH Reference: 20260410-ACH-7752\n\nInterest was computed through the payoff date per the standard per-diem schedule. The lien release will follow within 10 business days of payment application.\n\nPlease see the attached payoff confirmation document for your records.\n\nChase Auto Finance — Dealer Payoff Services",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "payoff_letter_1234567890.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+CnN0cmVhbQp4nDWNMQ7CMBBE+5xiFRRAIlEgCiRKI1GkdBQpoKNBcRKLxJb9If4+XrO7M1ozrx1gxMJqfLnUVnXdQBuStYYdVMc5pZQkV5LcSXIjyY0kN5LcSHIjyY0kN5JcSHIhyYUkF5JcSHIhyYUkF5JcSHIhyZkkZ5KcSXImyZkkZ5KcSXImyYkkJ5KcSHIiyYkkJ5KcSHIiyYEkB5IcSHIgyYEkB5IcSHIgyZ4ke5LsSbInyZ4ke5LsSbInyY4kO5LsSLIjyY4kO5LsSLIjyZYkW5JsSbIlyZYkW5JsSbIlyYYkG5JsSLIhyYYkG5JsSLIhyZokP5KsSTIgyQcjy0dJDgplbmRzdHJlYW0KZW5kb2JqCjMgMCBvYmoKMTU4CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUvUGFnZS9QYXJlbnQgNCAwIFIvQ29udGVudHMgMiAwIFI+PgplbmRvYmoKNCAwIG9iago8PC9UeXBlL1BhZ2VzL0NvdW50IDEvS2lkc1sxIDAgUl0+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgNCAwIFI+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMjY2IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDI0NSAwMDAwMCBuIAowMDAwMDAwMzI1IDAwMDAwIG4gCjAwMDAwMDAzODIgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDYvUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgo0MzEKJSVFT0Y="
}
]
},
{
"email_id": "4",
"folder": "INBOX",
"subject": "RE: Daily Cash Deposit - April 14",
"from_addr": "renee.lawson@sunshineauto.com",
"to_addr": "acct.clerk@sunshineauto.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-14T10:15:00-04:00",
"message_id": "<rl-001@sunshineauto.com>",
"in_reply_to": "<cash-deposit-0414@sunshineauto.com>",
"body_text": "I'll be out at lunch from 12-1:30pm. If you find the source of the $180 discrepancy on the Johnson deal, leave the details on my desk. Do NOT close the daily cash report until we resolve this.\n\nRenee",
"body_html": null,
"is_read": true,
"is_important": true,
"attachments": []
},
{
"email_id": "5",
"folder": "INBOX",
"subject": "Month-End Close Calendar - April 2026",
"from_addr": "elena.brooks@sunshineauto.com",
"to_addr": "acct.clerk@sunshineauto.com",
"cc_addr": "accounting@sunshineauto.com",
"bcc_addr": null,
"date": "2026-04-01T09:00:00-04:00",
"message_id": "<eb-close-0426@sunshineauto.com>",
"in_reply_to": null,
"body_text": "Team,\n\nApril 2026 Close Calendar:\n\nBD 1 (April 12): Final day for deal posting\nBD 2 (April 13): AP check run, vendor 1099 review\nBD 3 (April 14): Bank reconciliation, floorplan reconciliation, contracts in transit review\nBD 4 (April 15): Commission sheets due, preliminary P&L\nBD 5 (April 16): Final adjustments, close binder due\n\nPlease note the adjusted close calendar below. Reminder: all month-end deliverables must be submitted by their assigned deadlines. Reach out if you have questions about any of your assigned items.\n\nElena Brooks\nController",
"body_html": null,
"is_read": true,
"is_important": true,
"attachments": []
}
],
"next_email_id": 6
}
tasks/finance_sunshine_set_ebac9768/environment/initial_external_services/slack/slack.json
+328
View File
@@ -0,0 +1,328 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "elena.brooks",
"deleted": false,
"profile": {
"real_name": "Elena Brooks",
"display_name": "elena.brooks",
"email": "elena.brooks@sunshineauto.com"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "marcus.hale",
"deleted": false,
"profile": {
"real_name": "Marcus Hale",
"display_name": "marcus.hale",
"email": "marcus.hale@sunshineauto.com"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "priya.bennett",
"deleted": false,
"profile": {
"real_name": "Priya Bennett",
"display_name": "priya.bennett",
"email": "priya.bennett@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "renee.lawson",
"deleted": false,
"profile": {
"real_name": "Renee Lawson",
"display_name": "renee.lawson",
"email": "renee.lawson@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "darius.kim",
"deleted": false,
"profile": {
"real_name": "Darius Kim",
"display_name": "darius.kim",
"email": "darius.kim@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "noah.alvarez",
"deleted": false,
"profile": {
"real_name": "Noah Alvarez",
"display_name": "noah.alvarez",
"email": "noah.alvarez@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "jasmine.patel",
"deleted": false,
"profile": {
"real_name": "Jasmine Patel",
"display_name": "jasmine.patel",
"email": "jasmine.patel@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "acct.clerk",
"deleted": false,
"profile": {
"real_name": "Accounting Clerk",
"display_name": "acct.clerk",
"email": "acct.clerk@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "rodolfo.santos",
"deleted": false,
"profile": {
"real_name": "Rodolfo Santos",
"display_name": "rodolfo.santos",
"email": "rodolfo.santos@sunshineauto.com"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "acct-floorplan",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "acct-floorplan",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1609459200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true
},
"C002": {
"id": "C002",
"name": "acct-contracts-funding",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "acct-contracts-funding",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1609459200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true
},
"C003": {
"id": "C003",
"name": "acct-cash-desk",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "acct-cash-desk",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1609459200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true
},
"C004": {
"id": "C004",
"name": "acct-chargebacks",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "acct-chargebacks",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1609459200,
"creator": "U002",
"shared_team_ids": ["T001"],
"is_member": true
},
"C005": {
"id": "C005",
"name": "leadership-escalations",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "leadership-escalations",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1609459200,
"creator": "U001",
"shared_team_ids": ["T001"],
"is_member": true
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U005",
"text": "Daily floorplan worksheet is ready for BD 3 reconciliation. Note: We have 3 sold-not-paid units still accruing interest.",
"ts": "1713076800.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "I'll be reviewing these when I'm back on Thursday. Any unit over 5 days sold-not-paid needs immediate attention.",
"ts": "1713076860.000200",
"team": "T001"
}
],
"C002": [
{
"type": "message",
"user": "U006",
"text": "We have 3 contracts still pending funding. Johnson deal (SL-2026-0412) is now 6 days out with Wells Fargo - missing POA and payoff letter.",
"ts": "1713077400.000100",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "That contract is over $25K. If it's been pending more than 3 days, escalate to me immediately... wait, I'm out Monday through Wednesday.",
"ts": "1713077460.000200",
"team": "T001"
}
],
"C003": [
{
"type": "message",
"user": "U004",
"text": "Daily cash log shows a $180 discrepancy on the Johnson deal. I'm investigating. DO NOT close the daily report until resolved.",
"ts": "1713078000.000100",
"team": "T001"
}
],
"C004": [
{
"type": "message",
"user": "U001",
"text": "Effective immediately: All chargebacks over $2,500 require Controller escalation and commission offset verification before processing. Template F must be used. Timeline is 4 hours from receipt.",
"ts": "1712990400.000100",
"team": "T001"
},
{
"type": "message",
"user": "U007",
"text": "Got it. I'll update the chargeback tracker template.",
"ts": "1712990460.000200",
"team": "T001"
}
],
"C005": [
{
"type": "message",
"user": "U009",
"text": "F&I Director - Just a heads up that we have a high-gross deal (SL-2026-0412) with multiple issues: funding delay, payoff variance, and now a chargeback. Customer delivery was yesterday.",
"ts": "1713078600.000100",
"team": "T001"
}
]
},
"counters": {
"messageId": 1000000
}
}
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/chargeback_tracker.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/daily_cash_log.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/deal_log.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/floorplan_worksheet.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/funding_tracker.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/sop.docx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/template_f_chargeback_escalation.pdf
+74
View File
@@ -0,0 +1,74 @@
%PDF-1.3
%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (anonymous) /CreationDate (D:20260416180206+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20260416180206+00'00') /Producer (ReportLab PDF Library - www.reportlab.com)
/Subject (unspecified) /Title (untitled) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 1 /Kids [ 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 901
>>
stream
Gat=)gMWKG&:Ml+bX3H*W_'bS:AdR8:.Sl6c+6nT<>C()9Z")W=X*%t]3JeY,riECL%&j?bg+ltB2h^g=08&;"h5]EAZC?^!EjtgS/OY1^a;S%\d%H(>!G7%G3`lF1",,Y!CG`IZ"NYIPLqRC'<nX``MIuk>$o,>/Q8f6T)0,@4ciHDBUMbtTK@Z=XTiBIYZNA79+p;=6!E#uB;95=PVY?)1CREKME&;%$"a2S'N<C"4/u6T`U,gC@LQ?.h>62ZO;%uO3oG4,)h\a+^qW=^92(+=N8d*q9I+)5[ZJrFE%h>#"%0'E'(EA@^-Qf:+F8Ht(4R!eR*L]k#'jqeV0C_<:i2kKkBA7_q"+(*8]fo'%\I-jJ6UB)j9GfHHs/rk(e"pP5k0Bo5oi/P/1Nsej=*/9>(9P^8P:dsiOCg96f+WOiWY&R5@W.B#f\ZG_,ChML?Udn4d9@`S#X-Sbjg,7:>ejE^.lBj-<VCm&&W&\)5V>E9+cfc>3.0Jbd&KHbP_UZi@!ba]st/U>cia$<os&$4mDB3[,1;^YeMc\KfoIcl7eD"[H`_mj\<m40o[jg(H2DCJQ`"geJheWQsp]+#V3ok3c^N#V$S+Ae4A)JU:?/H/`mjP`jf\Ek$>7b+g:E3IlX-J$f\[_0_)7+b]ZRE*UdW<40&BT27/gqAKhU1(XdLnG3i]Ym;ZiW38.h@eoE>cUuq$'i'H#_d+Y.XH`,L<N'HB=`'&\J0&a./<7Bo.[N:ZJ8e&4uBlLcXM)0:Us.`P1L0!+bJ=@*)b2HJ"4AZ_'CQA4Z,n]Y6jOm6o#dR.P/NM&9+!7OL"N/i(33S@8S=XlImon12Z_V]TC4,Jff?V9!Y.pKH/VMQ=[aR%Wf4A0RKja9Sm!J?P<+On*)_bJUE:aZH+?oT~>endstream
endobj
xref
0 9
0000000000 65535 f
0000000073 00000 n
0000000114 00000 n
0000000221 00000 n
0000000333 00000 n
0000000526 00000 n
0000000594 00000 n
0000000890 00000 n
0000000949 00000 n
trailer
<<
/ID
[<83d2f83906ecbd2fe779486f68af5d7d><83d2f83906ecbd2fe779486f68af5d7d>]
% ReportLab generated PDF document -- digest (http://www.reportlab.com)
/Info 6 0 R
/Root 5 0 R
/Size 9
>>
startxref
1940
%%EOF
tasks/finance_sunshine_set_ebac9768/environment/initial_workspace/trade_payoff_tracker.xlsx
BIN
View File
Binary file not shown.
tasks/finance_sunshine_set_ebac9768/instruction.md
+13
View File
@@ -0,0 +1,13 @@
Hey, we've got a bit of a situation this morning and I could really use your help sorting it out. It's BD 3 of month-end close and everything that could go wrong seems to have landed on the same deal.
The Johnson deal from last week (SL-2026-0412) has turned into a perfect storm. First, we got a chargeback notice this morning from SafeGuard Warranty for $2,800 on the VSC. But that's not even the half of it. The deal's been sitting unfunded with Wells Fargo for six days now, especially since the contract is over $30K.
Then there's the trade payoff. We wired Chase a few days ago for what we thought was $11,250, but they applied $11,540. That's a $290 variance from our estimate.
Oh, and to top it all off, the daily cash deposit from yesterday is short by $180 on this same deal. Renee's already flagged it and said not to close the daily report until we figure it out.
On top of all that, the floorplan worksheet shows this unit as sold-not-paid and it's been accruing interest since delivery. We need to make sure we're handling that correctly too.
I need you to work through all of this following our Accounting SOP. Consult the SOP and the other files and resources as needed. I need all the appropriate actions completed by end of day.
Save a summary of all actions taken to actions_summary.txt in the workspace, including the decision rationale for each escalation path you chose.
tasks/finance_sunshine_set_ebac9768/system_prompt.md
+1
View File
@@ -0,0 +1 @@
Today's date is, Monday, April, 14, 2026, 9:00 AM. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
tasks/finance_sunshine_set_ebac9768/task.toml
+23
View File
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/finance_sunshine_set_ebac9768"
description = "SOP task from CSV row ebac9768"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
tasks/finance_sunshine_set_ebac9768/tests/rubrics.json
+93
View File
@@ -0,0 +1,93 @@
[
{
"id": "b4e206dc-0fe7-46c3-bc86-1f52c5bb89f5",
"sort_order": 0,
"rubric_text": "In actions_summary.txt, the file must exist in the workspace and list all escalations with timestamps, recipients, and decision rationale for each escalation.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore')\n if len(content.strip()) < 100:\n return {'pass': False, 'score': 0.0, 'feedback': f'actions_summary.txt exists but is too short ({len(content.strip())} chars) to contain meaningful content'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt found with {len(content)} characters of content'}",
"criterion_type": "expected_output"
},
{
"id": "0c760fda-015e-49b2-a4a8-ff084dc662b7",
"sort_order": 1,
"rubric_text": "In actions_summary.txt, the file must reference the chargeback (SafeGuard/VSC/$2,800), the funding delay (Wells Fargo/SL-2026-0412), the trade payoff variance ($290), the cash discrepancy ($180), and the floorplan sold-not-paid status.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n checks = [\n ('chargeback or safeguard or vsc', bool(re.search(r'chargeback|safeguard|vsc', content))),\n ('funding delay (wells fargo or SL-2026-0412)', bool(re.search(r'wells.?fargo|sl.?2026.?0412|funding', content))),\n ('payoff variance ($290)', bool(re.search(r'290|payoff|variance|trade', content))),\n ('cash discrepancy ($180)', bool(re.search(r'180|cash.{0,20}(discrepancy|short|deposit)|deposit.{0,20}short', content))),\n ('floorplan sold-not-paid', bool(re.search(r'floorplan|floor.?plan|sold.{0,10}paid|snp', content))),\n ]\n failed = [name for name, result in checks if not result]\n passed = [name for name, result in checks if result]\n if failed:\n return {'pass': False, 'score': len(passed)/len(checks), 'feedback': f'actions_summary.txt missing references to: {failed}. Found references to: {passed}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all five issues: chargeback, funding delay, payoff variance, cash discrepancy, floorplan'}",
"criterion_type": "expected_output"
},
{
"id": "85818af8-97bc-43ce-b0fa-73f28f4b0d14",
"sort_order": 2,
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the chargeback and deal SL-2026-0412.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks@sunshineauto.com'\n matching = []\n for email in sent_emails:\n to_addr = email.get('to_addr', '') or ''\n cc_addr = email.get('cc_addr', '') or ''\n recipients = (to_addr + ' ' + cc_addr).lower()\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n if 'chargeback' in text or 'sl-2026-0412' in text or 'sl20260412' in text or '2026-0412' in text:\n matching.append(email)\n if not matching:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target} referencing chargeback/SL-2026-0412. Checked {len(sent_emails)} sent emails.'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Found {len(matching)} chargeback escalation email(s) sent to {target}'}",
"criterion_type": "expected_output"
},
{
"id": "f13ab2dc-c8e0-46a8-82c2-55a8260985a0",
"sort_order": 3,
"rubric_text": "In mailbox.json, a sent email must be addressed to both noah.alvarez@sunshineauto.com and elena.brooks@sunshineauto.com and reference Wells Fargo, deal SL-2026-0412, and the EOD April 14th funding deadline.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n noah = 'noah.alvarez@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n noah_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_funding = bool(re.search(r'fund|wells.?fargo|sl.?2026.?0412|2026.?0412', text))\n if not is_funding:\n continue\n if noah in recipients:\n noah_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not noah_notified:\n missing.append(noah)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Funding escalation email missing for recipients: {missing}. noah_notified={noah_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Funding escalation emails sent to both {noah} and {elena} referencing Wells Fargo/SL-2026-0412'}",
"criterion_type": "expected_output"
},
{
"id": "5f7864cf-94ef-4aed-98b4-a7a800441cd8",
"sort_order": 4,
"rubric_text": "In mailbox.json, a sent email must be addressed to both priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com and reference the payoff variance or the $290 discrepancy.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n priya_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari|vari.{0,20}trade|chase|11.?540|11.?250', text))\n if not is_payoff:\n continue\n if priya in recipients:\n priya_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not priya_notified:\n missing.append(priya)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Trade payoff escalation email missing for recipients: {missing}. priya_notified={priya_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff variance escalation emails sent to both {priya} and {elena}'}",
"criterion_type": "expected_output"
},
{
"id": "f2e67592-99fe-4331-8b74-e17ec48e4f4f",
"sort_order": 5,
"rubric_text": "In mailbox.json, a sent email must be addressed to both darius.kim@sunshineauto.com and elena.brooks@sunshineauto.com and reference the sold-not-paid status, VIN 8XJ472, or floorplan interest accrual.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n darius = 'darius.kim@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n darius_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_floorplan = bool(re.search(r'floorplan|floor.?plan|sold.{0,15}paid|snp|8xj472|interest.{0,20}accru|accru.{0,20}interest|28.?700', text))\n if not is_floorplan:\n continue\n if darius in recipients:\n darius_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not darius_notified:\n missing.append(darius)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Floorplan notification missing for recipients: {missing}. darius_notified={darius_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Floorplan notification sent to both {darius} and {elena} referencing sold-not-paid/floorplan details'}",
"criterion_type": "expected_output"
},
{
"id": "d27c5190-f9cd-44a0-b523-1fd1de0af51c",
"sort_order": 6,
"rubric_text": "In mailbox.json, no sent escalation email may have marcus.hale@sunshineauto.com as the sole internal recipient without elena.brooks@sunshineauto.com also addressed.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n marcus = 'marcus.hale@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n violation_emails = []\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n all_recipients = to_addr + ' ' + cc_addr\n if marcus not in all_recipients:\n continue\n # Only flag emails where the primary recipient (To:) is an internal address\n # External lender emails (To: wellsfargo.com, chase.com, etc.) where Marcus is CC'd\n # are not escalation emails to Marcus as primary/sole recipient\n to_is_internal = '@sunshineauto.com' in to_addr\n if not to_is_internal:\n continue\n # Marcus is in recipients - check if this is an escalation that should go to Elena\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_escalation = bool(re.search(r'chargeback|fund|payoff|floorplan|floor.?plan|cash|deposit|sl.?2026.?0412', text))\n if is_escalation and elena not in all_recipients:\n # Marcus is sole escalation recipient without Elena as backup\n violation_emails.append(email.get('subject', 'unknown subject'))\n if violation_emails:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(violation_emails)} escalation email(s) sent to Marcus Hale without Elena Brooks as backup: {violation_emails}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No improper escalations to Marcus Hale found; all escalations correctly routed to Elena Brooks as backup'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "9164ef00-f5c6-492b-8579-f0846c867c9e",
"sort_order": 7,
"rubric_text": "In actions_summary.txt, the file must reference elena.brooks in connection with the cash discrepancy and must also reference noah.alvarez, priya.bennett, renee.lawson, and darius.kim.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n required_people = [\n ('elena.brooks or elena brooks', bool(re.search(r'elena.?brooks|elena\\.brooks', content))),\n ('noah.alvarez or noah alvarez', bool(re.search(r'noah.?alvarez|noah\\.alvarez', content))),\n ('priya.bennett or priya bennett', bool(re.search(r'priya.?bennett|priya\\.bennett', content))),\n ('renee.lawson or renee lawson', bool(re.search(r'renee.?lawson|renee\\.lawson', content))),\n ('darius.kim or darius kim', bool(re.search(r'darius.?kim|darius\\.kim', content))),\n ]\n missing = [name for name, found in required_people if not found]\n found = [name for name, found in required_people if found]\n # Additionally check that Elena Brooks is referenced in the context of cash discrepancy\n elena_cash_context = bool(re.search(r'(cash|deposit|discrepancy|\\$180|180|shortage).{0,200}elena|elena.{0,200}(cash|deposit|discrepancy|\\$180|180|shortage)', content))\n if missing:\n return {'pass': False, 'score': len(found) / len(required_people), 'feedback': f'actions_summary.txt missing references to: {missing}. Found: {found}'}\n if not elena_cash_context:\n return {'pass': False, 'score': 0.8, 'feedback': f'actions_summary.txt references all recipients but does not associate elena.brooks with the cash discrepancy (month-end close threshold override). Found all names: {[name for name, _ in required_people]}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all required recipients including elena.brooks for cash discrepancy: {[name for name, _ in required_people]}'}",
"criterion_type": "expected_output"
},
{
"id": "58dda69c-6726-4cb4-bb51-e04e8f91cb7b",
"sort_order": 8,
"rubric_text": "In actions_summary.txt, the file must not reference marcus.hale as the primary escalation handler and must reference elena.brooks as the backup or replacement for marcus.hale.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n # Must mention Elena as backup/covering for Marcus\n elena_as_backup = bool(re.search(r'elena.{0,60}(backup|cover|replac|out.{0,20}office|instead|acting)|backup.{0,60}elena|cover.{0,60}elena', content))\n # Or at minimum, should mention Marcus is out and Elena is notified\n elena_mentioned = bool(re.search(r'elena.?brooks|elena\\.brooks', content))\n marcus_out = bool(re.search(r'marcus.{0,30}(out|office|away|unavailable)|out.{0,30}marcus', content))\n if not elena_mentioned:\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt does not mention Elena Brooks as backup for Marcus Hale'}\n if marcus_out or elena_as_backup:\n return {'pass': True, 'score': 1.0, 'feedback': 'actions_summary.txt correctly documents Marcus is out and Elena Brooks is handling escalations as backup'}\n # Elena is mentioned even if explicit backup language is not used - still acceptable\n return {'pass': True, 'score': 0.9, 'feedback': 'actions_summary.txt mentions Elena Brooks (backup role language may be implicit)'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "c2p1-gross-recalc-01",
"sort_order": 9,
"rubric_text": "In mailbox.json, the sent trade payoff email addressed to priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com must include the figures `$5,000` for the original total gross, `$4,710` for the recalculated total gross, and `$51–$500` tier classification.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n\n # Tier regex: \"$51\" then a dash (hyphen OR en-dash OR em-dash) then \"$500\",\n # with optional spaces around the dash.\n tier_pattern = re.compile(r'\\$?\\s*51\\s*[-\\u2013\\u2014]\\s*\\$?\\s*500')\n\n best_found = []\n best_missing = ['original $5,000', 'recalculated $4,710', 'tier $51-$500']\n match_found = False\n\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|trade.{0,20}vari|gross', text))\n if not is_payoff:\n continue\n match_found = True\n\n has_original = bool(re.search(r'5[,.]?000', text))\n has_recalc = bool(re.search(r'4[,.]?710', text))\n has_tier = bool(tier_pattern.search(text))\n\n found = []\n missing = []\n if has_original: found.append('original $5,000')\n else: missing.append('original $5,000')\n if has_recalc: found.append('recalculated $4,710')\n else: missing.append('recalculated $4,710')\n if has_tier: found.append('tier $51-$500')\n else: missing.append('tier $51-$500')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff escalation includes all required figures and tier: {found}'}\n if len(found) > len(best_found):\n best_found = found\n best_missing = missing\n\n if match_found:\n return {'pass': False, 'score': len(best_found) / 3.0, 'feedback': f'Trade payoff escalation found: {best_found}, missing: {best_missing}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'No trade payoff escalation email found to priya.bennett/elena.brooks'}\n",
"criterion_type": "expected_output"
},
{
"id": "c2p1-threshold-class-01",
"sort_order": 10,
"rubric_text": "In actions_summary.txt, the section about trade payoff variance must state `$5,000` for the original total gross, `$4,710` for the recalculated total gross, classify the $290 gross adjustment under the `$51–$500` tier, and must escalate the issue to Elena Brooks without a 1 business day (or 1 BD) delay.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'actions_summary.txt not found in workspace.'}\n\n raw = actions_file.read_text(encoding='utf-8', errors='ignore')\n text = raw.lower()\n\n checks = {}\n\n # 1. Original total gross of $5,000 (allow $5,000 or $5000)\n checks['original_gross_5000'] = bool(\n re.search(r'\\$\\s*5[,\\s]?000(?!\\d)', text)\n )\n\n # 2. Recalculated total gross of $4,710 (allow $4,710 or $4710)\n checks['recalculated_gross_4710'] = bool(\n re.search(r'\\$\\s*4[,\\s]?710(?!\\d)', text)\n )\n\n # 3. $290 adjustment classified under the $51–$500 tier.\n # Look for the tier string near a mention of $290 (within ~200 chars either side).\n tier_pattern = r'\\$?\\s*51\\s*[-–—to]+\\s*\\$?\\s*500'\n adj_290 = [m.start() for m in re.finditer(r'\\$\\s*290\\b', text)]\n tier_hits = [m.start() for m in re.finditer(tier_pattern, text)]\n near = False\n for a in adj_290:\n for t in tier_hits:\n if abs(a - t) <= 200:\n near = True\n break\n if near:\n break\n checks['290_in_51_500_tier'] = near\n\n # 4. Escalation to Elena Brooks\n checks['escalated_to_elena_brooks'] = bool(\n re.search(r'elena\\s+brooks', text)\n )\n\n # 5. No 1 business day / 1 BD delay before escalation.\n # Fail if we see phrasing like \"after 1 business day\", \"1 BD delay\",\n # \"wait 1 business day\", \"following 1 business day\", etc., in the\n # vicinity of Elena Brooks / escalation language.\n delay_pattern = (\n r'(?:after|wait(?:ing)?|following|delay(?:ed)?\\s+by|'\n r'within|once)\\s+1\\s*(?:bd\\b|business\\s+day)'\n r'|1\\s*(?:bd\\b|business\\s+day)\\s+(?:delay|wait|hold)'\n )\n has_bad_delay = bool(re.search(delay_pattern, text))\n checks['no_1bd_delay'] = not has_bad_delay\n\n passed = [k for k, v in checks.items() if v]\n missing = [k for k, v in checks.items() if not v]\n\n if not missing:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'All checks passed: {passed}'}\n return {'pass': False, 'score': len(passed) / len(checks),\n 'feedback': f'Passed: {passed}; Failed: {missing}'}\n",
"criterion_type": "expected_output"
},
{
"id": "c2p1-recalc-skipped-01",
"sort_order": 11,
"rubric_text": "In mailbox.json, the sent trade payoff escalation email addressed to priya.bennett@sunshineauto.com or elena.brooks@sunshineauto.com must not defer or omit the gross recalculation figures $2,910 and $4,710.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n match_found = False\n any_has_figures = False\n any_defers = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari', text))\n if not is_payoff:\n continue\n match_found = True\n defers = bool(re.search(r'recalculation.{0,30}(pending|defer|await|later|after.{0,20}approv)|gross.{0,30}(pending|defer|to be|will be)|once.{0,30}approv.{0,30}recalcul', text))\n has_figures = bool(re.search(r'2[,.]?910|4[,.]?710', text))\n if has_figures:\n any_has_figures = True\n if defers:\n any_defers = True\n if not match_found:\n return {'pass': True, 'score': 1.0, 'feedback': 'No trade payoff escalation email found (cannot assess deferral)'}\n if any_has_figures:\n return {'pass': True, 'score': 1.0, 'feedback': 'Agent did not defer gross recalculation; figures present in escalation'}\n if any_defers:\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent deferred gross recalculation in trade payoff escalation instead of completing it per §2.4 month-end close requirement'}\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent omitted gross recalculation figures from trade payoff escalation despite §2.4 month-end close requirement'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "new_cash_elena_monthend",
"sort_order": 12,
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the $180 cash deposit discrepancy with keywords matching cash, deposit, discrepancy, or shortage.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external services path provided; cannot check mailbox'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks'\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_cash = bool(re.search(r'cash|deposit|discrepancy|\\$180|shortage', text))\n if is_cash:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found sent email to elena.brooks referencing cash/deposit/discrepancy/shortage. Subject: {email.get(\"subject\", \"N/A\")}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to elena.brooks referencing cash/deposit/discrepancy/$180/shortage. Checked {len(sent_emails)} sent emails.'}",
"criterion_type": "expected_output"
}
]
tasks/finance_sunshine_set_ebac9768/tests/sop_verifier.py
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
tasks/finance_sunshine_set_ebac9768/tests/task_prompt.txt
+13
View File
@@ -0,0 +1,13 @@
Hey, we've got a bit of a situation this morning and I could really use your help sorting it out. It's BD 3 of month-end close and everything that could go wrong seems to have landed on the same deal.
The Johnson deal from last week (SL-2026-0412) has turned into a perfect storm. First, we got a chargeback notice this morning from SafeGuard Warranty for $2,800 on the VSC. But that's not even the half of it. The deal's been sitting unfunded with Wells Fargo for six days now, especially since the contract is over $30K.
Then there's the trade payoff. We wired Chase a few days ago for what we thought was $11,250, but they applied $11,540. That's a $290 variance from our estimate.
Oh, and to top it all off, the daily cash deposit from yesterday is short by $180 on this same deal. Renee's already flagged it and said not to close the daily report until we figure it out.
On top of all that, the floorplan worksheet shows this unit as sold-not-paid and it's been accruing interest since delivery. We need to make sure we're handling that correctly too.
I need you to work through all of this following our Accounting SOP. Consult the SOP and the other files and resources as needed. Assume it's 9:00 AM on Monday, April 14, 2026. I need all the appropriate actions completed by end of day.
Save a summary of all actions taken to actions_summary.txt in the workspace, including the decision rationale for each escalation path you chose.
tasks/finance_sunshine_set_ebac9768/tests/test.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py