Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+308
@@ -0,0 +1,308 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "j.trevors@meridianpartners.com",
|
||||
"name": "Joan Trevors"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "ap@meridianpartners.com",
|
||||
"name": "Meridian Partners AP"
|
||||
},
|
||||
{
|
||||
"email": "t.okonkwo@meridianpartners.com",
|
||||
"name": "Tanya Okonkwo"
|
||||
},
|
||||
{
|
||||
"email": "r.whitfield@meridianpartners.com",
|
||||
"name": "Rachel Whitfield"
|
||||
},
|
||||
{
|
||||
"email": "d.nakamura@meridianpartners.com",
|
||||
"name": "David Nakamura"
|
||||
},
|
||||
{
|
||||
"email": "s.rivera@meridianpartners.com",
|
||||
"name": "S. Rivera"
|
||||
},
|
||||
{
|
||||
"email": "k.desantis@meridianpartners.com",
|
||||
"name": "K. DeSantis"
|
||||
},
|
||||
{
|
||||
"email": "m.chen@meridianpartners.com",
|
||||
"name": "M. Chen"
|
||||
},
|
||||
{
|
||||
"email": "j.carter@meridianpartners.com",
|
||||
"name": "J. Carter"
|
||||
},
|
||||
{
|
||||
"email": "b.singh@meridianpartners.com",
|
||||
"name": "B. Singh"
|
||||
},
|
||||
{
|
||||
"email": "hr@meridianpartners.com",
|
||||
"name": "Human Resources"
|
||||
},
|
||||
{
|
||||
"email": "office@meridianpartners.com",
|
||||
"name": "Office Management"
|
||||
},
|
||||
{
|
||||
"email": "t.okafor@meridianadvisory.com",
|
||||
"name": "Thomas Okafor"
|
||||
},
|
||||
{
|
||||
"email": "p.nair@meridiandigital.com",
|
||||
"name": "Priya Nair"
|
||||
},
|
||||
{
|
||||
"email": "m.webb@premierjanitor.com",
|
||||
"name": "M. Webb (Premier Janitorial)"
|
||||
},
|
||||
{
|
||||
"email": "r.torres@acmeoffice.com",
|
||||
"name": "R. Torres (Acme Office Solutions)"
|
||||
},
|
||||
{
|
||||
"email": "billing@riversideproperties.com",
|
||||
"name": "Riverside Properties LLC"
|
||||
},
|
||||
{
|
||||
"email": "billing@hartwell-associates.com",
|
||||
"name": "Hartwell & Associates LLP"
|
||||
},
|
||||
{
|
||||
"email": "c.morrison@morrisonwebb.com",
|
||||
"name": "C. Morrison (Morrison & Webb)"
|
||||
},
|
||||
{
|
||||
"email": "n.reeves@summitstrategy.com",
|
||||
"name": "N. Reeves (Summit Strategy Partners)"
|
||||
},
|
||||
{
|
||||
"email": "p.yuen@blueridgecg.com",
|
||||
"name": "P. Yuen (Blue Ridge Consulting)"
|
||||
},
|
||||
{
|
||||
"email": "orders@greatlakesfurniture.com",
|
||||
"name": "Great Lakes Furniture & Design"
|
||||
},
|
||||
{
|
||||
"email": "accounts@techflowsolutions.com",
|
||||
"name": "TechFlow Solutions"
|
||||
},
|
||||
{
|
||||
"email": "billing@vantagehr.com",
|
||||
"name": "Vantage HR Solutions"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "finance@meridianpartners.com",
|
||||
"name": "Finance Team",
|
||||
"members": [
|
||||
"j.trevors@meridianpartners.com",
|
||||
"t.okonkwo@meridianpartners.com",
|
||||
"r.whitfield@meridianpartners.com",
|
||||
"d.nakamura@meridianpartners.com",
|
||||
"s.rivera@meridianpartners.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "AP"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Premier Janitorial Services \u2013 Rate Adjustment Effective March 2026",
|
||||
"from_addr": "m.webb@premierjanitor.com",
|
||||
"to_addr": "ap@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-02-28T11:15:00Z",
|
||||
"message_id": "<msg-001@premierjanitor.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear Meridian Partners Accounts Payable,\n\nPlease be advised that effective March 1, 2026, Premier Janitorial Services will be adjusting our service rates by 3% to reflect increased labor and supply costs.\n\nYour March invoice (Month 6 of 6 under PO-2025-00051) will reflect the updated monthly rate of $2,884.00.\n\nWe appreciate your continued partnership and are happy to answer any questions.\n\nBest regards,\nMarcus Webb\nPremier Janitorial Services\n(773) 555-0114\nm.webb@premierjanitor.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Invoice SSP-2026-0031 \u2013 Summit Strategy Partners LLC",
|
||||
"from_addr": "n.reeves@summitstrategy.com",
|
||||
"to_addr": "ap@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-21T09:30:00Z",
|
||||
"message_id": "<msg-002@summitstrategy.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Good morning,\n\nPlease find attached invoice SSP-2026-0031 dated March 12, 2026 for $18,750.00 for consulting services rendered to Meridian Partners LLC.\n\nOur payment terms are Net 30. Please don\u2019t hesitate to reach out with any questions.\n\nThank you,\nNathan Reeves\nDirector of Client Services\nSummit Strategy Partners LLC\n(312) 555-0847\nn.reeves@summitstrategy.com",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "INBOX",
|
||||
"subject": "Invoice BRC-2026-0019 \u2013 Blue Ridge Consulting Group",
|
||||
"from_addr": "p.yuen@blueridgecg.com",
|
||||
"to_addr": "ap@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-18T14:20:00Z",
|
||||
"message_id": "<msg-003@blueridgecg.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nAttached please find invoice BRC-2026-0019 for $5,500.00 covering Q1 2026 management consulting services for Meridian Partners.\n\nPlease process at your earliest convenience per our 2/10 Net 30 terms.\n\nBest regards,\nPatricia Yuen\nBlue Ridge Consulting Group\n(312) 555-0389\np.yuen@blueridgecg.com",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Today\u2019s Batch \u2013 March 24 \u2013 Action Needed",
|
||||
"from_addr": "t.okonkwo@meridianpartners.com",
|
||||
"to_addr": "j.trevors@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-24T08:30:00Z",
|
||||
"message_id": "<msg-004@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Joan,\n\nBatch runs at 2pm today. Before then, please:\n\n1. Confirm the invoices queued for PAY-2026-0324 are cleared and ready to go\n2. Log any new invoices that came in today\n3. Do a scan of the open holds in the ledger and let me know if anything needs attention before I run the batch\n\nThanks,\nTanya\n\nTanya Okonkwo | AP Manager\nMeridian Partners, LLC\nt.okonkwo@meridianpartners.com",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": true,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "INBOX",
|
||||
"subject": "Benefits Open Enrollment \u2013 Deadline April 4, 2026",
|
||||
"from_addr": "hr@meridianpartners.com",
|
||||
"to_addr": "j.trevors@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-20T10:00:00Z",
|
||||
"message_id": "<msg-005@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear Team,\n\nThis is a reminder that the annual Benefits Open Enrollment period is now open. You have until April 4, 2026 to make any changes to your health, dental, and vision coverage for the upcoming plan year.\n\nPlease log into the benefits portal at hr.meridianpartners.com to review your options and confirm or update your selections.\n\nContact HR with any questions.\n\nHuman Resources\nMeridian Partners, LLC",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "6",
|
||||
"folder": "INBOX",
|
||||
"subject": "Kitchen Refrigerator Cleanout \u2013 Friday March 27",
|
||||
"from_addr": "office@meridianpartners.com",
|
||||
"to_addr": "j.trevors@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-22T13:30:00Z",
|
||||
"message_id": "<msg-006@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi everyone,\n\nJust a heads up \u2014 the kitchen refrigerator will be cleaned out this Friday, March 27th. Please remove any personal items by end of day Thursday or they will be discarded.\n\nThanks for your cooperation!\n\nOffice Management\nMeridian Partners, LLC",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "7",
|
||||
"folder": "Sent",
|
||||
"subject": "AP APPROVAL REQUIRED: Morrison & Webb Legal Group LLP - $3,200.00 - Invoice MW-2026-0018",
|
||||
"from_addr": "j.trevors@meridianpartners.com",
|
||||
"to_addr": "k.desantis@meridianpartners.com, t.okonkwo@meridianpartners.com",
|
||||
"cc_addr": "ap@meridianpartners.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-06T11:05:00Z",
|
||||
"message_id": "<msg-007@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "A non-PO invoice requires your approval before payment can be processed. Vendor: Morrison & Webb Legal Group LLP (V-00007). Invoice: MW-2026-0018, dated 2026-03-05. Amount: $3,200.00. Description: Legal services \u2013 February 2026. GL Code: 1000-300-6100. Please reply APPROVED or REJECTED with reason. If no response is received within 3 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "8",
|
||||
"folder": "Sent",
|
||||
"subject": "Invoice MW-2026-0018 \u2013 Pending Internal Approval",
|
||||
"from_addr": "j.trevors@meridianpartners.com",
|
||||
"to_addr": "c.morrison@morrisonwebb.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-07T14:45:00Z",
|
||||
"message_id": "<msg-008@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear C. Morrison,\n\nInvoice MW-2026-0018 for $3,200.00 is currently pending internal approval. We expect to process payment within 5 additional business days. We apologize for the delay.\n\nMeridian Partners Accounts Payable",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "9",
|
||||
"folder": "Sent",
|
||||
"subject": "Meridian Partners - Price Discrepancy on Invoice TFS-26-0089",
|
||||
"from_addr": "j.trevors@meridianpartners.com",
|
||||
"to_addr": "accounts@techflowsolutions.com",
|
||||
"cc_addr": "ap@meridianpartners.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-06T14:22:00Z",
|
||||
"message_id": "<msg-009@meridianpartners.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear TechFlow Solutions,\n\nWe are writing regarding Invoice TFS-26-0089. Our records indicate a discrepancy between the rate on this invoice and the rate on our purchase order.\n\nPO Rate (PO-2026-00007): $250.00/hr\nInvoiced Rate: $280.00/hr\nVariance: $30.00/hr\n\nPlease provide a corrected invoice or written justification for the rate difference within 5 business days.\n\nThank you,\nJ. Trevors\nAccounts Payable\nMeridian Partners, LLC\nap@meridianpartners.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "10",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Benefits Open Enrollment \u2013 Deadline April 4, 2026",
|
||||
"from_addr": "j.trevors@meridianpartners.com",
|
||||
"to_addr": "hr@meridianpartners.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-20T10:48:00Z",
|
||||
"message_id": "<msg-010@meridianpartners.com>",
|
||||
"in_reply_to": "<msg-005@meridianpartners.com>",
|
||||
"body_text": "Thanks for the reminder \u2014 will log in this week.\n\nJoan\n\n--- Original Message ---\nFrom: hr@meridianpartners.com\nDate: 2026-03-20 10:00\nSubject: Benefits Open Enrollment \u2013 Deadline April 4, 2026\n\nDear Team,\n\nThis is a reminder that the annual Benefits Open Enrollment period is now open...",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "11",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Invoice MW-2026-0018 \u2013 Pending Internal Approval",
|
||||
"from_addr": "j.trevors@meridianpartners.com",
|
||||
"to_addr": "k.desantis@meridianpartners.com, t.okonkwo@meridianpartners.com",
|
||||
"cc_addr": "ap@meridianpartners.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-05T14:50:00Z",
|
||||
"message_id": "<msg-011@meridianpartners.com>",
|
||||
"in_reply_to": "<msg-007@meridianpartners.com>",
|
||||
"body_text": "Hi Tanya and Karen,\n\nNo response has been received on the approval request for Morrison & Webb Invoice MW-2026-0018 ($3,200.00) sent March 2nd. The 3-business-day window has elapsed. I am placing the invoice on APPR hold in the register and have notified the vendor.\n\nPlease let me know if you would like me to take any further action.\n\nJ. Trevors\nAccounts Payable\nMeridian Partners, LLC",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 12
|
||||
}
|
||||
+489
@@ -0,0 +1,489 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "j.trevors",
|
||||
"deleted": false,
|
||||
"real_name": "Joan Trevors",
|
||||
"profile": {
|
||||
"real_name": "Joan Trevors",
|
||||
"display_name": "Joan Trevors",
|
||||
"real_name_normalized": "Joan Trevors",
|
||||
"display_name_normalized": "Joan Trevors",
|
||||
"email": "j.trevors@meridianpartners.com",
|
||||
"title": "AP Clerk"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "t.okonkwo",
|
||||
"deleted": false,
|
||||
"real_name": "Tanya Okonkwo",
|
||||
"profile": {
|
||||
"real_name": "Tanya Okonkwo",
|
||||
"display_name": "Tanya Okonkwo",
|
||||
"real_name_normalized": "Tanya Okonkwo",
|
||||
"display_name_normalized": "Tanya Okonkwo",
|
||||
"email": "t.okonkwo@meridianpartners.com",
|
||||
"title": "AP Manager"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "r.whitfield",
|
||||
"deleted": false,
|
||||
"real_name": "Rachel Whitfield",
|
||||
"profile": {
|
||||
"real_name": "Rachel Whitfield",
|
||||
"display_name": "Rachel Whitfield",
|
||||
"real_name_normalized": "Rachel Whitfield",
|
||||
"display_name_normalized": "Rachel Whitfield",
|
||||
"email": "r.whitfield@meridianpartners.com",
|
||||
"title": "Controller"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "d.nakamura",
|
||||
"deleted": false,
|
||||
"real_name": "David Nakamura",
|
||||
"profile": {
|
||||
"real_name": "David Nakamura",
|
||||
"display_name": "David Nakamura",
|
||||
"real_name_normalized": "David Nakamura",
|
||||
"display_name_normalized": "David Nakamura",
|
||||
"email": "d.nakamura@meridianpartners.com",
|
||||
"title": "CFO"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_primary_owner": true,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "s.rivera",
|
||||
"deleted": false,
|
||||
"real_name": "S. Rivera",
|
||||
"profile": {
|
||||
"real_name": "S. Rivera",
|
||||
"display_name": "S. Rivera",
|
||||
"real_name_normalized": "S. Rivera",
|
||||
"display_name_normalized": "S. Rivera",
|
||||
"email": "s.rivera@meridianpartners.com",
|
||||
"title": "AP Staff"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "k.desantis",
|
||||
"deleted": false,
|
||||
"real_name": "K. DeSantis",
|
||||
"profile": {
|
||||
"real_name": "K. DeSantis",
|
||||
"display_name": "K. DeSantis",
|
||||
"real_name_normalized": "K. DeSantis",
|
||||
"display_name_normalized": "K. DeSantis",
|
||||
"email": "k.desantis@meridianpartners.com",
|
||||
"title": "Professional Services Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T001",
|
||||
"name": "m.chen",
|
||||
"deleted": false,
|
||||
"real_name": "M. Chen",
|
||||
"profile": {
|
||||
"real_name": "M. Chen",
|
||||
"display_name": "M. Chen",
|
||||
"real_name_normalized": "M. Chen",
|
||||
"display_name_normalized": "M. Chen",
|
||||
"email": "m.chen@meridianpartners.com",
|
||||
"title": "Administration Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008",
|
||||
"team_id": "T001",
|
||||
"name": "j.carter",
|
||||
"deleted": false,
|
||||
"real_name": "J. Carter",
|
||||
"profile": {
|
||||
"real_name": "J. Carter",
|
||||
"display_name": "J. Carter",
|
||||
"real_name_normalized": "J. Carter",
|
||||
"display_name_normalized": "J. Carter",
|
||||
"email": "j.carter@meridianpartners.com",
|
||||
"title": "Operations Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U009": {
|
||||
"id": "U009",
|
||||
"team_id": "T001",
|
||||
"name": "b.singh",
|
||||
"deleted": false,
|
||||
"real_name": "B. Singh",
|
||||
"profile": {
|
||||
"real_name": "B. Singh",
|
||||
"display_name": "B. Singh",
|
||||
"real_name_normalized": "B. Singh",
|
||||
"display_name_normalized": "B. Singh",
|
||||
"email": "b.singh@meridianpartners.com",
|
||||
"title": "IT Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_app_user": false,
|
||||
"is_email_confirmed": true
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "ap-exceptions",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1705363200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ap-exceptions",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1742823000,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Invoice matching exceptions — price variances, quantity discrepancies",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Post all invoice exceptions here.",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "ap-approvals",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1705363200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ap-approvals",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1742825700,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Non-PO invoice approvals and over-threshold payment authorization",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "AP staff post approval requests for non-PO invoices and payment authorization items here.",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "ap-escalations",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1705363200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ap-escalations",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1742484600,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Escalations to Controller — fraud, disputes >$5K, holds >10 business days",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "For escalations requiring Controller or AP Manager intervention.",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "ap-payments",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1705363200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ap-payments",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1742824800,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Payment batch confirmations and payment failure notices",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "AP Manager posts batch complete confirmations and payment failure alerts. Batch runs every Tuesday and Thursday at 2:00 PM EST.",
|
||||
"creator": "U002",
|
||||
"last_set": 1705363200
|
||||
}
|
||||
},
|
||||
"C005": {
|
||||
"id": "C005",
|
||||
"name": "finance-month-end",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1705363200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "finance-month-end",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1742823900,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Month-end close status, accrual postings, reconciliation completion",
|
||||
"creator": "U003",
|
||||
"last_set": 1705363200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "AP Manager posts close status updates. All Finance staff.",
|
||||
"creator": "U003",
|
||||
"last_set": 1705363200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "PRICE VARIANCE: TechFlow Solutions Inc., V-00006, Invoice TFS-26-0089, PO Price $250.00/hr, Invoice Price $280.00/hr, Variance 12%",
|
||||
"ts": "1741270500.000100",
|
||||
"team": "T001",
|
||||
"reply_count": 1,
|
||||
"reply_users_count": 1,
|
||||
"latest_reply": "1741272300.000200",
|
||||
"reply_users": ["U002"]
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Hold placed on TFS-26-0089. No prior rate notice on file — no match on Step 1 check. Email sent to vendor per §6.1 Step 2b requesting corrected invoice or written justification. Response due 3/13.",
|
||||
"ts": "1741272300.000200",
|
||||
"thread_ts": "1741270500.000100",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "TechFlow Solutions TFS-26-0089 — no vendor response received as of 3/13 deadline. Hold unresolved, 10 business days elapsed. Escalating to #ap-escalations per §13.1.",
|
||||
"ts": "1742418000.000300",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Non-PO approval request submitted — Morrison & Webb Legal Group LLP (V-00007), Invoice MW-2026-0018, $3,200.00, GL 1000-300-6100. Email sent to K. DeSantis and T. Okonkwo per §5.3. Response required by 3/5.",
|
||||
"ts": "1740927600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "MW-2026-0018 placed on APPR hold — 3-day response window closed with no email reply. Vendor notified of delay per §5.3. Hold remains until both approvals received.",
|
||||
"ts": "1741190400.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C003": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Invoice on hold > 10 business days — TechFlow Solutions Inc., V-00006, Invoice TFS-26-0089. MATCH hold placed 3/6, rate variance 12% ($280.00/hr billed vs PO rate $250.00/hr). Vendor contacted 3/6 per §6.1 Step 2b, no response received as of today. Escalating to Controller per §13.1.",
|
||||
"ts": "1742418900.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Received. I will contact TechFlow directly — do NOT release the hold until you hear from me. Joan, add TFS-26-0089 to exceptions_tracker_2026.xlsx today with expected resolution date of 3/31.",
|
||||
"ts": "1742482800.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Confirmed — hold stays active. Joan, log it and keep the invoice out of batch until Rachel clears it.",
|
||||
"ts": "1742484600.000300",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C004": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "BATCH COMPLETE: 2026-03-19, Payments: 4, Total Amount: $21,100.00, Method: ACH 4 / Check 0 / Wire 0",
|
||||
"ts": "1742412600.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Batch run today at 2:00 PM EST. Queued: Acme Office Solutions (2026-1124), Vantage HR (VHR-2026-0041). All match approvals and payment authorizations must be confirmed by 1:30 PM. No holds, no IC vendors, no amounts over $50K in the batch.",
|
||||
"ts": "1742823000.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Confirmed — Acme (2026-1124) cleared and approved. VHR-2026-0041 match confirmed, queued. TechFlow TFS-26-0089 remains on MATCH hold — excluded from today's batch.",
|
||||
"ts": "1742824800.000300",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C005": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "AP CLOSE COMPLETE: February 2026",
|
||||
"ts": "1741204800.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Confirmed. February close reviewed and accepted. March close deadline is April 5th — keep on schedule. Note the open MATCH hold on TFS-26-0089; document status before close.",
|
||||
"ts": "1741206600.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Heads up for March close: TechFlow TFS-26-0089 escalated to Rachel on 3/19, still unresolved — logging as open item. New invoices received today — processing underway.",
|
||||
"ts": "1742823900.000300",
|
||||
"team": "T001"
|
||||
}
|
||||
]
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Find the invoice from Great Lakes Furniture & Design that we received today. It's an Office Supplies Account. First, if it should be added to the Invoice Register, update the Invoice Register through the "Status" column. Second, if an email should be sent, draft if for me and put it in Drafts. Do NOT send it - I will review it myself before sending. Third, if a Slack message is required, send it.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, March, 24, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/finance_meridian_partners_cc2fc143"
|
||||
description = "SOP task from CSV row cc2fc143"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
[
|
||||
{
|
||||
"id": "53c53e31-7fa0-4136-9885-e8eb58789e27",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, row 13 must contain invoice data for GLFD-0441 with vendor V-00009 (Great Lakes Furniture & Design Co.), invoice date 2026-03-20, received date 2026-03-24, due date 2026-04-19, type Non-PO, amount $7,900.00, GL code 1000-100-6200, and status Open through the 'Status' column",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n # Find Status column from header row\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n row_str = ' '.join(row_data).lower()\n\n has_invoice_id = 'glfd-0441' in row_str\n has_vendor_id = 'v-00009' in row_str\n has_vendor_name = 'great lakes furniture & design co' in row_str\n has_invoice_date = '2026-03-20' in row_str\n has_received_date = '2026-03-24' in row_str\n has_due_date = '2026-04-19' in row_str\n has_non_po = 'non-po' in row_str or 'non po' in row_str\n amount_found = '7,900' in row_str or '7900' in row_str\n has_gl = '1000-100-6200' in row_str\n\n if status_col is not None:\n status_val = ws.cell(row=13, column=status_col).value\n has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n else:\n has_open = 'open' in row_str # fallback if header not found\n\n checks = {\n 'invoice_id GLFD-0441': has_invoice_id,\n 'vendor_id V-00009': has_vendor_id,\n 'vendor_name Great Lakes': has_vendor_name,\n 'invoice_date 2026-03-20': has_invoice_date,\n 'received_date 2026-03-24': has_received_date,\n 'due_date 2026-04-19': has_due_date,\n 'type Non-PO': has_non_po,\n 'amount $7,900.00': amount_found,\n 'gl_code 1000-100-6200': has_gl,\n 'status Open': has_open\n }\n\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 in {fpath.name} sheet '{ws.title}' contains all required fields: {', '.join(passed)}\"}\n except Exception as e:\n continue\n\n # Partial feedback pass\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n row_str = ' '.join(row_data).lower()\n\n if status_col is not None:\n status_val = ws.cell(row=13, column=status_col).value\n has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n else:\n has_open = 'open' in row_str\n\n checks = {\n 'invoice_id GLFD-0441': 'glfd-0441' in row_str,\n 'vendor_id V-00009': 'v-00009' in row_str,\n 'vendor_name Great Lakes': 'great lakes' in row_str,\n 'invoice_date 2026-03-20': '2026-03-20' in row_str,\n 'received_date 2026-03-24': '2026-03-24' in row_str,\n 'due_date 2026-04-19': '2026-04-19' in row_str,\n 'type Non-PO': 'non-po' in row_str or 'non po' in row_str,\n 'amount $7,900.00': '7,900' in row_str or '7900' in row_str,\n 'gl_code 1000-100-6200': '1000-100-6200' in row_str,\n 'status Open': has_open\n }\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n if passed:\n return {\"pass\": False, \"score\": len(passed)/len(checks), \"feedback\": f\"Row 13 in {fpath.name} missing: {', '.join(failed)}. Found: {', '.join(passed)}. Row content: {row_data}\"}\n Line truncated
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "1bcd6b5d-4a16-42e7-9464-4a8a2c676eda",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In mailbox.json (Drafts folder), exactly one draft email must exist addressed to m.chen@meridianpartners.com and r.whitfield@meridianpartners.com with CC to ap@meridianpartners.com, subject 'AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441', and body containing only the following information and no other information: 'A non-PO invoice requires your approval before payment can be processed. Vendor: Great Lakes Furniture & Design Co. (V-00009). Invoice: GLFD-0441, dated 2026-03-20. Amount: $7,900.00. Description: Executive conference table, 10-person, walnut finish. GL Code: 1000-100-6200. Please reply APPROVED or REJECTED with reason. If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\nimport unicodedata\n\ndef normalize(s):\n s = unicodedata.normalize('NFKC', s)\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n s = s.replace('\\u2019', \"'\").replace('\\u2018', \"'\")\n s = s.replace('\\u201c', '\"').replace('\\u201d', '\"')\n s = re.sub(r'[();:!?]', '', s)\n s = re.sub(r'[,.](?!\\d)', '', s)\n s = s.lower()\n s = re.sub(r'\\s+', ' ', s).strip()\n return s\n\nEXPECTED_SUBJECT = \"AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441\"\n\nEXPECTED_BODY = (\n \"A non-PO invoice requires your approval before payment can be processed.\\n\"\n \"Vendor: Great Lakes Furniture & Design Co. (V-00009).\\n\"\n \"Invoice: GLFD-0441, dated 2026-03-20\\n\"\n \"Amount: $7,900.00.\\n\"\n \"Description: Executive conference table, 10-person, walnut finish.\\n\"\n \"GL Code: 1000-100-6200.\\n\"\n \"Please reply APPROVED or REJECTED with reason.\\n\"\n \"If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.\"\n)\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get('emails', [])\n drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n\n if not drafts:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in Drafts folder\"}\n\n # Pick best matching draft by subject keywords\n best_draft = drafts[0]\n best_score = -1\n for draft in drafts:\n subject = str(draft.get('subject', '')).lower()\n score = ('glfd-0441' in subject) + ('great lakes' in subject)\n if score > best_score:\n best_score = score\n best_draft = draft\n\n draft = best_draft\n\n to_raw = draft.get('to_addr', '') or draft.get('to', '') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join([str(x).lower().strip() for x in to_raw])\n else:\n to_addr = str(to_raw).lower()\n\n cc_raw = draft.get('cc_addr', '') or draft.get('cc', '') or ''\n if isinstance(cc_raw, list):\n cc_addr = ' '.join([str(x).lower().strip() for x in cc_raw])\n else:\n cc_addr = str(cc_raw).lower()\n\n subject = str(draft.get('subject', ''))\n body = str(draft.get('body_text', '') or draft.get('body', '') or '')\n\n checks = {\n 'to: m.chen@meridianpartners.com': 'm.chen@meridianpartners.com' in to_addr,\n 'to: r.whitfield@meridianpartners.com': 'r.whitfield@meridianpartners.com' in to_addr,\n 'cc: ap@meridianpartners.com': 'ap@meridianpartners.com' in cc_addr,\n 'subject exact match': normalize(subject) == normalize(EXPECTED_SUBJECT),\n 'body exact match': normalize(body) == normalize(EXPECTED_BODY),\n }\n\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n score = len(passed) / len(checks)\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Draft email verified: all {len(checks)} checks passed.\"}\n\n details = f\"Failed: {', '.join(failed)}. Passed: {', '.join(passed)}.\"\n if 'subject exact match' in failed:\n details += f\" Subject found: '{subject}'.\"\n if 'body exact match' in failed:\n norm_actual = normalize(body)\n norm_expected = normalize(EXPECTED_BODY)\n if norm_actual == norm_expected:\n details += \" Body: normalized content matches — difference is line-break/formatting only.\"\n else:\n actual_words = norm_actual.split()\n expected_words = norm_expected.split()\n divergence = None\n for i, (aw, ew) in enumerate(zip(actual_words, expected_words)):\n if aw != ew:\n start = max(0, i - 3)\n expected_ctx = ' '.join(expected_words[start:i+6])\n actual_ctx = ' '.join(actual_words[start:i+6])\n divergence = f\"First divergence at word {i+1}: expected '...{expected_ctx}...' | got '...{actual_ctx}...'\"\n break\n if divergence is None:\n divergence = f\"Word count differs: expected {len(expected_words)}, got {len(actual_words)}\"\n details += f\" Body diff: {divergLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e64b48a4-131c-4e84-b578-61ab0558ed20",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In slack_data.json, no new Slack messages must have been sent (the agent must not have sent any Slack messages as part of this task)",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport time\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n\n messages = data.get('messages', {})\n\n # Look for any messages that mention Great Lakes, GLFD-0441, or 7900 that might have been sent by the agent\n # The task says 'No new Slack messages are sent'\n # We check if there are any messages related to this invoice task\n task_related_messages = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = str(msg.get('text', '')).lower()\n if (\n 'great lakes' in text\n or 'glfd-0441' in text\n or 'glfd0441' in text\n or ('7,900' in text and 'furniture' in text)\n or ('7900' in text and 'furniture' in text)\n ):\n task_related_messages.append({'channel': channel_id, 'text': msg.get('text', '')[:100]})\n\n if task_related_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(task_related_messages)} Slack message(s) related to Great Lakes/GLFD-0441 that should not have been sent: {task_related_messages}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Slack messages related to Great Lakes Furniture or GLFD-0441 found, as expected\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774390462114",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "No fields are populated after the \"Status\" column in Row 13 of `ap_ledger.xlsx`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n if status_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find 'Status' column header in {fpath.name}\"}\n\n populated_after_status = []\n for col in range(status_col + 1, ws.max_column + 1):\n cell_value = ws.cell(row=13, column=col).value\n if cell_value is None:\n continue\n if cell_value == 0 or cell_value == 0.0:\n continue\n if str(cell_value).strip() == \"\" or str(cell_value).strip() == \"0\":\n continue\n col_header = ws.cell(row=1, column=col).value\n populated_after_status.append((col, col_header, cell_value))\n\n if not populated_after_status:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 has no populated fields after 'Status' (column {status_col})\"}\n else:\n details = \"; \".join([f\"Col {c} ('{h}'): '{v}'\" for c, h, v in populated_after_status])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 13 has {len(populated_after_status)} populated field(s) after 'Status': {details}\"}\n\n except Exception as e:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774486274859",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json`,no more than one email must exist in the Drafts folder, and no emails related to Great Lakes Furniture & Design or invoice GLFD-0441 must exist in the Sent folder.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get('emails', [])\n drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n sent = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n\n failures = []\n\n if len(drafts) != 1:\n failures.append(f\"Expected exactly 1 draft, found {len(drafts)}\")\n\n sent_related = [\n e for e in sent\n if any(kw in str(e.get('subject', '')).lower() or kw in str(e.get('body_text', '')).lower() or kw in str(e.get('body', '')).lower() or kw in str(e.get('to_addr', '')).lower()\n for kw in ['great lakes', 'glfd-0441', 'orders@greatlakesfurniture.com'])\n ]\n\n if sent_related:\n failures.append(f\"Found {len(sent_related)} sent email(s) related to Great Lakes/GLFD-0441\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(failures)}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Exactly 1 draft found and no related emails in Sent folder\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774487311749",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, the Invoice Register must contain no more than 13 rows of data (header row plus 12 invoice entries, including the new GLFD-0441 entry in row 13).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n extra_rows = []\n for row in range(14, ws.max_row + 1):\n if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n extra_rows.append(row)\n\n if extra_rows:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Data found beyond row 13 in {fpath.name}: rows {extra_rows}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Invoice Register contains no data beyond row 13\"}\n except Exception:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774487313895",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `vendor_master.xlsx`, no new vendor rows must have been added (the file must contain no more than 12 rows: 1 header row and 11 vendor rows).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n vendor_files = [f for f in xlsx_files if 'vendor' in f.name.lower()]\n if not vendor_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No vendor_master.xlsx found in workspace\"}\n\n for fpath in vendor_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb.worksheets[0]\n\n extra_rows = []\n for row in range(13, ws.max_row + 1):\n if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n extra_rows.append(row)\n\n if extra_rows:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"New vendor rows found in {fpath.name} beyond row 12: rows {extra_rows}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"vendor_master.xlsx contains no extra vendor rows\"}\n except Exception:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read vendor_master.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Find the invoice from Great Lakes Furniture & Design that we received today, March 24, 2026. It's an Office Supplies Account. First, if it should be added to the Invoice Register, update the Invoice Register through the "Status" column. Second, if an email should be sent, draft if for me and put it in Drafts. Do NOT send it - I will review it myself before sending. Third, if a Slack message is required, send it.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user