Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge
2026-06-24 12:44:34 -07:00
commit 25c9eda5ca
1800 changed files with 323575 additions and 0 deletions
@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,144 @@
{
"events": {
"evt-R1": {
"id": "evt-R1",
"summary": "Great Lakes Regional Sales Conference \u2014 Chicago",
"start": {
"dateTime": "2026-10-02T08:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-10-04T18:00:00-05:00",
"timeZone": "America/Chicago"
},
"location": "Marriott Chicago Magnificent Mile, 540 N Michigan Ave, Chicago, IL",
"description": "Annual Great Lakes Regional Sales Conference. All West Region AEs expected to attend.\n\nOrganizer: peter.kim@vanguardshield.com\nAttendees: r.okonkwo@vanguardshield.com, k.osei@vanguardshield.com, r.torres@vanguardshield.com\nRecurring: No"
},
"evt-R2": {
"id": "evt-R2",
"summary": "Client Meeting \u2014 Hartwell Group HQ",
"start": {
"dateTime": "2026-10-05T12:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-05T13:30:00-04:00",
"timeZone": "America/New_York"
},
"location": "Hartwell Group HQ, 200 State St, Boston, MA",
"description": "Quarterly account review with Hartwell Group procurement team.\n\nOrganizer: r.okonkwo@vanguardshield.com\nAttendees: r.okonkwo@vanguardshield.com, hartwell.procurement@hartwellgroup.com"
},
"evt-D1": {
"id": "evt-D1",
"summary": "Midwest Sales Team Kickoff \u2014 Detroit",
"start": {
"dateTime": "2026-10-01T09:00:00-04:00",
"timeZone": "America/Detroit"
},
"end": {
"dateTime": "2026-10-02T17:00:00-04:00",
"timeZone": "America/Detroit"
},
"location": "Westin Book Cadillac Detroit, 1114 Washington Blvd, Detroit, MI",
"description": "Q4 Midwest sales team strategy kickoff. Two-day offsite event.\n\nOrganizer: d.huang@vanguardshield.com\nAttendees: d.huang@vanguardshield.com, m.bassett@vanguardshield.com, t.chen@vanguardshield.com, a.reyes@vanguardshield.com"
},
"evt-D2": {
"id": "evt-D2",
"summary": "Client Meeting \u2014 Nexus Corp HQ",
"start": {
"dateTime": "2026-10-05T11:30:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-05T13:00:00-04:00",
"timeZone": "America/New_York"
},
"location": "Nexus Corp HQ, 1 Nexus Plaza, Columbus, OH",
"description": "Q4 pipeline review with Nexus Corp account team. Working lunch.\n\nOrganizer: d.huang@vanguardshield.com\nAttendees: d.huang@vanguardshield.com, nexus.procurement@nexuscorp.com"
},
"evt-V1": {
"id": "evt-V1",
"summary": "Client Meeting \u2014 Pinnacle Group Quarterly Review",
"start": {
"dateTime": "2026-10-05T12:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-05T13:30:00-04:00",
"timeZone": "America/New_York"
},
"location": "Foundry Bistro Hartford, 55 Asylum St, Hartford, CT",
"description": "Quarterly business review with Pinnacle Group team. Lunch meeting.\n\nOrganizer: v.frost@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, pinnacle.team@pinnaclegroup.com"
},
"evt-V2": {
"id": "evt-V2",
"summary": "Client Dinner \u2014 Meridian Solutions Account Team",
"start": {
"dateTime": "2026-10-06T18:30:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-06T20:30:00-04:00",
"timeZone": "America/New_York"
},
"location": "The Capital Grille Hartford, 230 Trumbull St, Hartford, CT",
"description": "New account dinner with Meridian Solutions procurement and finance leads.\n\nOrganizer: v.frost@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, mharris@vanguardshield.com (optional), meridian.team@meridiansolutions.com"
},
"evt-V3": {
"id": "evt-V3",
"summary": "Sales East Region \u2014 Team Lunch",
"start": {
"dateTime": "2026-10-07T12:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-07T13:00:00-04:00",
"timeZone": "America/New_York"
},
"location": "Harvest Kitchen Hartford, 88 Pratt St, Hartford, CT",
"description": "Weekly East Region sales team lunch. Internal agenda: pipeline review and forecast update.\n\nOrganizer: mharris@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, mharris@vanguardshield.com, t.nguyen@vanguardshield.com, s.okafor@vanguardshield.com"
},
"evt-A1": {
"id": "evt-A1",
"summary": "Sales Ops & HR \u2014 T&E Weekly Standup",
"start": {
"dateTime": "2026-10-08T14:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-08T14:30:00-04:00",
"timeZone": "America/New_York"
},
"location": "Teams \u2014 HR Ops channel",
"description": "Weekly standup: T&E batch status, any escalations, HR-Ops queue review.\n\nOrganizer: j.morales@vanguardshield.com\nAttendees: analyst@vanguardshield.com, j.morales@vanguardshield.com, hr.intake@vanguardshield.com"
},
"evt-DIST1": {
"id": "evt-DIST1",
"summary": "Q4 Sales Planning \u2014 All Hands",
"start": {
"dateTime": "2026-10-06T15:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-06T16:00:00-04:00",
"timeZone": "America/New_York"
},
"location": "Teams \u2014 Sales All Hands",
"description": "Company-wide Q4 sales planning call. Territory and quota updates from VP Sales.\n\nOrganizer: l.williams@vanguardshield.com\nAttendees: all-sales@vanguardshield.com"
},
"evt-A2": {
"id": "evt-A2",
"summary": "1:1 with J. Morales",
"start": {
"dateTime": "2026-10-09T10:00:00-04:00",
"timeZone": "America/New_York"
},
"end": {
"dateTime": "2026-10-09T10:30:00-04:00",
"timeZone": "America/New_York"
},
"location": "J. Morales office",
"description": "Biweekly check-in.\n\nOrganizer: j.morales@vanguardshield.com\nAttendees: analyst@vanguardshield.com, j.morales@vanguardshield.com\nRecurring: Yes (biweekly)"
}
}
}
@@ -0,0 +1,144 @@
{
"mailbox": {
"email": "analyst@vanguardshield.com",
"name": "Analyst \u2014 HR & Sales Ops"
},
"contacts": [
{
"email": "bsandoval@vanguardshield.com",
"name": "B. Sandoval"
},
{
"email": "hr.intake@vanguardshield.com",
"name": "HR Intake Bot"
},
{
"email": "j.morales@vanguardshield.com",
"name": "Jessica Morales"
},
{
"email": "l.williams@vanguardshield.com",
"name": "Lisa Williams"
},
{
"email": "hrops@vanguardshield.com",
"name": "HR Ops"
},
{
"email": "people@vanguardshield.com",
"name": "People Directory"
},
{
"email": "it-notifications@vanguardshield.com",
"name": "IT Notifications"
},
{
"email": "d.huang@vanguardshield.com",
"name": "Derek Huang"
},
{
"email": "training@vanguardshield.com",
"name": "Compliance Training Team"
}
],
"groups": [
{
"email": "all-staff@vanguardshield.com",
"name": "All Staff",
"members": [
"analyst@vanguardshield.com",
"j.morales@vanguardshield.com",
"l.williams@vanguardshield.com"
]
}
],
"folders": [],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "T&E reports for review \u2014 Oct 1\u20137 batch",
"from_addr": "j.morales@vanguardshield.com",
"to_addr": "analyst@vanguardshield.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-10-07T17:20:00-04:00",
"message_id": "<msg-oct-001@vanguardshield.com>",
"in_reply_to": null,
"body_text": "Hi \u2014\n\nJust a heads up that the Jira queue has items ready for your review. Please make sure everything\u2019s worked through before the afternoon standup Thursday. Let me know if anything escalates.\n\nThanks,\nJessica Morales\nHR Operations",
"body_html": null,
"is_read": true,
"is_important": true,
"attachments": []
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "My expense report for last week \u2014 any update?",
"from_addr": "d.huang@vanguardshield.com",
"to_addr": "analyst@vanguardshield.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-10-08T07:42:00-04:00",
"message_id": "<msg-oct-002@gmail.com>",
"in_reply_to": null,
"body_text": "Hi,\n\nJust checking in on my expense report for last week. We have a team off-site coming up and I want to make sure this cycle gets processed before the next submission window.\n\nThanks,\nDerek Huang\nRegional Sales Lead \u2014 Midwest",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "3",
"folder": "INBOX",
"subject": "FW: Q4 Sales Quota Alignment \u2014 Action Required by Oct 15",
"from_addr": "l.williams@vanguardshield.com",
"to_addr": "analyst@vanguardshield.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-10-07T09:15:00-04:00",
"message_id": "<msg-oct-003@vanguardshield.com>",
"in_reply_to": null,
"body_text": "Forwarding for your records.\n\nSales Ops and Finance are doing the Q4 territory quota reconciliation. If any T&E submissions from the sales team include notes about deal-related travel, please note those in standard processing.\n\nDeadline for Finance to receive quota data is Oct 15.\n\n\u2014 Lisa Williams, VP Sales\n\n---------- Forwarded message ----------\nFrom: CFO Office <cfo@vanguardshield.com>\nSubject: Q4 Sales Quota Alignment\n\nFinance and Sales Ops: Please ensure all Q4 territory quota assignments are reconciled and confirmed in Salesforce by October 15. Questions to finance.ops@.",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "4",
"folder": "INBOX",
"subject": "Scheduled maintenance \u2014 Jira downtime Oct 18\u201319",
"from_addr": "it-notifications@vanguardshield.com",
"to_addr": "all-staff@vanguardshield.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-10-07T08:00:00-04:00",
"message_id": "<msg-oct-004@vanguardshield.com>",
"in_reply_to": null,
"body_text": "Please be advised that Jira will be unavailable for scheduled maintenance from Saturday Oct 18 at 10 PM ET through Sunday Oct 19 at 6 AM ET. Please save any in-progress work before the maintenance window.\n\nIT Operations",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "5",
"folder": "INBOX",
"subject": "Reminder: Q4 compliance training due Oct 31",
"from_addr": "training@vanguardshield.com",
"to_addr": "analyst@vanguardshield.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-10-05T09:00:00-04:00",
"message_id": "<msg-oct-005@vanguardshield.com>",
"in_reply_to": null,
"body_text": "This is a reminder that your Q4 compliance training module is due by October 31, 2026. Please complete the online course at training.vanguardshield.com.\n\nTopics: Data privacy refresher, Anti-money laundering update, Claims handling best practices.\n\nCompliance Training Team",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
}
],
"next_email_id": 6
}
@@ -0,0 +1,766 @@
{
"products": {
"gid://shopify/Product/1001": {
"id": "gid://shopify/Product/1001",
"title": "VSM Branded Notebook Set (12-pack)",
"description": "Professional notebook set featuring VSM branding. Perfect for client meetings, internal use, or corporate gifts. Includes 12 high-quality notebooks with the Vanguard Shield logo.",
"descriptionHtml": "<p>Professional notebook set featuring VSM branding. Perfect for client meetings, internal use, or corporate gifts. Includes 12 high-quality notebooks with the Vanguard Shield logo.</p>",
"handle": "vsm-branded-notebook-set-12-pack",
"productType": "Branded Merchandise",
"vendor": "VSM Print Shop",
"tags": [
"branded",
"notebook",
"corporate",
"bulk",
"merchandise"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "48.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "48.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1001",
"url": "https://store.vanguardshield.com/images/notebook-set.jpg",
"altText": "VSM Branded Notebook Set (12-pack)",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1001",
"url": "https://store.vanguardshield.com/images/notebook-set.jpg",
"altText": "VSM Branded Notebook Set (12-pack)",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1001",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1001",
"title": "Default Title",
"price": {
"amount": "48.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "BRNB-12PK",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 250,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-09-20T14:30:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 250
},
"gid://shopify/Product/1002": {
"id": "gid://shopify/Product/1002",
"title": "Client Gift Set \u2014 Standard Edition",
"description": "Curated gift set for valued clients. Includes VSM branded pen, notebook, water bottle, and premium gift packaging. Ideal for client appreciation and business development.",
"descriptionHtml": "<p>Curated gift set for valued clients. Includes VSM branded pen, notebook, water bottle, and premium gift packaging. Ideal for client appreciation and business development.</p>",
"handle": "client-gift-set-standard-edition",
"productType": "Client Gift",
"vendor": "VSM Print Shop",
"tags": [
"client gift",
"corporate gift",
"standard",
"branded"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "95.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "95.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1002",
"url": "https://store.vanguardshield.com/images/gift-set-standard.jpg",
"altText": "Client Gift Set \u2014 Standard Edition",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1002",
"url": "https://store.vanguardshield.com/images/gift-set-standard.jpg",
"altText": "Client Gift Set \u2014 Standard Edition",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1002",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1002",
"title": "Default Title",
"price": {
"amount": "95.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "CGSP-STD",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 120,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-09-28T16:20:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 120
},
"gid://shopify/Product/1003": {
"id": "gid://shopify/Product/1003",
"title": "Client Gift Set \u2014 Premium Edition",
"description": "Our finest client gift set. Includes VSM branded leather portfolio, premium pen set, executive notebook, water bottle, and luxury gift packaging. Perfect for top-tier clients and special occasions.",
"descriptionHtml": "<p>Our finest client gift set. Includes VSM branded leather portfolio, premium pen set, executive notebook, water bottle, and luxury gift packaging. Perfect for top-tier clients and special occasions.</p>",
"handle": "client-gift-set-premium-edition",
"productType": "Client Gift",
"vendor": "VSM Print Shop",
"tags": [
"client gift",
"corporate gift",
"premium",
"executive",
"branded"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "175.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "175.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1003",
"url": "https://store.vanguardshield.com/images/gift-set-premium.jpg",
"altText": "Client Gift Set \u2014 Premium Edition",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1003",
"url": "https://store.vanguardshield.com/images/gift-set-premium.jpg",
"altText": "Client Gift Set \u2014 Premium Edition",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1003",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1003",
"title": "Default Title",
"price": {
"amount": "175.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "CGSP-PREM",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 85,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-10-04T18:23:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 85
},
"gid://shopify/Product/1004": {
"id": "gid://shopify/Product/1004",
"title": "VSM Branded Water Bottle (single)",
"description": "High-quality stainless steel water bottle with VSM logo. Keeps beverages cold for 24 hours or hot for 12 hours. Perfect for daily use or as a gift item.",
"descriptionHtml": "<p>High-quality stainless steel water bottle with VSM logo. Keeps beverages cold for 24 hours or hot for 12 hours. Perfect for daily use or as a gift item.</p>",
"handle": "vsm-branded-water-bottle-single",
"productType": "Branded Merchandise",
"vendor": "VSM Print Shop",
"tags": [
"branded",
"water bottle",
"merchandise",
"drinkware"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "12.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "12.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1004",
"url": "https://store.vanguardshield.com/images/water-bottle.jpg",
"altText": "VSM Branded Water Bottle",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1004",
"url": "https://store.vanguardshield.com/images/water-bottle.jpg",
"altText": "VSM Branded Water Bottle",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1004",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1004",
"title": "Default Title",
"price": {
"amount": "12.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "BRNW-BTLS",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 500,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-09-19T11:45:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 500
},
"gid://shopify/Product/1005": {
"id": "gid://shopify/Product/1005",
"title": "VSM Branded Pen & Portfolio Set",
"description": "Professional portfolio with VSM branding and matching pen. Features document pockets, card slots, and notepad. Ideal for client meetings and professional presentations.",
"descriptionHtml": "<p>Professional portfolio with VSM branding and matching pen. Features document pockets, card slots, and notepad. Ideal for client meetings and professional presentations.</p>",
"handle": "vsm-branded-pen-portfolio-set",
"productType": "Branded Merchandise",
"vendor": "VSM Print Shop",
"tags": [
"branded",
"portfolio",
"pen",
"professional",
"merchandise"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "28.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "28.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1005",
"url": "https://store.vanguardshield.com/images/pen-portfolio-set.jpg",
"altText": "VSM Branded Pen & Portfolio Set",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1005",
"url": "https://store.vanguardshield.com/images/pen-portfolio-set.jpg",
"altText": "VSM Branded Pen & Portfolio Set",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1005",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1005",
"title": "Default Title",
"price": {
"amount": "28.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "BRNP-PORT",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 175,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-09-15T15:20:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 175
},
"gid://shopify/Product/1006": {
"id": "gid://shopify/Product/1006",
"title": "Conference Table Banner (retractable)",
"description": "Professional retractable banner featuring VSM branding. Easy to set up and transport. Perfect for conferences, trade shows, and corporate events. Includes carrying case.",
"descriptionHtml": "<p>Professional retractable banner featuring VSM branding. Easy to set up and transport. Perfect for conferences, trade shows, and corporate events. Includes carrying case.</p>",
"handle": "conference-table-banner-retractable",
"productType": "Conference Materials",
"vendor": "VSM Print Shop",
"tags": [
"conference",
"banner",
"event",
"display",
"marketing"
],
"availableForSale": true,
"priceRange": {
"minVariantPrice": {
"amount": "85.00",
"currencyCode": "USD"
},
"maxVariantPrice": {
"amount": "85.00",
"currencyCode": "USD"
}
},
"featuredImage": {
"id": "gid://shopify/Image/1006",
"url": "https://store.vanguardshield.com/images/conference-banner.jpg",
"altText": "Conference Table Banner (retractable)",
"width": 800,
"height": 800
},
"images": [
{
"id": "gid://shopify/Image/1006",
"url": "https://store.vanguardshield.com/images/conference-banner.jpg",
"altText": "Conference Table Banner (retractable)",
"width": 800,
"height": 800
}
],
"options": [
{
"id": "opt1006",
"name": "Title",
"values": [
"Default Title"
]
}
],
"variants": [
{
"id": "gid://shopify/ProductVariant/1006",
"title": "Default Title",
"price": {
"amount": "85.00",
"currencyCode": "USD"
},
"availableForSale": true,
"sku": "CONF-BNRR",
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"quantityAvailable": 45,
"requiresShipping": true,
"taxable": false
}
],
"createdAt": "2026-01-15T10:00:00.000000Z",
"updatedAt": "2026-09-30T13:40:00.000000Z",
"publishedAt": "2026-01-15T10:00:00.000000Z",
"totalInventory": 45
}
},
"carts": {
"gid://shopify/Cart/c1001": {
"id": "gid://shopify/Cart/c1001",
"checkoutUrl": "https://store.vanguardshield.com/checkout/c1001",
"createdAt": "2026-10-10T10:15:30.000000Z",
"updatedAt": "2026-10-10T10:18:45.000000Z",
"lines": [
{
"id": "gid://shopify/CartLine/2001",
"quantity": 2,
"merchandise": {
"id": "gid://shopify/ProductVariant/1002",
"title": "Default Title",
"product": {
"id": "gid://shopify/Product/1002",
"title": "Client Gift Set \u2014 Standard Edition",
"handle": "client-gift-set-standard-edition"
},
"image": null,
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"price": {
"amount": "95.00",
"currencyCode": "USD"
}
},
"cost": {
"amountPerQuantity": {
"amount": "95.00",
"currencyCode": "USD"
},
"compareAtAmountPerQuantity": null,
"subtotalAmount": {
"amount": "190.00",
"currencyCode": "USD"
},
"totalAmount": {
"amount": "190.00",
"currencyCode": "USD"
}
},
"attributes": [],
"discountAllocations": []
},
{
"id": "gid://shopify/CartLine/2002",
"quantity": 5,
"merchandise": {
"id": "gid://shopify/ProductVariant/1004",
"title": "Default Title",
"product": {
"id": "gid://shopify/Product/1004",
"title": "VSM Branded Water Bottle (single)",
"handle": "vsm-branded-water-bottle-single"
},
"image": null,
"selectedOptions": [
{
"name": "Title",
"value": "Default Title"
}
],
"price": {
"amount": "12.00",
"currencyCode": "USD"
}
},
"cost": {
"amountPerQuantity": {
"amount": "12.00",
"currencyCode": "USD"
},
"compareAtAmountPerQuantity": null,
"subtotalAmount": {
"amount": "60.00",
"currencyCode": "USD"
},
"totalAmount": {
"amount": "60.00",
"currencyCode": "USD"
}
},
"attributes": [],
"discountAllocations": []
}
],
"cost": {
"subtotalAmount": {
"amount": "250.00",
"currencyCode": "USD"
},
"subtotalAmountEstimated": false,
"totalAmount": {
"amount": "250.00",
"currencyCode": "USD"
},
"totalAmountEstimated": false,
"totalTaxAmount": null,
"totalTaxAmountEstimated": false,
"checkoutChargeAmount": {
"amount": "250.00",
"currencyCode": "USD"
}
},
"buyerIdentity": {
"email": "m.richardson@vanguardshield.com",
"countryCode": "US"
},
"attributes": [
{
"key": "department",
"value": "Sales - Northeast Region"
}
],
"discountCodes": [],
"discountAllocations": [],
"appliedGiftCards": [],
"deliveryGroups": [],
"note": "For upcoming client meetings in Q4",
"totalQuantity": 7
}
},
"policies": [
{
"id": "gid://shopify/ShopPolicy/1",
"title": "Internal Ordering Policy",
"body": "<h2>VSM Internal Store Ordering Policy</h2><p>This store is exclusively for Vanguard Shield Mutual employees ordering branded merchandise, client gifts, and conference materials.</p><h3>Eligibility</h3><p>All VSM employees with valid employee IDs may place orders. Orders must be approved by department managers for quantities exceeding standard limits.</p><h3>Approval Requirements</h3><ul><li>Standard merchandise (under $100): No approval needed</li><li>Client gifts ($100-$500): Manager approval required</li><li>Bulk orders or premium gifts (over $500): Director approval required</li></ul><h3>Shipping</h3><p>Orders ship to VSM office locations or approved conference venues only. Allow 5-7 business days for standard delivery. Rush orders available with manager approval.</p><h3>Billing</h3><p>All orders are billed to department cost centers. Ensure accurate department coding during checkout.</p>",
"url": "https://store.vanguardshield.com/policies/ordering"
},
{
"id": "gid://shopify/ShopPolicy/2",
"title": "Returns and Exchanges",
"body": "<h2>Returns and Exchanges</h2><p>We accept returns on unused, unopened items within 30 days of delivery.</p><h3>Return Process</h3><ol><li>Contact store-admin@vanguardshield.com with your order number</li><li>Receive return authorization and shipping label</li><li>Package items securely and ship within 7 days</li></ol><h3>Non-Returnable Items</h3><ul><li>Personalized or custom-ordered items</li><li>Items ordered for specific conferences or events (within 14 days of event)</li><li>Opened client gift sets</li></ul><h3>Exchanges</h3><p>Defective items may be exchanged at any time. Contact us immediately if you receive damaged or incorrect merchandise.</p>",
"url": "https://store.vanguardshield.com/policies/returns"
},
{
"id": "gid://shopify/ShopPolicy/3",
"title": "Client Gift Guidelines",
"body": "<h2>Client Gift Guidelines</h2><p>VSM client gifts are designed to strengthen relationships while adhering to industry compliance standards.</p><h3>Gift Value Limits</h3><p>Standard Edition gifts are appropriate for routine client appreciation. Premium Edition gifts require Regional Director approval and are reserved for:</p><ul><li>Key account milestones (10+ year relationships)</li><li>Significant new business partnerships</li><li>Executive-level relationship building</li></ul><h3>Compliance</h3><p>All gifts must comply with recipient organization policies. Sales teams are responsible for verifying gift acceptance policies before ordering. Gifts to government or regulated entity employees may require additional approval.</p><h3>Documentation</h3><p>Retain order confirmations for expense reporting. Client gift expenses are tracked by the Finance team for compliance reporting.</p>",
"url": "https://store.vanguardshield.com/policies/client-gifts"
},
{
"id": "gid://shopify/ShopPolicy/4",
"title": "Frequently Asked Questions",
"body": "<h2>Frequently Asked Questions</h2><h3>How do I place an order?</h3><p>Browse products, add items to your cart, and complete checkout. Use your VSM email address and ensure department information is accurate for billing purposes.</p><h3>Can I ship to my home address?</h3><p>No. All orders must ship to VSM office locations or approved business venues. Contact store-admin@vanguardshield.com for special shipping requests.</p><h3>How long does shipping take?</h3><p>Standard shipping is 5-7 business days. Rush delivery (2-3 days) is available with manager approval for urgent orders.</p><h3>What if I need a product not listed in the store?</h3><p>Contact the VSM Print Shop team at store-admin@vanguardshield.com to discuss custom merchandise or special orders. Custom orders require 3-4 weeks lead time.</p><h3>Can I order in bulk?</h3><p>Yes. Bulk orders (20+ units) may qualify for volume discounts. Contact store-admin@vanguardshield.com for pricing and availability.</p><h3>Who do I contact with questions?</h3><p>Email store-admin@vanguardshield.com or call the Print Shop at extension 5500 during business hours (Monday-Friday, 8:00 AM - 5:00 PM ET).</p>",
"url": "https://store.vanguardshield.com/policies/faq"
}
],
"counters": {
"cart_id": 1001,
"line_id": 2002
},
"orders": {
"gid://shopify/Order/5001": {
"id": "gid://shopify/Order/5001",
"name": "#VSM-5001",
"createdAt": "2026-10-04T10:22:00.000000Z",
"updatedAt": "2026-10-04T10:22:00.000000Z",
"fulfillmentStatus": "FULFILLED",
"financialStatus": "PAID",
"totalPriceSet": {
"shopMoney": {
"amount": "175.00",
"currencyCode": "USD"
}
},
"customer": {
"email": "v.frost@vanguardshield.com",
"firstName": "Vanessa",
"lastName": "Frost",
"displayName": "Vanessa Frost"
},
"lineItems": [
{
"title": "Client Gift Set \u2014 Premium Edition",
"quantity": 1,
"variant": {
"id": "gid://shopify/ProductVariant/1003",
"title": "Default Title",
"sku": "CGSP-PREM",
"price": {
"amount": "175.00",
"currencyCode": "USD"
}
},
"originalTotalSet": {
"shopMoney": {
"amount": "175.00",
"currencyCode": "USD"
}
},
"id": "gid://shopify/ProductVariant/1003",
"price": {
"amount": "175.00",
"currencyCode": "USD"
},
"totalPrice": {
"amount": "175.00",
"currencyCode": "USD"
}
}
],
"note": "Q4 client engagement \u2014 Meridian Solutions account",
"tags": [
"sales-east",
"client-gift"
],
"shippingAddress": {
"address1": "100 Pearl St",
"city": "Hartford",
"province": "Connecticut",
"provinceCode": "CT",
"zip": "06103",
"country": "United States",
"countryCodeV2": "US"
},
"subtotalPrice": {
"amount": "175.00",
"currencyCode": "USD"
},
"totalPrice": {
"amount": "175.00",
"currencyCode": "USD"
}
},
"gid://shopify/Order/5002": {
"id": "gid://shopify/Order/5002",
"name": "#VSM-5002",
"createdAt": "2026-09-15T14:08:00.000000Z",
"updatedAt": "2026-09-15T14:08:00.000000Z",
"fulfillmentStatus": "FULFILLED",
"financialStatus": "PAID",
"totalPriceSet": {
"shopMoney": {
"amount": "48.00",
"currencyCode": "USD"
}
},
"customer": {
"email": "j.calloway@vanguardshield.com",
"firstName": "James",
"lastName": "Calloway",
"displayName": "James Calloway"
},
"lineItems": [
{
"title": "VSM Branded Notebook Set (12-pack)",
"quantity": 1,
"variant": {
"id": "gid://shopify/ProductVariant/1001",
"title": "Default Title",
"sku": "BRNB-12PK",
"price": {
"amount": "48.00",
"currencyCode": "USD"
}
},
"originalTotalSet": {
"shopMoney": {
"amount": "48.00",
"currencyCode": "USD"
}
},
"id": "gid://shopify/ProductVariant/1001",
"price": {
"amount": "48.00",
"currencyCode": "USD"
},
"totalPrice": {
"amount": "48.00",
"currencyCode": "USD"
}
}
],
"note": "Client gifts for Pacific Ventures meetings",
"tags": [
"sales-west",
"branded-merch"
],
"shippingAddress": {
"address1": "350 Mission St",
"city": "San Francisco",
"province": "California",
"provinceCode": "CA",
"zip": "94105",
"country": "United States",
"countryCodeV2": "US"
},
"subtotalPrice": {
"amount": "48.00",
"currencyCode": "USD"
},
"totalPrice": {
"amount": "48.00",
"currencyCode": "USD"
}
}
}
}
@@ -0,0 +1,360 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T_VANGUARD",
"name": "analyst",
"deleted": false,
"real_name": "Analyst",
"profile": {
"real_name": "Analyst",
"display_name": "Analyst",
"email": "analyst@vanguardshield.com",
"title": "Operations Analyst \u2014 HR & Sales"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T_VANGUARD",
"name": "pkim",
"deleted": false,
"real_name": "Peter Kim",
"profile": {
"real_name": "Peter Kim",
"display_name": "Peter Kim",
"email": "peter.kim@vanguardshield.com",
"title": "Sales Director \u2014 West Region"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T_VANGUARD",
"name": "mharris",
"deleted": false,
"real_name": "Maria Harris",
"profile": {
"real_name": "Maria Harris",
"display_name": "Maria Harris",
"email": "mharris@vanguardshield.com",
"title": "Sales Manager \u2014 East Region"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T_VANGUARD",
"name": "jmorales",
"deleted": false,
"real_name": "Jessica Morales",
"profile": {
"real_name": "Jessica Morales",
"display_name": "Jessica Morales",
"email": "j.morales@vanguardshield.com",
"title": "HR Operations Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T_VANGUARD",
"name": "hrops.bot",
"deleted": false,
"real_name": "HR Ops Bot",
"profile": {
"real_name": "HR Ops Bot",
"display_name": "HR Ops Bot",
"email": "hrops-bot@vanguardshield.com",
"title": "Automation Bot"
},
"is_admin": false,
"is_owner": false,
"is_bot": true,
"is_app_user": true
},
"U006": {
"id": "U006",
"team_id": "T_VANGUARD",
"name": "lwilliams",
"deleted": false,
"real_name": "Lisa Williams",
"profile": {
"real_name": "Lisa Williams",
"display_name": "Lisa Williams",
"email": "l.williams@vanguardshield.com",
"title": "VP Sales"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T_VANGUARD",
"name": "rokonkwo",
"deleted": false,
"real_name": "Rachel Okonkwo",
"profile": {
"real_name": "Rachel Okonkwo",
"display_name": "Rachel Okonkwo",
"email": "r.okonkwo@vanguardshield.com",
"title": "Account Executive \u2014 Sales West"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T_VANGUARD",
"name": "dhuang",
"deleted": false,
"real_name": "Derek Huang",
"profile": {
"real_name": "Derek Huang",
"display_name": "Derek Huang",
"email": "d.huang@vanguardshield.com",
"title": "Regional Sales Lead \u2014 Midwest"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T_VANGUARD",
"name": "mvance",
"deleted": false,
"real_name": "Marcus Vance",
"profile": {
"real_name": "Marcus Vance",
"display_name": "Marcus Vance",
"email": "m.vance@vanguardshield.com",
"title": "Finance Controller"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"CHROPS": {
"id": "CHROPS",
"name": "hr-ops-help",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "hr-ops-help",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T_VANGUARD",
"updated": 1791392400,
"creator": "U004",
"shared_team_ids": [
"T_VANGUARD"
],
"is_member": true,
"topic": {
"value": "HR operations help and manager lookup escalations",
"creator": "U004",
"last_set": 1640995200
},
"purpose": {
"value": "HR operations help and manager lookup escalations",
"creator": "U004",
"last_set": 1640995200
}
},
"CSALESOPS": {
"id": "CSALESOPS",
"name": "sales-ops",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "sales-ops",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T_VANGUARD",
"updated": 1791392400,
"creator": "U006",
"shared_team_ids": [
"T_VANGUARD"
],
"is_member": true,
"topic": {
"value": "Sales team coordination and ops updates",
"creator": "U006",
"last_set": 1640995200
},
"purpose": {
"value": "Sales team operational updates \u2014 territory, quotas, and scheduling",
"creator": "U006",
"last_set": 1640995200
}
},
"CFINAPPR": {
"id": "CFINAPPR",
"name": "finance-approvals",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1600000000,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "finance-approvals",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T_VANGUARD",
"updated": 1791392400,
"creator": "U009",
"shared_team_ids": [
"T_VANGUARD"
],
"is_member": true,
"topic": {
"value": "Finance approval threads \u2014 intercompany suspense and GL reclassifications",
"creator": "U009",
"last_set": 1600000000
},
"purpose": {
"value": "Finance approval threads \u2014 intercompany suspense and GL reclassifications",
"creator": "U009",
"last_set": 1600000000
}
},
"DU002": {
"id": "DU002",
"name": "pkim",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1640995200,
"is_archived": false,
"user": "U002",
"context_team_id": "T_VANGUARD",
"members": [
"U002"
]
}
},
"messages": {
"CHROPS": [
{
"type": "message",
"user": "U004",
"text": "Thanks @hrops.bot \u2014 OPS-HR-TE-0912 closed. Adding note to Amanda Chen's Jira employee record.",
"ts": "1790085600.000200",
"team": "T_VANGUARD"
}
],
"CSALESOPS": [
{
"type": "message",
"user": "U006",
"text": "Q4 kick-off is officially live. All territory assignments have been updated in Salesforce. Route through the standard portal.",
"ts": "1790845200.000200",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U008",
"text": "Got it \u2014 heading out for the kickoff today. Will submit my report when I\u2019m back.",
"ts": "1790870400.000300",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U007",
"text": "Off to the conference tomorrow morning. Will have it in on time.",
"ts": "1790888400.000100",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U006",
"text": "Quick note from my sync with the CFO: the Q4 expense policy is unchanged from Q3.",
"ts": "1791190800.000200",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U008",
"text": "All good on my end.",
"ts": "1791388800.000100",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U007",
"text": "All wrapped up here.",
"ts": "1791390600.000200",
"team": "T_VANGUARD"
}
],
"CFINAPPR": [
{
"type": "message",
"user": "U009",
"text": "Approved: Intercompany reclassification $7,250.00 \u2014 OPS-FIN-SUSPENSE-2026-09 line 8. GL 4510 \u2192 4520. https://jira.vanguardshield.com/OPS-FIN-SUSPENSE-2026-09",
"ts": "1790679600.000300",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U009",
"text": "Approved: Intercompany reclassification $3,800.00 \u2014 OPS-FIN-SUSPENSE-2026-09 line 14. GL 5100 \u2192 5200. https://jira.vanguardshield.com/OPS-FIN-SUSPENSE-2026-09",
"ts": "1790679600.000600",
"team": "T_VANGUARD"
},
{
"type": "message",
"user": "U009",
"text": "Sep month-end suspense review complete. Two items reclassified, three items pending Controller review from the Sep close package. Finance team: flag anything from the Oct 5 close package to this channel.",
"ts": "1791187200.000100",
"team": "T_VANGUARD"
}
],
"DU002": []
},
"counters": {
"messageId": 2000000
}
}
@@ -0,0 +1 @@
I've got three T&E reports that need processing. Can you work through each one following the attached SOP?
@@ -0,0 +1 @@
Today's date is, October, 8, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/insurance_vanguard_shield_mutual_90ff0751"
description = "SOP task from CSV row 90ff0751"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
(October 8th, 2026). I've got three T&E reports that need processing. Can you work through each one following the attached SOP?
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py