Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-R1": {
|
||||
"id": "evt-R1",
|
||||
"summary": "Great Lakes Regional Sales Conference \u2014 Chicago",
|
||||
"start": {
|
||||
"dateTime": "2026-10-02T08:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-04T18:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"location": "Marriott Chicago Magnificent Mile, 540 N Michigan Ave, Chicago, IL",
|
||||
"description": "Annual Great Lakes Regional Sales Conference. All West Region AEs expected to attend.\n\nOrganizer: peter.kim@vanguardshield.com\nAttendees: r.okonkwo@vanguardshield.com, k.osei@vanguardshield.com, r.torres@vanguardshield.com\nRecurring: No"
|
||||
},
|
||||
"evt-R2": {
|
||||
"id": "evt-R2",
|
||||
"summary": "Client Meeting \u2014 Hartwell Group HQ",
|
||||
"start": {
|
||||
"dateTime": "2026-10-05T12:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-05T13:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Hartwell Group HQ, 200 State St, Boston, MA",
|
||||
"description": "Quarterly account review with Hartwell Group procurement team.\n\nOrganizer: r.okonkwo@vanguardshield.com\nAttendees: r.okonkwo@vanguardshield.com, hartwell.procurement@hartwellgroup.com"
|
||||
},
|
||||
"evt-D1": {
|
||||
"id": "evt-D1",
|
||||
"summary": "Midwest Sales Team Kickoff \u2014 Detroit",
|
||||
"start": {
|
||||
"dateTime": "2026-10-01T09:00:00-04:00",
|
||||
"timeZone": "America/Detroit"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-02T17:00:00-04:00",
|
||||
"timeZone": "America/Detroit"
|
||||
},
|
||||
"location": "Westin Book Cadillac Detroit, 1114 Washington Blvd, Detroit, MI",
|
||||
"description": "Q4 Midwest sales team strategy kickoff. Two-day offsite event.\n\nOrganizer: d.huang@vanguardshield.com\nAttendees: d.huang@vanguardshield.com, m.bassett@vanguardshield.com, t.chen@vanguardshield.com, a.reyes@vanguardshield.com"
|
||||
},
|
||||
"evt-D2": {
|
||||
"id": "evt-D2",
|
||||
"summary": "Client Meeting \u2014 Nexus Corp HQ",
|
||||
"start": {
|
||||
"dateTime": "2026-10-05T11:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-05T13:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Nexus Corp HQ, 1 Nexus Plaza, Columbus, OH",
|
||||
"description": "Q4 pipeline review with Nexus Corp account team. Working lunch.\n\nOrganizer: d.huang@vanguardshield.com\nAttendees: d.huang@vanguardshield.com, nexus.procurement@nexuscorp.com"
|
||||
},
|
||||
"evt-V1": {
|
||||
"id": "evt-V1",
|
||||
"summary": "Client Meeting \u2014 Pinnacle Group Quarterly Review",
|
||||
"start": {
|
||||
"dateTime": "2026-10-05T12:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-05T13:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Foundry Bistro Hartford, 55 Asylum St, Hartford, CT",
|
||||
"description": "Quarterly business review with Pinnacle Group team. Lunch meeting.\n\nOrganizer: v.frost@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, pinnacle.team@pinnaclegroup.com"
|
||||
},
|
||||
"evt-V2": {
|
||||
"id": "evt-V2",
|
||||
"summary": "Client Dinner \u2014 Meridian Solutions Account Team",
|
||||
"start": {
|
||||
"dateTime": "2026-10-06T18:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-06T20:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "The Capital Grille Hartford, 230 Trumbull St, Hartford, CT",
|
||||
"description": "New account dinner with Meridian Solutions procurement and finance leads.\n\nOrganizer: v.frost@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, mharris@vanguardshield.com (optional), meridian.team@meridiansolutions.com"
|
||||
},
|
||||
"evt-V3": {
|
||||
"id": "evt-V3",
|
||||
"summary": "Sales East Region \u2014 Team Lunch",
|
||||
"start": {
|
||||
"dateTime": "2026-10-07T12:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-07T13:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Harvest Kitchen Hartford, 88 Pratt St, Hartford, CT",
|
||||
"description": "Weekly East Region sales team lunch. Internal agenda: pipeline review and forecast update.\n\nOrganizer: mharris@vanguardshield.com\nAttendees: v.frost@vanguardshield.com, mharris@vanguardshield.com, t.nguyen@vanguardshield.com, s.okafor@vanguardshield.com"
|
||||
},
|
||||
"evt-A1": {
|
||||
"id": "evt-A1",
|
||||
"summary": "Sales Ops & HR \u2014 T&E Weekly Standup",
|
||||
"start": {
|
||||
"dateTime": "2026-10-08T14:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-08T14:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Teams \u2014 HR Ops channel",
|
||||
"description": "Weekly standup: T&E batch status, any escalations, HR-Ops queue review.\n\nOrganizer: j.morales@vanguardshield.com\nAttendees: analyst@vanguardshield.com, j.morales@vanguardshield.com, hr.intake@vanguardshield.com"
|
||||
},
|
||||
"evt-DIST1": {
|
||||
"id": "evt-DIST1",
|
||||
"summary": "Q4 Sales Planning \u2014 All Hands",
|
||||
"start": {
|
||||
"dateTime": "2026-10-06T15:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-06T16:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Teams \u2014 Sales All Hands",
|
||||
"description": "Company-wide Q4 sales planning call. Territory and quota updates from VP Sales.\n\nOrganizer: l.williams@vanguardshield.com\nAttendees: all-sales@vanguardshield.com"
|
||||
},
|
||||
"evt-A2": {
|
||||
"id": "evt-A2",
|
||||
"summary": "1:1 with J. Morales",
|
||||
"start": {
|
||||
"dateTime": "2026-10-09T10:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-09T10:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "J. Morales office",
|
||||
"description": "Biweekly check-in.\n\nOrganizer: j.morales@vanguardshield.com\nAttendees: analyst@vanguardshield.com, j.morales@vanguardshield.com\nRecurring: Yes (biweekly)"
|
||||
}
|
||||
}
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "analyst@vanguardshield.com",
|
||||
"name": "Analyst \u2014 HR & Sales Ops"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "bsandoval@vanguardshield.com",
|
||||
"name": "B. Sandoval"
|
||||
},
|
||||
{
|
||||
"email": "hr.intake@vanguardshield.com",
|
||||
"name": "HR Intake Bot"
|
||||
},
|
||||
{
|
||||
"email": "j.morales@vanguardshield.com",
|
||||
"name": "Jessica Morales"
|
||||
},
|
||||
{
|
||||
"email": "l.williams@vanguardshield.com",
|
||||
"name": "Lisa Williams"
|
||||
},
|
||||
{
|
||||
"email": "hrops@vanguardshield.com",
|
||||
"name": "HR Ops"
|
||||
},
|
||||
{
|
||||
"email": "people@vanguardshield.com",
|
||||
"name": "People Directory"
|
||||
},
|
||||
{
|
||||
"email": "it-notifications@vanguardshield.com",
|
||||
"name": "IT Notifications"
|
||||
},
|
||||
{
|
||||
"email": "d.huang@vanguardshield.com",
|
||||
"name": "Derek Huang"
|
||||
},
|
||||
{
|
||||
"email": "training@vanguardshield.com",
|
||||
"name": "Compliance Training Team"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "all-staff@vanguardshield.com",
|
||||
"name": "All Staff",
|
||||
"members": [
|
||||
"analyst@vanguardshield.com",
|
||||
"j.morales@vanguardshield.com",
|
||||
"l.williams@vanguardshield.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "T&E reports for review \u2014 Oct 1\u20137 batch",
|
||||
"from_addr": "j.morales@vanguardshield.com",
|
||||
"to_addr": "analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-10-07T17:20:00-04:00",
|
||||
"message_id": "<msg-oct-001@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi \u2014\n\nJust a heads up that the Jira queue has items ready for your review. Please make sure everything\u2019s worked through before the afternoon standup Thursday. Let me know if anything escalates.\n\nThanks,\nJessica Morales\nHR Operations",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": true,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "My expense report for last week \u2014 any update?",
|
||||
"from_addr": "d.huang@vanguardshield.com",
|
||||
"to_addr": "analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-10-08T07:42:00-04:00",
|
||||
"message_id": "<msg-oct-002@gmail.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi,\n\nJust checking in on my expense report for last week. We have a team off-site coming up and I want to make sure this cycle gets processed before the next submission window.\n\nThanks,\nDerek Huang\nRegional Sales Lead \u2014 Midwest",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "INBOX",
|
||||
"subject": "FW: Q4 Sales Quota Alignment \u2014 Action Required by Oct 15",
|
||||
"from_addr": "l.williams@vanguardshield.com",
|
||||
"to_addr": "analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-10-07T09:15:00-04:00",
|
||||
"message_id": "<msg-oct-003@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Forwarding for your records.\n\nSales Ops and Finance are doing the Q4 territory quota reconciliation. If any T&E submissions from the sales team include notes about deal-related travel, please note those in standard processing.\n\nDeadline for Finance to receive quota data is Oct 15.\n\n\u2014 Lisa Williams, VP Sales\n\n---------- Forwarded message ----------\nFrom: CFO Office <cfo@vanguardshield.com>\nSubject: Q4 Sales Quota Alignment\n\nFinance and Sales Ops: Please ensure all Q4 territory quota assignments are reconciled and confirmed in Salesforce by October 15. Questions to finance.ops@.",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Scheduled maintenance \u2014 Jira downtime Oct 18\u201319",
|
||||
"from_addr": "it-notifications@vanguardshield.com",
|
||||
"to_addr": "all-staff@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-10-07T08:00:00-04:00",
|
||||
"message_id": "<msg-oct-004@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Please be advised that Jira will be unavailable for scheduled maintenance from Saturday Oct 18 at 10 PM ET through Sunday Oct 19 at 6 AM ET. Please save any in-progress work before the maintenance window.\n\nIT Operations",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "INBOX",
|
||||
"subject": "Reminder: Q4 compliance training due Oct 31",
|
||||
"from_addr": "training@vanguardshield.com",
|
||||
"to_addr": "analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-10-05T09:00:00-04:00",
|
||||
"message_id": "<msg-oct-005@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "This is a reminder that your Q4 compliance training module is due by October 31, 2026. Please complete the online course at training.vanguardshield.com.\n\nTopics: Data privacy refresher, Anti-money laundering update, Claims handling best practices.\n\nCompliance Training Team",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 6
|
||||
}
|
||||
+1155
File diff suppressed because it is too large.
Load diff
+766
@@ -0,0 +1,766 @@
|
||||
{
|
||||
"products": {
|
||||
"gid://shopify/Product/1001": {
|
||||
"id": "gid://shopify/Product/1001",
|
||||
"title": "VSM Branded Notebook Set (12-pack)",
|
||||
"description": "Professional notebook set featuring VSM branding. Perfect for client meetings, internal use, or corporate gifts. Includes 12 high-quality notebooks with the Vanguard Shield logo.",
|
||||
"descriptionHtml": "<p>Professional notebook set featuring VSM branding. Perfect for client meetings, internal use, or corporate gifts. Includes 12 high-quality notebooks with the Vanguard Shield logo.</p>",
|
||||
"handle": "vsm-branded-notebook-set-12-pack",
|
||||
"productType": "Branded Merchandise",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"branded",
|
||||
"notebook",
|
||||
"corporate",
|
||||
"bulk",
|
||||
"merchandise"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1001",
|
||||
"url": "https://store.vanguardshield.com/images/notebook-set.jpg",
|
||||
"altText": "VSM Branded Notebook Set (12-pack)",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1001",
|
||||
"url": "https://store.vanguardshield.com/images/notebook-set.jpg",
|
||||
"altText": "VSM Branded Notebook Set (12-pack)",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1001",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1001",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "BRNB-12PK",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 250,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-09-20T14:30:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 250
|
||||
},
|
||||
"gid://shopify/Product/1002": {
|
||||
"id": "gid://shopify/Product/1002",
|
||||
"title": "Client Gift Set \u2014 Standard Edition",
|
||||
"description": "Curated gift set for valued clients. Includes VSM branded pen, notebook, water bottle, and premium gift packaging. Ideal for client appreciation and business development.",
|
||||
"descriptionHtml": "<p>Curated gift set for valued clients. Includes VSM branded pen, notebook, water bottle, and premium gift packaging. Ideal for client appreciation and business development.</p>",
|
||||
"handle": "client-gift-set-standard-edition",
|
||||
"productType": "Client Gift",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"client gift",
|
||||
"corporate gift",
|
||||
"standard",
|
||||
"branded"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "95.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "95.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1002",
|
||||
"url": "https://store.vanguardshield.com/images/gift-set-standard.jpg",
|
||||
"altText": "Client Gift Set \u2014 Standard Edition",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1002",
|
||||
"url": "https://store.vanguardshield.com/images/gift-set-standard.jpg",
|
||||
"altText": "Client Gift Set \u2014 Standard Edition",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1002",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1002",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "95.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "CGSP-STD",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 120,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-09-28T16:20:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 120
|
||||
},
|
||||
"gid://shopify/Product/1003": {
|
||||
"id": "gid://shopify/Product/1003",
|
||||
"title": "Client Gift Set \u2014 Premium Edition",
|
||||
"description": "Our finest client gift set. Includes VSM branded leather portfolio, premium pen set, executive notebook, water bottle, and luxury gift packaging. Perfect for top-tier clients and special occasions.",
|
||||
"descriptionHtml": "<p>Our finest client gift set. Includes VSM branded leather portfolio, premium pen set, executive notebook, water bottle, and luxury gift packaging. Perfect for top-tier clients and special occasions.</p>",
|
||||
"handle": "client-gift-set-premium-edition",
|
||||
"productType": "Client Gift",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"client gift",
|
||||
"corporate gift",
|
||||
"premium",
|
||||
"executive",
|
||||
"branded"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1003",
|
||||
"url": "https://store.vanguardshield.com/images/gift-set-premium.jpg",
|
||||
"altText": "Client Gift Set \u2014 Premium Edition",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1003",
|
||||
"url": "https://store.vanguardshield.com/images/gift-set-premium.jpg",
|
||||
"altText": "Client Gift Set \u2014 Premium Edition",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1003",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1003",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "CGSP-PREM",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 85,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-10-04T18:23:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 85
|
||||
},
|
||||
"gid://shopify/Product/1004": {
|
||||
"id": "gid://shopify/Product/1004",
|
||||
"title": "VSM Branded Water Bottle (single)",
|
||||
"description": "High-quality stainless steel water bottle with VSM logo. Keeps beverages cold for 24 hours or hot for 12 hours. Perfect for daily use or as a gift item.",
|
||||
"descriptionHtml": "<p>High-quality stainless steel water bottle with VSM logo. Keeps beverages cold for 24 hours or hot for 12 hours. Perfect for daily use or as a gift item.</p>",
|
||||
"handle": "vsm-branded-water-bottle-single",
|
||||
"productType": "Branded Merchandise",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"branded",
|
||||
"water bottle",
|
||||
"merchandise",
|
||||
"drinkware"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "12.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "12.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1004",
|
||||
"url": "https://store.vanguardshield.com/images/water-bottle.jpg",
|
||||
"altText": "VSM Branded Water Bottle",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1004",
|
||||
"url": "https://store.vanguardshield.com/images/water-bottle.jpg",
|
||||
"altText": "VSM Branded Water Bottle",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1004",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1004",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "12.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "BRNW-BTLS",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 500,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-09-19T11:45:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 500
|
||||
},
|
||||
"gid://shopify/Product/1005": {
|
||||
"id": "gid://shopify/Product/1005",
|
||||
"title": "VSM Branded Pen & Portfolio Set",
|
||||
"description": "Professional portfolio with VSM branding and matching pen. Features document pockets, card slots, and notepad. Ideal for client meetings and professional presentations.",
|
||||
"descriptionHtml": "<p>Professional portfolio with VSM branding and matching pen. Features document pockets, card slots, and notepad. Ideal for client meetings and professional presentations.</p>",
|
||||
"handle": "vsm-branded-pen-portfolio-set",
|
||||
"productType": "Branded Merchandise",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"branded",
|
||||
"portfolio",
|
||||
"pen",
|
||||
"professional",
|
||||
"merchandise"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "28.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "28.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1005",
|
||||
"url": "https://store.vanguardshield.com/images/pen-portfolio-set.jpg",
|
||||
"altText": "VSM Branded Pen & Portfolio Set",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1005",
|
||||
"url": "https://store.vanguardshield.com/images/pen-portfolio-set.jpg",
|
||||
"altText": "VSM Branded Pen & Portfolio Set",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1005",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1005",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "28.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "BRNP-PORT",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 175,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-09-15T15:20:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 175
|
||||
},
|
||||
"gid://shopify/Product/1006": {
|
||||
"id": "gid://shopify/Product/1006",
|
||||
"title": "Conference Table Banner (retractable)",
|
||||
"description": "Professional retractable banner featuring VSM branding. Easy to set up and transport. Perfect for conferences, trade shows, and corporate events. Includes carrying case.",
|
||||
"descriptionHtml": "<p>Professional retractable banner featuring VSM branding. Easy to set up and transport. Perfect for conferences, trade shows, and corporate events. Includes carrying case.</p>",
|
||||
"handle": "conference-table-banner-retractable",
|
||||
"productType": "Conference Materials",
|
||||
"vendor": "VSM Print Shop",
|
||||
"tags": [
|
||||
"conference",
|
||||
"banner",
|
||||
"event",
|
||||
"display",
|
||||
"marketing"
|
||||
],
|
||||
"availableForSale": true,
|
||||
"priceRange": {
|
||||
"minVariantPrice": {
|
||||
"amount": "85.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"maxVariantPrice": {
|
||||
"amount": "85.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"featuredImage": {
|
||||
"id": "gid://shopify/Image/1006",
|
||||
"url": "https://store.vanguardshield.com/images/conference-banner.jpg",
|
||||
"altText": "Conference Table Banner (retractable)",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
},
|
||||
"images": [
|
||||
{
|
||||
"id": "gid://shopify/Image/1006",
|
||||
"url": "https://store.vanguardshield.com/images/conference-banner.jpg",
|
||||
"altText": "Conference Table Banner (retractable)",
|
||||
"width": 800,
|
||||
"height": 800
|
||||
}
|
||||
],
|
||||
"options": [
|
||||
{
|
||||
"id": "opt1006",
|
||||
"name": "Title",
|
||||
"values": [
|
||||
"Default Title"
|
||||
]
|
||||
}
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"id": "gid://shopify/ProductVariant/1006",
|
||||
"title": "Default Title",
|
||||
"price": {
|
||||
"amount": "85.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"availableForSale": true,
|
||||
"sku": "CONF-BNRR",
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"quantityAvailable": 45,
|
||||
"requiresShipping": true,
|
||||
"taxable": false
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-01-15T10:00:00.000000Z",
|
||||
"updatedAt": "2026-09-30T13:40:00.000000Z",
|
||||
"publishedAt": "2026-01-15T10:00:00.000000Z",
|
||||
"totalInventory": 45
|
||||
}
|
||||
},
|
||||
"carts": {
|
||||
"gid://shopify/Cart/c1001": {
|
||||
"id": "gid://shopify/Cart/c1001",
|
||||
"checkoutUrl": "https://store.vanguardshield.com/checkout/c1001",
|
||||
"createdAt": "2026-10-10T10:15:30.000000Z",
|
||||
"updatedAt": "2026-10-10T10:18:45.000000Z",
|
||||
"lines": [
|
||||
{
|
||||
"id": "gid://shopify/CartLine/2001",
|
||||
"quantity": 2,
|
||||
"merchandise": {
|
||||
"id": "gid://shopify/ProductVariant/1002",
|
||||
"title": "Default Title",
|
||||
"product": {
|
||||
"id": "gid://shopify/Product/1002",
|
||||
"title": "Client Gift Set \u2014 Standard Edition",
|
||||
"handle": "client-gift-set-standard-edition"
|
||||
},
|
||||
"image": null,
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"price": {
|
||||
"amount": "95.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"cost": {
|
||||
"amountPerQuantity": {
|
||||
"amount": "95.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"compareAtAmountPerQuantity": null,
|
||||
"subtotalAmount": {
|
||||
"amount": "190.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalAmount": {
|
||||
"amount": "190.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"attributes": [],
|
||||
"discountAllocations": []
|
||||
},
|
||||
{
|
||||
"id": "gid://shopify/CartLine/2002",
|
||||
"quantity": 5,
|
||||
"merchandise": {
|
||||
"id": "gid://shopify/ProductVariant/1004",
|
||||
"title": "Default Title",
|
||||
"product": {
|
||||
"id": "gid://shopify/Product/1004",
|
||||
"title": "VSM Branded Water Bottle (single)",
|
||||
"handle": "vsm-branded-water-bottle-single"
|
||||
},
|
||||
"image": null,
|
||||
"selectedOptions": [
|
||||
{
|
||||
"name": "Title",
|
||||
"value": "Default Title"
|
||||
}
|
||||
],
|
||||
"price": {
|
||||
"amount": "12.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"cost": {
|
||||
"amountPerQuantity": {
|
||||
"amount": "12.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"compareAtAmountPerQuantity": null,
|
||||
"subtotalAmount": {
|
||||
"amount": "60.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalAmount": {
|
||||
"amount": "60.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"attributes": [],
|
||||
"discountAllocations": []
|
||||
}
|
||||
],
|
||||
"cost": {
|
||||
"subtotalAmount": {
|
||||
"amount": "250.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"subtotalAmountEstimated": false,
|
||||
"totalAmount": {
|
||||
"amount": "250.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalAmountEstimated": false,
|
||||
"totalTaxAmount": null,
|
||||
"totalTaxAmountEstimated": false,
|
||||
"checkoutChargeAmount": {
|
||||
"amount": "250.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"buyerIdentity": {
|
||||
"email": "m.richardson@vanguardshield.com",
|
||||
"countryCode": "US"
|
||||
},
|
||||
"attributes": [
|
||||
{
|
||||
"key": "department",
|
||||
"value": "Sales - Northeast Region"
|
||||
}
|
||||
],
|
||||
"discountCodes": [],
|
||||
"discountAllocations": [],
|
||||
"appliedGiftCards": [],
|
||||
"deliveryGroups": [],
|
||||
"note": "For upcoming client meetings in Q4",
|
||||
"totalQuantity": 7
|
||||
}
|
||||
},
|
||||
"policies": [
|
||||
{
|
||||
"id": "gid://shopify/ShopPolicy/1",
|
||||
"title": "Internal Ordering Policy",
|
||||
"body": "<h2>VSM Internal Store Ordering Policy</h2><p>This store is exclusively for Vanguard Shield Mutual employees ordering branded merchandise, client gifts, and conference materials.</p><h3>Eligibility</h3><p>All VSM employees with valid employee IDs may place orders. Orders must be approved by department managers for quantities exceeding standard limits.</p><h3>Approval Requirements</h3><ul><li>Standard merchandise (under $100): No approval needed</li><li>Client gifts ($100-$500): Manager approval required</li><li>Bulk orders or premium gifts (over $500): Director approval required</li></ul><h3>Shipping</h3><p>Orders ship to VSM office locations or approved conference venues only. Allow 5-7 business days for standard delivery. Rush orders available with manager approval.</p><h3>Billing</h3><p>All orders are billed to department cost centers. Ensure accurate department coding during checkout.</p>",
|
||||
"url": "https://store.vanguardshield.com/policies/ordering"
|
||||
},
|
||||
{
|
||||
"id": "gid://shopify/ShopPolicy/2",
|
||||
"title": "Returns and Exchanges",
|
||||
"body": "<h2>Returns and Exchanges</h2><p>We accept returns on unused, unopened items within 30 days of delivery.</p><h3>Return Process</h3><ol><li>Contact store-admin@vanguardshield.com with your order number</li><li>Receive return authorization and shipping label</li><li>Package items securely and ship within 7 days</li></ol><h3>Non-Returnable Items</h3><ul><li>Personalized or custom-ordered items</li><li>Items ordered for specific conferences or events (within 14 days of event)</li><li>Opened client gift sets</li></ul><h3>Exchanges</h3><p>Defective items may be exchanged at any time. Contact us immediately if you receive damaged or incorrect merchandise.</p>",
|
||||
"url": "https://store.vanguardshield.com/policies/returns"
|
||||
},
|
||||
{
|
||||
"id": "gid://shopify/ShopPolicy/3",
|
||||
"title": "Client Gift Guidelines",
|
||||
"body": "<h2>Client Gift Guidelines</h2><p>VSM client gifts are designed to strengthen relationships while adhering to industry compliance standards.</p><h3>Gift Value Limits</h3><p>Standard Edition gifts are appropriate for routine client appreciation. Premium Edition gifts require Regional Director approval and are reserved for:</p><ul><li>Key account milestones (10+ year relationships)</li><li>Significant new business partnerships</li><li>Executive-level relationship building</li></ul><h3>Compliance</h3><p>All gifts must comply with recipient organization policies. Sales teams are responsible for verifying gift acceptance policies before ordering. Gifts to government or regulated entity employees may require additional approval.</p><h3>Documentation</h3><p>Retain order confirmations for expense reporting. Client gift expenses are tracked by the Finance team for compliance reporting.</p>",
|
||||
"url": "https://store.vanguardshield.com/policies/client-gifts"
|
||||
},
|
||||
{
|
||||
"id": "gid://shopify/ShopPolicy/4",
|
||||
"title": "Frequently Asked Questions",
|
||||
"body": "<h2>Frequently Asked Questions</h2><h3>How do I place an order?</h3><p>Browse products, add items to your cart, and complete checkout. Use your VSM email address and ensure department information is accurate for billing purposes.</p><h3>Can I ship to my home address?</h3><p>No. All orders must ship to VSM office locations or approved business venues. Contact store-admin@vanguardshield.com for special shipping requests.</p><h3>How long does shipping take?</h3><p>Standard shipping is 5-7 business days. Rush delivery (2-3 days) is available with manager approval for urgent orders.</p><h3>What if I need a product not listed in the store?</h3><p>Contact the VSM Print Shop team at store-admin@vanguardshield.com to discuss custom merchandise or special orders. Custom orders require 3-4 weeks lead time.</p><h3>Can I order in bulk?</h3><p>Yes. Bulk orders (20+ units) may qualify for volume discounts. Contact store-admin@vanguardshield.com for pricing and availability.</p><h3>Who do I contact with questions?</h3><p>Email store-admin@vanguardshield.com or call the Print Shop at extension 5500 during business hours (Monday-Friday, 8:00 AM - 5:00 PM ET).</p>",
|
||||
"url": "https://store.vanguardshield.com/policies/faq"
|
||||
}
|
||||
],
|
||||
"counters": {
|
||||
"cart_id": 1001,
|
||||
"line_id": 2002
|
||||
},
|
||||
"orders": {
|
||||
"gid://shopify/Order/5001": {
|
||||
"id": "gid://shopify/Order/5001",
|
||||
"name": "#VSM-5001",
|
||||
"createdAt": "2026-10-04T10:22:00.000000Z",
|
||||
"updatedAt": "2026-10-04T10:22:00.000000Z",
|
||||
"fulfillmentStatus": "FULFILLED",
|
||||
"financialStatus": "PAID",
|
||||
"totalPriceSet": {
|
||||
"shopMoney": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"customer": {
|
||||
"email": "v.frost@vanguardshield.com",
|
||||
"firstName": "Vanessa",
|
||||
"lastName": "Frost",
|
||||
"displayName": "Vanessa Frost"
|
||||
},
|
||||
"lineItems": [
|
||||
{
|
||||
"title": "Client Gift Set \u2014 Premium Edition",
|
||||
"quantity": 1,
|
||||
"variant": {
|
||||
"id": "gid://shopify/ProductVariant/1003",
|
||||
"title": "Default Title",
|
||||
"sku": "CGSP-PREM",
|
||||
"price": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"originalTotalSet": {
|
||||
"shopMoney": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"id": "gid://shopify/ProductVariant/1003",
|
||||
"price": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalPrice": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
}
|
||||
],
|
||||
"note": "Q4 client engagement \u2014 Meridian Solutions account",
|
||||
"tags": [
|
||||
"sales-east",
|
||||
"client-gift"
|
||||
],
|
||||
"shippingAddress": {
|
||||
"address1": "100 Pearl St",
|
||||
"city": "Hartford",
|
||||
"province": "Connecticut",
|
||||
"provinceCode": "CT",
|
||||
"zip": "06103",
|
||||
"country": "United States",
|
||||
"countryCodeV2": "US"
|
||||
},
|
||||
"subtotalPrice": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalPrice": {
|
||||
"amount": "175.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"gid://shopify/Order/5002": {
|
||||
"id": "gid://shopify/Order/5002",
|
||||
"name": "#VSM-5002",
|
||||
"createdAt": "2026-09-15T14:08:00.000000Z",
|
||||
"updatedAt": "2026-09-15T14:08:00.000000Z",
|
||||
"fulfillmentStatus": "FULFILLED",
|
||||
"financialStatus": "PAID",
|
||||
"totalPriceSet": {
|
||||
"shopMoney": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"customer": {
|
||||
"email": "j.calloway@vanguardshield.com",
|
||||
"firstName": "James",
|
||||
"lastName": "Calloway",
|
||||
"displayName": "James Calloway"
|
||||
},
|
||||
"lineItems": [
|
||||
{
|
||||
"title": "VSM Branded Notebook Set (12-pack)",
|
||||
"quantity": 1,
|
||||
"variant": {
|
||||
"id": "gid://shopify/ProductVariant/1001",
|
||||
"title": "Default Title",
|
||||
"sku": "BRNB-12PK",
|
||||
"price": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"originalTotalSet": {
|
||||
"shopMoney": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
},
|
||||
"id": "gid://shopify/ProductVariant/1001",
|
||||
"price": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalPrice": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
}
|
||||
],
|
||||
"note": "Client gifts for Pacific Ventures meetings",
|
||||
"tags": [
|
||||
"sales-west",
|
||||
"branded-merch"
|
||||
],
|
||||
"shippingAddress": {
|
||||
"address1": "350 Mission St",
|
||||
"city": "San Francisco",
|
||||
"province": "California",
|
||||
"provinceCode": "CA",
|
||||
"zip": "94105",
|
||||
"country": "United States",
|
||||
"countryCodeV2": "US"
|
||||
},
|
||||
"subtotalPrice": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
},
|
||||
"totalPrice": {
|
||||
"amount": "48.00",
|
||||
"currencyCode": "USD"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "analyst",
|
||||
"deleted": false,
|
||||
"real_name": "Analyst",
|
||||
"profile": {
|
||||
"real_name": "Analyst",
|
||||
"display_name": "Analyst",
|
||||
"email": "analyst@vanguardshield.com",
|
||||
"title": "Operations Analyst \u2014 HR & Sales"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "pkim",
|
||||
"deleted": false,
|
||||
"real_name": "Peter Kim",
|
||||
"profile": {
|
||||
"real_name": "Peter Kim",
|
||||
"display_name": "Peter Kim",
|
||||
"email": "peter.kim@vanguardshield.com",
|
||||
"title": "Sales Director \u2014 West Region"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "mharris",
|
||||
"deleted": false,
|
||||
"real_name": "Maria Harris",
|
||||
"profile": {
|
||||
"real_name": "Maria Harris",
|
||||
"display_name": "Maria Harris",
|
||||
"email": "mharris@vanguardshield.com",
|
||||
"title": "Sales Manager \u2014 East Region"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "jmorales",
|
||||
"deleted": false,
|
||||
"real_name": "Jessica Morales",
|
||||
"profile": {
|
||||
"real_name": "Jessica Morales",
|
||||
"display_name": "Jessica Morales",
|
||||
"email": "j.morales@vanguardshield.com",
|
||||
"title": "HR Operations Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "hrops.bot",
|
||||
"deleted": false,
|
||||
"real_name": "HR Ops Bot",
|
||||
"profile": {
|
||||
"real_name": "HR Ops Bot",
|
||||
"display_name": "HR Ops Bot",
|
||||
"email": "hrops-bot@vanguardshield.com",
|
||||
"title": "Automation Bot"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": true,
|
||||
"is_app_user": true
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "lwilliams",
|
||||
"deleted": false,
|
||||
"real_name": "Lisa Williams",
|
||||
"profile": {
|
||||
"real_name": "Lisa Williams",
|
||||
"display_name": "Lisa Williams",
|
||||
"email": "l.williams@vanguardshield.com",
|
||||
"title": "VP Sales"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "rokonkwo",
|
||||
"deleted": false,
|
||||
"real_name": "Rachel Okonkwo",
|
||||
"profile": {
|
||||
"real_name": "Rachel Okonkwo",
|
||||
"display_name": "Rachel Okonkwo",
|
||||
"email": "r.okonkwo@vanguardshield.com",
|
||||
"title": "Account Executive \u2014 Sales West"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "dhuang",
|
||||
"deleted": false,
|
||||
"real_name": "Derek Huang",
|
||||
"profile": {
|
||||
"real_name": "Derek Huang",
|
||||
"display_name": "Derek Huang",
|
||||
"email": "d.huang@vanguardshield.com",
|
||||
"title": "Regional Sales Lead \u2014 Midwest"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U009": {
|
||||
"id": "U009",
|
||||
"team_id": "T_VANGUARD",
|
||||
"name": "mvance",
|
||||
"deleted": false,
|
||||
"real_name": "Marcus Vance",
|
||||
"profile": {
|
||||
"real_name": "Marcus Vance",
|
||||
"display_name": "Marcus Vance",
|
||||
"email": "m.vance@vanguardshield.com",
|
||||
"title": "Finance Controller"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"CHROPS": {
|
||||
"id": "CHROPS",
|
||||
"name": "hr-ops-help",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "hr-ops-help",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T_VANGUARD",
|
||||
"updated": 1791392400,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": [
|
||||
"T_VANGUARD"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "HR operations help and manager lookup escalations",
|
||||
"creator": "U004",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "HR operations help and manager lookup escalations",
|
||||
"creator": "U004",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"CSALESOPS": {
|
||||
"id": "CSALESOPS",
|
||||
"name": "sales-ops",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "sales-ops",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T_VANGUARD",
|
||||
"updated": 1791392400,
|
||||
"creator": "U006",
|
||||
"shared_team_ids": [
|
||||
"T_VANGUARD"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Sales team coordination and ops updates",
|
||||
"creator": "U006",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Sales team operational updates \u2014 territory, quotas, and scheduling",
|
||||
"creator": "U006",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"CFINAPPR": {
|
||||
"id": "CFINAPPR",
|
||||
"name": "finance-approvals",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1600000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "finance-approvals",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T_VANGUARD",
|
||||
"updated": 1791392400,
|
||||
"creator": "U009",
|
||||
"shared_team_ids": [
|
||||
"T_VANGUARD"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Finance approval threads \u2014 intercompany suspense and GL reclassifications",
|
||||
"creator": "U009",
|
||||
"last_set": 1600000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Finance approval threads \u2014 intercompany suspense and GL reclassifications",
|
||||
"creator": "U009",
|
||||
"last_set": 1600000000
|
||||
}
|
||||
},
|
||||
"DU002": {
|
||||
"id": "DU002",
|
||||
"name": "pkim",
|
||||
"is_channel": false,
|
||||
"is_group": false,
|
||||
"is_im": true,
|
||||
"is_mpim": false,
|
||||
"is_private": true,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"user": "U002",
|
||||
"context_team_id": "T_VANGUARD",
|
||||
"members": [
|
||||
"U002"
|
||||
]
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"CHROPS": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U004",
|
||||
"text": "Thanks @hrops.bot \u2014 OPS-HR-TE-0912 closed. Adding note to Amanda Chen's Jira employee record.",
|
||||
"ts": "1790085600.000200",
|
||||
"team": "T_VANGUARD"
|
||||
}
|
||||
],
|
||||
"CSALESOPS": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U006",
|
||||
"text": "Q4 kick-off is officially live. All territory assignments have been updated in Salesforce. Route through the standard portal.",
|
||||
"ts": "1790845200.000200",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U008",
|
||||
"text": "Got it \u2014 heading out for the kickoff today. Will submit my report when I\u2019m back.",
|
||||
"ts": "1790870400.000300",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "Off to the conference tomorrow morning. Will have it in on time.",
|
||||
"ts": "1790888400.000100",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U006",
|
||||
"text": "Quick note from my sync with the CFO: the Q4 expense policy is unchanged from Q3.",
|
||||
"ts": "1791190800.000200",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U008",
|
||||
"text": "All good on my end.",
|
||||
"ts": "1791388800.000100",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U007",
|
||||
"text": "All wrapped up here.",
|
||||
"ts": "1791390600.000200",
|
||||
"team": "T_VANGUARD"
|
||||
}
|
||||
],
|
||||
"CFINAPPR": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U009",
|
||||
"text": "Approved: Intercompany reclassification $7,250.00 \u2014 OPS-FIN-SUSPENSE-2026-09 line 8. GL 4510 \u2192 4520. https://jira.vanguardshield.com/OPS-FIN-SUSPENSE-2026-09",
|
||||
"ts": "1790679600.000300",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U009",
|
||||
"text": "Approved: Intercompany reclassification $3,800.00 \u2014 OPS-FIN-SUSPENSE-2026-09 line 14. GL 5100 \u2192 5200. https://jira.vanguardshield.com/OPS-FIN-SUSPENSE-2026-09",
|
||||
"ts": "1790679600.000600",
|
||||
"team": "T_VANGUARD"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U009",
|
||||
"text": "Sep month-end suspense review complete. Two items reclassified, three items pending Controller review from the Sep close package. Finance team: flag anything from the Oct 5 close package to this channel.",
|
||||
"ts": "1791187200.000100",
|
||||
"team": "T_VANGUARD"
|
||||
}
|
||||
],
|
||||
"DU002": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 2000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
I've got three T&E reports that need processing. Can you work through each one following the attached SOP?
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, October, 8, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/insurance_vanguard_shield_mutual_90ff0751"
|
||||
description = "SOP task from CSV row 90ff0751"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
[
|
||||
{
|
||||
"id": "eccaf388-6d88-479e-ae0a-734020b8b053",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In external_data/final/jira_state.json, issue OPS-3 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references VSM Branded Water Bottles, water bottle, or Oct 1, AND flags the expense as missing business justification/unauthorized vendor.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Try multiple paths to find jira_state.json\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'external_data' / 'final' / 'jira_state.json')\n\n jira_data = None\n found_path = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n found_path = str(p)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in any expected location'}\n\n issue_id = 'OPS-3'\n issue = None\n\n if isinstance(jira_data, dict):\n if 'issues' in jira_data:\n issues_obj = jira_data['issues']\n if isinstance(issues_obj, dict):\n if issue_id in issues_obj:\n issue = issues_obj[issue_id]\n else:\n for key, val in issues_obj.items():\n if isinstance(val, dict) and (val.get('id') == issue_id or val.get('key') == issue_id):\n issue = val\n break\n elif isinstance(issues_obj, list):\n for iss in issues_obj:\n if isinstance(iss, dict) and (iss.get('id') == issue_id or iss.get('key') == issue_id):\n issue = iss\n break\n\n if issue is None and issue_id in jira_data:\n issue = jira_data[issue_id]\n\n if issue is None:\n for key, val in jira_data.items():\n if isinstance(val, dict):\n if val.get('id') == issue_id or val.get('key') == issue_id:\n issue = val\n break\n elif isinstance(jira_data, list):\n for iss in jira_data:\n if isinstance(iss, dict) and (iss.get('id') == issue_id or iss.get('key') == issue_id):\n issue = iss\n break\n\n # Collect comments from BOTH the issue object AND the top-level comments dict\n comments = []\n\n # 1) Comments stored on the issue itself\n if issue is not None:\n issue_comments = issue.get('comments', [])\n if isinstance(issue_comments, list):\n comments.extend(issue_comments)\n fields = issue.get('fields', {})\n if isinstance(fields, dict):\n comment_field = fields.get('comment', {})\n if isinstance(comment_field, dict):\n inner = comment_field.get('comments', [])\n if isinstance(inner, list):\n comments.extend(inner)\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n # 2) Comments stored in the top-level 'comments' collection, keyed by issue id\n if isinstance(jira_data, dict):\n top_comments = jira_data.get('comments')\n if isinstance(top_comments, dict):\n entry = top_comments.get(issue_id)\n if isinstance(entry, list):\n comments.extend(entry)\n elif isinstance(entry, dict):\n # in case it's wrapped, e.g. {'comments': [...]}\n inner = entry.get('comments')\n if isinstance(inner, list):\n comments.extend(inner)\n\n if issue is None and not comments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Issue {issue_id} not found and no top-level comments for it either (loaded from {found_path}).'}\n\n def extract_adf_text(node):\n \"\"\"Recursively pull every 'text' string out of an Atlassian Document Format dict.\"\"\"\n out = []\n if isinstance(node, dict):\n if isinstance(node.get('text'), str):\n out.append(node['text'])\n for v in node.values():\n if isinstance(v, (dict, list)):\n out.extend(extract_adf_text(v))\n elif isinstance(node, list):\n for item in node:\n out.extend(extract_adf_text(item))\n return out\n\n def get_comment_text(c):\n if isinstance(c, str):\n return c\n if isinstance(c, dict):\n for field in ['body', 'text', 'comment', 'content', 'message']:\n if field in c:\n val = c[field]\n if isinstance(val, str):\n return val\n if isinstance(val, (dict, list)):\n pieces = extract_adf_text(val)\n if pieces:\n return ' '.join(pieces)\n return json.dumps(c)\n return str(c)\n\n def has_justification(textLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "c713cced-31be-4221-8419-072a071ecab7",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In jira_state.json, issue OPS-3 must NOT contain any comment where\nthe text contains ('Capital Grille' or 'Marriott' or 'Hartwell Group')\nAND contains 'Missing business justification'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n\n jira_data = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n\n # Collect all issues\n all_issues = []\n if isinstance(jira_data, dict):\n if 'issues' in jira_data:\n issues_val = jira_data['issues']\n if isinstance(issues_val, list):\n all_issues = issues_val\n elif isinstance(issues_val, dict):\n all_issues = list(issues_val.values())\n else:\n all_issues = list(jira_data.values())\n elif isinstance(jira_data, list):\n all_issues = jira_data\n\n # Find issue OPS-3\n target_issue = None\n for iss in all_issues:\n if not isinstance(iss, dict):\n continue\n issue_key = str(iss.get('key', iss.get('id', iss.get('issue_key', '')))).strip()\n if issue_key.upper() == 'OPS-3':\n target_issue = iss\n break\n\n if target_issue is None:\n # Issue not found at all - that means no violating comment exists, so pass\n return {'pass': True, 'score': 1.0, 'feedback': 'Issue OPS-3 not found in jira_state.json, so no violating comment exists. PASS.'}\n\n # Get comments from the issue\n comments = []\n if 'comments' in target_issue:\n c = target_issue['comments']\n if isinstance(c, list):\n comments = c\n elif isinstance(c, dict):\n comments = list(c.values())\n # Also check nested fields structure\n if 'fields' in target_issue and isinstance(target_issue['fields'], dict):\n fields = target_issue['fields']\n if 'comment' in fields:\n comment_field = fields['comment']\n if isinstance(comment_field, dict) and 'comments' in comment_field:\n comments.extend(comment_field['comments'] if isinstance(comment_field['comments'], list) else list(comment_field['comments'].values()))\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n vendor_keywords = ['capital grille', 'marriott', 'hartwell group']\n justification_keyword = 'missing business justification'\n\n violating_comments = []\n for comment in comments:\n if isinstance(comment, dict):\n text = str(comment.get('body', comment.get('text', comment.get('content', '')))).lower()\n elif isinstance(comment, str):\n text = comment.lower()\n else:\n continue\n\n has_vendor = any(kw in text for kw in vendor_keywords)\n has_justification = justification_keyword in text\n\n if has_vendor and has_justification:\n violating_comments.append(text[:200])\n\n if len(violating_comments) > 0:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'FAIL: Issue OPS-3 contains {len(violating_comments)} comment(s) that mention a vendor keyword (Capital Grille/Marriott/Hartwell Group) AND \"Missing business justification\". Violating comment excerpts: {violating_comments}'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'PASS: Issue OPS-3 found with {len(comments)} comment(s), none of which contain both a vendor keyword and \"Missing business justification\".'\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "6a839ad9-3949-4bf7-9002-63258e6e5e43",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references Sullivan's Steakhouse or Oct 3, AND flags the expense as missing business justification/unauthorized vendor.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef _extract_adf_text(node):\n \"\"\"Recursively extract plain text from an Atlassian Document Format node.\"\"\"\n if node is None:\n return ''\n if isinstance(node, str):\n return node\n if isinstance(node, list):\n return ' '.join(_extract_adf_text(n) for n in node)\n if isinstance(node, dict):\n parts = []\n # Direct text field (ADF text nodes have type='text' and a 'text' key)\n if isinstance(node.get('text'), str):\n parts.append(node['text'])\n # Recurse into 'content' which holds child nodes\n if 'content' in node:\n parts.append(_extract_adf_text(node['content']))\n return ' '.join(p for p in parts if p)\n return ''\n\n\ndef _comment_to_text(comment):\n \"\"\"Turn a comment entry (string or dict) into a single text string.\"\"\"\n if isinstance(comment, str):\n return comment\n if not isinstance(comment, dict):\n return ''\n # Try common text-bearing fields\n for field in ['body', 'text', 'content', 'message', 'comment']:\n if field in comment:\n val = comment[field]\n if isinstance(val, str):\n return val\n if isinstance(val, dict) or isinstance(val, list):\n extracted = _extract_adf_text(val)\n if extracted:\n return extracted\n # Fallback: stringify whole comment\n return json.dumps(comment)\n\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in multiple possible locations\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'external_data' / 'final' / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n\n jira_data = None\n jira_path_used = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n jira_path_used = str(p)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'jira_state.json not found in any expected location. Tried: '\n + str([str(p) for p in paths_to_try])}\n\n ISSUE_KEY = 'OPS-4'\n\n # ---- Collect all comments associated with the issue from any location ----\n collected_comments = []\n\n if isinstance(jira_data, dict):\n # 1) Top-level 'comments' collection keyed by issue ID (the case the old verifier missed)\n top_comments = jira_data.get('comments')\n if isinstance(top_comments, dict):\n entry = top_comments.get(ISSUE_KEY)\n if isinstance(entry, list):\n collected_comments.extend(entry)\n elif entry:\n collected_comments.append(entry)\n elif isinstance(top_comments, list):\n # Possibly a flat list with issue references on each comment\n for c in top_comments:\n if isinstance(c, dict):\n ref = c.get('issue') or c.get('issueKey') or c.get('issue_id') or c.get('key')\n if ref == ISSUE_KEY:\n collected_comments.append(c)\n\n # 2) Find the issue object itself (various shapes)\n target_issue = None\n issues_obj = jira_data.get('issues')\n if isinstance(issues_obj, dict):\n target_issue = issues_obj.get(ISSUE_KEY)\n elif isinstance(issues_obj, list):\n for issue in issues_obj:\n if isinstance(issue, dict):\n k = issue.get('key') or issue.get('id') or ''\n if k == ISSUE_KEY:\n target_issue = issue\n break\n if target_issue is None and ISSUE_KEY in jira_data:\n target_issue = jira_data[ISSUE_KEY]\n\n # 3) Pull comments embedded in the issue itself\n if isinstance(target_issue, dict):\n if 'comments' in target_issue:\n c = target_issue['comments']\n if isinstance(c, list):\n collected_comments.extend(c)\n elif c:\n collected_comments.append(c)\n if 'fields' in target_issue and isinstance(target_issue['fields'], dict):\n fields = target_issue['fields']\n cf = fields.get('comment')\n if isinstance(cf, dict) and isinstance(cf.get('comments'), list):\n collected_comments.extend(cf['comments'])\n elif isinstance(cf, list):\n collected_comments.extend(cf)\n if isinstance(fields.get('comments'), list):\n collected_comments.extend(fields['comments'])\n\n if len(collected_commeLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880280596",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references Hudson's Coffee or Oct 4, AND flags the expense as missing business justification/unauthorized vendor.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text_from_adf(node):\n \"\"\"Walk an Atlassian Document Format dict/list and return concatenated text.\n If given a plain string, returns it. If given anything else, returns ''.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n if isinstance(node, list):\n return \" \".join(extract_text_from_adf(n) for n in node)\n if isinstance(node, dict):\n parts = []\n # ADF text nodes have a \"text\" string field\n if isinstance(node.get(\"text\"), str):\n parts.append(node[\"text\"])\n # Recurse into \"content\" (ADF's child container)\n if \"content\" in node:\n parts.append(extract_text_from_adf(node[\"content\"]))\n return \" \".join(p for p in parts if p)\n return \"\"\n\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in multiple locations\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n candidates.append(Path(external_services_path) / \"final\" / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\")\n for base in [external_services_path, workspace_path]:\n if base:\n for p in Path(base).rglob(\"jira_state.json\"):\n candidates.append(p)\n\n for c in candidates:\n if c and c.exists():\n jira_path = c\n break\n\n if jira_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Could not find jira_state.json in workspace or external services path.\"}\n\n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n\n ISSUE_KEY = \"OPS-4\"\n\n # --- Locate the issue object (may or may not exist; comments can live elsewhere) ---\n issue = None\n if isinstance(jira_data, dict):\n if \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, dict):\n if ISSUE_KEY in issues:\n issue = issues[ISSUE_KEY]\n else:\n for v in issues.values():\n if isinstance(v, dict) and (\n v.get(\"key\") == ISSUE_KEY or v.get(\"id\") == ISSUE_KEY or v.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = v\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and (\n item.get(\"key\") == ISSUE_KEY or item.get(\"id\") == ISSUE_KEY or item.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = item\n break\n if issue is None and ISSUE_KEY in jira_data:\n issue = jira_data[ISSUE_KEY]\n elif isinstance(jira_data, list):\n for item in jira_data:\n if isinstance(item, dict) and (\n item.get(\"key\") == ISSUE_KEY or item.get(\"id\") == ISSUE_KEY or item.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = item\n break\n\n # --- Collect comments from BOTH locations ---\n comments = []\n\n # Location A: top-level \"comments\" collection keyed by issue ID\n if isinstance(jira_data, dict) and isinstance(jira_data.get(\"comments\"), dict):\n top_comments = jira_data[\"comments\"].get(ISSUE_KEY)\n if isinstance(top_comments, list):\n comments.extend(top_comments)\n elif isinstance(top_comments, dict):\n comments.append(top_comments)\n\n # Location B: inside the issue object itself\n if isinstance(issue, dict):\n if \"comments\" in issue:\n c = issue[\"comments\"]\n if isinstance(c, list):\n comments.extend(c)\n elif isinstance(c, dict):\n comments.append(c)\n elif \"comment\" in issue:\n c = issue[\"comment\"]\n if isinstance(c, list):\n comments.extend(c)\n elif isinstance(c, dict):\n if \"comments\" in c and isinstance(c[\"comments\"], list):\n comments.extend(c[\"comments\"])\n else:\n comments.append(c)\n elif \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n fc = issue[\"fields\"].get(\"comment\")\n if isinstance(fc, dict) and isinstance(fc.get(\"comments\"), list):\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880300763",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must NOT contain any comment where Nexus Corp or Oct 5 appears within the same sentence as Missing business justification, meaning the Nexus Corp / Oct 5 line is itself being flagged for rejection, not merely mentioned as a supported line within a broader rejection comment.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef _extract_adf_text(node):\n \"\"\"Recursively pull all 'text' fields out of an Atlassian Document Format body.\"\"\"\n parts = []\n if isinstance(node, dict):\n if isinstance(node.get(\"text\"), str):\n parts.append(node[\"text\"])\n for v in node.values():\n if isinstance(v, (dict, list)):\n parts.extend(_extract_adf_text(v))\n elif isinstance(node, list):\n for item in node:\n parts.extend(_extract_adf_text(item))\n return parts\n\ndef _comment_text(comment):\n \"\"\"Return a plain-text string for one comment entry.\"\"\"\n if isinstance(comment, str):\n return comment\n if not isinstance(comment, dict):\n return \"\"\n body = comment.get(\"body\")\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n return \" \".join(_extract_adf_text(body))\n if isinstance(body, list):\n return \" \".join(_extract_adf_text(body))\n # Fallbacks just in case\n for field in (\"text\", \"content\", \"message\"):\n val = comment.get(field)\n if isinstance(val, str):\n return val\n return \"\"\n\ndef _split_sentences(text):\n \"\"\"Split on sentence terminators and newlines. Keeps things simple.\"\"\"\n # Replace newlines with a period so list items / line breaks count as sentence boundaries.\n normalized = re.sub(r'[\\r\\n]+', '. ', text)\n # Split on . ! ? followed by space or end of string.\n sentences = re.split(r'(?<=[.!?])\\s+', normalized)\n return [s.strip() for s in sentences if s.strip()]\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n candidates = []\n ws = Path(workspace_path)\n candidates.append(ws / \"jira_state.json\")\n candidates.append(ws / \"external_data\" / \"final\" / \"jira_state.json\")\n if external_services_path:\n esp = Path(external_services_path)\n candidates.append(esp / \"jira_state.json\")\n candidates.append(esp / \"final\" / \"jira_state.json\")\n candidates.append(esp / \"external_data\" / \"final\" / \"jira_state.json\")\n\n jira_path = None\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n if jira_path is None:\n found = list(ws.rglob(\"jira_state.json\"))\n if external_services_path:\n found += list(Path(external_services_path).rglob(\"jira_state.json\"))\n if found:\n jira_path = found[0]\n\n if jira_path is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"jira_state.json not found; forbidden comment cannot exist, passing by default.\"}\n\n try:\n with open(jira_path, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Could not parse jira_state.json ({e}); passing by default.\"}\n\n # Comments live at jira_data[\"comments\"][\"OPS-4\"]\n comments = []\n if isinstance(jira_data, dict):\n top_comments = jira_data.get(\"comments\")\n if isinstance(top_comments, dict):\n entry = top_comments.get(\"OPS-4\")\n if isinstance(entry, list):\n comments = entry\n\n if not comments:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"No comments found for OPS-4; forbidden comment does not exist.\"}\n\n violating = []\n for i, comment in enumerate(comments):\n text = _comment_text(comment)\n if not text:\n continue\n\n for sentence in _split_sentences(text):\n s_lower = sentence.lower()\n has_trigger = (\"nexus corp\" in s_lower) or (\"oct 5\" in s_lower) or (\"october 5\" in s_lower)\n has_missing_just = \"missing business justification\" in s_lower\n if has_trigger and has_missing_just:\n violating.append((i, sentence[:200]))\n break # one violating sentence per comment is enough\n\n if violating:\n details = \"; \".join([f\"Comment #{idx}: '{snippet}'\" for idx, snippet in violating])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (f\"FAIL: Found {len(violating)} comment(s) on OPS-4 where \"\n f\"('Nexus Corp' or 'Oct 5') appears in the same sentence as \"\n f\"'Missing business justification'. {details}\")\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (f\"PASS: Checked {len(comments)} comment(s) on OPS-4; \"\n f\"no single sentence contains both ('Nexus Corp'/'Oct 5') and \"\n f\"'Missing business justification'.\")\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880314655",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In external_data/final/jira_state.json, a standalone issue must exist under the top-level 'issues' object with 'Manager Lookup Needed' and 'T&E Exception' in its fields.summary, and 'OPS-4' in the text content of its fields.description (search nested content[].content[].text if fields.description is a document object; optionally also 'Derek Huang' or 'EMP-4507')",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n jira_data = None\n checked_paths = []\n\n # Try external_services_path first\n if external_services_path:\n for candidate in [\n Path(external_services_path) / \"jira_state.json\",\n Path(external_services_path) / \"final\" / \"jira_state.json\",\n ]:\n checked_paths.append(str(candidate))\n if candidate.exists():\n try:\n with open(candidate, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n # Also try workspace_path\n if jira_data is None:\n for candidate in [\n Path(workspace_path) / \"jira_state.json\",\n Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\",\n ]:\n checked_paths.append(str(candidate))\n if candidate.exists():\n try:\n with open(candidate, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n # Try glob as last resort\n if jira_data is None:\n for p in Path(workspace_path).rglob(\"jira_state.json\"):\n checked_paths.append(str(p))\n try:\n with open(p, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n if jira_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found. Checked: {checked_paths}\"}\n\n # Extract issues from the top-level 'issues' object\n issues = []\n if isinstance(jira_data, dict):\n if \"issues\" in jira_data:\n raw_issues = jira_data[\"issues\"]\n if isinstance(raw_issues, list):\n issues = raw_issues\n elif isinstance(raw_issues, dict):\n issues = list(raw_issues.values())\n else:\n for k, v in jira_data.items():\n if isinstance(v, dict):\n issues.append(v)\n elif isinstance(jira_data, list):\n issues = jira_data\n\n if not issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No issues found in jira_state.json. Top-level keys: {list(jira_data.keys()) if isinstance(jira_data, dict) else 'N/A (list)'}\"}\n\n def extract_text_from_description(desc):\n \"\"\"Extract all text from a description field, handling both string and ADF document objects.\"\"\"\n if isinstance(desc, str):\n return desc\n if isinstance(desc, dict):\n # Atlassian Document Format (ADF) - nested content[].content[].text\n texts = []\n _extract_text_recursive(desc, texts)\n return \" \".join(texts)\n return \"\"\n\n def _extract_text_recursive(node, texts):\n \"\"\"Recursively extract text from an ADF node.\"\"\"\n if isinstance(node, dict):\n if \"text\" in node and isinstance(node[\"text\"], str):\n texts.append(node[\"text\"])\n if \"content\" in node and isinstance(node[\"content\"], list):\n for child in node[\"content\"]:\n _extract_text_recursive(child, texts)\n # Also check other possible nested structures\n for key in [\"value\", \"body\"]:\n if key in node:\n _extract_text_recursive(node[key], texts)\n elif isinstance(node, list):\n for item in node:\n _extract_text_recursive(item, texts)\n\n def get_summary(issue):\n \"\"\"Get summary from an issue, checking nested 'fields' dict and flat keys.\"\"\"\n val = \"\"\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n for key_variant in [\"summary\", \"Summary\", \"SUMMARY\"]:\n if key_variant in issue[\"fields\"]:\n candidate = issue[\"fields\"][key_variant]\n if isinstance(candidate, str):\n return candidate\n for key_variant in [\"summary\", \"Summary\", \"SUMMARY\", \"title\", \"Title\", \"name\", \"Name\"]:\n if key_variant in issue and isinstance(issue[key_variant], str):\n return issue[key_variant]\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n for alt in [\"title\", \"Title\", \"name\", \"Name\"]:\n if alt in issue[\"fields\"] and isinstance(issue[\"fields\"][alt], str):\n return issue[\"fields\"][alt]\n return val\n\n def get_description_text(issue):\n \"\"\"Get full text of descriptioLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880364263",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In external_data/final/jira_state.json, the issue keyed 'OPS-5' under the top-level 'issues' object must NOT contain any comment where the text contains ('Foundry Bistro' or 'Capital Grille' or 'Harvest Kitchen') AND contains 'Missing business justification'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Rubric: In external_data/final/jira_state.json, the issue keyed 'OPS-5'\n # under the top-level 'issues' object must NOT contain any comment where the text\n # contains ('Foundry Bistro' or 'Capital Grille' or 'Harvest Kitchen') AND contains\n # 'Missing business justification'\n\n jira_data = None\n checked_paths = []\n\n # Build candidate paths in order of priority\n candidates = []\n\n # The rubric says \"external_data/final/jira_state.json\" which is relative to workspace\n candidates.append(Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n\n # Also try external_services_path (jira is an external service)\n if external_services_path is not None:\n candidates.insert(0, Path(external_services_path) / \"jira_state.json\")\n\n for p in candidates:\n checked_paths.append(str(p))\n if p.exists():\n try:\n with open(p, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse {p}: {e}\"}\n\n if jira_data is None:\n # All three runs failed, so we should not pass by default.\n # The file not being found likely means the agent didn't produce expected output.\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at any of: {checked_paths}. Cannot verify the rubric without this file.\"}\n\n target_key = \"OPS-5\"\n issue = None\n\n # The rubric says: \"the issue keyed 'OPS-5' under the top-level 'issues' object\"\n if \"issues\" in jira_data:\n issues_obj = jira_data[\"issues\"]\n if isinstance(issues_obj, dict):\n if target_key in issues_obj:\n issue = issues_obj[target_key]\n elif isinstance(issues_obj, list):\n for iss in issues_obj:\n if isinstance(iss, dict):\n key = iss.get(\"key\") or iss.get(\"issue_key\") or iss.get(\"id\") or \"\"\n if str(key) == target_key:\n issue = iss\n break\n\n # Fallback: top-level key\n if issue is None and target_key in jira_data:\n issue = jira_data[target_key]\n\n # Fallback: recursive search\n if issue is None:\n def find_issue(obj, target):\n if isinstance(obj, dict):\n if str(obj.get(\"key\", \"\")) == target or str(obj.get(\"issue_key\", \"\")) == target or str(obj.get(\"id\", \"\")) == target:\n return obj\n for v in obj.values():\n result = find_issue(v, target)\n if result is not None:\n return result\n elif isinstance(obj, list):\n for item in obj:\n result = find_issue(item, target)\n if result is not None:\n return result\n return None\n issue = find_issue(jira_data, target_key)\n\n if issue is None:\n # Issue not found - since this is checking a negative (must NOT contain),\n # if the issue doesn't exist, the forbidden comment can't exist.\n # But since all 3 runs failed, this likely means the agent didn't process correctly.\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue {target_key} not found in jira_state.json. The agent may not have processed the T&E reports correctly.\"}\n\n # Now look for comments on this issue\n comments = []\n if isinstance(issue, dict):\n # Try multiple possible comment field locations\n for ckey in [\"comments\", \"comment\", \"fields\"]:\n if ckey in issue:\n val = issue[ckey]\n if ckey == \"fields\" and isinstance(val, dict):\n val = val.get(\"comment\", val.get(\"comments\", []))\n if isinstance(val, dict):\n val = val.get(\"comments\", [])\n if isinstance(val, list):\n comments = val\n break\n elif isinstance(val, dict) and \"comments\" in val:\n comments = val[\"comments\"] if isinstance(val[\"comments\"], list) else []\n break\n\n restaurant_names = [\"foundry bistro\", \"capital grille\", \"harvest kitchen\"]\n forbidden_phrase = \"missing business justification\"\n\n found_forbidden = []\n for c in comments:\n text = \"\"\n if isinstance(c, str):\n text = c\n elif isinstance(c, dict):\n for tkey in [\"body\", \"text\", \"content\", \"message\"]:\n candiLine truncated
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880383123",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `slack_data.json`, there must be at least one message in a channel where `is_im` is true and the channel's `user` field equals `U002`, where that message contains both `[T&E Review Flag]` and `EMP-3312`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for candidate in candidates:\n if candidate.exists():\n slack_path = candidate\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n channels_obj = slack_data.get('channels', {})\n messages_obj = slack_data.get('messages', {})\n \n matching_messages = []\n \n for channel_id, channel in channels_obj.items():\n if not isinstance(channel, dict):\n continue\n \n is_im = channel.get('is_im', False)\n if is_im is not True:\n continue\n \n user_field = str(channel.get('user', '')).strip()\n if user_field != 'U002':\n continue\n \n channel_messages = messages_obj.get(channel_id, [])\n if not isinstance(channel_messages, list):\n continue\n \n for msg in channel_messages:\n if isinstance(msg, dict):\n text = ''\n for field in ['text', 'message', 'content', 'body']:\n if field in msg:\n text += ' ' + str(msg[field])\n msg_str = json.dumps(msg)\n elif isinstance(msg, str):\n text = msg\n msg_str = msg\n else:\n continue\n \n has_flag = '[T&E Review Flag]' in text or '[T&E Review Flag]' in msg_str\n has_emp = 'EMP-3312' in text or 'EMP-3312' in msg_str\n \n if has_flag and has_emp:\n matching_messages.append(msg_str[:200])\n \n if matching_messages:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found {len(matching_messages)} matching message(s) in a DM channel with user U002 containing both '[T&E Review Flag]' and 'EMP-3312'. First match: {matching_messages[0]}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No message found in a DM channel (is_im=true, user=U002) containing both '[T&E Review Flag]' and 'EMP-3312'. Checked all channels in slack_data.json.\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775880399839",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `slack_data.json`, there must NOT be any message in any channel where `is_im` is true whose message text contains '[T&E Review Flag]' AND contains at least one of 'OPS-4', 'EMP-4507', or 'Derek Huang'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n \n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None and external_services_path is not None:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No slack_data.json found; condition is trivially satisfied (no violating messages possible).\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n violating_messages = []\n keywords = ['OPS-4', 'EMP-4507', 'Derek Huang']\n flag_text = '[T&E Review Flag]'\n \n channels_obj = slack_data.get('channels', {})\n messages_obj = slack_data.get('messages', {})\n \n for channel_id, channel in channels_obj.items():\n if not isinstance(channel, dict):\n continue\n \n is_im = channel.get(\"is_im\", False)\n if is_im is not True:\n continue\n \n channel_messages = messages_obj.get(channel_id, [])\n if not isinstance(channel_messages, list):\n continue\n \n for msg in channel_messages:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\") or \"\"\n if not isinstance(text, str):\n text = str(text)\n \n if flag_text in text:\n for kw in keywords:\n if kw.lower() in text.lower():\n violating_messages.append({\n \"channel\": channel.get(\"name\", channel_id),\n \"text_snippet\": text[:200],\n \"matched_keyword\": kw\n })\n break\n \n if violating_messages:\n details = \"; \".join(\n f\"Channel '{v['channel']}': matched '{v['matched_keyword']}' in message: '{v['text_snippet']}'\" \n for v in violating_messages\n )\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_messages)} violating message(s) in IM channels containing '[T&E Review Flag]' and a prohibited keyword. Details: {details}\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No messages found in IM channels that contain both '[T&E Review Flag]' and any of 'OPS-4', 'EMP-4507', or 'Derek Huang'.\"\n }",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776019924720",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In external_data/final/jira_state.json, issue OPS-5 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text contains T&E review complete AND all lines supported AND OPS-5 (case-insensitive; em-dash or hyphen accepted as separator between phrases).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n candidates = [\n Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\",\n ]\n if external_services_path:\n candidates.append(Path(external_services_path) / \"final\" / \"jira_state.json\")\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n\n jira_path = None\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n\n if jira_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find jira_state.json. Checked: {[str(c) for c in candidates]}\"}\n\n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n\n issue_key = \"OPS-5\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key)\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue '{issue_key}' not found in issues object.\"}\n\n # Collect comments from BOTH locations:\n # 1) inside the issue object (flat or nested ADF)\n # 2) the top-level comments dict keyed by issue ID\n comments = []\n\n if isinstance(issue, dict):\n if \"comments\" in issue and isinstance(issue[\"comments\"], list):\n comments.extend(issue[\"comments\"])\n elif \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n comment_field = issue[\"fields\"].get(\"comment\", {})\n if isinstance(comment_field, dict) and \"comments\" in comment_field:\n comments.extend(comment_field[\"comments\"])\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n # Top-level comments collection\n top_level_comments = data.get(\"comments\", {})\n if isinstance(top_level_comments, dict):\n bucket = top_level_comments.get(issue_key)\n if isinstance(bucket, list):\n comments.extend(bucket)\n elif isinstance(bucket, dict):\n # In case it's wrapped, e.g. {\"comments\": [...]}\n inner = bucket.get(\"comments\")\n if isinstance(inner, list):\n comments.extend(inner)\n elif isinstance(top_level_comments, list):\n # Fallback: a flat list where each entry tags its issue\n for c in top_level_comments:\n if isinstance(c, dict) and c.get(\"issue\") == issue_key:\n comments.append(c)\n\n if not comments:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue '{issue_key}' has no comments in either the issue object or the top-level comments collection.\"}\n\n def extract_text(node):\n \"\"\"Recursively extract text from an Atlassian Document Format node or plain string.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n texts = []\n if \"text\" in node and isinstance(node[\"text\"], str):\n texts.append(node[\"text\"])\n for key in (\"content\", \"body\", \"value\"):\n if key in node:\n texts.append(extract_text(node[key]))\n return \" \".join(texts)\n if isinstance(node, list):\n return \" \".join(extract_text(item) for item in node)\n return \"\"\n\n def get_comment_text(comment):\n \"\"\"Get text from a comment object, handling both plain strings and ADF bodies.\"\"\"\n if isinstance(comment, str):\n return comment\n if isinstance(comment, dict):\n body = comment.get(\"body\")\n if body is not None:\n return extract_text(body)\n for field in (\"text\", \"content\", \"message\"):\n val = comment.get(field)\n if isinstance(val, str):\n return val\n return extract_text(comment)\n return str(comment)\n\n def normalize(text):\n text = text.lower()\n text = re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\u2010\\u2011\\uFE58\\uFE63\\uFF0D]', '-', text)\n return text\n\n required_phrases = ['t&e review complete', 'all lines supported', 'ops-hr-te-1106']\n\n for comment in comments:\n text = get_comment_text(comment)\n norm = normalize(text)\n if all(phrase in norm for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying comment on {issue_key} containing all required phrases. Comment text: {text[:300]}\"}\n\n comment_texts = [get_comment_text(c)[:200] for c in comments]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No comment on {issue_key} contained all three required phraLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776019964232",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `slack_data.json`, there must NOT be any message in any channel where `is_im` is true whose message text contains '[T&E Review Flag]' AND contains at least one of 'OPS-5', 'EMP-5021', or 'Vanessa Frost'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check external services for slack_data.json\n # It could be in workspace or external services path\n slack_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found in any expected location. Since no Slack data exists, the negative condition is trivially satisfied (no violating messages exist).\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Navigate the slack data structure\n # Typical structures: {\"channels\": [...]} or {\"ims\": [...]} or list of channels\n channels = []\n if isinstance(slack_data, dict):\n # Could have 'channels', 'ims', or be keyed by channel id\n if 'channels' in slack_data:\n channels = slack_data['channels'] if isinstance(slack_data['channels'], list) else list(slack_data['channels'].values())\n if 'ims' in slack_data:\n ims = slack_data['ims'] if isinstance(slack_data['ims'], list) else list(slack_data['ims'].values())\n channels.extend(ims)\n # If neither, try treating all values as channels\n if not channels:\n for key, val in slack_data.items():\n if isinstance(val, (dict, list)):\n if isinstance(val, dict):\n channels.append(val)\n elif isinstance(val, list):\n channels.extend(val)\n elif isinstance(slack_data, list):\n channels = slack_data\n \n violating_messages = []\n keywords = ['OPS-5', 'EMP-5021', 'Vanessa Frost']\n keywords_lower = [k.lower() for k in keywords]\n flag_text = '[T&E Review Flag]'.lower()\n \n def check_messages(messages, channel_info=\"\"):\n if not isinstance(messages, list):\n return\n for msg in messages:\n if not isinstance(msg, dict):\n continue\n text = msg.get('text', '') or msg.get('message', '') or msg.get('content', '') or ''\n text_lower = text.lower()\n if flag_text in text_lower:\n for i, kw_lower in enumerate(keywords_lower):\n if kw_lower in text_lower:\n violating_messages.append({\n 'channel_info': channel_info,\n 'keyword': keywords[i],\n 'text_snippet': text[:200]\n })\n break\n \n for ch in channels:\n if not isinstance(ch, dict):\n continue\n is_im = ch.get('is_im', False)\n if is_im is True or is_im == 'true' or str(is_im).lower() == 'true':\n # This is an IM channel - check its messages\n msgs = ch.get('messages', []) or ch.get('history', []) or []\n ch_name = ch.get('name', ch.get('id', 'unknown'))\n check_messages(msgs, channel_info=f\"IM channel: {ch_name}\")\n \n if violating_messages:\n details = \"; \".join([f\"Found '{v['keyword']}' with '[T&E Review Flag]' in {v['channel_info']}: \\\"{v['text_snippet']}...\\\"\" for v in violating_messages])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Found {len(violating_messages)} violating message(s) in IM channels. {details}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: No messages found in IM channels containing '[T&E Review Flag]' along with any of 'OPS-5', 'EMP-5021', or 'Vanessa Frost'.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
(October 8th, 2026). I've got three T&E reports that need processing. Can you work through each one following the attached SOP?
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user