Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+617
@@ -0,0 +1,617 @@
|
||||
{
|
||||
"issues": {
|
||||
"OPS-5": {
|
||||
"id": "10001",
|
||||
"key": "OPS-5",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-5",
|
||||
"fields": {
|
||||
"summary": "Q1 2026 International Wire Transfer Population \u2014 Dual-Authorization Audit",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Population workbook: Wire_Population_Q1_2026.xlsx (available in workspace)\nQuarter: Q1 2026 (January 1 \u2013 March 31, 2026)\nAudit scope: All international wire transfers initiated in Q1 2026 requiring dual-authorization review under SOP \u00a73.3.\nInitiated by: Treasury Operations"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10001",
|
||||
"name": "Task",
|
||||
"description": "A task",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 0
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10001",
|
||||
"name": "In Review",
|
||||
"description": "Audit in progress",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "indeterminate",
|
||||
"name": "In Progress",
|
||||
"colorName": "yellow"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "2",
|
||||
"name": "High"
|
||||
},
|
||||
"assignee": null,
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2026-04-01T08:00:00.000Z",
|
||||
"updated": "2026-04-10T09:00:00.000Z",
|
||||
"labels": [
|
||||
"quarterly-audit",
|
||||
"wire-transfers",
|
||||
"Q1-2026"
|
||||
],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
},
|
||||
"OPS-246": {
|
||||
"id": "10002",
|
||||
"key": "OPS-246",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-246",
|
||||
"fields": {
|
||||
"summary": "Queue Roles & Assignees",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Compliance-Alerts Queue \u2014 Role Assignments\n\nThis issue is pinned to the Compliance-Alerts board. It records the current named individual responsible for each queue role. Update this issue when role assignments change.\n\nChief Compliance Officer (CCO): diana.walsh\n\nAssign all new Compliance-Alerts tickets to the individual listed for the CCO role. If the role is vacant or the named individual is unavailable, follow the \u00a71.3 escalation path (create an IT-Support ticket titled \"Missing Queue Assignee \u2013 [Queue] \u2013 [Role]\")."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10002",
|
||||
"name": "Epic",
|
||||
"description": "Pinned reference issue",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 1
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10002",
|
||||
"name": "Active",
|
||||
"description": "Standing reference issue \u2014 do not close",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "indeterminate",
|
||||
"name": "In Progress",
|
||||
"colorName": "yellow"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "2",
|
||||
"name": "High"
|
||||
},
|
||||
"assignee": {
|
||||
"accountId": "diana.walsh",
|
||||
"accountType": "atlassian",
|
||||
"emailAddress": "diana.walsh@vanguardshield.com",
|
||||
"displayName": "Diana Walsh",
|
||||
"active": true,
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2025-01-01T08:00:00.000Z",
|
||||
"updated": "2026-01-01T08:00:00.000Z",
|
||||
"labels": [
|
||||
"pinned",
|
||||
"queue-roles",
|
||||
"compliance-alerts"
|
||||
],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
},
|
||||
"OPS-1": {
|
||||
"id": "10003",
|
||||
"key": "OPS-1",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-1",
|
||||
"fields": {
|
||||
"summary": "International Wire INTL-2026-0214 \u2014 Pending Dual-Authorization Review",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Wire ID: INTL-2026-0214\nPayee: Pemberton Global Reinsurance Ltd.\nBeneficiary Bank: HSBC Bank plc, London\nAmount: $445,000.00\nDate Initiated: 2026-03-03\nQuarter: Q1 2026\nStatus: Pending dual-authorization audit per SOP \u00a73.3."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10001",
|
||||
"name": "Task",
|
||||
"description": "A task",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 0
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10001",
|
||||
"name": "Open",
|
||||
"description": "Awaiting audit",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "new",
|
||||
"name": "To Do",
|
||||
"colorName": "blue-gray"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "3",
|
||||
"name": "Medium"
|
||||
},
|
||||
"assignee": null,
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2026-03-03T10:00:00.000Z",
|
||||
"updated": "2026-04-01T08:00:00.000Z",
|
||||
"labels": [],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
},
|
||||
"OPS-2": {
|
||||
"id": "10004",
|
||||
"key": "OPS-2",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-2",
|
||||
"fields": {
|
||||
"summary": "International Wire INTL-2026-0219 \u2014 Pending Dual-Authorization Review",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Wire ID: INTL-2026-0219\nPayee: Heckmann Actuarial Partners GmbH\nBeneficiary Bank: Deutsche Bank AG, Frankfurt\nAmount: $178,500.00\nDate Initiated: 2026-03-10\nQuarter: Q1 2026\nStatus: Pending dual-authorization audit per SOP \u00a73.3."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10001",
|
||||
"name": "Task",
|
||||
"description": "A task",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 0
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10001",
|
||||
"name": "Open",
|
||||
"description": "Awaiting audit",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "new",
|
||||
"name": "To Do",
|
||||
"colorName": "blue-gray"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "3",
|
||||
"name": "Medium"
|
||||
},
|
||||
"assignee": null,
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2026-03-10T10:00:00.000Z",
|
||||
"updated": "2026-04-01T08:00:00.000Z",
|
||||
"labels": [],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
},
|
||||
"OPS-3": {
|
||||
"id": "10005",
|
||||
"key": "OPS-3",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-3",
|
||||
"fields": {
|
||||
"summary": "International Wire INTL-2026-0223 \u2014 Pending Dual-Authorization Review",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Wire ID: INTL-2026-0223\nPayee: Langford Risk Solutions B.V.\nBeneficiary Bank: ING Bank N.V., Amsterdam\nAmount: $623,000.00\nDate Initiated: 2026-03-17\nQuarter: Q1 2026\nStatus: Pending dual-authorization audit per SOP \u00a73.3."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10001",
|
||||
"name": "Task",
|
||||
"description": "A task",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 0
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10001",
|
||||
"name": "Open",
|
||||
"description": "Awaiting audit",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "new",
|
||||
"name": "To Do",
|
||||
"colorName": "blue-gray"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "3",
|
||||
"name": "Medium"
|
||||
},
|
||||
"assignee": null,
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2026-03-17T10:00:00.000Z",
|
||||
"updated": "2026-04-01T08:00:00.000Z",
|
||||
"labels": [],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
},
|
||||
"OPS-4": {
|
||||
"id": "10006",
|
||||
"key": "OPS-4",
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/issue/OPS-4",
|
||||
"fields": {
|
||||
"summary": "International Wire INTL-2026-0228 \u2014 Pending Dual-Authorization Review",
|
||||
"description": {
|
||||
"type": "doc",
|
||||
"version": 1,
|
||||
"content": [
|
||||
{
|
||||
"type": "paragraph",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Wire ID: INTL-2026-0228\nPayee: Carlisle Treaty Services S.A.\nBeneficiary Bank: BNP Paribas S.A., Paris\nAmount: $312,000.00\nDate Initiated: 2026-03-24\nQuarter: Q1 2026\nStatus: Pending dual-authorization audit per SOP \u00a73.3."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"issuetype": {
|
||||
"id": "10001",
|
||||
"name": "Task",
|
||||
"description": "A task",
|
||||
"subtask": false,
|
||||
"hierarchyLevel": 0
|
||||
},
|
||||
"project": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"status": {
|
||||
"id": "10001",
|
||||
"name": "Open",
|
||||
"description": "Awaiting audit",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "new",
|
||||
"name": "To Do",
|
||||
"colorName": "blue-gray"
|
||||
}
|
||||
},
|
||||
"priority": {
|
||||
"id": "3",
|
||||
"name": "Medium"
|
||||
},
|
||||
"assignee": null,
|
||||
"reporter": null,
|
||||
"creator": null,
|
||||
"created": "2026-03-24T10:00:00.000Z",
|
||||
"updated": "2026-04-01T08:00:00.000Z",
|
||||
"labels": [],
|
||||
"components": [],
|
||||
"fixVersions": [],
|
||||
"versions": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"sprints": {},
|
||||
"comments": {},
|
||||
"projects": {
|
||||
"OPS": {
|
||||
"id": "10001",
|
||||
"key": "OPS",
|
||||
"name": "Vanguard Shield Operations",
|
||||
"description": "Operations project for treasury wire tracking, compliance queue management, and audit issues",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/OPS"
|
||||
},
|
||||
"COMP": {
|
||||
"id": "10002",
|
||||
"key": "COMP",
|
||||
"name": "Compliance-Alerts",
|
||||
"description": "Compliance escalation queue for wire control exceptions and other compliance alerts. Queue roles and assignees are maintained in pinned issue OPS-246 (Queue Roles & Assignees) on this board.",
|
||||
"projectTypeKey": "software",
|
||||
"simplified": false,
|
||||
"self": "https://api.atlassian.com/ex/jira/vanguardshield/rest/api/3/project/COMP"
|
||||
}
|
||||
},
|
||||
"fields": [
|
||||
{
|
||||
"id": "summary",
|
||||
"key": "summary",
|
||||
"name": "Summary",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "description",
|
||||
"key": "description",
|
||||
"name": "Description",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "status",
|
||||
"key": "status",
|
||||
"name": "Status",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "priority",
|
||||
"key": "priority",
|
||||
"name": "Priority",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "assignee",
|
||||
"key": "assignee",
|
||||
"name": "Assignee",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "reporter",
|
||||
"key": "reporter",
|
||||
"name": "Reporter",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "created",
|
||||
"key": "created",
|
||||
"name": "Created",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "updated",
|
||||
"key": "updated",
|
||||
"name": "Updated",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "labels",
|
||||
"key": "labels",
|
||||
"name": "Labels",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
},
|
||||
{
|
||||
"id": "issuetype",
|
||||
"key": "issuetype",
|
||||
"name": "Issue Type",
|
||||
"custom": false,
|
||||
"searchable": true
|
||||
}
|
||||
],
|
||||
"linkTypes": [
|
||||
{
|
||||
"id": "10001",
|
||||
"name": "Blocks",
|
||||
"inward": "is blocked by",
|
||||
"outward": "blocks"
|
||||
},
|
||||
{
|
||||
"id": "10002",
|
||||
"name": "Cloners",
|
||||
"inward": "is cloned by",
|
||||
"outward": "clones"
|
||||
},
|
||||
{
|
||||
"id": "10003",
|
||||
"name": "Duplicate",
|
||||
"inward": "is duplicated by",
|
||||
"outward": "duplicates"
|
||||
},
|
||||
{
|
||||
"id": "10004",
|
||||
"name": "Relates",
|
||||
"inward": "relates to",
|
||||
"outward": "relates to"
|
||||
}
|
||||
],
|
||||
"counters": {
|
||||
"issueId": 10007,
|
||||
"sprintId": 1000,
|
||||
"commentId": 1
|
||||
},
|
||||
"users": {
|
||||
"diana.walsh": {
|
||||
"accountId": "diana.walsh",
|
||||
"displayName": "Diana Walsh",
|
||||
"accountType": "atlassian",
|
||||
"emailAddress": "diana.walsh@vanguardshield.com"
|
||||
}
|
||||
},
|
||||
"boards": {},
|
||||
"statuses": {
|
||||
"1": {
|
||||
"id": "1",
|
||||
"name": "In Review",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "indeterminate",
|
||||
"name": "In Progress",
|
||||
"colorName": "yellow"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"id": "2",
|
||||
"name": "Active",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "indeterminate",
|
||||
"name": "In Progress",
|
||||
"colorName": "yellow"
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"id": "3",
|
||||
"name": "Open",
|
||||
"statusCategory": {
|
||||
"id": 2,
|
||||
"key": "new",
|
||||
"name": "To Do",
|
||||
"colorName": "blue-gray"
|
||||
}
|
||||
}
|
||||
},
|
||||
"workflow": {
|
||||
"In Review": [
|
||||
{
|
||||
"id": "1",
|
||||
"name": "To In Review",
|
||||
"to": "In Review"
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"name": "To Active",
|
||||
"to": "Active"
|
||||
},
|
||||
{
|
||||
"id": "3",
|
||||
"name": "To Open",
|
||||
"to": "Open"
|
||||
}
|
||||
],
|
||||
"Active": [
|
||||
{
|
||||
"id": "4",
|
||||
"name": "To In Review",
|
||||
"to": "In Review"
|
||||
},
|
||||
{
|
||||
"id": "5",
|
||||
"name": "To Active",
|
||||
"to": "Active"
|
||||
},
|
||||
{
|
||||
"id": "6",
|
||||
"name": "To Open",
|
||||
"to": "Open"
|
||||
}
|
||||
],
|
||||
"Open": [
|
||||
{
|
||||
"id": "7",
|
||||
"name": "To In Review",
|
||||
"to": "In Review"
|
||||
},
|
||||
{
|
||||
"id": "8",
|
||||
"name": "To Active",
|
||||
"to": "Active"
|
||||
},
|
||||
{
|
||||
"id": "9",
|
||||
"name": "To Open",
|
||||
"to": "Open"
|
||||
}
|
||||
]
|
||||
},
|
||||
"defaultStatusValue": "Open"
|
||||
}
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "thomas.breck",
|
||||
"deleted": false,
|
||||
"real_name": "Thomas Breck",
|
||||
"profile": {
|
||||
"real_name": "Thomas Breck",
|
||||
"display_name": "thomas.breck",
|
||||
"email": "thomas.breck@vanguardshield.com",
|
||||
"title": "Treasury Operations Analyst"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "claire.overton",
|
||||
"deleted": false,
|
||||
"real_name": "Claire Overton",
|
||||
"profile": {
|
||||
"real_name": "Claire Overton",
|
||||
"display_name": "claire.overton",
|
||||
"email": "claire.overton@vanguardshield.com",
|
||||
"title": "VP, Treasury Operations"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "rachel.kim",
|
||||
"deleted": false,
|
||||
"real_name": "Rachel Kim",
|
||||
"profile": {
|
||||
"real_name": "Rachel Kim",
|
||||
"display_name": "rachel.kim",
|
||||
"email": "rachel.kim@vanguardshield.com",
|
||||
"title": "Chief Financial Officer"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "peter.walsh",
|
||||
"deleted": false,
|
||||
"real_name": "Peter Walsh",
|
||||
"profile": {
|
||||
"real_name": "Peter Walsh",
|
||||
"display_name": "peter.walsh",
|
||||
"email": "peter.walsh@vanguardshield.com",
|
||||
"title": "EVP, Corporate Finance"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "nadine.cole",
|
||||
"deleted": false,
|
||||
"real_name": "Nadine Cole",
|
||||
"profile": {
|
||||
"real_name": "Nadine Cole",
|
||||
"display_name": "nadine.cole",
|
||||
"email": "nadine.cole@vanguardshield.com",
|
||||
"title": "AVP, Enterprise Risk"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "steven.ford",
|
||||
"deleted": false,
|
||||
"real_name": "Steven Ford",
|
||||
"profile": {
|
||||
"real_name": "Steven Ford",
|
||||
"display_name": "steven.ford",
|
||||
"email": "steven.ford@vanguardshield.com",
|
||||
"title": "Chief Operating Officer"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "treasury-wires",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "treasury-wires",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U006",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Wire transfer approval workflow",
|
||||
"creator": "U006",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for wire transfer submissions and approvals per corporate policy",
|
||||
"creator": "U006",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "compliance-urgent",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "compliance-urgent",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U006",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Urgent compliance alerts and escalations",
|
||||
"creator": "U006",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "High-priority compliance notifications requiring immediate attention",
|
||||
"creator": "U006",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0214. Payee: Pemberton Global Reinsurance Ltd. Amount: $445,000.00. Please review and approve per policy. Jira: <https://vanguardshield.atlassian.net/browse/OPS-1|OPS-1>",
|
||||
"ts": "1772528400.000100",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1772529600.000300",
|
||||
"reply_users": [
|
||||
"U002",
|
||||
"U003"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1772529000.000200",
|
||||
"thread_ts": "1772528400.000100",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Approved.",
|
||||
"ts": "1772529600.000300",
|
||||
"thread_ts": "1772528400.000100",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0219. Payee: Heckmann Actuarial Partners GmbH. Amount: $178,500.00. Please review and approve per policy. Jira: OPS-2",
|
||||
"ts": "1773133200.000400",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1773134400.000600",
|
||||
"reply_users": [
|
||||
"U004",
|
||||
"U005"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U004",
|
||||
"text": "Approved.",
|
||||
"ts": "1773133800.000500",
|
||||
"thread_ts": "1773133200.000400",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Approved.",
|
||||
"ts": "1773134400.000600",
|
||||
"thread_ts": "1773133200.000400",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0223. Payee: Langford Risk Solutions B.V. Amount: $623,000.00. Please review and approve per policy. Jira: <https://vanguardshield.atlassian.net/browse/OPS-3|OPS-3>",
|
||||
"ts": "1773738000.000700",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1773739200.000900",
|
||||
"reply_users": [
|
||||
"U002",
|
||||
"U006"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1773738600.000800",
|
||||
"thread_ts": "1773738000.000700",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U006",
|
||||
"text": "LGTM",
|
||||
"ts": "1773739200.000900",
|
||||
"thread_ts": "1773738000.000700",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0228. Payee: Carlisle Treaty Services S.A. Amount: $312,000.00. Please review and approve per policy. Jira: <https://vanguardshield.atlassian.net/browse/OPS-4|OPS-4>",
|
||||
"ts": "1774342800.001000",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 1,
|
||||
"latest_reply": "1774343400.001100",
|
||||
"reply_users": [
|
||||
"U002"
|
||||
],
|
||||
"reply_users_count": 1
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1774343400.001100",
|
||||
"thread_ts": "1774342800.001000",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
}
|
||||
],
|
||||
"C002": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000100
|
||||
}
|
||||
}
|
||||
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
You are acting as a junior operations analyst at Vanguard Shield Mutual. Complete the Q1 '26 international wire transfer dual-authorization audit, strictly following high-risk wire transfer auditing instructions in SOP.pdf.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 10, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/insurance_vanguard_shield_mutual_4321b8e9"
|
||||
description = "SOP task from CSV row 4321b8e9"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
It is April 10, 2026. You are acting as a junior operations analyst at Vanguard Shield Mutual. Complete the Q1 '26 international wire transfer dual-authorization audit, strictly following high-risk wire transfer auditing instructions in SOP.pdf.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user