Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-h01": {
|
||||
"summary": "New Year\u2019s Day",
|
||||
"start": {
|
||||
"date": "2026-01-01"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-01-02"
|
||||
},
|
||||
"id": "evt-h01"
|
||||
},
|
||||
"evt-h02": {
|
||||
"summary": "Martin Luther King Jr. Day",
|
||||
"start": {
|
||||
"date": "2026-01-19"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-01-20"
|
||||
},
|
||||
"id": "evt-h02"
|
||||
},
|
||||
"evt-h03": {
|
||||
"summary": "Presidents\u2019 Day",
|
||||
"start": {
|
||||
"date": "2026-02-16"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-02-17"
|
||||
},
|
||||
"id": "evt-h03"
|
||||
},
|
||||
"evt-h04": {
|
||||
"summary": "Memorial Day",
|
||||
"start": {
|
||||
"date": "2026-05-25"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-05-26"
|
||||
},
|
||||
"id": "evt-h04"
|
||||
},
|
||||
"evt-h05": {
|
||||
"summary": "Independence Day (Observed)",
|
||||
"start": {
|
||||
"date": "2026-07-03"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-07-04"
|
||||
},
|
||||
"id": "evt-h05"
|
||||
},
|
||||
"evt-h06": {
|
||||
"summary": "Labor Day",
|
||||
"start": {
|
||||
"date": "2026-09-07"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-09-08"
|
||||
},
|
||||
"id": "evt-h06"
|
||||
},
|
||||
"evt-h07": {
|
||||
"summary": "Thanksgiving Day",
|
||||
"start": {
|
||||
"date": "2026-11-26"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-11-27"
|
||||
},
|
||||
"id": "evt-h07"
|
||||
},
|
||||
"evt-h08": {
|
||||
"summary": "Day after Thanksgiving",
|
||||
"start": {
|
||||
"date": "2026-11-27"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-11-28"
|
||||
},
|
||||
"id": "evt-h08"
|
||||
},
|
||||
"evt-h09": {
|
||||
"summary": "Winter Break",
|
||||
"start": {
|
||||
"date": "2026-12-24"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-12-31"
|
||||
},
|
||||
"description": "University Closed - Multi-day all-day event",
|
||||
"id": "evt-h09"
|
||||
},
|
||||
"evt-onb-10487": {
|
||||
"summary": "Orientation \u2013 Sandie Otis",
|
||||
"start": {
|
||||
"dateTime": "2026-01-01T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-01T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10487"
|
||||
},
|
||||
"evt-onb-10769": {
|
||||
"summary": "Orientation \u2013 Valenka Kerwin",
|
||||
"start": {
|
||||
"dateTime": "2026-01-05T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-05T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10769"
|
||||
},
|
||||
"evt-onb-10949": {
|
||||
"summary": "Orientation \u2013 Moyra Barny",
|
||||
"start": {
|
||||
"dateTime": "2026-01-08T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-08T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10949"
|
||||
},
|
||||
"evt-onb-10965": {
|
||||
"summary": "Orientation \u2013 Joby Kalil",
|
||||
"start": {
|
||||
"dateTime": "2026-01-09T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-09T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10965"
|
||||
},
|
||||
"evt-onb-10777": {
|
||||
"summary": "Orientation \u2013 Delilah Gregg",
|
||||
"start": {
|
||||
"dateTime": "2026-01-13T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-13T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10777"
|
||||
},
|
||||
"evt-onb-10419": {
|
||||
"summary": "Orientation \u2013 Kameko Gene",
|
||||
"start": {
|
||||
"dateTime": "2026-01-13T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-13T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10419"
|
||||
},
|
||||
"evt-onb-10722": {
|
||||
"summary": "Orientation \u2013 Tammie Burt",
|
||||
"start": {
|
||||
"dateTime": "2026-01-14T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-14T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10722"
|
||||
},
|
||||
"evt-onb-10787": {
|
||||
"summary": "Orientation \u2013 Neysa Winn",
|
||||
"start": {
|
||||
"dateTime": "2026-01-15T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-15T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10787"
|
||||
},
|
||||
"evt-onb-10451": {
|
||||
"summary": "Orientation \u2013 Natalya Martie",
|
||||
"start": {
|
||||
"dateTime": "2026-01-16T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-16T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10451"
|
||||
},
|
||||
"evt-onb-11240": {
|
||||
"summary": "Orientation \u2013 Myriam Scot",
|
||||
"start": {
|
||||
"dateTime": "2026-01-22T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-22T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-11240"
|
||||
},
|
||||
"evt-onb-10566": {
|
||||
"summary": "Orientation \u2013 Hanna Saunders",
|
||||
"start": {
|
||||
"dateTime": "2026-01-29T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-01-29T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10566"
|
||||
},
|
||||
"evt-onb-10783": {
|
||||
"summary": "Orientation \u2013 Annabell Gaylor",
|
||||
"start": {
|
||||
"dateTime": "2026-02-02T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-02T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10783"
|
||||
},
|
||||
"evt-onb-10000": {
|
||||
"summary": "Orientation \u2013 Amalita Rhett",
|
||||
"start": {
|
||||
"dateTime": "2026-02-08T09:00:00-07:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-08T12:00:00-07:00"
|
||||
},
|
||||
"id": "evt-onb-10000"
|
||||
},
|
||||
"evt-onb-10003": {
|
||||
"summary": "Orientation \u2013 Felita Demott",
|
||||
"start": {
|
||||
"dateTime": "2026-03-22T09:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-22T12:00:00-06:00"
|
||||
},
|
||||
"id": "evt-onb-10003"
|
||||
},
|
||||
"evt-onb-10002": {
|
||||
"summary": "Orientation \u2013 Bryna Marmaduke",
|
||||
"start": {
|
||||
"dateTime": "2026-03-24T09:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-24T12:00:00-06:00"
|
||||
},
|
||||
"id": "evt-onb-10002"
|
||||
},
|
||||
"evt-onb-10004": {
|
||||
"summary": "Orientation \u2013 Stefanie Heinrik",
|
||||
"start": {
|
||||
"dateTime": "2026-03-27T09:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-27T12:00:00-06:00"
|
||||
},
|
||||
"id": "evt-onb-10004"
|
||||
},
|
||||
"evt-onb-10005": {
|
||||
"summary": "Orientation \u2013 Adriena Rogerio",
|
||||
"start": {
|
||||
"dateTime": "2026-03-28T09:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-28T12:00:00-06:00"
|
||||
},
|
||||
"id": "evt-onb-10005"
|
||||
},
|
||||
"evt-onb-10001": {
|
||||
"summary": "Orientation \u2013 Cory Reinhold",
|
||||
"start": {
|
||||
"dateTime": "2026-03-29T09:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-29T12:00:00-06:00"
|
||||
},
|
||||
"id": "evt-onb-10001"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1251
File diff suppressed because it is too large
Load Diff
+1989
File diff suppressed because it is too large
Load Diff
+367
@@ -0,0 +1,367 @@
|
||||
{
|
||||
"users": {
|
||||
"U_WHITFIELD": {
|
||||
"id": "U_WHITFIELD",
|
||||
"team_id": "T_CRESTWOOD",
|
||||
"name": "j.whitfield",
|
||||
"deleted": false,
|
||||
"real_name": "Dr. James Whitfield",
|
||||
"profile": {
|
||||
"real_name": "Dr. James Whitfield",
|
||||
"display_name": "JamesW",
|
||||
"email": "j.whitfield@crestwood.edu",
|
||||
"title": "VP of HR"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true
|
||||
},
|
||||
"U_ASHFORD": {
|
||||
"id": "U_ASHFORD",
|
||||
"team_id": "T_CRESTWOOD",
|
||||
"name": "n.ashford",
|
||||
"deleted": false,
|
||||
"real_name": "Nicole Ashford",
|
||||
"profile": {
|
||||
"real_name": "Nicole Ashford",
|
||||
"display_name": "NicoleA",
|
||||
"email": "hr@crestwood.edu",
|
||||
"title": "HR Director"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": false
|
||||
},
|
||||
"U_PARK": {
|
||||
"id": "U_PARK",
|
||||
"team_id": "T_CRESTWOOD",
|
||||
"name": "j.park",
|
||||
"deleted": false,
|
||||
"real_name": "Jenna Park",
|
||||
"profile": {
|
||||
"real_name": "Jenna Park",
|
||||
"display_name": "JennaP",
|
||||
"email": "j.park@crestwood.edu",
|
||||
"title": "IT Access Coordinator"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U_OKONKWO": {
|
||||
"id": "U_OKONKWO",
|
||||
"name": "Okonkwo"
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C_ONBOARDING": {
|
||||
"id": "C_ONBOARDING",
|
||||
"name": "hr-onboarding",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "New hire processing and tracking"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_OFFBOARDING": {
|
||||
"id": "C_OFFBOARDING",
|
||||
"name": "hr-offboarding",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "Resignations and COBRA"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_LEAVE": {
|
||||
"id": "C_LEAVE",
|
||||
"name": "hr-leave-requests",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "FMLA and PTO disputes"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_PERFORMANCE": {
|
||||
"id": "C_PERFORMANCE",
|
||||
"name": "hr-performance",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "PIPs and disciplinary actions"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_RECRUITING": {
|
||||
"id": "C_RECRUITING",
|
||||
"name": "hr-recruiting",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "Hiring and screening updates"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_ESCALATIONS": {
|
||||
"id": "C_ESCALATIONS",
|
||||
"name": "hr-escalations",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "P1 Issues and Legal Threats"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
},
|
||||
"C_GENERAL": {
|
||||
"id": "C_GENERAL",
|
||||
"name": "hr-general",
|
||||
"is_channel": true,
|
||||
"purpose": {
|
||||
"value": "Policy and team announcements"
|
||||
},
|
||||
"created": 1738569600,
|
||||
"is_archived": false,
|
||||
"is_member": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C_ONBOARDING": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Sandie Otis, Staff, Information Technology, Downtown, Start Date: 01/01/2026, Jira: ONBOARD-12",
|
||||
"ts": "1766999100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Sandie Otis, 10487, 01/22/2026",
|
||||
"ts": "1769097600.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Valenka Kerwin, Staff, Employee Relations, West, Start Date: 01/05/2026, Jira: ONBOARD-9",
|
||||
"ts": "1767171900.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Valenka Kerwin, 10769, 01/25/2026",
|
||||
"ts": "1769356800.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Moyra Barny, Faculty, English, West, Start Date: 01/08/2026, Jira: ONBOARD-1",
|
||||
"ts": "1767603900.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Moyra Barny, 10949, 02/01/2026",
|
||||
"ts": "1769961600.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Joby Kalil, Staff, Human Resources, West, Start Date: 01/09/2026, Jira: ONBOARD-8",
|
||||
"ts": "1767690300.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Joby Kalil, 10965, 02/01/2026",
|
||||
"ts": "1769961600.000201"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Delilah Gregg, Faculty, Physics, Main, Start Date: 01/13/2026, Jira: ONBOARD-6",
|
||||
"ts": "1767863100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Delilah Gregg, 10777, 02/05/2026",
|
||||
"ts": "1770307200.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Kameko Gene, Staff, Recruiting, Downtown, Start Date: 01/13/2026, Jira: ONBOARD-5",
|
||||
"ts": "1767863100.000101"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Kameko Gene, 10419, 02/04/2026",
|
||||
"ts": "1770220800.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Tammie Burt, Staff, Payroll, West, Start Date: 01/14/2026, Jira: ONBOARD-2",
|
||||
"ts": "1767949500.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Tammie Burt, 10722, 02/04/2026",
|
||||
"ts": "1770220800.000201"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Neysa Winn, Staff, Recruiting, Main, Start Date: 01/15/2026, Jira: ONBOARD-18",
|
||||
"ts": "1768208700.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Neysa Winn, 10787, 02/12/2026",
|
||||
"ts": "1770912000.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Natalya Martie, Staff, Information Technology, Main, Start Date: 01/16/2026, Jira: ONBOARD-11",
|
||||
"ts": "1768295100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Natalya Martie, 10451, 02/06/2026",
|
||||
"ts": "1770393600.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Myriam Scot, Faculty, Engineering, Main, Start Date: 01/22/2026, Jira: ONBOARD-17",
|
||||
"ts": "1768813500.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Myriam Scot, 11240, 02/17/2026",
|
||||
"ts": "1771344000.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Hanna Saunders, Staff, Human Resources, Downtown, Start Date: 01/29/2026, Jira: ONBOARD-16",
|
||||
"ts": "1769418300.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Hanna Saunders, 10566, 02/18/2026",
|
||||
"ts": "1771430400.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Annabell Gaylor, Staff, Benefits & Leave, Main, Start Date: 02/02/2026, Jira: ONBOARD-4",
|
||||
"ts": "1769591100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Annabell Gaylor, 10783, 02/22/2026",
|
||||
"ts": "1771776000.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Amalita Rhett, Staff, Mathematics, Main, Start Date: 02/08/2026, Jira: ONBOARD-14",
|
||||
"ts": "1770195900.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "ONBOARDING COMPLETE: Amalita Rhett, 10000, 03/01/2026",
|
||||
"ts": "1772380800.000200"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Felita Demott, Staff, History, Main, Start Date: 03/22/2026, Jira: ONBOARD-3",
|
||||
"ts": "1773824700.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Bryna Marmaduke, Staff, Biology, Main, Start Date: 03/24/2026, Jira: ONBOARD-10",
|
||||
"ts": "1773911100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Stefanie Heinrik, Staff, Engineering, West, Start Date: 03/27/2026, Jira: ONBOARD-7",
|
||||
"ts": "1774343100.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Adriena Rogerio, Staff, Art, Downtown, Start Date: 03/28/2026, Jira: ONBOARD-15",
|
||||
"ts": "1774429500.000100"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_OKONKWO",
|
||||
"text": "NEW HIRE: Cory Reinhold, Staff, Art, West, Start Date: 03/29/2026, Jira: ONBOARD-13",
|
||||
"ts": "1774429500.000101"
|
||||
}
|
||||
],
|
||||
"C_OFFBOARDING": [],
|
||||
"C_LEAVE": [],
|
||||
"C_PERFORMANCE": [],
|
||||
"C_RECRUITING": [],
|
||||
"C_ESCALATIONS": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U_ASHFORD",
|
||||
"text": "LEGAL THREAT: Ardisj Parke, URGENT: Dispute regarding denied PTO request \u2013 do not reply.",
|
||||
"ts": "1775748000.000100",
|
||||
"team": "T_CRESTWOOD"
|
||||
}
|
||||
],
|
||||
"C_GENERAL": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 5000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
I've been out of the office for the past 2 days and I need to be today's HR coordinator. Please look at and resolve any unread email. However, on matters of off boarding, don't do SOP 12.3, 12.2, or updates to Jira on 12.2. My name is Derek Okonkwo.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 9, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/hr_crestwood_university_2071c562"
|
||||
description = "SOP task from CSV row 2071c562"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
I've been out of the office for the past 2 days and I need to be today's HR coordinator. Please look at and resolve any unread email. However, on matters of off boarding, don't do SOP 12.3, 12.2, or updates to Jira on 12.2. Today is April 9th, 2026 and my name is Derek Okonkwo.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user