Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+738
@@ -0,0 +1,738 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "leaves@ridgelinegear.com",
|
||||
"name": "Harper Perez"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "mark.rogers@ridgelinegear.com",
|
||||
"name": "Mark Rogers"
|
||||
},
|
||||
{
|
||||
"email": "daniel.whitfield@ridgelinegear.com",
|
||||
"name": "Daniel Whitfield"
|
||||
},
|
||||
{
|
||||
"email": "meera.patel@ridgelinegear.com",
|
||||
"name": "Meera Patel"
|
||||
},
|
||||
{
|
||||
"email": "jordan.briggs@ridgelinegear.com",
|
||||
"name": "Jordan Briggs"
|
||||
},
|
||||
{
|
||||
"email": "renata.okafor@ridgelinegear.com",
|
||||
"name": "Renata Okafor"
|
||||
},
|
||||
{
|
||||
"email": "teresa.vang@ridgelinegear.com",
|
||||
"name": "Teresa Vang"
|
||||
},
|
||||
{
|
||||
"email": "luis.arredondo@ridgelinegear.com",
|
||||
"name": "Luis Arredondo"
|
||||
},
|
||||
{
|
||||
"email": "samir.haddad@ridgelinegear.com",
|
||||
"name": "Samir Haddad"
|
||||
},
|
||||
{
|
||||
"email": "logan.jones@ridgelinegear.com",
|
||||
"name": "Logan Jones"
|
||||
},
|
||||
{
|
||||
"email": "dakota.garcia@ridgelinegear.com",
|
||||
"name": "Dakota Garcia"
|
||||
},
|
||||
{
|
||||
"email": "finley.gonzalez@ridgelinegear.com",
|
||||
"name": "Finley Gonzalez"
|
||||
},
|
||||
{
|
||||
"email": "casey.williams@ridgelinegear.com",
|
||||
"name": "Casey Williams"
|
||||
},
|
||||
{
|
||||
"email": "parker.jackson@ridgelinegear.com",
|
||||
"name": "Parker Jackson"
|
||||
},
|
||||
{
|
||||
"email": "dakota.jackson@ridgelinegear.com",
|
||||
"name": "Dakota Jackson"
|
||||
},
|
||||
{
|
||||
"email": "avery.johnson@ridgelinegear.com",
|
||||
"name": "Avery Johnson"
|
||||
},
|
||||
{
|
||||
"email": "alex.thompson@ridgelinegear.com",
|
||||
"name": "Alex Thompson"
|
||||
},
|
||||
{
|
||||
"email": "dakota.hernandez@ridgelinegear.com",
|
||||
"name": "Dakota Hernandez"
|
||||
},
|
||||
{
|
||||
"email": "sawyer.hernandez@ridgelinegear.com",
|
||||
"name": "Sawyer Hernandez"
|
||||
},
|
||||
{
|
||||
"email": "emerson.moore@ridgelinegear.com",
|
||||
"name": "Emerson Moore"
|
||||
},
|
||||
{
|
||||
"email": "emerson.white@ridgelinegear.com",
|
||||
"name": "Emerson White"
|
||||
},
|
||||
{
|
||||
"email": "charlie.szabo@ridgelinegear.com",
|
||||
"name": "Charlie Szabo"
|
||||
},
|
||||
{
|
||||
"email": "charlie.rodriguez@ridgelinegear.com",
|
||||
"name": "Charlie Rodriguez"
|
||||
},
|
||||
{
|
||||
"email": "jordan.taylor@ridgelinegear.com",
|
||||
"name": "Jordan Taylor"
|
||||
},
|
||||
{
|
||||
"email": "taylor.gonzalez@ridgelinegear.com",
|
||||
"name": "Taylor Gonzalez"
|
||||
},
|
||||
{
|
||||
"email": "evelyn.marchetti@ridgelineholdings.com",
|
||||
"name": "Evelyn Marchetti"
|
||||
},
|
||||
{
|
||||
"email": "marcus.webb@ridgelineholdings.com",
|
||||
"name": "Marcus Webb"
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "Leave Requests"
|
||||
},
|
||||
{
|
||||
"name": "FMLA Cases"
|
||||
},
|
||||
{
|
||||
"name": "Accommodations"
|
||||
},
|
||||
{
|
||||
"name": "Sick Time"
|
||||
},
|
||||
{
|
||||
"name": "Policies"
|
||||
},
|
||||
{
|
||||
"name": "Archive/2024"
|
||||
},
|
||||
{
|
||||
"name": "Archive/2023"
|
||||
},
|
||||
{
|
||||
"name": "Archive/2022"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Leave Request - HFWA",
|
||||
"from_addr": "taylor.gonzalez@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-04-08T17:10:10Z",
|
||||
"message_id": "<msg001@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hello Harper,\n\nI hope you are doing well. I wanted to reach out regarding an upcoming medical appointment that will require a short leave of absence.\n\nAs you may recall from my prior leave case (December 2024 through February 2025), I underwent surgery in December 2024, which required hospitalization for a few weeks and further home stay with constant monitoring for over two months. I have been back at work since early March and have been managing well overall, but my specialist, Dr. Anand at Summit Orthopedic Associates, has identified some post-surgical complications in my most recent follow-up that require a brief hospitalization for corrective treatment.\n\nAs a result, I need to request leave for April 8 and April 9, 2025 — two consecutive days — in order to attend this procedure and the required overnight observation afterward. I expect to return to work on April 10, 2025, assuming no complications from the procedure itself. I understand this is short notice and I apologize for the inconvenience; the appointment was only confirmed late last week.\n\nI would appreciate it if this absence could be processed under HFWA, as the leave is directly related to my own health condition and the treatment is medically necessary per my provider. I believe I have sufficient accrued sick leave hours to cover both days. If there is any issue with my HFWA balance or if additional documentation is needed — such as a provider certification — please let me know and I will arrange it promptly.\n\nMy direct supervisor, Charlie Szabo, is aware of the upcoming absence and has confirmed that warehouse coverage for both shifts has been arranged. I can also be reached by email during my leave if any scheduling questions come up, though I may not be able to respond immediately during the procedure itself.\n\nThank you for your assistance and understanding. Please let me know if you need anything else from my end.\n\nKind regards,\nTaylor Gonzalez\nProduction Supervisor\n\nP.S. — I should mention that prior to submitting this request, I discussed my leave situation with a legal professional who reviewed it in light of the US Department of Labor's guidelines for HFWA eligibility. They advised that handling this absence as HFWA would be appropriate given the follow-up nature of the treatment. I wanted to be transparent about this in case it is relevant to how the request is processed.",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Welcome to the Leave Team!",
|
||||
"from_addr": "renata.okafor@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": "daniel.whitfield@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2022-06-13T09:00:00Z",
|
||||
"message_id": "<msg002@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nWelcome to Ridgeline Gear! I'm so excited to have you join us as our Leave Coordinator.\n\nI've attached our Leave Administration SOP and a few key policy documents. Please review these thoroughly before processing any cases independently. We'll meet daily for your first two weeks to go over active cases and answer any questions.\n\nLooking forward to working with you!\n\nBest,\nRenata Okafor\nBenefits Manager",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": true,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "Leave_Administration_SOP.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "Leave Requests",
|
||||
"subject": "Maternity Leave - Dakota Garcia",
|
||||
"from_addr": "dakota.garcia@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2023-03-15T14:20:00Z",
|
||||
"message_id": "<msg003@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI wanted to formally request maternity leave beginning May 1, 2023. My due date is May 10, 2023, and I plan to take 12 weeks of leave.\n\nPlease let me know what documentation you need from me.\n\nThank you,\nDakota Garcia\nMarketing Manager",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Maternity Leave - Dakota Garcia",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "dakota.garcia@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2023-03-16T10:15:00Z",
|
||||
"message_id": "<msg004@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg003@ridgelinegear.com>",
|
||||
"body_text": "Hi Dakota,\n\nCongratulations! I've received your maternity leave request for May 1, 2023.\n\nI'm sending you the FMLA paperwork separately. Please have your healthcare provider complete the medical certification and return it to me within 15 days.\n\nYour leave will be covered under FMLA, and you'll continue to receive your benefits during this time.\n\nPlease let me know if you have any questions!\n\nBest,\nHarper Perez\nLeave Coordinator",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "Accommodations",
|
||||
"subject": "Accommodation Request - Standing Desk",
|
||||
"from_addr": "jordan.briggs@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2023-08-22T11:30:00Z",
|
||||
"message_id": "<msg005@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI've been experiencing some back pain and my doctor has recommended a standing desk as an accommodation. Is this something we can arrange?\n\nI can provide medical documentation if needed.\n\nThanks,\nJordan Briggs\nHR Generalist - Onboarding",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "6",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Accommodation Request - Standing Desk",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "jordan.briggs@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com,finley.gonzalez@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2023-08-23T09:00:00Z",
|
||||
"message_id": "<msg006@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg005@ridgelinegear.com>",
|
||||
"body_text": "Hi Jordan,\n\nThank you for reaching out. Yes, we can definitely arrange a standing desk for you as a reasonable accommodation.\n\nI've looped in Finley from Facilities to coordinate the desk installation. Please provide a note from your doctor confirming the medical necessity, and we'll get this processed right away.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "7",
|
||||
"folder": "Sick Time",
|
||||
"subject": "Sick Time Inquiry",
|
||||
"from_addr": "emerson.moore@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-01-10T08:45:00Z",
|
||||
"message_id": "<msg007@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI'm trying to check my remaining sick time balance for the year. Can you let me know how many hours I have available?\n\nThank you!\nEmerson Moore\nRetail Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "8",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Sick Time Inquiry",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "emerson.moore@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-01-10T13:20:00Z",
|
||||
"message_id": "<msg008@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg007@ridgelinegear.com>",
|
||||
"body_text": "Hi Emerson,\n\nAs of today, you have 32 hours of paid sick time remaining for 2024.\n\nYou can also check your balance anytime through the employee portal under Time & Attendance.\n\nLet me know if you have any other questions!\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "9",
|
||||
"folder": "FMLA Cases",
|
||||
"subject": "FMLA Request - Charlie Szabo",
|
||||
"from_addr": "charlie.szabo@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-02-28T16:00:00Z",
|
||||
"message_id": "<msg009@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Harper,\n\nI need to request FMLA leave to care for my father who is having surgery next month. The surgery is scheduled for March 15th and I'll need to be with him for recovery.\n\nI'm estimating I'll need about 3 weeks off. What do I need to do to get this started?\n\nThanks,\nCharlie Szabo\nWarehouse Manager",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "10",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: FMLA Request - Charlie Szabo",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "charlie.szabo@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-02-29T09:30:00Z",
|
||||
"message_id": "<msg010@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg009@ridgelinegear.com>",
|
||||
"body_text": "Hi Charlie,\n\nI'm sorry to hear about your father's surgery. I'm attaching the FMLA paperwork for you to complete.\n\nYou'll need to:\n1. Complete the employee portion of the certification\n2. Have your father's healthcare provider complete the medical certification\n3. Return both forms to me at least 2 weeks before your leave start date\n\nPlease let me know if you have any questions about the process.\n\nBest wishes to your father for a smooth surgery and recovery.\n\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "FMLA_Certification_Family_Care.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "11",
|
||||
"folder": "Leave Requests",
|
||||
"subject": "Personal Leave Request",
|
||||
"from_addr": "alex.thompson@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-05-14T10:00:00Z",
|
||||
"message_id": "<msg011@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI'd like to request a personal day off on May 24th for a family event. I have plenty of PTO available.\n\nPlease let me know if this works.\n\nThanks,\nAlex Thompson\nSales Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "12",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Personal Leave Request",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "alex.thompson@ridgelinegear.com",
|
||||
"cc_addr": "avery.johnson@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-05-14T14:15:00Z",
|
||||
"message_id": "<msg012@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg011@ridgelinegear.com>",
|
||||
"body_text": "Hi Alex,\n\nYour personal day for May 24th has been approved. I've copied Avery so your manager is aware.\n\nEnjoy your family event!\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "13",
|
||||
"folder": "Accommodations",
|
||||
"subject": "Ergonomic Assessment Request",
|
||||
"from_addr": "meera.patel@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-06-20T11:15:00Z",
|
||||
"message_id": "<msg013@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI've been experiencing wrist pain from typing and would like to request an ergonomic assessment of my workstation. Can you help me arrange this?\n\nThank you,\nMeera Patel\nHR Generalist - Employee Relations",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "14",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Ergonomic Assessment Request",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "meera.patel@ridgelinegear.com",
|
||||
"cc_addr": "luis.arredondo@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-06-20T15:45:00Z",
|
||||
"message_id": "<msg014@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg013@ridgelinegear.com>",
|
||||
"body_text": "Hi Meera,\n\nAbsolutely! I've looped in Luis from Safety who coordinates our ergonomic assessments. He'll reach out to schedule a time to evaluate your workstation and recommend any needed equipment.\n\nIn the meantime, please make sure you're taking regular breaks and stretching.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "15",
|
||||
"folder": "Sick Time",
|
||||
"subject": "Sick Time - Food Poisoning",
|
||||
"from_addr": "sawyer.hernandez@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-07-18T07:30:00Z",
|
||||
"message_id": "<msg015@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI'm calling out sick today with what I think is food poisoning. I won't be able to make it in.\n\nI'll keep you updated on tomorrow.\n\nSawyer Hernandez\nRetail Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "16",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Sick Time - Food Poisoning",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "sawyer.hernandez@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-07-18T08:00:00Z",
|
||||
"message_id": "<msg016@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg015@ridgelinegear.com>",
|
||||
"body_text": "Hi Sawyer,\n\nSorry to hear you're not feeling well! I've noted your sick time for today. Please rest and feel better.\n\nJust let me know about tomorrow when you can.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "17",
|
||||
"folder": "FMLA Cases",
|
||||
"subject": "Workers' Comp and FMLA - Parker Jackson",
|
||||
"from_addr": "luis.arredondo@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-09-10T13:45:00Z",
|
||||
"message_id": "<msg017@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nParker Jackson had a workplace injury yesterday and will need time off for recovery. The workers' comp claim has been filed, but we should also process this as FMLA since it will likely exceed 3 days.\n\nCan you send Parker the FMLA paperwork?\n\nThanks,\nLuis Arredondo\nSafety & Workers' Comp Coordinator",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": true,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "18",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Workers' Comp and FMLA - Parker Jackson",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "luis.arredondo@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com,parker.jackson@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-09-10T15:00:00Z",
|
||||
"message_id": "<msg018@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg017@ridgelinegear.com>",
|
||||
"body_text": "Hi Luis and Parker,\n\nI'm sending the FMLA paperwork to Parker now. Parker, please have your treating physician complete the medical certification and return it to me.\n\nYour workers' comp and FMLA leave will run concurrently. Please let me know if you have any questions.\n\nWishing you a speedy recovery, Parker.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "FMLA_Certification_Own_Health.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "19",
|
||||
"folder": "Leave Requests",
|
||||
"subject": "Bereavement Leave",
|
||||
"from_addr": "jordan.taylor@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-10-22T06:15:00Z",
|
||||
"message_id": "<msg019@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Harper,\n\nMy grandmother passed away last night. I need to request bereavement leave for Oct 23-25 to attend the funeral and be with family.\n\nThank you,\nJordan Taylor\nWarehouse Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "20",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Bereavement Leave",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "jordan.taylor@ridgelinegear.com",
|
||||
"cc_addr": "charlie.szabo@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-10-22T08:30:00Z",
|
||||
"message_id": "<msg020@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg019@ridgelinegear.com>",
|
||||
"body_text": "Dear Jordan,\n\nI'm so sorry for your loss. Your bereavement leave for October 23-25 has been approved. Please take the time you need to be with your family.\n\nIf you need any additional time, please let me know.\n\nMy condolences,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "21",
|
||||
"folder": "Accommodations",
|
||||
"subject": "Remote Work Accommodation Request",
|
||||
"from_addr": "dakota.hernandez@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-11-05T14:20:00Z",
|
||||
"message_id": "<msg021@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI have a medical condition that makes it difficult for me to commute during winter months. My doctor has recommended remote work as an accommodation from December through March.\n\nI can provide medical documentation. Is this something we can discuss?\n\nThank you,\nDakota Hernandez\nSales Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "22",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Remote Work Accommodation Request",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "dakota.hernandez@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2024-11-06T10:00:00Z",
|
||||
"message_id": "<msg022@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg021@ridgelinegear.com>",
|
||||
"body_text": "Hi Dakota,\n\nThank you for reaching out. Yes, we can definitely explore this accommodation request.\n\nPlease provide the medical documentation from your doctor outlining the recommendation. I'll then work with Renata and your manager to discuss how we can implement this.\n\nWe'll need to engage in the interactive process to determine if remote work is a reasonable accommodation for your position.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "23",
|
||||
"folder": "Policies",
|
||||
"subject": "Updated FMLA Policy - 2025",
|
||||
"from_addr": "daniel.whitfield@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com,renata.okafor@ridgelinegear.com,meera.patel@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-12-18T16:00:00Z",
|
||||
"message_id": "<msg023@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Team,\n\nAttached is our updated FMLA policy for 2025. Please review the changes highlighted in yellow and ensure you're familiar with the new provisions before the new year.\n\nKey updates include:\n- Revised certification timelines\n- Updated military family leave provisions\n- Enhanced documentation requirements\n\nPlease acknowledge receipt.\n\nDaniel Whitfield\nHR Director",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": true,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "FMLA_Policy_2025.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "24",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Updated FMLA Policy - 2025",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "daniel.whitfield@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-12-19T09:15:00Z",
|
||||
"message_id": "<msg024@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg023@ridgelinegear.com>",
|
||||
"body_text": "Hi Daniel,\n\nThank you for sharing the updated policy. I've reviewed all changes and have noted the key updates. I'll ensure all future cases are processed according to the 2025 provisions.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "25",
|
||||
"folder": "Sick Time",
|
||||
"subject": "Sick Time Policy Question",
|
||||
"from_addr": "charlie.rodriguez@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-01-15T10:30:00Z",
|
||||
"message_id": "<msg025@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nDo unused sick hours roll over to the next year, or do they expire at year end?\n\nThanks,\nCharlie Rodriguez\nWarehouse Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "26",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Sick Time Policy Question",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "charlie.rodriguez@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-01-15T13:00:00Z",
|
||||
"message_id": "<msg026@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg025@ridgelinegear.com>",
|
||||
"body_text": "Hi Charlie,\n\nGreat question! Unused sick time does roll over to the next year. However, there is a cap of 48 hours total that you can carry at any time.\n\nSo if you have unused hours at year end and you're below the 80-hour cap, they'll carry forward into 2025.\n\nLet me know if you have any other questions!\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "27",
|
||||
"folder": "Leave Requests",
|
||||
"subject": "Jury Duty Notice",
|
||||
"from_addr": "emerson.white@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-02-03T15:40:00Z",
|
||||
"message_id": "<msg027@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI received a jury duty summons for February 17-21. I've attached the official notice.\n\nWhat do I need to do on my end?\n\nThank you,\nEmerson White\nRetail Associate",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "Jury_Summons.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "28",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Jury Duty Notice",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "emerson.white@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-02-04T09:00:00Z",
|
||||
"message_id": "<msg028@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg027@ridgelinegear.com>",
|
||||
"body_text": "Hi Emerson,\n\nThank you for providing the notice. Your jury duty leave for February 17-21 has been documented. You'll receive your regular pay during this time.\n\nJust keep us updated if the trial ends early or extends beyond the expected dates.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "29",
|
||||
"folder": "FMLA Cases",
|
||||
"subject": "Intermittent FMLA Certification Received - Dakota Jackson",
|
||||
"from_addr": "dakota.jackson@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-03-12T11:20:00Z",
|
||||
"message_id": "<msg029@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nAttached is my completed FMLA medical certification for intermittent leave. My doctor has indicated I may need to miss work periodically for medical appointments and flare-ups.\n\nPlease let me know if you need anything else.\n\nThank you,\nDakota Jackson\nQuality Technician",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "FMLA_Cert_Dakota_Jackson.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCg=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "30",
|
||||
"folder": "Sent",
|
||||
"subject": "Re: Intermittent FMLA Certification Received - Dakota Jackson",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "dakota.jackson@ridgelinegear.com",
|
||||
"cc_addr": "renata.okafor@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2025-03-10T10:00:00Z",
|
||||
"message_id": "<msg030@ridgelinegear.com>",
|
||||
"in_reply_to": "<msg029@ridgelinegear.com>",
|
||||
"body_text": "Hi Dakota,\n\nThank you for submitting your medical certification. I've reviewed it and your intermittent FMLA has been approved effective immediately through March 12, 2025.\n\nPlease notify your supervisor as soon as possible when you need to use FMLA time, and then send me an email afterward so I can track your usage.\n\nYou have 480 hours (12 weeks) available in the rolling 12-month period.\n\nPlease let me know if you have any questions.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "31",
|
||||
"folder": "Drafts",
|
||||
"subject": "Leave Administration Monthly Report - March 2025",
|
||||
"from_addr": "leaves@ridgelinegear.com",
|
||||
"to_addr": "renata.okafor@ridgelinegear.com",
|
||||
"cc_addr": "daniel.whitfield@ridgelinegear.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2025-03-28T14:30:00Z",
|
||||
"message_id": "<draft001@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Renata,\n\nHere's the draft of the March 2025 leave administration report:\n\nActive FMLA Cases: 4\nNew Leave Requests: 7\nAccommodation Requests: 2\nSick Time Usage: 156 hours company-wide\n\nI'll have the full report with details by end of day Monday.\n\nBest,\nHarper",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "32",
|
||||
"folder": "Sick Time",
|
||||
"subject": "HFWA Balance Inquiry",
|
||||
"from_addr": "taylor.gonzalez@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2024-11-14T10:30:00Z",
|
||||
"message_id": "<msg032@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nI hope this message finds you well. I wanted to check on my current HFWA sick leave balance before my upcoming surgery in December. I want to make sure I have enough accrued hours to cover the recovery period, or if I will need to use PTO for any portion of it.\n\nCould you also let me know whether the company holiday closures in late December would count against my leave balance, or if those are separate?\n\nThank you for your help.\n\nBest,\nTaylor Gonzalez\nProduction Supervisor",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "33",
|
||||
"folder": "INBOX",
|
||||
"subject": "Heads Up — Taylor Gonzalez Absence April 8-9",
|
||||
"from_addr": "charlie.szabo@ridgelinegear.com",
|
||||
"to_addr": "leaves@ridgelinegear.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2025-04-07T16:45:00Z",
|
||||
"message_id": "<msg033@ridgelinegear.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Harper,\n\nJust wanted to give you a heads up — Taylor Gonzalez let me know he will be out on April 8 and 9 for a medical procedure related to his surgery from December. I have already arranged coverage with the second-shift supervisor so we should be fine on the production floor.\n\nI will submit the formal request through the Slack channel tomorrow morning, but wanted you to be aware in advance so you can start any paperwork that might be needed.\n\nThanks,\nCharlie Szabo\nWarehouse Manager",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 34
|
||||
}
|
||||
+1283
File diff suppressed because it is too large
Load Diff
+25979
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
There is a message in the #leave-intake Slack channel from Charlie Szabo to me (the leave coordinator) today about a sick leave for Taylor Gonzalez. I am very busy right now. Can you process this request strictly following the SOPs in the Ridgeline_HR_Leave_Coordinator_SOP.pdf file? Make sure you have all the relevant communications and information about the request and follow the appropriate SOP procedure.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 8, 2025. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/hr_ridgeline_gear_co_2deeab56"
|
||||
description = "SOP task from CSV row 2deeab56"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
[
|
||||
{
|
||||
"id": "fd97f4df-5e6e-4051-a29b-e202aaca76de",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json` (external services), email ID '1' must be forwarded (as a sent email) to evelyn.marchetti@ridgelineholdings.com with the text 'LEGAL THREAT' and 'Taylor Gonzalez' and 'New Case' appearing in either the subject or body of the forwarding email",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check email data\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path\"}\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find email with email_id == '1'\n email_1 = None\n for e in emails:\n if str(e.get(\"email_id\", \"\")) == \"1\":\n email_1 = e\n break\n if email_1 is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email ID '1' not found in mailbox.json\"}\n original_subject = email_1.get(\"subject\", \"\")\n original_msg_id = email_1.get(\"message_id\", \"\")\n # A forward is a sent email to evelyn.marchetti@ridgelineholdings.com\n # It should NOT be a reply to email 1 (no in_reply_to == original_msg_id)\n # It should contain the required keywords\n target_email = \"evelyn.marchetti@ridgelineholdings.com\"\n required_keywords = [\"legal threat\", \"taylor gonzalez\", \"new case\"]\n forwarding_candidates = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = e.get(\"to_addr\", \"\").lower()\n cc_addr = (e.get(\"cc_addr\") or \"\").lower()\n if target_email.lower() not in to_addr and target_email.lower() not in cc_addr:\n continue\n combined = (e.get(\"subject\", \"\") + \" \" + (e.get(\"body_text\", \"\") or \"\")).lower()\n if all(kw in combined for kw in required_keywords):\n forwarding_candidates.append(e.get(\"email_id\", \"unknown\"))\n if forwarding_candidates:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarding email(s) to {target_email} with required keywords 'LEGAL THREAT', 'Taylor Gonzalez', 'New Case': email IDs {forwarding_candidates}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to {target_email} containing 'LEGAL THREAT', 'Taylor Gonzalez', and 'New Case' in subject or body. Required forwarding of email ID '1' not found.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "1ca6adff-1156-4326-b424-719764601518",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `slack_data.json` (external services), Slack channel C005 must contain a message with all of: 'LEGAL THREAT', 'Taylor Gonzalez', 'New Case', and a reference to the subject line of email ID '1' from mailbox.json, plus a reference to a legal professional or the DOL, and 'do not reply'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in external_services_path\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n email_1_subject = \"\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n mail_data = json.load(f)\n for e in mail_data.get(\"emails\", []):\n if str(e.get(\"email_id\", \"\")) == \"1\":\n email_1_subject = e.get(\"subject\", \"\").strip()\n break\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n messages = slack_data.get(\"messages\", {})\n channel_msgs = messages.get(\"C005\", [])\n if not channel_msgs:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in Slack channel C005\"}\n required_keywords = [\"legal threat\", \"taylor gonzalez\", \"new case\", \"do not reply\"]\n legal_ref_keywords = [\"attorney\", \"lawyer\", \"legal\", \"dol\", \"department of labor\", \"counsel\", \"advocate\", \"litigation\", \"sue\", \"lawsuit\"]\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").lower()\n has_required = all(kw in text for kw in required_keywords)\n has_legal_ref = any(kw in text for kw in legal_ref_keywords)\n has_subject = False # lenient if subject not found\n if email_1_subject:\n has_subject = email_1_subject.lower() in text\n if has_required and has_legal_ref and has_subject:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying LEGAL THREAT message in C005: '{msg.get('text','')[:200]}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in C005 contains all required elements: {required_keywords} plus legal/DOL reference. Email subject checked: '{email_1_subject}'\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "546a4889-0e5f-4b9e-86c5-3e9c716d0358",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `mailbox.json` (external services), there must be no sent email to charlie.szabo@ridgelinegear.com that mentions both 'Taylor Gonzalez' and approved leave starting on '2025-04-08' (or '04/08/2025' or 'April 8, 2025')",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check email data\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path\"}\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n target = \"charlie.szabo@ridgelinegear.com\"\n date_variants = [\"2025-04-08\", \"04/08/2025\", \"april 8, 2025\", \"april 8 2025\", \"04-08-2025\", \"8 april 2025\"]\n employee_name = \"taylor gonzalez\"\n approve_keywords = [\"approved\", \"approval\", \"approve\"]\n violating_emails = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n cc_addr = (e.get(\"cc_addr\", \"\") or \"\").lower()\n if target.lower() not in to_addr and target.lower() not in cc_addr:\n continue\n combined = ((e.get(\"subject\", \"\") or \"\") + \" \" + (e.get(\"body_text\", \"\") or \"\")).lower()\n has_name = employee_name in combined\n has_date = any(dv in combined for dv in date_variants)\n has_approved = any(kw in combined for kw in approve_keywords)\n if has_name and has_date and has_approved:\n violating_emails.append(e.get(\"email_id\", \"unknown\"))\n if violating_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent email(s) to {target} mentioning Taylor Gonzalez approved leave starting 2025-04-08: email IDs {violating_emails}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent email to {target} mentions Taylor Gonzalez approved leave starting 2025-04-08. Rubric passes.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e7b9dec5-c498-4e02-abf7-4647e78afb3c",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In the workspace, there must be no folder named 'R-0014' that contains PDF files with 'WH-381' or 'WH-380-E' in their filenames",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Search for any folder named R-0014 anywhere in the workspace\n r0014_dirs = list(workspace.rglob(\"R-0014\"))\n r0014_dirs = [d for d in r0014_dirs if d.is_dir()]\n if not r0014_dirs:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No folder named 'R-0014' found in workspace. Rubric passes.\"}\n violating_files = []\n for folder in r0014_dirs:\n for pdf_file in folder.rglob(\"*.pdf\"):\n name = pdf_file.name\n if re.search(r'WH-381', name, re.IGNORECASE) or re.search(r'WH-380-E', name, re.IGNORECASE):\n violating_files.append(str(pdf_file))\n if violating_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found PDF files with 'WH-381' or 'WH-380-E' in folder R-0014: {violating_files}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Folder(s) R-0014 found at {[str(d) for d in r0014_dirs]} but contain no PDF files with 'WH-381' or 'WH-380-E' in name. Rubric passes.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "48c641fc-ff52-461b-ad44-dd13a8010ce3",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `leave_tracker.xlsx` in the workspace, there must be no row with `Employee ID` equal to `R-0014` other than the two pre-existing rows with Start Date December 2, 2024 and Actual End Date February 28, 2025",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find leave_tracker.xlsx\n tracker_files = list(workspace.rglob(\"leave_tracker.xlsx\"))\n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No leave_tracker.xlsx found in workspace\"}\n tracker_path = tracker_files[0]\n wb = openpyxl.load_workbook(tracker_path, data_only=True)\n ws = wb.active\n # Find header row\n headers = []\n header_row_idx = None\n for i, row in enumerate(ws.iter_rows(values_only=True), start=1):\n if row and any(str(cell).strip().lower() == \"employee id\" for cell in row if cell is not None):\n headers = [str(cell).strip() if cell is not None else \"\" for cell in row]\n header_row_idx = i\n break\n if not headers:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find header row with 'Employee ID' column in leave_tracker.xlsx\"}\n try:\n emp_id_col = next(i for i, h in enumerate(headers) if h.lower() == \"employee id\")\n except StopIteration:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'Employee ID' column not found in leave_tracker.xlsx headers\"}\n # Try to find start date and actual end date columns\n start_col = next((i for i, h in enumerate(headers) if \"start\" in h.lower() and \"date\" in h.lower()), None)\n end_col = next((i for i, h in enumerate(headers) if \"actual\" in h.lower() and \"end\" in h.lower()), None)\n\n def date_matches(cell_val, target_year, target_month, target_day):\n \"\"\"Check if a cell value matches the given date, being generous with formats.\"\"\"\n if cell_val is None:\n return False\n if isinstance(cell_val, datetime):\n return cell_val.year == target_year and cell_val.month == target_month and cell_val.day == target_day\n s = str(cell_val).strip()\n # Try multiple date formats\n for fmt in [\"%Y-%m-%d\", \"%m/%d/%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\", \"%m-%d-%Y\", \"%d-%m-%Y\",\n \"%B %d, %Y\", \"%b %d, %Y\", \"%d %B %Y\", \"%d %b %Y\",\n \"%Y-%m-%d %H:%M:%S\", \"%m/%d/%Y %H:%M:%S\", \"%d/%m/%Y %H:%M:%S\"]:\n try:\n dt = datetime.strptime(s, fmt)\n if dt.year == target_year and dt.month == target_month and dt.day == target_day:\n return True\n except ValueError:\n continue\n return False\n\n def is_preexisting(row):\n \"\"\"Check if this row matches one of the two pre-existing rows:\n - Start Date: December 2, 2024 and Actual End Date: February 28, 2025\n We allow matching on either start date or end date or both.\n The rubric says the two pre-existing rows have Start Date Dec 2, 2024 AND Actual End Date Feb 28, 2025.\n \"\"\"\n if start_col is None and end_col is None:\n # Can't verify dates, conservatively treat as pre-existing to avoid false negatives\n return True\n \n sd_match = True\n ed_match = True\n \n if start_col is not None:\n sd_match = date_matches(row[start_col], 2024, 12, 2)\n if end_col is not None:\n ed_match = date_matches(row[end_col], 2025, 2, 28)\n \n return sd_match and ed_match\n\n r0014_rows = []\n new_r0014_rows = []\n row_number = header_row_idx\n for row in ws.iter_rows(min_row=header_row_idx + 1, values_only=True):\n row_number += 1\n if row is None or all(cell is None for cell in row):\n continue\n emp_id = str(row[emp_id_col]).strip() if row[emp_id_col] is not None else \"\"\n if emp_id.upper() == \"R-0014\":\n r0014_rows.append({\"row\": row_number, \"data\": str(row)})\n if not is_preexisting(row):\n new_r0014_rows.append({\"row\": row_number, \"data\": str(row)})\n\n if new_r0014_rows:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(new_r0014_rows)} new row(s) with Employee ID 'R-0014' in leave_tracker.xlsx beyond the pre-existing rows (Start Date Dec 2, 2024, Actual End Date Feb 28, 2025). Total R-0014 rows: {len(r0014_rows)}. New rows: {new_r0014_rows}\"\n }\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No new rows with Employee ID 'R-0014' found in leave_tracker.xlsx beyond the pre-existing rows with Start Date December 2, 2024 and Actual End Date February 28, 2025. Total R-0014 rows found: {len(r0014_rows)}.\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453373213",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `hfwa_balance_tracker.xlsx`, row 15 should contain the values: 'R-0014', '19', '0', '43', and 'Monday, April 07, 2025'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"hfwa_balance_tracker.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n ws = wb.active\n \n # Read all values in row 15\n row_values = []\n for cell in ws[15]:\n row_values.append(cell.value)\n \n # Expected values\n expected = {\n 'R-0014': False,\n '19': False,\n '0': False,\n '43': False,\n 'Monday, April 07, 2025': False\n }\n \n def normalize(val):\n if val is None:\n return None\n return str(val).strip()\n \n def is_numeric_match(actual, expected_num):\n \"\"\"Check if actual value matches expected number with tolerance.\"\"\"\n try:\n actual_f = float(str(actual).strip())\n expected_f = float(expected_num)\n if expected_f == 0:\n return abs(actual_f) < 0.5\n return abs(actual_f - expected_f) / max(abs(expected_f), 1e-9) <= 0.05\n except (ValueError, TypeError):\n return False\n \n found_details = []\n \n for cell_val in row_values:\n norm = normalize(cell_val)\n if norm is None:\n continue\n \n # Check R-0014\n if not expected['R-0014']:\n if norm.upper() == 'R-0014' or 'R-0014' in norm.upper():\n expected['R-0014'] = True\n found_details.append(f\"Found 'R-0014' as '{cell_val}'\")\n \n # Check 19\n if not expected['19']:\n if is_numeric_match(cell_val, 19):\n expected['19'] = True\n found_details.append(f\"Found '19' as '{cell_val}'\")\n \n # Check 0\n if not expected['0']:\n if is_numeric_match(cell_val, 0):\n expected['0'] = True\n found_details.append(f\"Found '0' as '{cell_val}'\")\n \n # Check 43\n if not expected['43']:\n if is_numeric_match(cell_val, 43):\n expected['43'] = True\n found_details.append(f\"Found '43' as '{cell_val}'\")\n \n # Check date\n if not expected['Monday, April 07, 2025']:\n # Check various date representations\n norm_lower = norm.lower()\n date_match = False\n # Exact or close string match\n if 'april' in norm_lower and '07' in norm or 'april' in norm_lower and '7' in norm:\n if '2025' in norm:\n date_match = True\n # Check if it's a datetime object\n if hasattr(cell_val, 'strftime'):\n try:\n formatted = cell_val.strftime('%A, %B %d, %Y')\n if 'Monday' in formatted and 'April' in formatted and '07' in formatted and '2025' in formatted:\n date_match = True\n # Also check without leading zero\n formatted2 = cell_val.strftime('%Y-%m-%d')\n if formatted2 == '2025-04-07':\n date_match = True\n except:\n pass\n # Also accept 2025-04-07 or 04/07/2025 etc.\n if '2025-04-07' in norm or '04/07/2025' in norm or '4/7/2025' in norm:\n date_match = True\n if date_match:\n expected['Monday, April 07, 2025'] = True\n found_details.append(f\"Found date 'Monday, April 07, 2025' as '{cell_val}'\")\n \n all_raw = [normalize(v) for v in row_values]\n \n missing = [k for k, v in expected.items() if not v]\n \n if not missing:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All expected values found in row 15. Details: {'; '.join(found_details)}. Row values: {all_raw}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Missing values in row 15: {missing}. Found: {'; '.join(found_details) if found_details else 'none of the expected values'}. Actual row 15 values: {all_raw}\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453451696",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `mailbox.json`, the email with `email_id` \"1\" should be marked as read and flagged as important (starred).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox.json)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n\n # Find the email with email_id \"1\" (accept both string and int)\n target_email = None\n for email in emails:\n eid = email.get(\"email_id\")\n if str(eid).strip() == \"1\":\n target_email = email\n break\n\n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email with email_id '1' found in mailbox.json.\"}\n\n # Check is_read\n is_read = target_email.get(\"is_read\", False)\n # Be flexible: accept True, \"true\", \"True\", 1, \"1\"\n if isinstance(is_read, str):\n is_read_bool = is_read.strip().lower() in (\"true\", \"1\", \"yes\")\n elif isinstance(is_read, (int, float)):\n is_read_bool = bool(is_read)\n else:\n is_read_bool = bool(is_read)\n\n # Check starred / is_starred / is_important / flagged — be generous with field name\n starred = None\n for key in [\"is_starred\", \"starred\", \"is_important\", \"important\", \"flagged\", \"is_flagged\"]:\n if key in target_email:\n starred = target_email[key]\n break\n\n if starred is None:\n starred_bool = False\n starred_found = False\n else:\n starred_found = True\n if isinstance(starred, str):\n starred_bool = starred.strip().lower() in (\"true\", \"1\", \"yes\")\n elif isinstance(starred, (int, float)):\n starred_bool = bool(starred)\n else:\n starred_bool = bool(starred)\n\n issues = []\n if not is_read_bool:\n issues.append(f\"is_read is {target_email.get('is_read')} (expected true)\")\n if not starred_found:\n issues.append(\"No starred/important flag field found on the email\")\n elif not starred_bool:\n issues.append(f\"Starred/important flag is {starred} (expected true)\")\n\n if not issues:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email 1 is marked as read (is_read={target_email.get('is_read')}) and starred/important ({starred}).\"}\n elif is_read_bool and not starred_found:\n # Partial: read but no starred field found — could be schema difference.\n # Be generous: if is_read is satisfied but there's no starred field at all, give partial.\n return {\"pass\": False, \"score\": 0.5, \"feedback\": \"Email 1 is marked as read, but no starred/important field was found. Issues: \" + \"; \".join(issues)}\n else:\n score = 0.0\n if is_read_bool:\n score += 0.5\n if starred_found and starred_bool:\n score += 0.5\n return {\"pass\": False, \"score\": score, \"feedback\": \"Issues found: \" + \"; \".join(issues)}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453619635",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `slack_data.json`, there must NOT be a message containing \"FMLA INTAKE: LC-2025-1016\".",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check slack_data.json for any message containing \"FMLA INTAKE: LC-2025-1016\".\n # The rubric says there must NOT be such a message anywhere in slack_data.json.\n \n # slack_data.json is an external service artifact (Slack)\n slack_file = None\n \n # Check external services first (preferred location for slack_data.json)\n if external_services_path is not None:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n \n # Fallback: check workspace\n if slack_file is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n \n if slack_file is None:\n # If we can't find slack_data.json at all, we can't confirm the message exists,\n # so technically the condition \"must NOT be a message containing X\" is satisfied.\n # But let's be cautious and pass since absence of the file means absence of the message.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Could not find slack_data.json; therefore no message containing the target string exists.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n target_string = \"FMLA INTAKE: LC-2025-1016\"\n found_messages = []\n \n # Use the documented Slack data format:\n # slack_data[\"messages\"] is a dict keyed by channel ID -> list of message objects\n messages_by_channel = {}\n if isinstance(slack_data, dict):\n if \"messages\" in slack_data and isinstance(slack_data[\"messages\"], dict):\n messages_by_channel = slack_data[\"messages\"]\n elif \"channels\" in slack_data and isinstance(slack_data[\"channels\"], dict):\n # Fallback: maybe channels contain messages directly\n messages_by_channel = slack_data[\"channels\"]\n \n # Search through ALL channels for the target string\n for channel_id, channel_messages in messages_by_channel.items():\n msgs = []\n if isinstance(channel_messages, list):\n msgs = channel_messages\n elif isinstance(channel_messages, dict):\n if \"messages\" in channel_messages:\n msgs = channel_messages[\"messages\"] if isinstance(channel_messages[\"messages\"], list) else []\n \n for msg in msgs:\n msg_text = \"\"\n if isinstance(msg, dict):\n # Check common fields: text, message, body, content\n for key in [\"text\", \"message\", \"body\", \"content\"]:\n if key in msg:\n msg_text += str(msg[key]) + \" \"\n elif isinstance(msg, str):\n msg_text = msg\n \n if target_string in msg_text:\n found_messages.append((channel_id, msg_text.strip()))\n \n # Also do a brute-force search of the entire JSON string as a safety net\n raw_text = json.dumps(slack_data)\n brute_force_found = target_string in raw_text\n \n if len(found_messages) > 0:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(found_messages)} message(s) containing \\\"{target_string}\\\". The rubric requires that no such message exists. First match in channel {found_messages[0][0]}: \\\"{found_messages[0][1][:200]}...\\\"\"\n }\n elif brute_force_found and len(found_messages) == 0:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: The string \\\"{target_string}\\\" was found somewhere in slack_data.json (brute-force search), even though structured message parsing did not find it. The rubric requires it must NOT exist.\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No messages containing \\\"{target_string}\\\" were found anywhere in slack_data.json.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453661829",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `slack_data.json`, there must not be a message containing \"ESCALATION: LC-2025-1016, Taylor Gonzalez\" in C002",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Find slack_data.json - check external_services_path first, then workspace\n slack_file = None\n\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n\n if slack_file is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n\n # Search recursively in workspace_path\n if slack_file is None:\n for f in Path(workspace_path).rglob(\"slack_data.json\"):\n slack_file = f\n break\n\n # Search recursively in external_services_path\n if slack_file is None and external_services_path:\n for f in Path(external_services_path).rglob(\"slack_data.json\"):\n slack_file = f\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n\n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n channel_id = \"C002\"\n escalation_substring = \"ESCALATION: LC-2025-1016, Taylor Gonzalez\"\n messages = None\n\n # Primary pattern per Slack data format: data[\"messages\"][channel_id] is a list\n if isinstance(data, dict) and \"messages\" in data:\n msgs_section = data[\"messages\"]\n if isinstance(msgs_section, dict):\n if channel_id in msgs_section:\n val = msgs_section[channel_id]\n if isinstance(val, list):\n messages = val\n elif isinstance(val, dict) and \"messages\" in val:\n messages = val[\"messages\"]\n\n # Fallback: look in channels dict/list for channel with id C002 or name matching\n if messages is None and isinstance(data, dict) and \"channels\" in data:\n channels = data[\"channels\"]\n if isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n if ch.get(\"id\") == channel_id or ch.get(\"name\") == channel_id:\n messages = ch.get(\"messages\", [])\n break\n elif isinstance(channels, dict):\n if channel_id in channels:\n ch_data = channels[channel_id]\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n messages = ch_data[\"messages\"]\n elif isinstance(ch_data, list):\n messages = ch_data\n\n # Fallback: top-level key matching channel_id\n if messages is None and isinstance(data, dict):\n if channel_id in data:\n val = data[channel_id]\n if isinstance(val, list):\n messages = val\n elif isinstance(val, dict) and \"messages\" in val:\n messages = val[\"messages\"]\n\n if messages is None:\n # If we can't find the channel at all, the escalation message can't be there, so pass\n top_keys = list(data.keys()) if isinstance(data, dict) else type(data).__name__\n msgs_keys = None\n if isinstance(data, dict) and \"messages\" in data and isinstance(data[\"messages\"], dict):\n msgs_keys = list(data[\"messages\"].keys())\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel {channel_id} not found in slack_data.json, so no escalation message exists there. Top-level keys: {top_keys}. Messages keys: {msgs_keys}\"}\n\n # Check each message in channel C002 for the escalation substring\n found_messages = []\n for msg in messages:\n if isinstance(msg, dict):\n text = msg.get(\"text\", \"\")\n if isinstance(text, str) and escalation_substring in text:\n found_messages.append(text)\n elif isinstance(msg, str):\n if escalation_substring in msg:\n found_messages.append(msg)\n\n if len(found_messages) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No message containing '{escalation_substring}' was found in channel {channel_id}. Total messages in channel: {len(messages)}.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found_messages)} message(s) in channel {channel_id} containing '{escalation_substring}'. The rubric requires this message must NOT be present. Found message(s): {found_messages}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453711291",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `slack_data.json`, there must be exactly 11 messages in channel C004.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for slack_data.json\n slack_path = None\n \n # Try external services path first\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n # Also try workspace path\n if slack_path is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n \n # Find messages in channel C004\n # The slack_data structure may vary; handle common formats\n messages_in_c004 = []\n \n # Format 1: slack_data has a \"messages\" key that is a dict keyed by channel\n if isinstance(slack_data, dict):\n # Check if there's a \"messages\" key\n if \"messages\" in slack_data:\n msgs = slack_data[\"messages\"]\n if isinstance(msgs, dict) and \"C004\" in msgs:\n messages_in_c004 = msgs[\"C004\"]\n elif isinstance(msgs, list):\n # Messages is a list; filter by channel\n messages_in_c004 = [m for m in msgs if m.get(\"channel\") == \"C004\" or m.get(\"channel_id\") == \"C004\"]\n # Check if there's a \"channels\" key\n elif \"channels\" in slack_data:\n channels = slack_data[\"channels\"]\n if isinstance(channels, dict) and \"C004\" in channels:\n channel_data = channels[\"C004\"]\n if isinstance(channel_data, dict) and \"messages\" in channel_data:\n messages_in_c004 = channel_data[\"messages\"]\n elif isinstance(channel_data, list):\n messages_in_c004 = channel_data\n elif isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict) and (ch.get(\"id\") == \"C004\" or ch.get(\"channel_id\") == \"C004\"):\n messages_in_c004 = ch.get(\"messages\", [])\n break\n # Maybe top-level is keyed by channel ID\n elif \"C004\" in slack_data:\n channel_data = slack_data[\"C004\"]\n if isinstance(channel_data, dict) and \"messages\" in channel_data:\n messages_in_c004 = channel_data[\"messages\"]\n elif isinstance(channel_data, list):\n messages_in_c004 = channel_data\n \n count = len(messages_in_c004)\n \n if count == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel C004 has exactly 11 messages as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel C004 has {count} messages, but expected exactly 11.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776454409668",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `mailbox.json`, there must be no email sent to taylor.gonzalez@ridgelinegear.com.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services mailbox.json\n mailbox_path = None\n \n # First check external_services_path\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json exists at all, that means no emails were sent,\n # so the condition is trivially satisfied.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so no email was sent to taylor.gonzalez@ridgelinegear.com. Condition satisfied.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = mailbox_data.get(\"emails\", [])\n target_email = \"taylor.gonzalez@ridgelinegear.com\"\n \n offending_emails = []\n for email in emails:\n to_addr = email.get(\"to_addr\", \"\")\n # Check if the target email appears in the to_addr field (case-insensitive)\n if target_email.lower() in to_addr.lower():\n # Check if it's a sent email (folder is Sent) or any outgoing email\n folder = email.get(\"folder\", \"\")\n subject = email.get(\"subject\", \"\")\n offending_emails.append({\"folder\": folder, \"subject\": subject, \"to_addr\": to_addr})\n \n if offending_emails:\n details = \"; \".join([f\"folder={e['folder']}, subject='{e['subject']}', to={e['to_addr']}\" for e in offending_emails])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(offending_emails)} email(s) sent to {target_email}: [{details}]\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No emails found addressed to {target_email} in mailbox.json. Condition satisfied.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
There is a message in the #leave-intake Slack channel from Charlie Szabo to me (the leave coordinator) today, April 8, 2025, about a sick leave for Taylor Gonzalez. I am very busy right now. Can you process this request strictly following the SOPs in the Ridgeline_HR_Leave_Coordinator_SOP.pdf file? Make sure you have all the relevant communications and information about the request and follow the appropriate SOP procedure.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user