Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"events": {
|
||||
"CIG_2026_0341": {
|
||||
"id": "CIG_2026_0341",
|
||||
"summary": "IVIG Visit — CIG_2026_0341",
|
||||
"start": {
|
||||
"dateTime": "2026-10-05T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-10-05T12:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Home IVIG infusion for Margaret Thornton. Nurse: Patricia Nolan, RN. Drug: Privigen 40g IV. Attendees: pnolan.rn@homeinfusionkc.com",
|
||||
"location": "4218 Ridgeview Ct, Kansas City, KS 66112"
|
||||
}
|
||||
}
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "billing@careig.com",
|
||||
"name": "CareIG Billing"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "lchoi@careig.com",
|
||||
"name": "Lisa Choi"
|
||||
},
|
||||
{
|
||||
"email": "rkim@careig.com",
|
||||
"name": "Rachel Kim"
|
||||
},
|
||||
{
|
||||
"email": "pnolan.rn@homeinfusionkc.com",
|
||||
"name": "Patricia Nolan"
|
||||
},
|
||||
{
|
||||
"email": "evasquez@kcimunocare.com",
|
||||
"name": "Dr. Elena Vasquez"
|
||||
},
|
||||
{
|
||||
"email": "m.thornton64@gmail.com",
|
||||
"name": "Margaret Thornton"
|
||||
},
|
||||
{
|
||||
"email": "intake@careig.com",
|
||||
"name": "CareIG Intake"
|
||||
},
|
||||
{
|
||||
"email": "pa@careig.com",
|
||||
"name": "CareIG PA"
|
||||
},
|
||||
{
|
||||
"email": "compliance@careig.com",
|
||||
"name": "CareIG Compliance"
|
||||
},
|
||||
{
|
||||
"email": "legal@careig.com",
|
||||
"name": "CareIG Legal"
|
||||
},
|
||||
{
|
||||
"email": "pa.review@sunflowerhealthks.com",
|
||||
"name": "Sunflower Health PA"
|
||||
},
|
||||
{
|
||||
"email": "claims@sunflowerhealthks.com",
|
||||
"name": "Sunflower Health Claims"
|
||||
},
|
||||
{
|
||||
"email": "appeals@sunflowerhealthks.com",
|
||||
"name": "Sunflower Health Appeals"
|
||||
}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "CareIG — Consent Form Required | Thornton",
|
||||
"from_addr": "intake@careig.com",
|
||||
"to_addr": "m.thornton64@gmail.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-09-02T11:45:00-05:00",
|
||||
"message_id": "<msg001-careig-20260902@careig.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear Margaret,\n\nThank you for speaking with us today. Dr. Vasquez has referred you to CareIG Specialty Pharmacy for home IVIG infusion therapy.\n\nBefore we can proceed with verifying your insurance and scheduling your first visit, we need a signed Consent for Treatment and Financial Responsibility form. Please find the form attached.\n\nYou may sign and return it by:\n- Replying to this email with the signed form attached\n- Faxing to (816) 555-0200\n\nOnce we verify your insurance, we will also contact you with your estimated out-of-pocket costs before scheduling.\n\nIf you have any questions, please call us at (816) 555-0200.\n\nLisa Choi — CareIG Specialty Pharmacy, Intake\nintake@careig.com | (816) 555-0200",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "PA REQUEST CareIG | Thornton | MemberID: SHC-884729103",
|
||||
"from_addr": "pa@careig.com",
|
||||
"to_addr": "pa.review@sunflowerhealthks.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-09-12T09:15:00-05:00",
|
||||
"message_id": "<msg002-careig-20260912@careig.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "To Whom It May Concern,\n\nPlease find attached the Prior Authorization request packet for the following patient:\n\nPatient: Margaret Thornton\nDOB: 1961-06-15\nMember ID: SHC-884729103\nGroup: GRP-20145\n\nRequesting Provider: CareIG Specialty Pharmacy\nNPI: 1234567893\nTax ID: 47-3821056\n\nPrescribing Physician: Dr. Elena Vasquez, NPI: 1528074836\n\nRequested Drug: Privigen 40g IV every 4 weeks\nDiagnosis: D83.9\n\nAttached: pa_submission_packet_SunflowerHealth.pdf\n\nPlease direct questions to pa@careig.com or (816) 555-0200.\n\nRachel Kim — CareIG Specialty Pharmacy, Prior Authorization\npa@careig.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": "pa_submission_packet_SunflowerHealth.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"content_base64": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlL1BhZ2UvUGFyZW50IDIgMCBSL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Db250ZW50cyA0IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L0xlbmd0aCA0OD4+CnN0cmVhbQpCVAovRjEgMTIgVGYKNzIgNzIwIFRkCihQQSBTdWJtaXNzaW9uIFBhY2tldCkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iagoxIDAgb2JqCjw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+CmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+CmVuZG9iagp4cmVmCjAgNQowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAyMDggMDAwMDAgbiAKMDAwMDAwMDI1NyAwMDAwMCBuIAowMDAwMDAwMDE1IDAwMDAwIG4gCjAwMDAwMDAxMDQgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDUvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgozMTQKJUVPRgo="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "INBOX",
|
||||
"subject": "VISIT ASSIGNMENT CIG_2026_0341 | 2026-10-05",
|
||||
"from_addr": "intake@careig.com",
|
||||
"to_addr": "pnolan.rn@homeinfusionkc.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-09-28T15:20:00-05:00",
|
||||
"message_id": "<msg003-careig-20260928@careig.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear Patricia,\n\nYou have been assigned to the following IVIG home infusion visit:\n\nCase ID: CIG_2026_0341\nPatient: Margaret Thornton\nAddress: 4218 Ridgeview Ct, Kansas City, KS 66112\nDate: 2026-10-05 at 9:00 AM\nEstimated Duration: 3.5 hours\n\nDrug: Privigen\nOrdered Dose: 40g\nRoute: IV\nLot Number: LG26K0489\nNDC: 44206-0455-10\nInfusion Start Rate: 30 mL/hr\n\nPre-Medication Orders: Diphenhydramine 50mg IV prior to infusion\n\nEmergency Contact: (913) 555-0147\nContraindications: No known drug allergies. No history of hyperprolinemia.\n\nPlease confirm receipt of this assignment by replying to this email.\n\nRachel Kim — CareIG Specialty Pharmacy\nintake@careig.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Re: VISIT ASSIGNMENT CIG_2026_0341 | 2026-10-05",
|
||||
"from_addr": "pnolan.rn@homeinfusionkc.com",
|
||||
"to_addr": "intake@careig.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-09-28T16:05:00-05:00",
|
||||
"message_id": "<msg004-pnolan-20260928@homeinfusionkc.com>",
|
||||
"in_reply_to": "<msg003-careig-20260928@careig.com>",
|
||||
"body_text": "Hi Rachel,\n\nConfirmed. I have the visit on my schedule for 10/05 at 9:00 AM.\n\nI will pick up the Privigen and supplies from the delivery at the patient address beforehand. Please let me know if anything changes.\n\nPatricia Nolan, RN\n(913) 555-0283",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 5
|
||||
}
|
||||
+464
@@ -0,0 +1,464 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "lchoi",
|
||||
"deleted": false,
|
||||
"real_name": "Lisa Choi",
|
||||
"profile": {
|
||||
"real_name": "Lisa Choi",
|
||||
"display_name": "lchoi",
|
||||
"email": "lchoi@careig.com",
|
||||
"title": "Intake Representative"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "rkim",
|
||||
"deleted": false,
|
||||
"real_name": "Rachel Kim",
|
||||
"profile": {
|
||||
"real_name": "Rachel Kim",
|
||||
"display_name": "rkim",
|
||||
"email": "rkim@careig.com",
|
||||
"title": "Benefits Verification Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "pnolan",
|
||||
"deleted": false,
|
||||
"real_name": "Patricia Nolan",
|
||||
"profile": {
|
||||
"real_name": "Patricia Nolan",
|
||||
"display_name": "pnolan",
|
||||
"email": "pnolan.rn@homeinfusionkc.com",
|
||||
"title": "Infusion Nurse"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "dpark",
|
||||
"deleted": false,
|
||||
"real_name": "David Park",
|
||||
"profile": {
|
||||
"real_name": "David Park",
|
||||
"display_name": "dpark",
|
||||
"email": "dpark@careig.com",
|
||||
"title": "Clinical Coordinator"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "jmartinez",
|
||||
"deleted": false,
|
||||
"real_name": "Julio Martinez",
|
||||
"profile": {
|
||||
"real_name": "Julio Martinez",
|
||||
"display_name": "jmartinez",
|
||||
"email": "jmartinez@careig.com",
|
||||
"title": "Billing Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006",
|
||||
"team_id": "T001",
|
||||
"name": "tnguyen",
|
||||
"deleted": false,
|
||||
"real_name": "Tanya Nguyen",
|
||||
"profile": {
|
||||
"real_name": "Tanya Nguyen",
|
||||
"display_name": "tnguyen",
|
||||
"email": "tnguyen@careig.com",
|
||||
"title": "AR Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007",
|
||||
"team_id": "T001",
|
||||
"name": "swright",
|
||||
"deleted": false,
|
||||
"real_name": "Sandra Wright",
|
||||
"profile": {
|
||||
"real_name": "Sandra Wright",
|
||||
"display_name": "swright",
|
||||
"email": "swright@careig.com",
|
||||
"title": "Director of Revenue Cycle"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "intake-team",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "intake-team",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Intake workflow notifications \u2014 BVS assignments, PA submissions, formulary issues, financial hardship flags",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Intake workflow notifications \u2014 BVS assignments, PA submissions, formulary issues, financial hardship flags",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "intake-alerts",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "intake-alerts",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Intake hold alerts \u2014 missing NPI, unreachable patients, consent issues, PA holds, drug mismatches",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Intake hold alerts \u2014 missing NPI, unreachable patients, consent issues, PA holds, drug mismatches",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "intake-mgmt",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "intake-mgmt",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Intake management escalations \u2014 OON pharmacy decisions, formulary unresolved, PA delays",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Intake management escalations \u2014 OON pharmacy decisions, formulary unresolved, PA delays",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "clinical-scheduling",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "clinical-scheduling",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Clinical workflow \u2014 PA approvals, scheduling, supply holds, visit completion, unsigned notes",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Clinical workflow \u2014 PA approvals, scheduling, supply holds, visit completion, unsigned notes",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C005": {
|
||||
"id": "C005",
|
||||
"name": "billing",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "billing",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Billing workflow \u2014 claim submissions, AR follow-ups, claim holds, timely filing risks",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Billing workflow \u2014 claim submissions, AR follow-ups, claim holds, timely filing risks",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C006": {
|
||||
"id": "C006",
|
||||
"name": "billing-appeals",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "billing-appeals",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Denial and appeal notifications \u2014 PA denials, denial filings, appeal submissions",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Denial and appeal notifications \u2014 PA denials, denial filings, appeal submissions",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C007": {
|
||||
"id": "C007",
|
||||
"name": "billing-mgmt",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "billing-mgmt",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Billing management escalations \u2014 underpayments, benefit type issues, write-off requests, collections, L2 denials",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Billing management escalations \u2014 underpayments, benefit type issues, write-off requests, collections, L2 denials",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
},
|
||||
"C008": {
|
||||
"id": "C008",
|
||||
"name": "compliance",
|
||||
"is_channel": false,
|
||||
"is_group": true,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": true,
|
||||
"created": 1640995200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "compliance",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1640995200,
|
||||
"creator": "U007",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Compliance alerts \u2014 legal threats, fraud reports (restricted access)",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Compliance alerts \u2014 legal threats, fraud reports (restricted access)",
|
||||
"creator": "U007",
|
||||
"last_set": 1640995200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "BVS ASSIGNED CIG_2026_0341 | Payer: Sunflower Health Commercial PPO | Drug: Privigen | Dose: 40g",
|
||||
"ts": "1788461600.000000",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "PA SUBMITTED CIG_2026_0341 | Payer: Sunflower Health Commercial PPO | Drug: Privigen | Ref: SHC-PA-20260912-7803",
|
||||
"ts": "1789325100.000000",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [],
|
||||
"C003": [],
|
||||
"C004": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "PA APPROVED CIG_2026_0341 | Drug: Privigen | Dose: 40g | PA#: PA-2026-SHC-41092 | Exp: 2027-03-20 | Drug Source: CareIG",
|
||||
"ts": "1790021200.000000",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U004",
|
||||
"text": "SCHEDULED CIG_2026_0341 | Date: 2026-10-05 | Nurse: Patricia Nolan, RN",
|
||||
"ts": "1790710700.000000",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "VISIT COMPLETE CIG_2026_0341 | DOS: 2026-10-05",
|
||||
"ts": "1791320200.000000",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C005": [],
|
||||
"C006": [],
|
||||
"C007": [],
|
||||
"C008": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
+1729
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
I (Julio Martinez) believe we’re ready to advance Margaret’s case to the next step. Please do that for me by following the SOP. Complete all steps possible and needed, including any file updates, communications, and logging specified by the SOP.
|
||||
|
||||
All case files and reference files are in the workspace. And you can use my information if you need it for anything.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, October, 8, 2026, 09:30 AM CDT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/medical_careig_specialty_pharmacy_08ae3378"
|
||||
description = "SOP task from CSV row 08ae3378"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
[Current datetime: 2026-10-08, 09:30 AM CDT]
|
||||
[Current user: Julio Martinez]
|
||||
|
||||
I believe we’re ready to advance Margaret’s case to the next step. Please do that for me by following the SOP. Complete all steps possible and needed, including any file updates, communications, and logging specified by the SOP.
|
||||
|
||||
All case files and reference files are in the workspace. And you can use my information if you need it for anything.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user