Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,128 @@
{
"mailbox": {
"email": "appeals-intake@mojavecrest.com",
"name": "Mojave Crest Appeals Intake"
},
"contacts": [
{
"email": "ur@desertspringsrecovery.org",
"name": "Utilization Review - Desert Springs Recovery Center"
},
{
"email": "intake@canyonridgereview.com",
"name": "Canyon Ridge"
},
{
"email": "member.services@mojavecrest.com",
"name": "Member Services - Mojave Crest Assurance Company"
},
{
"email": "records@sunriseimaging.com",
"name": "Radiology Records - Sunrise Imaging Center"
}
],
"folders": [
{
"name": "Active Appeals"
},
{
"name": "Pending Review"
},
{
"name": "Archive/2026-Q1"
}
],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "Concurrent Care Appeal — Auth AUTH-2026-00441 — Termination 2026-04-16",
"from_addr": "ur@desertspringsrecovery.org",
"to_addr": "appeals-intake@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-14T09:47:00-07:00",
"message_id": "<AUTH-00441-appeal-20260414@desertspringsrecovery.org>",
"in_reply_to": null,
"body_text": "EXPEDITED CONCURRENT CARE APPEAL REQUEST\n\nTO: Mojave Crest Assurance Company Appeals & Grievances Department\nFROM: Utilization Review Department, Desert Springs Recovery Center\nDATE: April 14, 2026\nRE: Expedited Appeal - Concurrent Care Authorization Termination\n\n═══════════════════════════════════════════════════════════\nMEMBER AND AUTHORIZATION INFORMATION\n═══════════════════════════════════════════════════════════\n\nMember ID: MBR-2026-04471\nGroup ID: GRP-00892\nAuthorization ID: AUTH-2026-00441\nClaim ID: CLM-2026-03917\n\nAuthorized Service: Inpatient Substance Use Disorder Residential Treatment\nFacility: Desert Springs Recovery Center\nFacility NPI: 1497263850\n\nAdmission Date: March 18, 2026\nStay Dates: March 18, 2026 through present (ongoing)\nTotal Billed Charges: $88,500.00 (to date)\n\n═══════════════════════════════════════════════════════════\nTERMINATION NOTICE DETAILS\n═══════════════════════════════════════════════════════════\n\nMCA Termination Notice Date: April 13, 2026\nTermination Effective Date/Time: April 16, 2026 at 12:00 PM PT\nMedical Policy Cited: MP-018-v2.1-20260101\n (\"Residential SUD Treatment - Medical Necessity Criteria\")\n\n═══════════════════════════════════════════════════════════\nBASIS FOR APPEAL\n═══════════════════════════════════════════════════════════\n\nDesert Springs Recovery Center hereby submits an EXPEDITED APPEAL of the concurrent care authorization termination issued April 13, 2026, on the following grounds:\n\n1. CLINICAL APPROPRIATENESS\n The member continues to meet medical necessity criteria for residential SUD treatment. Clinical deterioration and relapse risk remain elevated. Premature discharge poses immediate threat to member safety and treatment outcomes.\n\n2. MENTAL HEALTH PARITY VIOLATION\n MCA's application of Medical Policy MP-018-v2.1 to residential SUD treatment imposes criteria that are MATERIALLY MORE RESTRICTIVE than those applied to comparable medical/surgical inpatient rehabilitation services, including:\n\n • Stricter functional improvement timelines\n • More rigid discharge-readiness benchmarks\n • Higher burden of proof for continued stay justification\n • Narrower recognition of co-occurring condition complexity\n\n This differential standard violates:\n - Mental Health Parity and Addiction Equity Act (MHPAEA), 42 U.S.C. § 300gg-26\n - Nevada Revised Statutes § 695B.255 (Mental Health and Substance Use Disorder Parity)\n\n3. EXPEDITED REVIEW JUSTIFICATION\n With termination effective April 16 at 12:00 PM PT (48 hours from receipt of this appeal), the member faces:\n - Immediate loss of medically necessary care\n - Unsafe discharge during critical treatment phase\n - Increased relapse and hospitalization risk\n\n═══════════════════════════════════════════════════════════\nREQUEST\n═══════════════════════════════════════════════════════════\n\nDesert Springs Recovery Center respectfully requests:\n\n1. EXPEDITED concurrent care appeal review pursuant to NAC 695B.530\n2. Extension of authorization coverage pending appeal resolution\n3. Independent clinical review by qualified SUD specialist\n4. Parity compliance assessment of MP-018-v2.1 application\n\n\n═══════════════════════════════════════════════════════════\nTREATING PHYSICIAN CERTIFICATION OF EXPEDITED REVIEW NEED\n═══════════════════════════════════════════════════════════\n\nI, Dr. Marcus Odera, MD, FASAM, am the attending physician of record for\nthe member identified above (MBR-2026-04471) at Desert Springs Recovery\nCenter. I hereby certify, in writing, pursuant to 42 C.F.R. § 438.210 and\nNevaLine truncated
"body_html": null,
"is_read": false,
"is_important": true,
"attachments": [
{
"filename": "termination_notice_AUTH-2026-00441.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDEKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KPj4KZW5kb2JqCjMgMCBvYmoKPDwKL1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovUmVzb3VyY2VzIDw8Ci9Gb250IDw8Ci9GMSA0IDAgUgo+Pgo+PgovQ29udGVudHMgNSAwIFIKPj4KZW5kb2JqCjQgMCBvYmoKPDwKL1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9CYXNlRm9udCAvVGltZXMtUm9tYW4KPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0xlbmd0aCA0NAo+PgpzdHJlYW0KQlQKL0YxIDEyIFRmCjEwMCA3MDAgVGQKKFRlcm1pbmF0aW9uIE5vdGljZSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDA2NCAwMDAwMCBuIAowMDAwMDAwMTQ1IDAwMDAwIG4gCjAwMDAwMDAyNjMgMDAwMDAgbiAKMDAwMDAwMDM0OCAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9TaXplIDYKL1Jvb3QgMSAwIFIKPj4Kc3RhcnR4cmVmCjQ0MQolJUVPRgo="
},
{
"filename": "clinical_notes_2026-04-13.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDEKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KPj4KZW5kb2JqCjMgMCBvYmoKPDwKL1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovUmVzb3VyY2VzIDw8Ci9Gb250IDw8Ci9GMSA0IDAgUgo+Pgo+PgovQ29udGVudHMgNSAwIFIKPj4KZW5kb2JqCjQgMCBvYmoKPDwKL1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9CYXNlRm9udCAvVGltZXMtUm9tYW4KPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0xlbmd0aCA0Mwo+PgpzdHJlYW0KQlQKL0YxIDEyIFRmCjEwMCA3MDAgVGQKKENsaW5pY2FsIE5vdGVzKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAxNSAwMDAwMCBuIAowMDAwMDAwMDY0IDAwMDAwIG4gCjAwMDAwMDAxNDUgMDAwMDAgbiAKMDAwMDAwMDI2MyAwMDAwMCBuIAowMDAwMDAwMzQ4IDAwMDAwIG4gCnRyYWlsZXIKPDwKL1NpemUgNgovUm9vdCAxIDAgUgo+PgpzdGFydHhyZWYKNDQwCiUlRU9GCg=="
},
{
"filename": "provider_appeal_letter_2026-04-14.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDEKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KPj4KZW5kb2JqCjMgMCBvYmoKPDwKL1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovUmVzb3VyY2VzIDw8Ci9Gb250IDw8Ci9GMSA0IDAgUgo+Pgo+PgovQ29udGVudHMgNSAwIFIKPj4KZW5kb2JqCjQgMCBvYmoKPDwKL1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9CYXNlRm9udCAvVGltZXMtUm9tYW4KPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0xlbmd0aCA0Mgo+PgpzdHJlYW0KQlQKL0YxIDEyIFRmCjEwMCA3MDAgVGQKKEFwcGVhbCBMZXR0ZXIpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDE1IDAwMDAwIG4gCjAwMDAwMDAwNjQgMDAwMDAgbiAKMDAwMDAwMDE0NSAwMDAwMCBuIAowMDAwMDAwMjYzIDAwMDAwIG4gCjAwMDAwMDAzNDggMDAwMDAgbiAKdHJhaWxlcgo8PAovU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0MzkKJSVFT0YK"
}
]
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "Case Acknowledgment - CASE-2026-00086 - External Review Accepted",
"from_addr": "intake@canyonridgereview.com",
"to_addr": "appeals-intake@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-12T14:22:00Z",
"message_id": "<IRO-ACK-00086-20260412@canyonridgereview.com>",
"in_reply_to": null,
"body_text": "Mojave Crest Assurance Company Appeals & Grievances Department:\n\nThis correspondence confirms Canyon Ridge's receipt and acceptance of the following case for external review:\n\nCASE ID: CASE-2026-00086\nMember ID: MBR-2026-09341\nDate Forwarded: April 10, 2026\nReview Type: Standard External Review\nService Category: Outpatient Diagnostic Imaging\n\nCASE STATUS:\n✓ Case file received complete\n✓ Clinical reviewer assigned\n✓ Review in progress\n\nExpected determination date: May 25, 2026\n\nNo additional information is required from Mojave Crest Assurance Company at this time. Canyon Ridge will notify all parties upon completion of the independent medical review.\n\nIf you have questions regarding this case, please reference CASE-2026-00086 in all correspondence.\n\nSincerely,\n\nCanyon Ridge\nCase Management Department\nPhone: (877) 555-0198\nintake@canyonridgereview.com\n\n---\nThis is an informational notice. No action is required.",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "3",
"folder": "INBOX",
"subject": "Status Inquiry - CASE-2026-00088",
"from_addr": "member.services@mojavecrest.com",
"to_addr": "appeals-intake@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-11T10:15:00Z",
"message_id": "<MS-FWD-88-20260411@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Appeals Intake Team,\n\nMember Services has received an inquiry from the member below requesting a status update on their pending appeal. Please see forwarded message:\n\n--- FORWARDED MESSAGE ---\nFrom: Member (MBR-2026-17203)\nReceived: April 11, 2026 9:47 AM\n\nHello,\n\nI submitted an appeal three weeks ago regarding the denial of my inpatient medical/surgical stay (Case # CASE-2026-00088). I haven't heard anything back and my doctor is asking when we can proceed.\n\nCan you please let me know what the status is and when I can expect a decision?\n\nThank you\n--- END FORWARDED MESSAGE ---\n\nCASE DETAILS:\nCase ID: CASE-2026-00088\nMember ID: MBR-2026-17203\nAppeal Received: March 21, 2026\nService: Inpatient Medical/Surgical Stay\n\nMember Services has logged this inquiry as an inbound contact (Ticket #MS-56721). No new appeal intake action is required.\n\nPlease provide status update when available so we can respond to member.\n\nThanks,\nMember Services Team\nMojave Crest Assurance Company",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "4",
"folder": "INBOX",
"subject": "Supplemental Radiology Records - CASE-2026-00086",
"from_addr": "records@sunriseimaging.com",
"to_addr": "appeals-intake@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-01T13:22:00Z",
"message_id": "<SUPP-DOC-086-20260401@sunriseimaging.com>",
"in_reply_to": null,
"body_text": "Mojave Crest Assurance Company Appeals Department:\n\nRE: Supplemental Radiology Records Submission\n CASE-2026-00086 / Member ID MBR-2026-09341\n\nPer your request dated March 31, 2026, Sunrise Imaging Center is providing supplemental documentation in support of the pending appeal for outpatient advanced imaging services (PET/CT Whole Body, CPT 78816).\n\nENCLOSED DOCUMENTATION:\n• Complete radiology interpretation report (March 5, 2026)\n• Ordering physician referral and clinical indication letter\n• Facility accreditation documentation (ACR)\n\nThese records should be appended to the existing case file for CASE-2026-00086. Please contact our records department at (702) 555-0214 if additional documentation is required.\n\nRadiology Records Department\nSunrise Imaging Center\nNPI: 1623847291\nPhone: (702) 555-0213\nFax: (702) 555-0214\nrecords@sunriseimaging.com",
"body_html": null,
"is_read": false,
"is_important": false,
"attachments": [
{
"filename": "supplemental_radiology_records_CASE-2026-00086.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDEKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KPj4KZW5kb2JqCjMgMCBvYmoKPDwKL1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovUmVzb3VyY2VzIDw8Ci9Gb250IDw8Ci9GMSA0IDAgUgo+Pgo+PgovQ29udGVudHMgNSAwIFIKPj4KZW5kb2JqCjQgMCBvYmoKPDwKL1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9CYXNlRm9udCAvVGltZXMtUm9tYW4KPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0xlbmd0aCA1Mgo+PgpzdHJlYW0KQlQKL0YxIDEyIFRmCjEwMCA3MDAgVGQKKFN1cHBsZW1lbnRhbCBSZWNvcmRzKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAxNSAwMDAwMCBuIAowMDAwMDAwMDY0IDAwMDAwIG4gCjAwMDAwMDAxNDUgMDAwMDAgbiAKMDAwMDAwMDI2MyAwMDAwMCBuIAowMDAwMDAwMzQ4IDAwMDAwIG4gCnRyYWlsZXIKPDwKL1NpemUgNgovUm9vdCAxIDAgUgo+PgpzdGFydHhyZWYKNDQ5CiUlRU9GCg=="
}
]
}
],
"next_email_id": 5
}
@@ -0,0 +1,753 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "treyes",
"deleted": false,
"real_name": "Tamra Reyes",
"profile": {
"real_name": "Tamra Reyes",
"display_name": "treyes",
"email": "t.reyes@mojavecrest.com",
"title": "Director of Appeals & Grievances"
},
"is_admin": true,
"is_owner": true,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "dwhitfield",
"deleted": false,
"real_name": "Darnell Whitfield",
"profile": {
"real_name": "Darnell Whitfield",
"display_name": "dwhitfield",
"email": "d.whitfield@mojavecrest.com",
"title": "Appeals Supervisor"
},
"is_admin": true,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "pannadurai",
"deleted": false,
"real_name": "Priya Annadurai",
"profile": {
"real_name": "Priya Annadurai",
"display_name": "pannadurai",
"email": "p.annadurai@mojavecrest.com",
"title": "Senior Appeals Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "hbergstrom",
"deleted": false,
"real_name": "Dr. Helena Bergstrom",
"profile": {
"real_name": "Dr. Helena Bergstrom",
"display_name": "hbergstrom",
"email": "h.bergstrom@mojavecrest.com",
"title": "Medical Director - Medical/Surgical"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "aontario",
"deleted": false,
"real_name": "Dr. Ayla Ontario",
"profile": {
"real_name": "Dr. Ayla Ontario",
"display_name": "aontario",
"email": "a.ontario@mojavecrest.com",
"title": "Medical Director - Behavioral Health / Parity Committee Chair"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "malvarado",
"deleted": false,
"real_name": "Marisol Alvarado",
"profile": {
"real_name": "Marisol Alvarado",
"display_name": "malvarado",
"email": "m.alvarado@mojavecrest.com",
"title": "Lead Clinical Nurse - Appeals"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "jhenning",
"deleted": false,
"real_name": "Dr. Jules Henning",
"profile": {
"real_name": "Dr. Jules Henning",
"display_name": "jhenning",
"email": "j.henning@mojavecrest.com",
"title": "Pharmacy Director"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "kdufree",
"deleted": false,
"real_name": "Kendel Dufree",
"profile": {
"real_name": "Kendel Dufree",
"display_name": "kdufree",
"email": "k.dufree@mojavecrest.com",
"title": "Compliance Officer"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "parity-committee",
"deleted": false,
"real_name": "Parity Review Committee",
"profile": {
"real_name": "Parity Review Committee",
"display_name": "parity-committee",
"email": "parity@mojavecrest.com",
"title": "Committee Group Handle"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U010": {
"id": "U010",
"team_id": "T001",
"name": "coordinator",
"deleted": false,
"real_name": "Appeals Coordinator (Test Model)",
"profile": {
"real_name": "Appeals Coordinator (Test Model)",
"display_name": "coordinator",
"email": "coordinator@mojavecrest.com",
"title": "Appeals Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "appeals-intake",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "appeals-intake",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "New appeal intake logging and routing questions",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "Case log notifications and intake coordination",
"creator": "U002",
"last_set": 1704067200
}
},
"C002": {
"id": "C002",
"name": "appeals-clinical-queue",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "appeals-clinical-queue",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Clinical reviewer case assignments",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "Format B-1 routing to clinical reviewers",
"creator": "U002",
"last_set": 1704067200
}
},
"C003": {
"id": "C003",
"name": "appeals-urgent",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "appeals-urgent",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U002",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Expedited appeals requiring immediate attention",
"creator": "U002",
"last_set": 1704067200
},
"purpose": {
"value": "72-hour expedited case coordination",
"creator": "U002",
"last_set": 1704067200
}
},
"C004": {
"id": "C004",
"name": "appeals-parity",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "appeals-parity",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U005",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Mental Health & Substance Use Disorder Parity Act compliance",
"creator": "U005",
"last_set": 1704067200
},
"purpose": {
"value": "General parity policy discussion - case-specific notifications via DM only",
"creator": "U005",
"last_set": 1704067200
}
},
"C005": {
"id": "C005",
"name": "appeals-compliance",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "appeals-compliance",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U008",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Regulatory compliance and audit coordination",
"creator": "U008",
"last_set": 1704067200
},
"purpose": {
"value": "Compliance events, audits, and regulatory tracking",
"creator": "U008",
"last_set": 1704067200
}
},
"C006": {
"id": "C006",
"name": "appeals-general",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1609459200,
"is_archived": false,
"is_general": true,
"unlinked": 0,
"name_normalized": "appeals-general",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1704067200,
"creator": "U001",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "General appeals team discussion and announcements",
"creator": "U001",
"last_set": 1704067200
},
"purpose": {
"value": "Team coordination, schedules, and non-case discussion",
"creator": "U001",
"last_set": 1704067200
}
},
"DM_U010_U002_1": {
"id": "DM_U010_U002_1",
"name": "dwhitfield",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"is_archived": false,
"is_member": true,
"user": "U002",
"created": 1768924800,
"context_team_id": "T001",
"members": [
"U002",
"U010"
]
},
"DM_U010_U002_2": {
"id": "DM_U010_U002_2",
"name": "dwhitfield",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"is_archived": false,
"is_member": true,
"user": "U002",
"created": 1769097600,
"context_team_id": "T001",
"members": [
"U002",
"U010"
]
},
"DM_U010_U005_1": {
"id": "DM_U010_U005_1",
"name": "aontario",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"is_archived": false,
"is_member": true,
"user": "U005",
"created": 1776208800,
"context_team_id": "T001",
"members": [
"U005",
"U010"
]
},
"DM_U010_U009_1": {
"id": "DM_U010_U009_1",
"name": "parity-committee",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"is_archived": false,
"is_member": true,
"user": "U009",
"created": 1776208800,
"context_team_id": "T001",
"members": [
"U009",
"U010"
]
},
"DM_U003_U005_1": {
"id": "DM_U003_U005_1",
"name": "DM_U003_U005_1",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"members": [
"U003",
"U005"
],
"user": "U003"
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U003",
"text": "\ud83d\udccb New appeal logged: CASE-2026-00084 | Standard | M/S | Received 2026-01-20",
"ts": "1768924800.000100",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "\ud83d\udccb New appeal logged: CASE-2026-00085 | Standard | BH | Received 2026-01-20",
"ts": "1769011200.000200",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "Quick question - member submitted appeal via both fax and portal. Same documents. Do we log as duplicate or single case with dual receipt notation?",
"ts": "1769097600.000300",
"team": "T001",
"reply_count": 1,
"latest_reply": "1769098200.000400",
"reply_users": [
"U002"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U002",
"text": "Single case, dual receipt notation in the intake log. Use the earlier timestamp as official receipt time.",
"ts": "1769098200.000400",
"team": "T001",
"thread_ts": "1769097600.000300",
"parent_user_id": "U003"
},
{
"type": "message",
"user": "U003",
"text": "\ud83d\udccb New appeal logged: CASE-2026-00087 | Standard | Administrative | Received 2026-01-22",
"ts": "1769184000.000500",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "Heads up -- the Q1 audit sample Kendel mentioned includes a few from December. If anyone worked those cases, make sure the case folders are complete in SharePoint.",
"ts": "1769270400.000600",
"team": "T001"
}
],
"C002": [
{
"type": "message",
"user": "U003",
"text": "\ud83d\udce4 **FORMAT B-1 Clinical Routing**\n\n**Case ID:** CASE-2026-00084\n**Type:** Standard Appeal - Medical/Surgical\n**Service:** Inpatient rehabilitation stay extension\n**Assigned to:** @malvarado\n**Due Date:** 2026-02-19\n**Documents:** Available in SharePoint folder CASE-2026-00084\n\nPlease acknowledge receipt and confirm review timeline.",
"ts": "1769011200.000100",
"team": "T001",
"reply_count": 1,
"latest_reply": "1769012100.000200",
"reply_users": [
"U006"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U006",
"text": "Received. Will have initial review completed by EOD Monday 1/27.",
"ts": "1769012100.000200",
"team": "T001",
"thread_ts": "1769011200.000100",
"parent_user_id": "U003"
}
],
"C003": [
{
"type": "message",
"user": "U003",
"text": "\ud83d\udea8 **EXPEDITED APPEAL**\n\n**Case ID:** CASE-2026-00082\n**Service:** Emergency department follow-up surgery\n**Member Status:** Pre-surgical clearance obtained, surgery scheduled 1/23\n**Assigned to:** @hbergstrom\n**Due:** 72 hours from receipt (expires 2026-01-23 11:00 AM)\n\nClinical records uploaded. Member's surgeon provided supporting letter citing immediate medical necessity.",
"ts": "1768924800.000100",
"team": "T001",
"reply_count": 2,
"latest_reply": "1768935300.000300",
"reply_users": [
"U003",
"U004"
],
"reply_users_count": 2
},
{
"type": "message",
"user": "U004",
"text": "Reviewed. Overturning denial - medical necessity clearly documented. Will call surgeon's office directly to confirm. Determination letter being finalized now.",
"ts": "1768932000.000200",
"team": "T001",
"thread_ts": "1768924800.000100",
"parent_user_id": "U003"
},
{
"type": "message",
"user": "U003",
"text": "\u2705 RESOLVED - Overturn issued, member and provider notified by phone and written determination sent 2026-01-21 10:45 AM.",
"ts": "1768935300.000300",
"team": "T001",
"thread_ts": "1768924800.000100",
"parent_user_id": "U003"
}
],
"C004": [
{
"type": "message",
"user": "U005",
"text": "Reminder: The 2026 parity comparative analysis report is due to our compliance team by end of Q1. I'll be scheduling committee review sessions in late February. If anyone has specific case patterns or utilization management concerns to raise, please send them my way by Feb 10.",
"ts": "1769011200.000100",
"team": "T001"
},
{
"type": "message",
"user": "U005",
"text": "Also flagging that CMS updated the MHPAEA FAQs last week - Section VII now has additional guidance on NQTLs for medication-assisted treatment. Worth a read if you're working on SUD cases. Link in our policy library under Federal Guidance > Parity.",
"ts": "1769184000.000200",
"team": "T001"
}
],
"C005": [
{
"type": "message",
"user": "U008",
"text": "\ud83d\udd0d **Compliance Event - RESOLVED**\n\n**Event ID:** CE-2026-003\n**Issue:** Acknowledgment letter for CASE-2026-00079 sent 1 business day late due to mail room delay\n**Resolution:** Member contacted via phone same day, written acknowledgment expedited via email with member consent\n**Corrective Action:** Mail room supervisor notified, daily pickup confirmation process implemented\n**Status:** Closed 2026-01-21\n\nNo regulatory report required (single incident, no member harm, proactive outreach completed).",
"ts": "1769011200.000100",
"team": "T001",
"reply_count": 1,
"latest_reply": "1769014800.000200",
"reply_users": [
"U002"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U002",
"text": "Thanks for the quick turnaround on that, Kendel. Glad we caught it early.",
"ts": "1769014800.000200",
"team": "T001",
"thread_ts": "1769011200.000100",
"parent_user_id": "U008"
},
{
"type": "message",
"user": "U008",
"text": "\ud83d\udcca **Routine Audit Notice**\n\nQ1 2026 appeals audit sample has been selected. Reviewing 25 cases from December 2025 for TAT compliance, acknowledgment accuracy, and clinical documentation standards.\n\nAudit fieldwork: Feb 3-14\nPreliminary findings: Feb 21\nFinal report: Feb 28\n\nNo action required from coordinators at this time. I'll reach out individually if I need clarification on specific case files.",
"ts": "1769270400.000300",
"team": "T001"
}
],
"C006": [
{
"type": "message",
"user": "U002",
"text": "\ud83d\udcc5 **MCA Appeals & Grievances - 2026 Holiday Schedule**\n\nThe following federal holidays are observed by MCA. Please plan case timelines accordingly - TAT calculations exclude these dates:\n\n\u2022 **New Year's Day** - Wednesday, January 1, 2026\n\u2022 **Martin Luther King Jr. Day** - Monday, January 19, 2026\n\u2022 **Presidents' Day** - Monday, February 16, 2026\n\u2022 **Memorial Day** - Monday, May 25, 2026\n\u2022 **Juneteenth** - Friday, June 19, 2026\n\u2022 **Independence Day** - Friday, July 3, 2026 (observed)\n\u2022 **Labor Day** - Monday, September 7, 2026\n\u2022 **Thanksgiving** - Thursday, November 26, 2026\n\u2022 **Christmas Day** - Friday, December 25, 2026\n\nReminder: If a deadline falls on a holiday or weekend, it rolls to the next business day per CFR 42 \u00a7438.408.",
"ts": "1769184000.000100",
"team": "T001",
"reply_count": 1,
"latest_reply": "1769185800.000200",
"reply_users": [
"U006"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U006",
"text": "Thanks Darnell! Bookmarking this for reference.",
"ts": "1769185800.000200",
"team": "T001",
"thread_ts": "1769184000.000100",
"parent_user_id": "U002"
},
{
"type": "message",
"user": "U003",
"text": "Hope everyone has a great week! Anyone planning anything fun this weekend? \u2615",
"ts": "1769270400.000300",
"team": "T001",
"reply_count": 1,
"latest_reply": "1769271300.000400",
"reply_users": [
"U006"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U006",
"text": "Yes! Planning to actually use my new hiking boots finally \ud83d\ude0a",
"ts": "1769271300.000400",
"team": "T001",
"thread_ts": "1769270400.000300",
"parent_user_id": "U003"
}
],
"DM_U010_U002_1": [
{
"type": "message",
"user": "U010",
"text": "Hi Darnell - quick question on CASE-2026-00078. Member submitted additional clinical records after initial review was completed. Do I need to reopen or can I route as supplemental documentation?",
"ts": "1768924800.000100",
"team": "T001",
"channel": "DM_U010_U002_1"
},
{
"type": "message",
"user": "U002",
"text": "If the determination hasn't been issued yet, route as supplemental to the assigned reviewer. If determination already went out, this would be a new appeal - check the date on the member's cover letter to confirm their intent.",
"ts": "1768926600.000200",
"team": "T001",
"channel": "DM_U010_U002_1"
},
{
"type": "message",
"user": "U010",
"text": "Got it - determination is still in draft. I'll forward to Dr. Bergstrom as supplemental. Thanks!",
"ts": "1768927200.000300",
"team": "T001",
"channel": "DM_U010_U002_1"
}
],
"DM_U010_U002_2": [
{
"type": "message",
"user": "U010",
"text": "Darnell - need your input on CASE-2026-00081. This is a pharmacy appeal but the denial was based on medical necessity, not formulary. Should this go to Dr. Henning or to a medical reviewer?",
"ts": "1769097600.000100",
"team": "T001",
"channel": "DM_U010_U002_2"
},
{
"type": "message",
"user": "U002",
"text": "Good catch. Medical necessity determinations require physician review even for pharmacy cases. Route to Dr. Henning first for formulary/clinical guideline assessment, but flag that we may need a medical director consult if it hinges on diagnosis-specific coverage criteria.",
"ts": "1769099400.000200",
"team": "T001",
"channel": "DM_U010_U002_2"
},
{
"type": "message",
"user": "U010",
"text": "Perfect, will route to Dr. Henning with that note. Thanks for the clarification!",
"ts": "1769100000.000300",
"team": "T001",
"channel": "DM_U010_U002_2"
},
{
"type": "message",
"user": "U002",
"text": "\u2705 Resolved - case routed appropriately.",
"ts": "1769101200.000400",
"team": "T001",
"channel": "DM_U010_U002_2"
}
],
"DM_U010_U005_1": [],
"DM_U010_U009_1": [],
"DM_U003_U005_1": [
{
"type": "message",
"user": "U003",
"ts": "1769184600.000100",
"thread_ts": "1769184600.000100",
"text": "Hi Dr. Ontario - routing CASE-2026-00085 to you directly. BH appeal (IOP additional sessions) with a parity flag raised. Case folder in SharePoint. Due Date: 2026-02-19. Let me know if you need anything."
}
]
},
"counters": {
"messageId": 1000100
}
}
@@ -0,0 +1 @@
You are an A&G Coordinator at Mojave Crest Assurance. Check your intake queue and process any new appeals per SOP. Process each so that the case is logged, provider is acknowledged, appropriate parties are notified. Do not complete any action in relation to other emails in the inbox that do not represent a new appeal. This instruction supersedes any instruction provided in the SOP for this given task.
@@ -0,0 +1 @@
Today's date is, April, 14, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/insurance_mojave_crest_assurance_company_5122b4da"
description = "SOP task from CSV row 5122b4da"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,86 @@
[
{
"id": "847ec739-0f1c-4b09-baea-e72a664b9d57",
"sort_order": 0,
"rubric_text": "In `jira_state.json`, a new Jira ticket (beyond pre-existing issues MCA_APPEALS-1 through MCA_APPEALS-5) must exist with the summary or description text containing at least 3 of the following terms: MBR-2026-04471, 88500, parity, Dr. Ontario, concurrent care. Note: the MCP tool assigns the key based on project issue count, so the key may be MCA-APPEALS-6, MCA-6, CASE-1, or similar — the exact key is not checked.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n # Pre-existing issues that were present before the agent started\n pre_existing_keys = {\n \"MCA_APPEALS-1\", \"MCA_APPEALS-2\", \"MCA_APPEALS-3\",\n \"MCA_APPEALS-4\", \"MCA_APPEALS-5\"\n }\n\n issues = data.get(\"issues\", {})\n\n # Find any new issue (not in the pre-existing set) — the MCP tool assigns\n # the key based on project issue count, so it may be MCA-APPEALS-6, MCA-6,\n # CASE-1, or similar depending on which project key the agent used.\n new_issues = {k: v for k, v in issues.items() if k not in pre_existing_keys}\n\n if not new_issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n \"No new Jira ticket found beyond pre-existing issues \"\n \"(MCA_APPEALS-1 through 90). A new Jira ticket must be \"\n \"created with required case information.\"\n )\n }\n\n # Take the first (should be only one) new issue found\n ticket_key, ticket = next(iter(new_issues.items()))\n\n # Collect all text from the ticket recursively\n def collect_text(obj):\n texts = []\n if isinstance(obj, str):\n texts.append(obj)\n elif isinstance(obj, dict):\n for v in obj.values():\n texts.extend(collect_text(v))\n elif isinstance(obj, list):\n for item in obj:\n texts.extend(collect_text(item))\n elif obj is not None:\n texts.append(str(obj))\n return texts\n\n all_text = \" \".join(collect_text(ticket)).lower()\n\n # Check for required terms (case-insensitive)\n # Rubric: at least 3 of these 5 terms\n required_terms = {\n \"MBR-2026-04471\": [\"mbr-2026-04471\", \"mbr202604471\", \"mbr 2026 04471\", \"04471\"],\n \"88500\": [\"88500\", \"88,500\", \"88500.00\", \"$88,500\", \"$88500\"],\n \"parity\": [\"parity\", \"mhpaea\"],\n \"Dr. Ontario\": [\"ontario\"],\n \"concurrent care\": [\"concurrent\"],\n }\n\n found_terms = []\n missing_terms = []\n for term_label, variants in required_terms.items():\n term_found = any(variant in all_text for variant in variants)\n if term_found:\n found_terms.append(term_label)\n else:\n missing_terms.append(term_label)\n\n # Rubric requires at least 3 of 5 terms\n min_required = 3\n total_terms = len(required_terms)\n score = len(found_terms) / total_terms\n\n if len(found_terms) >= min_required:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"New Jira ticket ({ticket_key}) found with {len(found_terms)}/{total_terms} \"\n f\"required terms present: {', '.join(found_terms)}.\"\n + (f\" Missing: {', '.join(missing_terms)}.\" if missing_terms else \"\")\n )\n }\n else:\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": (\n f\"New Jira ticket ({ticket_key}) found but only {len(found_terms)}/{min_required} \"\n f\"required terms present (need at least {min_required}). \"\n f\"Found: {', '.join(found_terms) if found_terms else 'none'}. \"\n f\"Missing: {', '.join(missing_terms)}.\"\n )\n }\n",
"criterion_type": "expected_output"
},
{
"id": "986fba7a-6a01-4681-bb62-af638f93c172",
"sort_order": 1,
"rubric_text": "In `slack_data.json`, a DM must exist to @parity-committee containing Format B-3 fields: Case # MCA-APPEALS-91, Service Classification, Comparable M/S Classification, Flag Indicator, Case Folder, and Decision Deadline of 2026-04-16 12:00 PM PT. This notification must be a DM (not a public channel post in #appeals-parity).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find parity-committee user IDs using various name fields\n parity_committee_user_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n if \"parity\" in name or \"parity\" in real_name or \"parity\" in display or \"parity-committee\" in name:\n parity_committee_user_ids.add(uid)\n\n # Identify DM channels to parity-committee\n dm_channel_ids_to_parity = set()\n for channel_id, cinfo in channels.items():\n is_im = cinfo.get(\"is_im\", False)\n is_mpim = cinfo.get(\"is_mpim\", False)\n is_dm = is_im or is_mpim or str(channel_id).startswith(\"D\")\n if is_dm:\n dm_user = cinfo.get(\"user\", \"\")\n if dm_user in parity_committee_user_ids:\n dm_channel_ids_to_parity.add(channel_id)\n ch_name = cinfo.get(\"name\", \"\").lower()\n if \"parity\" in ch_name:\n dm_channel_ids_to_parity.add(channel_id)\n\n found_b3 = False\n found_msg = None\n best_missing = None\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n\n channel_info = channels.get(channel_id, {})\n channel_name = channel_info.get(\"name\", \"\").lower()\n is_im = channel_info.get(\"is_im\", False)\n is_mpim = channel_info.get(\"is_mpim\", False)\n is_dm = is_im or is_mpim or str(channel_id).startswith(\"D\")\n\n # Skip public #appeals-parity channel - rubric says must be DM not public channel\n is_public_appeals_parity = (\"appeals-parity\" in channel_name) and not is_dm\n if is_public_appeals_parity:\n continue\n\n is_parity_dm_channel = channel_id in dm_channel_ids_to_parity\n dm_user = channel_info.get(\"user\", \"\")\n if dm_user in parity_committee_user_ids:\n is_parity_dm_channel = True\n if \"parity\" in channel_name and is_dm:\n is_parity_dm_channel = True\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n # Must reference MCA-APPEALS-91 (the rubric case number)\n # Accept various formats: MCA-APPEALS-91, MCA-APPEALS-0091, CASE-2026-00091, etc.\n has_case_ref = False\n case_patterns = [\n \"mca-appeals-91\", \"mca-appeals-091\", \"mca-appeals-0091\",\n \"mca appeals 91\", \"mca appeals 091\", \"mca appeals 0091\",\n \"case-2026-00091\", \"case-2026-0091\", \"case-2026-091\", \"case-2026-91\",\n \"case 2026 00091\",\n \"appeals-91\", \"appeals-091\", \"appeals-0091\"\n ]\n for pat in case_patterns:\n if pat in text_lower:\n has_case_ref = True\n break\n # Also try regex for variations\n if not has_case_ref:\n if re.search(r'mca[\\-\\s]?appeals[\\-\\s]?0*91\\b', text_lower):\n has_case_ref = True\n if re.search(r'case[\\-\\s]?2026[\\-\\s]?0*91\\b', text_lower):\n has_case_ref = True\n\n if not has_case_ref:\n continue\n\n # Check if directed to parity-committee\n mentions_parity = (\"parity-committee\" in text_lower or\n \"parity_committee\" in text_lower or\n \"parity committee\" in text_lower or\n \"@parity\" in text_lower or\n any(uid in text for uid in parity_committee_user_ids))\n\n directed_to_parity = is_parity_dm_channel or mentions_parity\n\n if not directed_to_parity:\n continue\n\n # Check for B-3 required fields\n current_missing = []\n\n # 1. Case # MCA-APPEALS-91 - already confirmed above\n\n # 2. Service Classification\n has_service_class = any(term in text_lower for term in [\n \"service cLine truncated
"criterion_type": "expected_output"
},
{
"id": "104af031-8aec-49dd-8e46-ef40afa79a2a",
"sort_order": 2,
"rubric_text": "In `slack_data.json`, a DM must exist to @aontario referencing MCA-APPEALS-91 and requesting parity clinical routing. Use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []). DM channels store the other party's user ID in user, not members.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find aontario user id(s)\n aontario_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n email = uinfo.get(\"profile\", {}).get(\"email\", \"\").lower()\n if \"aontario\" in name or \"aontario\" in display or \"ontario\" in real_name or \"aontario\" in email:\n aontario_ids.add(uid)\n\n # Find DM channels involving aontario\n # Per rubric: use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []).\n # DM channels store the other party's user ID in \"user\", not \"members\".\n aontario_channels = set()\n for cid, cinfo in channels.items():\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False) or str(cid).startswith(\"D\")\n cname = cinfo.get(\"name\", \"\").lower()\n\n # Primary: use \"user\" field for DM channels (per rubric)\n dm_user = cinfo.get(\"user\", \"\")\n if is_dm and dm_user in aontario_ids:\n aontario_channels.add(cid)\n\n # Fallback: check members for mpim or other formats\n members = cinfo.get(\"members\", [])\n if is_dm and any(uid in members for uid in aontario_ids):\n aontario_channels.add(cid)\n\n # Also match by channel name\n if \"aontario\" in cname or \"ontario\" in cname:\n aontario_channels.add(cid)\n\n # Build a pattern to match various forms of the appeal reference:\n # MCA-APPEALS-91, CASE-2026-00091, case 91, appeal 91, etc.\n appeal_patterns = [\n r\"mca[\\-\\s]?appeals[\\-\\s]?0*91\",\n r\"case[\\-\\s]?2026[\\-\\s]?0*91\",\n r\"appeal[s]?[\\-\\s#:]*0*91\",\n r\"\\b0*91\\b\"\n ]\n\n def text_references_appeal(text):\n t = text.lower()\n # Direct string checks\n if \"mca-appeals-91\" in t or \"mca-appeals-091\" in t:\n return True\n if \"case-2026-00091\" in t or \"case-2026-0091\" in t or \"case-2026-091\" in t or \"case-2026-91\" in t:\n return True\n for pat in appeal_patterns:\n if re.search(pat, t):\n return True\n return False\n\n def text_has_parity_clinical_routing(text):\n t = text.lower()\n # Check for parity clinical routing - be loose\n has_parity = \"parity\" in t\n has_clinical = \"clinical\" in t\n has_routing = \"routing\" in t or \"route\" in t or \"review\" in t or \"assign\" in t or \"forward\" in t or \"refer\" in t\n # Accept: parity + clinical, parity + routing, clinical + routing, or any two\n if has_parity and (has_clinical or has_routing):\n return True\n if has_clinical and has_routing:\n return True\n # Also accept just \"parity\" mentioned alongside appeal reference (loose)\n if has_parity:\n return True\n # Accept clinical routing without parity as well (loose)\n if has_clinical and has_routing:\n return True\n return False\n\n found = False\n found_msg = None\n debug_candidates = []\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n if not text_references_appeal(text):\n continue\n\n # Check if this message is directed to aontario\n is_aontario_channel = channel_id in aontario_channels\n mentions_aontario = \"aontario\" in text_lower or \"ontario\" in text_lower or any(uid in text for uid in aontario_ids)\n\n is_to_aontario = is_aontario_channel or mentions_aontario\n\n if not is_to_aontario:\n debug_candidates.append(f\"channel={channel_id}, is_dm={is_dm}, text_preview={text[:100]}\")\n continue\n\n # CheLine truncated
"criterion_type": "expected_output"
},
{
"id": "c5731f16-8ccc-49ca-9334-988b66fb349a",
"sort_order": 3,
"rubric_text": "In `slack_data.json`, a DM must exist to @dwhitfield containing Format B-2 escalation with: Case # MCA-APPEALS-91, Type = Concurrent, Deadline = 2026-04-16 12:00 PM PT, and all three triggers: 'Exceeds Coordinator Authority' ($88,500 > $25K), parity flag raised, and concurrent care termination within 72 hours. Use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []). DM channels store the other party's user ID in user, not members. Check it is a DM, do not use cinfo.get(\"members\", []), I repeat do not use this, instead use cinfo.get(\"user\", \"\") in the code.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find dwhitfield user IDs\n dwhitfield_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n if \"dwhitfield\" in name or \"whitfield\" in real_name or \"dwhitfield\" in display:\n dwhitfield_ids.add(uid)\n\n # Find DM channels involving dwhitfield using cinfo.get(\"user\", \"\") per rubric\n dwhitfield_dm_channels = set()\n for cid, cinfo in channels.items():\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False) or str(cid).startswith(\"D\")\n if is_dm:\n dm_user = cinfo.get(\"user\", \"\")\n if dm_user in dwhitfield_ids:\n dwhitfield_dm_channels.add(cid)\n\n found = False\n found_msg = None\n missing_fields = []\n best_missing = None\n\n # Normalize the case number patterns we accept\n # Rubric says: Case # MCA-APPEALS-91\n # Accept various representations: MCA-APPEALS-91, CASE-2026-00091, case 91, etc.\n def has_case_ref(text_lower, text):\n if \"mca-appeals-91\" in text_lower:\n return True\n if \"mca-appeals-0091\" in text_lower:\n return True\n if \"mca-appeals-00091\" in text_lower:\n return True\n if \"case-2026-00091\" in text_lower:\n return True\n if \"case-2026-0091\" in text_lower:\n return True\n # Flexible: any mention of appeals and 91 together\n if \"appeals\" in text_lower and (\"91\" in text or \"-91\" in text):\n return True\n return False\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n if not has_case_ref(text_lower, text):\n continue\n\n # Must be a DM to dwhitfield or mention dwhitfield\n is_dwhitfield_channel = channel_id in dwhitfield_dm_channels\n mentions_dwhitfield = \"dwhitfield\" in text_lower or \"whitfield\" in text_lower or any(uid in text for uid in dwhitfield_ids)\n\n # Accept if: known DM channel to dwhitfield, OR it's a DM that mentions dwhitfield,\n # OR it's any channel that mentions dwhitfield (be lenient)\n if not (is_dwhitfield_channel or mentions_dwhitfield):\n continue\n\n # Check B-2 fields\n missing = []\n\n # Type: Concurrent\n if \"concurrent\" not in text_lower:\n missing.append(\"Type=Concurrent\")\n\n # Deadline 2026-04-16 12:00 PM PT (lenient: just need 2026-04-16 or April 16)\n if \"2026-04-16\" not in text and \"april 16\" not in text_lower and \"04/16/2026\" not in text and \"4/16/2026\" not in text and \"04-16-2026\" not in text and \"apr 16\" not in text_lower:\n missing.append(\"Deadline 2026-04-16\")\n\n # Trigger 1: Exceeds coordinator authority / $25K / $88,500\n has_exceed = (\"exceed\" in text_lower or \"coordinator authority\" in text_lower or\n \"25,000\" in text or \"25000\" in text or \"25k\" in text_lower or\n \"88,500\" in text or \"88500\" in text or \"$88\" in text or\n \"authority\" in text_lower)\n if not has_exceed:\n missing.append(\"Trigger: Exceeds Coordinator Authority\")\n\n # Trigger 2: Parity flag\n has_parity = \"parity\" in text_lower\n if not has_parity:\n missing.append(\"Trigger: Parity flag\")\n\n # Trigger 3: Concurrent care termination within 72 hours\n has_concurrent_72 = (\"72\" in text and (\"concurrent\" in text_lower or \"termination\" in text_lower or \"hour\" in text_lower)) or \\\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "9d0cb234-202f-4f1c-8cad-e01a15904e8d",
"sort_order": 4,
"rubric_text": "In `mailbox.json`, a sent email must exist to Desert Springs Recovery Center (the triggering sender) referencing MCA-APPEALS-91, using Template A-1 with expedited header block containing exactly: 'Expedited Basis: Treating Physician Certification' and 'Decision Due By: 2026-04-16 12:00 PM PT'.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n mail_path = Path(external_services_path) / \"mailbox.json\"\n if not mail_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mail_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() in [\"sent\", \"sent items\", \"sent mail\"]]\n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.json\"}\n\n # We need to find a sent email that:\n # 1. Is addressed to Desert Springs Recovery Center\n # 2. References MCA-APPEALS-91 (which maps to CASE-2026-00091 or similar)\n # 3. Uses Template A-1\n # 4. Contains expedited header block with:\n # - 'Expedited Basis: Treating Physician Certification'\n # - 'Decision Due By: 2026-04-16 12:00 PM PT'\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n body_full = email.get(\"body_text\", \"\") or \"\"\n body_lower = body_full.lower()\n to_addr = (email.get(\"to_addr\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n combined_lower = body_lower + \" \" + subject + \" \" + to_addr\n\n # Check reference to MCA-APPEALS-91 (accept various formats)\n has_case_ref = False\n case_patterns = [\n \"mca-appeals-91\", \"mca-appeals-091\", \"mca-appeals-0091\",\n \"case-2026-00091\", \"case-2026-0091\", \"case-2026-091\", \"case-2026-91\",\n \"appeals-91\", \"appeals-091\", \"appeals-0091\", \"appeals-00091\",\n \"91\"\n ]\n # Be generous: check for any reasonable reference\n for pat in case_patterns[:-1]: # skip bare \"91\" unless nothing else matches\n if pat in combined_lower:\n has_case_ref = True\n break\n if not has_case_ref:\n # Check with regex for appeal/case number 91\n if re.search(r'(mca|case|appeal)[^a-z0-9]*0*91', combined_lower):\n has_case_ref = True\n if not has_case_ref:\n continue\n\n # Check Desert Springs Recovery Center as recipient\n is_to_desert_springs = False\n if \"desert\" in to_addr and \"spring\" in to_addr:\n is_to_desert_springs = True\n elif \"desertspring\" in to_addr:\n is_to_desert_springs = True\n elif \"desert springs\" in combined_lower:\n is_to_desert_springs = True\n elif \"desert\" in to_addr:\n is_to_desert_springs = True # generous\n if not is_to_desert_springs:\n continue\n\n # Now check the specific required content\n missing = []\n\n # Check for expedited basis: treating physician certification\n if \"treating physician certification\" in body_lower:\n pass # exact match\n elif \"treating physician\" in body_lower and \"certification\" in body_lower:\n pass # close enough\n elif \"treating physician\" in body_lower:\n pass # lenient\n else:\n missing.append(\"Expedited Basis: Treating Physician Certification\")\n\n # Check for Decision Due By: 2026-04-16 12:00 PM PT\n has_date = (\"2026-04-16\" in body_full or \"04/16/2026\" in body_full or \n \"april 16\" in body_lower or \"04-16-2026\" in body_full or\n \"4/16/2026\" in body_full or \"april 16, 2026\" in body_lower)\n if not has_date:\n missing.append(\"Decision Due By date (2026-04-16)\")\n\n has_time = (\"12:00 pm\" in body_lower or \"12:00pm\" in body_lower or \n \"12 pm\" in body_lower or \"12pm\" in body_lower or\n \"noon\" in body_lower or \"12:00 PM\" in body_full or\n \"12:00\" in body_full)\n if not has_time:\n missing.append(\"Decision Due By time (12:00 PM)\")\n\n # Check for expedited keyword\n if \"expedited\" not in body_lower:\n missing.append(\"Expedited header/keyword\")\n\n # Check for Template A-1 reference (lenient - could be in subject or body)\n has_template = (\"a-1\" in combined_lower or \"template a\" in combined_lower or\n \"a1\" in combined_lower or \"template\" in combined_lower)\n if not has_template:\n # Don't require explicit template mention if the content matches the template format\n pass # Be lenient - the template content matters more than naming it\n\n # Allow up to Line truncated
"criterion_type": "expected_output"
},
{
"id": "fa4a3f2f-ed67-452a-affd-07783e53daa3",
"sort_order": 5,
"rubric_text": "In `slack_data.json`, no Format B-3 parity notification must be posted in any public channel named #appeals-parity (the correct destination is a DM to @parity-committee only, per SOP s.8.3).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n # Find #appeals-parity channel\n parity_channel_ids = set()\n for cid, cinfo in channels.items():\n cname = cinfo.get(\"name\", \"\").lower()\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False)\n if \"appeals-parity\" in cname and not is_dm:\n parity_channel_ids.add(cid)\n if not parity_channel_ids:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No #appeals-parity channel found; no erroneous public parity posts possible\"}\n for channel_id in parity_channel_ids:\n msgs = messages.get(channel_id, [])\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\").lower()\n if \"case-2026-00091\" in text:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found B-3 parity notification posted in public #appeals-parity channel (should be DM only): '{msg.get('text','')[:200]}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Format B-3 parity notification found in public #appeals-parity channel; correctly routed as DM only\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "1d4b39bd-d917-40e4-b7ff-875ea8d42055",
"sort_order": 6,
"rubric_text": "In `slack_data.json`, no clinical routing messages for CASE-2026-00091 must be sent to @malvarado or @hbergstrom, and no Format B-1 must be posted in #appeals-clinical-queue for CASE-2026-00091 (parity flag overrides standard clinical routing per SOP s.8.3).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n # Find malvarado and hbergstrom user IDs\n bad_reviewer_ids = set()\n bad_reviewer_names = [\"malvarado\", \"hbergstrom\", \"alvarado\", \"bergstrom\"]\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n for bad_name in bad_reviewer_names:\n if bad_name in name or bad_name in real_name or bad_name in display:\n bad_reviewer_ids.add(uid)\n # Find #appeals-clinical-queue channel\n clinical_channel_ids = set()\n for cid, cinfo in channels.items():\n cname = cinfo.get(\"name\", \"\").lower()\n if \"clinical-queue\" in cname or \"clinical_queue\" in cname:\n clinical_channel_ids.add(cid)\n violations = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n if \"case-2026-00091\" not in text_lower:\n continue\n # Check for malvarado/hbergstrom mentions\n for bad_name in bad_reviewer_names:\n if bad_name in text_lower:\n violations.append(f\"Message to/mentioning {bad_name} references CASE-2026-00091: '{text[:150]}'\")\n # Check for bad reviewer user IDs\n for uid in bad_reviewer_ids:\n if uid in text:\n violations.append(f\"Message mentions bad reviewer UID {uid} for CASE-2026-00091\")\n # Check for B-1 in clinical queue\n if channel_id in clinical_channel_ids:\n violations.append(f\"Found message in #appeals-clinical-queue referencing CASE-2026-00091 (should not be routed via B-1): '{text[:150]}'\")\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Incorrect clinical routing violations found: \" + \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No routing to @malvarado or @hbergstrom, no B-1 in #appeals-clinical-queue for CASE-2026-00091\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "0977af41-e16d-4dee-854a-3ec1f2afd70d",
"sort_order": 7,
"rubric_text": "In `slack_data.json`, no public Slack channel post must contain the member ID 'MBR-2026-04471' or other PHI for this case (PHI must not appear in any public channel for CASE-2026-00091).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n phi_patterns = [\"mbr-2026-04471\", \"mbr2026004471\"]\n violations = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n if is_dm:\n continue # DMs are private; only check public channels\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\").lower()\n for phi in phi_patterns:\n if phi in text:\n channel_name = channel_info.get(\"name\", channel_id)\n violations.append(f\"PHI '{phi}' found in public channel #{channel_name}: '{msg.get('text','')[:150]}'\")\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PHI found in public Slack channels: \" + \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No PHI (MBR-2026-04471) found in any public Slack channel\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "0f5ecf91-0302-45ec-a471-f8333d976cb0",
"sort_order": 8,
"rubric_text": "In jira_state.json, exactly one new Jira issue must have been created by the task session (any key not matching the five pre-existing issues). No additional issues beyond this one new case may exist. The five pre-existing issues MCA_APPEALS-1, MCA_APPEALS-2, MCA_APPEALS-3, MCA_APPEALS-4, and MCA_APPEALS-5 must all still be present and must not have been modified. MCA_APPEALS-2 must have status 'Closed' and must not have been reopened.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n\n # Define the exact pre-existing issue keys (present before agent started)\n pre_existing_keys = {\n \"MCA_APPEALS-1\", \"MCA_APPEALS-2\", \"MCA_APPEALS-3\",\n \"MCA_APPEALS-4\", \"MCA_APPEALS-5\"\n }\n\n violations = []\n\n # Check all 5 pre-existing issues are still present\n for key in sorted(pre_existing_keys):\n if key not in issues:\n violations.append(\n f\"{key} is missing; all five pre-existing issues must still be present\"\n )\n\n # Find new issues (any key not in the pre-existing set)\n # The MCP tool assigns keys based on project issue count, so the new issue\n # may be MCA-APPEALS-6, MCA-6, CASE-1, or similar.\n new_issue_keys = [k for k in issues.keys() if k not in pre_existing_keys]\n\n # Check that exactly one new issue was created\n if len(new_issue_keys) == 0:\n violations.append(\n \"No new Jira issue was created; exactly one new issue must be present\"\n )\n elif len(new_issue_keys) > 1:\n violations.append(\n f\"More than one new Jira issue was created ({new_issue_keys}); \"\n \"only one new issue is allowed\"\n )\n\n # Check MCA_APPEALS-2 status: must be 'Closed' and must not have been reopened\n case_87 = issues.get(\"MCA_APPEALS-2\")\n if case_87:\n fields = case_87.get(\"fields\", case_87)\n # Try multiple possible field names for status\n status_raw = None\n for skey in [\"Status\", \"status\"]:\n if skey in fields:\n status_raw = fields[skey]\n break\n if status_raw is None and \"Status\" in case_87:\n status_raw = case_87[\"Status\"]\n if status_raw is None and \"status\" in case_87:\n status_raw = case_87[\"status\"]\n\n # status could be a dict with 'name' key or a string\n if isinstance(status_raw, dict):\n status_str = str(\n status_raw.get(\"name\", status_raw.get(\"statusCategory\", {}).get(\"name\", \"\"))\n ).strip().lower()\n else:\n status_str = str(status_raw).strip().lower() if status_raw else \"\"\n\n if \"closed\" not in status_str:\n violations.append(\n f\"MCA_APPEALS-2 must have status 'Closed' but found status: '{status_raw}'\"\n )\n if \"open\" in status_str or \"intake\" in status_str or \"reopen\" in status_str:\n violations.append(f\"MCA_APPEALS-2 was reopened (status: {status_raw})\")\n\n # Check for incorrect duplicate/link references that would indicate modification\n for link_key in [\"DuplicateOf\", \"duplicate_of\", \"duplicateOf\", \"Links\", \"links\", \"linkedIssues\"]:\n link_val = str(fields.get(link_key, case_87.get(link_key, \"\"))).lower()\n if \"91\" in link_val and (\"case-2026-00091\" in link_val or \"mca-appeals-91\" in link_val):\n violations.append(\n \"MCA_APPEALS-2 was incorrectly linked/flagged as duplicate of \"\n \"the new appeal case (must not have been modified)\"\n )\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Ticket violations: \" + \"; \".join(violations)}\n\n new_key = new_issue_keys[0] if new_issue_keys else \"unknown\"\n found_keys = sorted(list(issues.keys()))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"All checks passed. Found expected tickets: {found_keys}. \"\n f\"Exactly one new issue ({new_key}) was created. \"\n f\"All five pre-existing issues (MCA_APPEALS-1 through 90) are present and unmodified. \"\n f\"MCA_APPEALS-2 remains in Closed state.\"\n ),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776341102360",
"sort_order": 9,
"rubric_text": "In `slack_data.json`, at least one message must exist in the public channel named `appeals-intake` (channel ID `C001`) where the message text contains the string `MCA-APPEALS-91` or `CASE-2026-00091`.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks Slack data, which is an external service\n # Try both external_services_path and workspace_path for the file\n slack_file = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for candidate in candidates:\n if candidate.exists():\n slack_file = candidate\n break\n \n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in external_services_path or workspace_path.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n target_channel_id = \"C001\"\n target_channel_name = \"appeals-intake\"\n target_strings = [\"MCA-APPEALS-91\", \"CASE-2026-00091\"]\n \n messages_found = []\n \n # Primary pattern per Slack data format: messages dict keyed by channel ID\n messages_section = slack_data.get(\"messages\", {})\n if isinstance(messages_section, dict):\n for key in [target_channel_id, target_channel_name]:\n if key in messages_section:\n val = messages_section[key]\n if isinstance(val, list):\n messages_found.extend(val)\n \n # Pattern 2: channels as dict with embedded messages\n if not messages_found:\n channels = slack_data.get(\"channels\", {})\n if isinstance(channels, dict):\n channel_data = channels.get(target_channel_id, channels.get(target_channel_name, None))\n if channel_data is not None:\n if isinstance(channel_data, dict):\n msgs = channel_data.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_found.extend(msgs)\n elif isinstance(channel_data, list):\n messages_found.extend(channel_data)\n elif isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n ch_id = ch.get(\"id\", ch.get(\"channel_id\", \"\"))\n ch_name = ch.get(\"name\", ch.get(\"channel_name\", \"\"))\n if ch_id == target_channel_id or ch_name == target_channel_name:\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_found.extend(msgs)\n \n # Pattern 3: flat list of messages with channel info\n if not messages_found:\n if isinstance(slack_data.get(\"messages\"), list):\n for msg in slack_data[\"messages\"]:\n if isinstance(msg, dict):\n msg_channel = msg.get(\"channel\", msg.get(\"channel_id\", \"\"))\n if msg_channel in [target_channel_id, target_channel_name]:\n messages_found.append(msg)\n \n if not messages_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found in channel '{target_channel_name}' (ID: {target_channel_id}) in slack_data.json. Slack data keys: {list(slack_data.keys())}\"}\n \n # Check if any message contains either target string\n for msg in messages_found:\n text = \"\"\n if isinstance(msg, dict):\n text = msg.get(\"text\", msg.get(\"message\", msg.get(\"content\", \"\")))\n elif isinstance(msg, str):\n text = msg\n text_str = str(text)\n for target_string in target_strings:\n if target_string in text_str:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in appeals-intake (C001) containing '{target_string}': {text_str[:200]}\"}\n \n # Collect all message texts for debugging\n all_texts = []\n for msg in messages_found:\n if isinstance(msg, dict):\n all_texts.append(str(msg.get(\"text\", msg.get(\"message\", msg.get(\"content\", \"\"))))[:100])\n else:\n all_texts.append(str(msg)[:100])\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(messages_found)} message(s) in appeals-intake (C001), but none contained 'MCA-APPEALS-91' or 'CASE-2026-00091'. Message previews: {all_texts[:5]}\"}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776341121156",
"sort_order": 10,
"rubric_text": "In `mailbox.json`, the total number of sent emails must be exactly one. No outbound email must have been sent to any of the following addresses: `intake@canyonridgereview.com`, `member.services@mojavecrest.com`, or `records@sunriseimaging.com`.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json for sent emails.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, \"r\") as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n emails = mailbox_data.get(\"emails\", [])\n\n # Collect all sent emails\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() in (\"sent\", \"sent\")]\n\n forbidden_addresses = {\n \"intake@canyonridgereview.com\",\n \"member.services@mojavecrest.com\",\n \"records@sunriseimaging.com\"\n }\n\n # Check total count of sent emails\n if len(sent_emails) != 1:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected exactly 1 sent email, but found {len(sent_emails)} sent emails.\"\n }\n\n # Check that no sent email was sent to any forbidden address\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n # Handle possible comma-separated or semicolon-separated recipients\n recipients = [r.strip().lower() for r in to_addr.replace(\";\", \",\").split(\",\") if r.strip()]\n for recipient in recipients:\n if recipient in forbidden_addresses:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Sent email (subject: '{email.get('subject', '')}') was sent to forbidden address '{recipient}'.\"\n }\n\n sent_to = sent_emails[0].get(\"to_addr\", \"\")\n sent_subject = sent_emails[0].get(\"subject\", \"\")\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Exactly 1 sent email found (to: '{sent_to}', subject: '{sent_subject}'). No forbidden addresses were recipients.\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776341142822",
"sort_order": 11,
"rubric_text": "The file `appeals_log_master.xlsx` in the workspace filesystem contains exactly one header row and exactly five data rows, for a total of exactly six rows. No rows have been added beyond these five data rows.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"appeals_log_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'appeals_log_master.xlsx' not found at {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n total_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n wb.close()\n \n if total_rows == 6:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"File 'appeals_log_master.xlsx' contains exactly {total_rows} non-empty rows (1 header + 5 data rows), as expected.\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'appeals_log_master.xlsx' contains {total_rows} non-empty rows, but expected exactly 6 (1 header + 5 data rows). ws.max_row reports {ws.max_row}.\"\n }\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading 'appeals_log_master.xlsx': {str(e)}\"}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
The date is April 14, 2026. You are an A&G Coordinator at Mojave Crest Assurance. Check your intake queue and process any new appeals per SOP. Process each so that the case is logged, provider is acknowledged, appropriate parties are notified. Do not complete any action in relation to other emails in the inbox that do not represent a new appeal. This instruction supersedes any instruction provided in the SOP for this given task.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py