Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,145 @@
{
"events": {
"evt-001": {
"summary": "MCA Observed Holiday \u2014 Presidents' Day",
"start": {
"date": "2026-02-16"
},
"end": {
"date": "2026-02-17"
},
"description": "MCA-observed holiday. Office closed.",
"id": "evt-001"
},
"evt-002": {
"summary": "CASE-2026-00201 \u2014 Pre-service decision DEADLINE",
"start": {
"dateTime": "2026-02-20T08:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-20T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00201 | Member VA-09371-14 | Pre-Service Standard | 30-day decision deadline falls on Saturday Feb 21 \u2014 last day to send is today (Friday). Reviewer determination received 2/12.",
"id": "evt-002"
},
"evt-003": {
"summary": "CASE-2026-00196 \u2014 Acknowledgment deadline",
"start": {
"dateTime": "2026-02-23T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-23T17:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00196 | Member VA-73245-13 | Pre-Service Standard | Acknowledgment deadline \u2014 receipt Feb 13.",
"id": "evt-003"
},
"evt-004": {
"summary": "CASE-2026-00202 \u2014 Acknowledgment deadline",
"start": {
"dateTime": "2026-02-24T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-24T17:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00202 | Member VA-64821-09 | Post-Service Standard | Acknowledgment deadline \u2014 receipt Feb 17.",
"id": "evt-004"
},
"evt-005": {
"summary": "CASE-2026-00203 \u2014 Acknowledgment deadline",
"start": {
"dateTime": "2026-02-24T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-24T17:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00203 | Member VA-18734-05 | Post-Service Standard | Acknowledgment deadline \u2014 receipt Feb 17.",
"id": "evt-005"
},
"evt-006": {
"summary": "CASE-2026-00199 \u2014 Toll window expires (missing info deadline)",
"start": {
"dateTime": "2026-02-24T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-24T17:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00199 | Member VA-33605-07 | Tolling window expires 2/24 \u2014 if records not received, clock restarts and case proceeds on available info.",
"id": "evt-006"
},
"evt-007": {
"summary": "CASE-2026-00204 \u2014 Acknowledgment deadline",
"start": {
"dateTime": "2026-02-25T17:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-25T17:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00204 | Member VA-29567-08 | Pre-Service Standard | Acknowledgment deadline \u2014 receipt Feb 18.",
"id": "evt-007"
},
"evt-008": {
"summary": "Appeals Unit Weekly Standup",
"start": {
"dateTime": "2026-02-19T09:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-19T09:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "Weekly team standup \u2014 Darnell Whitfield facilitating. Conference Room B or Zoom.",
"id": "evt-008"
},
"evt-009": {
"summary": "CASE-2026-00197 \u2014 Reviewer status check",
"start": {
"dateTime": "2026-02-26T10:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-02-26T10:30:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "CASE-2026-00197 | Member VA-11742-08 | Follow up with M. Alvarado if no status update received. Decision due March 15.",
"id": "evt-009"
},
"evt-010": {
"summary": "Parity Assessment Training \u2014 All Coordinators Required",
"start": {
"dateTime": "2026-03-04T14:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-03-04T16:00:00-08:00",
"timeZone": "America/Los_Angeles"
},
"description": "MHPAEA parity assessment training led by Dr. Ayla Ontario. Attendance required for all Appeals Coordinators. Conference Room A.",
"id": "evt-010"
},
"evt-011": {
"summary": "Q1 Appeals Performance Review \u2014 Leadership",
"start": {
"dateTime": "2026-03-10T10:00:00-07:00",
"timeZone": "America/Los_Angeles"
},
"end": {
"dateTime": "2026-03-10T11:30:00-07:00",
"timeZone": "America/Los_Angeles"
},
"description": "Q1 review with Tamra Reyes and unit supervisors. Coordinators not required \u2014 leadership only.",
"id": "evt-011"
}
}
}
@@ -0,0 +1,372 @@
{
"mailbox": {
"email": "jordan.mills@mojavecrest.com",
"name": "Jordan Mills"
},
"contacts": [
{
"email": "d.whitfield@mojavecrest.com",
"name": "Darnell Whitfield"
},
{
"email": "m.alvarado@mojavecrest.com",
"name": "Marisol Alvarado"
},
{
"email": "a.ontario@mojavecrest.com",
"name": "Dr. Ayla Ontario"
},
{
"email": "h.bergstrom@mojavecrest.com",
"name": "Dr. Helena Bergstrom"
},
{
"email": "t.reyes@mojavecrest.com",
"name": "Tamra Reyes"
},
{
"email": "k.dufree@mojavecrest.com",
"name": "Kendel Dufree"
},
{
"email": "p.annadurai@mojavecrest.com",
"name": "Priya Annadurai"
},
{
"email": "appeals-intake@mojavecrest.com",
"name": "MCA Appeals Intake"
},
{
"email": "provider.relations@mojavecrest.com",
"name": "MCA Provider Relations"
},
{
"email": "billing@swshoulderelbow.com",
"name": "Southwest Shoulder & Elbow Institute (Billing)"
},
{
"email": "mwebb@webbassociateslegal.com",
"name": "Marcus Webb, Esq. \u2014 Webb & Associates LLP"
},
{
"email": "h.reyes.hector@gmail.com",
"name": "Hector Reyes (member)"
},
{
"email": "d.park.personal@gmail.com",
"name": "David Park (member)"
},
{
"email": "l.hassan.personal@gmail.com",
"name": "Leila Hassan (member)"
},
{
"email": "h.tanaka.personal@gmail.com",
"name": "Hiroshi Tanaka (member)"
},
{
"email": "billing@cactusvalleymed.com",
"name": "Cactus Valley Medical Center (Billing)"
},
{
"email": "intake@canyonridgereview.com",
"name": "Canyon Ridge Review Services"
},
{
"email": "appeals@mojavecrest.com",
"name": "MCA Appeals General"
},
{
"email": "m.soto.personal@gmail.com",
"name": "Mar\u00eda Soto (member)"
}
],
"folders": [
{
"name": "Archive \u2014 Jan 2026"
},
{
"name": "Appeals \u2014 Active"
},
{
"name": "Archive \u2014 Feb 2026"
}
],
"next_email_id": 16,
"emails": [
{
"email_id": "1",
"folder": "Archive \u2014 Feb 2026",
"subject": "RE: CASE-2026-00188 \u2014 Determination Sent",
"from_addr": "h.bergstrom@mojavecrest.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-06T15:22:00-08:00",
"message_id": "<msg-188-det@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Jordan,\n\nClinical worksheet for CASE-2026-00188 is uploaded to the case folder. Decision: overturn \u2014 auth criteria are met. The member's clinical documentation adequately supports medical necessity for the ESI series.\n\nYou're clear to send the favorable determination letter.\n\n\u2014 Helena",
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "2",
"folder": "Archive \u2014 Feb 2026",
"subject": "RE: CASE-2026-00193 \u2014 Grievance Resolution",
"from_addr": "d.whitfield@mojavecrest.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-11T11:44:00-08:00",
"message_id": "<msg-193-res@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Jordan,\n\nGrievance investigation complete for CASE-2026-00193. I reviewed the call logs from December. Hold times were within SLA \u2014 the member's experience was frustrating but the metrics were compliant. No corrective action warranted. Draft Template A-10 with 'Resolved without Action,' and run it by me before you send. Staff conduct grievances always get my review.\n\n\u2014 Darnell",
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "3",
"folder": "Appeals \u2014 Active",
"subject": "CASE-2026-00201 \u2014 Clinical Determination Ready",
"from_addr": "m.alvarado@mojavecrest.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-12T14:08:00-08:00",
"message_id": "<msg-201-det@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Jordan,\n\nI've completed my clinical review for CASE-2026-00201 (Tanaka, Hiroshi \u2014 ACDF pre-service appeal).\n\nDecision: Overturn. The member's surgeon submitted supplemental clinical documentation and updated MRI findings that I did not have in the original denial packet. The documentation now demonstrates failure of conservative care over a 6-month period and neurological compromise at C5-C6 consistent with clinical criteria for the requested service. Auth criteria are met.\n\nMy worksheet is in the case folder. You're clear to proceed with the favorable determination.\n\nRemember the decision deadline is Feb 21 \u2014 you don't have a lot of runway here.\n\n\u2014 Marisol",
"is_read": false,
"is_important": true,
"attachments": []
},
{
"email_id": "5",
"folder": "INBOX",
"subject": "URGENT: Nevada DOI Bulletin NV-2026-03 \u2014 Response Required by March 15",
"from_addr": "k.dufree@mojavecrest.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": "d.whitfield@mojavecrest.com",
"bcc_addr": null,
"date": "2026-02-17T09:02:00-08:00",
"message_id": "<msg-doi-bulletin@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Team,\n\nThe Nevada Division of Insurance has issued Bulletin NV-2026-03 regarding updated documentation requirements for internal appeal decisions. Compliance is drafting our formal response and will circulate for unit review by March 1.\n\nFor now, no changes to your day-to-day appeals handling procedures. Existing SOP remains in effect. I will send updated instructions if anything changes.\n\nNo immediate action required on your end. Just wanted to loop you in ahead of the all-hands on March 3.\n\nKendel\nCompliance Officer\nMojave Crest Assurance Company",
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "6",
"folder": "INBOX",
"subject": "Appeal \u2014 Denied Shoulder Surgery Claim \u2014 Mar\u00eda Soto (VA-64821-09)",
"from_addr": "m.soto.personal@gmail.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-17T16:08:00-08:00",
"message_id": "<msg-202-intake@swshoulderelbow.com>",
"in_reply_to": null,
"body_text": "To Mojave Crest Assurance Company,\n\nMy name is Mar\u00eda Soto. My member ID is VA-64821-09 and my group number is G-14827. I am writing to appeal the denial of my claim for shoulder surgery.\n\nI had an arthroscopic shoulder repair on February 3, 2026 (CPT 29827, billed amount $38,400.00). The claim was denied with the reason 'prior authorization not obtained for date of service.' However, I did have a prior authorization for this procedure. My surgeon's office told me a prior authorization was approved, and I scheduled the surgery based on that. I was not told that the authorization had expired or that I needed a new one.\n\nI believe this denial is in error and I am asking you to reconsider. I am attaching a copy of the Explanation of Benefits and my surgeon's notes.\n\nSincerely,\nMar\u00eda Soto\nMember ID: VA-64821-09\nDate of Birth: 07/14/1971\nPhone: (480) 555-0312",
"is_read": false,
"is_important": false,
"attachments": [
{
"filename": "EOB_Soto_CLM-2026-014209.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgRU9CIGRvY3VtZW50"
}
]
},
{
"email_id": "7",
"folder": "INBOX",
"subject": "Appeal and Complaint \u2014 Hector Reyes \u2014 Member ID VA-18734-05",
"from_addr": "h.reyes.hector@gmail.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-17T15:52:00-08:00",
"message_id": "<msg-203-intake@gmail.com>",
"in_reply_to": null,
"body_text": "To Mojave Crest Assurance,\n\nI am writing for two reasons.\n\nFirst, I want to formally appeal the denial of my MRI claim. I received an Explanation of Benefits dated February 9, 2026 stating that my knee MRI (date of service February 2, 2026) was denied as not medically necessary. I disagree with this decision. My orthopedist, Dr. Singh, ordered this MRI because I have had persistent knee pain for over a year and completed 11 sessions of physical therapy with only partial improvement. The MRI is needed to determine whether I require surgery. I have attached my doctor's notes and the PT discharge summary.\n\nSecond, I need to formally complain about how I was treated when I called your member services line on February 10, 2026. The representative I spoke with was dismissive, interrupted me several times when I tried to explain my situation, and at one point said 'I don't know why you're calling us, this is a medical decision.' That is completely unacceptable. I want this on record and I want to know what action will be taken.\n\nThank you,\nHector Reyes\nMember ID: VA-18734-05\nDate of Birth: 06/03/1984\nPhone: (702) 555-0371",
"is_read": false,
"is_important": false,
"attachments": [
{
"filename": "DrSingh_clinical_notes_2026-02-02.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgY2xpbmljYWwgbm90ZXM="
},
{
"filename": "PT_discharge_summary_Reyes.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgUFQgZGlzY2hhcmdl"
}
]
},
{
"email_id": "8",
"folder": "INBOX",
"subject": "Notice of Appeal and Reservation of Legal Rights \u2014 Sandra Kimura (VA-29567-08)",
"from_addr": "mwebb@webbassociateslegal.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-18T09:33:00-08:00",
"message_id": "<msg-204-intake@webbassociateslegal.com>",
"in_reply_to": null,
"body_text": "RE: Sandra Kimura (Member ID VA-29567-08) \u2014 Prior Authorization Denial \u2014 Lumbar Spine MRI (CPT 72148)\n\nDear Mojave Crest Assurance Company Appeals Unit,\n\nThis firm represents Sandra Kimura in connection with the denial of prior authorization for lumbar spine MRI with contrast (CPT 72148), denial issued February 10, 2026. Pursuant to the Vista Access PPO Evidence of Coverage, we are filing this formal appeal on Ms. Kimura's behalf and requesting internal appeal review.\n\nGROUNDS FOR APPEAL:\nMs. Kimura has chronic low back pain with facet arthropathy confirmed on prior MRI (2025). Her treating physician, Dr. Thomas Reeve of Sunrise Neurology & Spine, has documented a 7-month course of conservative treatment including physical therapy, NSAIDs, and facet injections without sustained relief. The denial's reliance on 'conservative care not exhausted' is factually unsupported by the clinical record.\n\nSupportive documentation is enclosed, including Dr. Reeve's letter of medical necessity dated February 14, 2026.\n\nPLEASE BE ADVISED: If Mojave Crest Assurance Company fails to issue a favorable determination within five (5) business days of the date of this letter, Webb & Associates LLP will seek injunctive relief and compensatory damages, including reasonable attorney's fees, in Clark County District Court. This letter serves as formal notice of that intent.\n\nPlease direct all future correspondence in this matter to the undersigned at the address below. Do not contact Ms. Kimura directly.\n\nRespectfully,\nMarcus Webb, Esq.\nWebb & Associates LLP\n3820 Howard Hughes Pkwy, Suite 600, Las Vegas, NV 89169\nmwebb@webbassociateslegal.com | (702) 555-0840",
"is_read": false,
"is_important": true,
"attachments": [
{
"filename": "Kimura_Appeal_Webb_20260218.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgYXBwZWFsIGxldHRlcg=="
},
{
"filename": "DrReeve_LMN_Kimura_20260214.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgbGV0dGVyIG9mIG1lZA=="
}
]
},
{
"email_id": "9",
"folder": "INBOX",
"subject": "Late Appeal \u2014 Lumbar MRI Denial \u2014 David Park (VA-51104-11)",
"from_addr": "d.park.personal@gmail.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-19T08:22:00-08:00",
"message_id": "<msg-205-intake@gmail.com>",
"in_reply_to": null,
"body_text": "To Mojave Crest Assurance,\n\nMy name is David Park. My member ID is VA-51104-11 and my group is G-31047. I am writing to appeal a prior authorization denial I received in July 2025.\n\nThe denial was for a lumbar spine MRI (denial letter dated July 22, 2025). I was told I had 180 days to appeal. I know that deadline has passed and I am sorry for filing late, but I have a serious reason for the delay.\n\nShortly after receiving the denial letter, I was hospitalized with a cardiac event. I was admitted on July 28, 2025 and discharged on September 10, 2025 after more than six weeks in the hospital. During that time I was not able to manage mail, make phone calls, or handle any personal business. After my discharge I was in cardiac rehabilitation for several months and was still not fully able to manage my affairs.\n\nI have attached a letter from my cardiologist confirming my hospitalization dates, and my discharge summary. I am asking you to please consider my circumstances and allow me to appeal despite the late filing.\n\nSincerely,\nDavid Park\nMember ID: VA-51104-11\nDate of Birth: 11/08/1959\nPhone: (702) 555-0824",
"is_read": false,
"is_important": false,
"attachments": [
{
"filename": "DrHadley_hospitalization_confirmation.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgaG9zcGl0YWxpemF0aW9u"
},
{
"filename": "discharge_summary_Park_20250910.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgZGlzY2hhcmdlIHN1bW1hcnk="
}
]
},
{
"email_id": "10",
"folder": "INBOX",
"subject": "Appeal of Authorization Denial \u2014 Leila Hassan (VA-73245-13)",
"from_addr": "l.hassan.personal@gmail.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-13T15:47:00-08:00",
"message_id": "<msg-196-intake@gmail.com>",
"in_reply_to": null,
"body_text": "To Mojave Crest Assurance Company Appeals,\n\nI am writing to appeal the denial of my prior authorization request for a spinal cord stimulator trial (CPT 63650). The denial letter is dated January 28, 2026.\n\nI have been living with severe chronic pain for three years. I have tried physical therapy (two full courses), medication management with multiple drug classes, lumbar facet injections (two rounds), and a course of occupational therapy. Nothing has provided lasting relief. My pain management physician, Dr. Nadia Petrova at the Advanced Pain Institute of Nevada, has determined that a spinal cord stimulator trial is the appropriate next step in my care and submitted documentation to support this.\n\nI believe the denial was in error and that my clinical documentation meets your criteria. I am including Dr. Petrova's medical necessity letter and my treatment history records.\n\nPlease let me know what happens next.\n\nLeila Hassan\nMember ID: VA-73245-13\nDate of Birth: 05/29/1968\nPhone: (702) 555-0193",
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "DrPetrova_LMN_Hassan_20260211.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgbWVkaWNhbCBuZWNlc3NpdHk="
},
{
"filename": "Hassan_treatment_history.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgdHJlYXRtZW50IGhpc3Rvcnk="
}
]
},
{
"email_id": "11",
"folder": "INBOX",
"subject": "Payment Status Inquiry \u2014 Claim CLM-2026-001822 \u2014 Park, David",
"from_addr": "billing@cactusvalleymed.com",
"to_addr": "jordan.mills@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-18T10:42:00-08:00",
"message_id": "<msg-billing-inquiry@cactusvalley.com>",
"in_reply_to": null,
"body_text": "Hello,\n\nThis is the billing department at Cactus Valley Medical Center. We are following up on claim CLM-2026-001822 for patient David Park (Member ID VA-51104-11). The claim was for a January 9, 2026 office visit (CPT 99213) in the amount of $195.00.\n\nOur records show this claim was submitted January 14, 2026. We have not yet received payment or an EOB. Could you please confirm the status and expected payment date?\n\nThank you,\nBilling Department\nCactus Valley Medical Center\nbilling@cactusvalleymed.com | (702) 555-0631",
"is_read": false,
"is_important": false,
"attachments": []
},
{
"email_id": "12",
"folder": "Sent",
"subject": "Mojave Crest Appeals Case #CASE-2026-00188 - Appeal Decision",
"from_addr": "jordan.mills@mojavecrest.com",
"to_addr": "h.bergstrom@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-06T16:55:00-08:00",
"message_id": "<msg-188-sent@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Dr. Bergstrom,\n\nAttached is my draft Template A-6 for CASE-2026-00188 per your determination. Please confirm the determination language looks correct before I send to the member.\n\nJordan Mills\nAppeals & Grievances Coordinator",
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "13",
"folder": "Sent",
"subject": "Mojave Crest Appeals Case #CASE-2026-00201 - Acknowledgment of Your Appeal",
"from_addr": "jordan.mills@mojavecrest.com",
"to_addr": "h.tanaka.personal@gmail.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-01-27T09:48:00-08:00",
"message_id": "<msg-201-ack-sent@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Dear Mr. Tanaka,\n\nThis letter confirms that Mojave Crest Assurance Company has received your appeal regarding the denial of prior authorization for anterior cervical discectomy and fusion (ACDF, CPT 22551).\n\nYour case details:\n Case Number: CASE-2026-00201\n Member ID: VA-09371-14\n Date Received: January 22, 2026\n Appeal Type: Pre-Service Standard\n Decision Due By: February 21, 2026\n\nI will be your Appeals Coordinator throughout this case. I may contact you if additional information is needed to complete the review.\n\nIf you have questions about your case, please reply to this email or call the Appeals Unit at (702) 555-0148, referencing your Case Number above.\n\nSincerely,\nJordan Mills\nAppeals & Grievances Coordinator\nMojave Crest Assurance Company\nCase #: CASE-2026-00201",
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "14",
"folder": "Sent",
"subject": "Mojave Crest Appeals Case #CASE-2026-00199 - Information Needed",
"from_addr": "jordan.mills@mojavecrest.com",
"to_addr": "l.sutter@desertsunplasticsurgery.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-09T10:14:00-08:00",
"message_id": "<msg-199-a2-sent2@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Dear Dr. Sutter's Office,\n\nI am writing regarding appeal Case #CASE-2026-00199. In order to complete our review of the benefit determination for CPT 17999, I need the following additional information:\n\n 1. Operative report or surgical notes from the procedure performed on January 30, 2026, documenting the nature and medical basis for the procedure\n 2. Pre-operative clinical photographs documenting the condition that prompted the procedure\n 3. The treating provider's letter explaining the reconstructive \u2014 rather than cosmetic \u2014 basis for the procedure, referencing the specific diagnosis and condition being treated\n\nPlease provide this information within 15 calendar days of the date of this letter (deadline: February 24, 2026). If we do not receive the requested information within that period, your appeal will proceed on the information currently available.\n\nSincerely,\nJordan Mills\nAppeals & Grievances Coordinator\nMojave Crest Assurance Company\nPhone: (702) 555-0148\nCase #: CASE-2026-00199",
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "15",
"folder": "Sent",
"subject": "Claim Reprocessing Request \u2014 CASE-2026-00201 \u2014 Appeals Overturn",
"from_addr": "jordan.mills@mojavecrest.com",
"to_addr": "claims-adjustment@mojavecrest.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-13T11:22:00-08:00",
"message_id": "<msg-201-reprocess@mojavecrest.com>",
"in_reply_to": null,
"body_text": "Claims Adjustment Team,\n\nPlease open a Claim Reprocessing Request for the following case:\n\n Case ID: CASE-2026-00201\n Member ID: VA-09371-14\n Determination: Appeals Overturn\n Reviewer: M. Alvarado (RN)\n Determination date: 2026-02-12\n Service: ACDF CPT 22551\n Billed charges: $62,800.00\n\nClinical worksheet is attached to the case folder in the Appeals repository. Please return the Reprocessing Worksheet to me via Slack DM within 2 business days per SOP.\n\nJordan Mills\nAppeals & Grievances Coordinator\nCase #: CASE-2026-00201",
"is_read": true,
"is_important": false,
"attachments": []
}
]
}
@@ -0,0 +1,630 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "TMCA",
"name": "jordan.mills",
"real_name": "Jordan Mills",
"profile": {
"email": "jordan.mills@mojavecrest.com",
"title": "Appeals & Grievances Coordinator",
"display_name": "Jordan Mills"
}
},
"U002": {
"id": "U002",
"team_id": "TMCA",
"name": "dwhitfield",
"real_name": "Darnell Whitfield",
"profile": {
"email": "d.whitfield@mojavecrest.com",
"title": "Appeals Supervisor",
"display_name": "Darnell Whitfield"
}
},
"U003": {
"id": "U003",
"team_id": "TMCA",
"name": "aontario",
"real_name": "Dr. Ayla Ontario",
"profile": {
"email": "a.ontario@mojavecrest.com",
"title": "Medical Director, Behavioral Health",
"display_name": "Dr. Ontario"
}
},
"U004": {
"id": "U004",
"team_id": "TMCA",
"name": "malvarado",
"real_name": "Marisol Alvarado",
"profile": {
"email": "m.alvarado@mojavecrest.com",
"title": "Lead RN Reviewer",
"display_name": "M. Alvarado"
}
},
"U005": {
"id": "U005",
"team_id": "TMCA",
"name": "pannadurai",
"real_name": "Priya Annadurai",
"profile": {
"email": "p.annadurai@mojavecrest.com",
"title": "Senior Appeals Coordinator",
"display_name": "P. Annadurai"
}
},
"U006": {
"id": "U006",
"team_id": "TMCA",
"name": "kdufree",
"real_name": "Kendel Dufree",
"profile": {
"email": "k.dufree@mojavecrest.com",
"title": "Compliance Officer",
"display_name": "K. Dufree"
}
},
"U007": {
"id": "U007",
"team_id": "TMCA",
"name": "treyes",
"real_name": "Tamra Reyes",
"profile": {
"email": "t.reyes@mojavecrest.com",
"title": "Director, Appeals & Grievances",
"display_name": "T. Reyes"
}
},
"U008": {
"id": "U008",
"team_id": "TMCA",
"name": "hbergstrom",
"real_name": "Dr. Helena Bergstrom",
"profile": {
"email": "h.bergstrom@mojavecrest.com",
"title": "Medical Director, Medical/Surgical",
"display_name": "Dr. Bergstrom"
}
},
"U009": {
"id": "U009",
"team_id": "TMCA",
"name": "claims-adj",
"real_name": "Claims Adjustment Team",
"profile": {
"email": "claims-adjustment@mojavecrest.com",
"title": "Claims Adjustment",
"display_name": "claims-adj"
}
},
"U010": {
"id": "U010",
"team_id": "TMCA",
"name": "jhenning",
"real_name": "Dr. Jules Henning",
"profile": {
"email": "j.henning@mojavecrest.com",
"title": "Pharmacy Director",
"display_name": "Dr. Henning"
}
}
},
"channels": {
"C001": {
"id": "C001",
"name": "appeals-intake",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "New case logging, general unit announcements, routing questions",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "For logging new appeals cases and general Appeals unit announcements. Do NOT post PHI.",
"creator": "U007",
"last_set": 1735689600
}
},
"C002": {
"id": "C002",
"name": "appeals-clinical-queue",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "Clinical case routing \u2014 RN Reviewer and Medical Director teams",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Route clinical cases here. Case ID only \u2014 no PHI.",
"creator": "U007",
"last_set": 1735689600
}
},
"C003": {
"id": "C003",
"name": "appeals-urgent",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "Expedited cases, at-risk deadlines, compliance events",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Expedited and time-sensitive cases only.",
"creator": "U007",
"last_set": 1735689600
}
},
"C004": {
"id": "C004",
"name": "appeals-parity",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "MHPAEA parity compliance \u2014 general discussion only",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "General parity discussion and announcements. Parity flag notifications go to @parity-committee DM \u2014 NOT here.",
"creator": "U007",
"last_set": 1735689600
}
},
"C005": {
"id": "C005",
"name": "appeals-compliance",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "Regulatory compliance events, DOI notifications, audit requests",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Compliance events only. Use Format B-4.",
"creator": "U007",
"last_set": 1735689600
}
},
"C006": {
"id": "C006",
"name": "appeals-general",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"TMCA"
],
"topic": {
"value": "General team chat, questions, non-case discussion",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "General questions, kudos, team news. Holiday schedule and SOP updates posted here.",
"creator": "U007",
"last_set": 1735689600
}
},
"D001": {
"id": "D001",
"name": "DM-jordan-darnell",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U001",
"shared_team_ids": [
"TMCA"
],
"members": [
"U001",
"U002"
],
"user": "U002"
},
"D002": {
"id": "D002",
"name": "DM-jordan-priya",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U001",
"shared_team_ids": [
"TMCA"
],
"members": [
"U001",
"U005"
],
"user": "U005"
},
"D003": {
"id": "D003",
"name": "DM-jordan-kdufree",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U001",
"shared_team_ids": [
"TMCA"
],
"members": [
"U001",
"U006"
],
"user": "U006"
},
"D004": {
"id": "D004",
"name": "DM-jordan-parity-committee",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U001",
"shared_team_ids": [
"TMCA"
],
"members": [
"U001",
"U003"
],
"user": "U003"
},
"D005": {
"id": "D005",
"name": "DM-jordan-claimsadj",
"is_channel": false,
"is_group": false,
"is_im": true,
"is_mpim": false,
"is_private": true,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"is_member": true,
"context_team_id": "TMCA",
"updated": 1735689600,
"creator": "U001",
"shared_team_ids": [
"TMCA"
],
"members": [
"U001",
"U009"
],
"user": "U009"
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "Good morning team \u2014 CASE-2026-00197 in queue, post-service standard, Jan 14 receipt. Ack sent 1/20. Routing to @malvarado for clinical review.",
"ts": "1768928640.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U004",
"text": "Got it \u2014 pulling the case file for CASE-2026-00197. Will have a look this week.",
"ts": "1768930260.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CASE-2026-00201 logged \u2014 pre-service standard. Receipt 1/22. Ack sent 1/27. Routed to @malvarado for clinical review. Decision deadline 2/21 \u2014 keep an eye on this one.",
"ts": "1769536320.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CASE-2026-00188 closed \u2014 overturn. Favorable determination sent 2/6. Closing in log.",
"ts": "1770656700.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CASE-2026-00199 logged \u2014 post-service standard, member VA-33605-07. Receipt 2/2. Ack sent 2/6. Clock paused \u2014 Template A-2 sent 2/9 pending clinical records. 15-day window expires 2/24.",
"ts": "1770661080.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CASE-2026-00193 closed \u2014 grievance resolved without action. Template A-10 sent 2/11. Closing in log.",
"ts": "1770847500.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U005",
"text": "Hey @jordan.mills \u2014 welcome back. Three intake emails came in while you were out Tuesday and Wednesday. I created stub log entries for all three so they don't age without a case ID: CASE-2026-00202, CASE-2026-00203, and CASE-2026-00204. Didn't process any of them further \u2014 all yours to work. Also, fresh email arrived this morning at 8:22 AM \u2014 not yet logged, you'll see it in inbox.",
"ts": "1771520280.000000",
"team": "TMCA"
}
],
"C002": [
{
"type": "message",
"user": "U001",
"text": "CLINICAL ROUTING\nCase #: CASE-2026-00188\nType: Pre-Service Standard\nService Classification: Medical/Surgical\nStandard of Review: Medical Necessity\nParity Flag: NO\nCase Folder: /appeals-repo/2026/CASE-2026-00188/",
"ts": "1768414500.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U004",
"text": "Picked up CASE-2026-00188. Reviewing the ESI criteria and clinical records.",
"ts": "1768417320.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CLINICAL ROUTING\nCase #: CASE-2026-00197\nType: Post-Service Standard\nService Classification: Medical/Surgical\nStandard of Review: Medical Necessity\nParity Flag: NO\nCase Folder: /appeals-repo/2026/CASE-2026-00197/",
"ts": "1768928820.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U004",
"text": "Picked up CASE-2026-00197. Will start clinical review of the knee arthroplasty records this week.",
"ts": "1768930380.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U004",
"text": "CASE-2026-00188 \u2014 RN review complete. Uphold. Member's clinical records do not demonstrate failure of conservative care meeting clinical criteria for the requested service. Referring to Medical Director per protocol.",
"ts": "1768932900.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CLINICAL ROUTING \u2014 ESCALATION\nCase #: CASE-2026-00188\nType: Pre-Service Standard\nService Classification: Medical/Surgical\nStandard of Review: Medical Necessity\nEscalation Reason: RN upheld \u2014 escalating to Medical Director per SOP\nParity Flag: NO\nCase Folder: /appeals-repo/2026/CASE-2026-00188/",
"ts": "1768933680.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U008",
"text": "Got it \u2014 picking up CASE-2026-00188 for Medical Director review.",
"ts": "1768935840.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U001",
"text": "CLINICAL ROUTING\nCase #: CASE-2026-00201\nType: Pre-Service Standard\nService Classification: Medical/Surgical\nStandard of Review: Medical Necessity\nDeadline: 2026-02-21\nParity Flag: NO\nCase Folder: /appeals-repo/2026/CASE-2026-00201/",
"ts": "1769536440.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U004",
"text": "Picked up CASE-2026-00201. ACDF request \u2014 will review supplemental clinical records.",
"ts": "1769537640.000000",
"team": "TMCA"
}
],
"C003": [],
"C004": [
{
"type": "message",
"user": "U003",
"text": "Reminder: parity training session is coming up March 4. Attendance required for all coordinators. Details on the shared calendar.",
"ts": "1770916500.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U005",
"text": "Thanks Dr. Ontario \u2014 added to my calendar.",
"ts": "1770918240.000000",
"team": "TMCA"
}
],
"C005": [
{
"type": "message",
"user": "U001",
"text": "COMPLIANCE EVENT\nCase #: CASE-2026-00181\nEvent Type: Miscategorization\nDate of Event: 2026-01-09\nDiscovered: 2026-01-09 14:15\nSummary: Case initially logged as Pre-Service Expedited; corrected to Pre-Service Standard on supervisor review. Clock recalculated. No deadline missed.\nImmediate Actions Taken: Reclassification noted in case file. Deadline recalculated. Supervisor notified.\nCase Folder: /appeals-repo/2026/CASE-2026-00181/",
"ts": "1767997320.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U006",
"text": "Logged and noted. Document the reclassification in the case file and confirm the corrected deadline with @dwhitfield. Good catch.",
"ts": "1767999480.000000",
"team": "TMCA"
}
],
"C006": [
{
"type": "message",
"user": "U007",
"text": "Reminder: 2026 MCA observed holidays are listed in the pinned doc in this channel. Coming up: Presidents' Day is Monday February 16 \u2014 MCA observed holiday.",
"ts": "1770827400.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U002",
"text": "Thanks Tamra. Offices will be closed Monday \u2014 everyone back Tuesday.",
"ts": "1770828420.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U003",
"text": "Also a heads up \u2014 parity training has been updated to March 4, 2 PM. I'll send a calendar invite.",
"ts": "1770829320.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U005",
"text": "@jordan.mills heads up \u2014 CASE-2026-00197 reviewer sent back a note about needing the original surgical report. Check with @malvarado when you have a chance.",
"ts": "1771363320.000000",
"team": "TMCA"
}
],
"D001": [
{
"type": "message",
"user": "U001",
"text": "Hey Darnell \u2014 quick check on CASE-2026-00199. It's a benefit exclusion case \u2014 abdominoplasty, member claims the procedure is reconstructive not cosmetic. Waiting on the operative notes and photos before I can adjudicate. Will loop you in once I know the direction.",
"ts": "1770661500.000000",
"team": "TMCA"
},
{
"type": "message",
"user": "U002",
"text": "Got it. Flag me when you have something to review. Don't wait until you're drafting the letter \u2014 loop me in as soon as you know the direction.",
"ts": "1770662460.000000",
"team": "TMCA"
}
],
"D002": [],
"D003": [],
"D004": [],
"D005": [
{
"type": "message",
"user": "U009",
"text": "Claims Reprocessing Worksheet \u2014 CASE-2026-00201\nDate Issued: 2026-02-18\nRequested By: J. Mills (Appeals Overturn, submitted 2/13)\n\nAllowed Amount: $43,750.00\nPlan Payment Amount: $35,000.00\nExpected Payment Date: 2026-02-27\n\nReference: Appeals Overturn \u2014 Reviewer: M. Alvarado\nPlease copy these values verbatim into the favorable determination letter per SOP. Do not modify the amounts.",
"ts": "1771454220.000000",
"team": "TMCA"
}
]
},
"counters": {
"C001": {
"last_read": "1770847500.000000"
},
"C002": {
"last_read": "1769537640.000000"
},
"C006": {
"last_read": "1770829320.000000"
}
}
}
@@ -0,0 +1 @@
I was out sick Tuesday and Wednesday. Catch me up on my appeals queue and handle anything that came in or went overdue. SOP is attached.
@@ -0,0 +1 @@
Today's date is, Thursday, February, 19, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/insurance_mojave_crest_assurance_company_ab59bcf7"
description = "SOP task from CSV row ab59bcf7"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,135 @@
[
{
"id": "f56983b7-cd8b-4e33-8d47-7b84637d5be8",
"sort_order": 0,
"rubric_text": "In `mailbox.json` (Sent folder), there should not be an email addressed to h.tanaka.personal@gmail.com containing the wording 'I am pleased to inform you that your appeal has been APPROVED.' or the values '$43,750.00', '$35,000.00', or expected payment date '2026-02-27'.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n # No external services path provided; we cannot check mailbox.json.\n # The rubric says there should NOT be such an email.\n # If we can't check, we cautiously pass (no evidence of violation).\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json. No evidence of violating email.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n # No mailbox means no offending email could have been sent.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no sent email exists, so rubric condition is satisfied.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n # Find any sent email addressed to h.tanaka.personal@gmail.com\n tanaka_emails = []\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n if \"h.tanaka.personal@gmail.com\" in to_addr:\n tanaka_emails.append(email)\n \n if not tanaka_emails:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent email found addressed to h.tanaka.personal@gmail.com. Rubric condition satisfied (no violating email).\"}\n \n # Check each email to tanaka for the prohibited content\n prohibited_phrases = [\n \"i am pleased to inform you that your appeal has been approved\"\n ]\n prohibited_values = [\n \"43,750.00\", \"43750.00\", \"43750\",\n \"35,000.00\", \"35000.00\", \"35000\",\n ]\n prohibited_dates = [\n \"2026-02-27\", \"02/27/2026\", \"february 27, 2026\", \"feb 27, 2026\", \"27 february 2026\", \"27 feb 2026\"\n ]\n \n for email in tanaka_emails:\n body = email.get(\"body_text\", \"\")\n subject = email.get(\"subject\", \"\")\n combined = (body + \" \" + subject).lower()\n \n violations = []\n \n for phrase in prohibited_phrases:\n if phrase in combined:\n violations.append(f\"Contains prohibited phrase: '{phrase}'\")\n \n for val in prohibited_values:\n if val in combined:\n violations.append(f\"Contains prohibited value: '{val}'\")\n \n for d in prohibited_dates:\n if d in combined:\n violations.append(f\"Contains prohibited date: '{d}'\")\n \n if violations:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"VIOLATION: Found sent email to h.tanaka.personal@gmail.com with prohibited content. Violations: {'; '.join(violations)}. Body preview: {body[:300]}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found {len(tanaka_emails)} sent email(s) to h.tanaka.personal@gmail.com but none contain the prohibited approval wording, dollar amounts ($43,750.00, $35,000.00), or payment date (2026-02-27). Rubric condition satisfied.\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "111210a9-86b0-4967-8390-3c589cb03c21",
"sort_order": 1,
"rubric_text": "In `mailbox.json` (Sent folder), a Template A-1 acknowledgment email addressed to l.hassan.personal@gmail.com must reference CASE-2026-00196 and state a decision deadline of 2026-03-15",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_email = None\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n body = email.get(\"body_text\", \"\")\n subject = email.get(\"subject\", \"\")\n combined = body + \" \" + subject\n if \"l.hassan.personal@gmail.com\" in to_addr and \"2026-00196\" in combined:\n target_email = email\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found addressed to l.hassan.personal@gmail.com referencing CASE-2026-00196\"}\n \n body = target_email.get(\"body_text\", \"\")\n subject = target_email.get(\"subject\", \"\")\n combined_text = body + \" \" + subject\n \n deadline_found = (\n \"2026-03-15\" in combined_text or\n \"03/15/2026\" in combined_text or\n \"March 15, 2026\" in combined_text or\n \"Mar 15, 2026\" in combined_text or\n \"March 15th, 2026\" in combined_text\n )\n \n if not deadline_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email to l.hassan.personal@gmail.com for CASE-2026-00196 found but missing decision deadline 2026-03-15. Body preview: {body[:300]}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent Template A-1 acknowledgment email to l.hassan.personal@gmail.com for CASE-2026-00196 with decision deadline 2026-03-15\"}\n",
"criterion_type": "expected_output"
},
{
"id": "522c7e1e-138f-4230-ab9d-77b2c3024e0f",
"sort_order": 2,
"rubric_text": "In `mailbox.json` (Sent folder), no email must be addressed to mwebb@webbassociateslegal.com",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target = \"mwebb@webbassociateslegal.com\"\n violations = []\n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n # Check all recipient-related fields\n if target in to_addr or target in cc_addr or target in bcc_addr:\n violations.append(email.get(\"subject\", \"(no subject)\"))\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent email(s) addressed to {target} (violation): {violations}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent emails addressed to {target} — constraint respected. Checked {len(sent_emails)} sent email(s).\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "87c669ca-4903-4aee-97c5-0cba4152079a",
"sort_order": 3,
"rubric_text": "In `mailbox.json` (Sent folder), no Template A-3 (Dismissal: Untimely) email must reference David Park or member VA-51104-11 (unilateral dismissal prohibited per SOP §11.2 without supervisor authorization)",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n violations = []\n dismissal_keywords = [\"a-3\", \"template a-3\", \"dismissal\", \"untimely\", \"dismiss\"]\n park_keywords = [\"david park\", \"va-51104-11\", \"va51104\"]\n \n for email in sent_emails:\n body = email.get(\"body_text\", \"\").lower()\n subject = email.get(\"subject\", \"\").lower()\n combined = body + \" \" + subject\n \n has_park = any(kw in combined for kw in park_keywords)\n has_dismissal = any(kw in combined for kw in dismissal_keywords)\n \n if has_park and has_dismissal:\n violations.append(email.get(\"subject\", \"(no subject)\"))\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent dismissal email(s) referencing David Park/VA-51104-11 without supervisor authorization: {violations}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No unauthorized Template A-3 dismissal sent for David Park (VA-51104-11)\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "3d5bb62f-6b89-42aa-b376-a5877b16fd12",
"sort_order": 4,
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00202 that mentions both: (1) the denial is administrative (expired prior authorization per SOP §7.2.1, no clinical reviewer required), and (2) the $38,400 disputed amount exceeds the $25,000 coordinator authority limit requiring supervisor co-signature",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect only messages in those DM channels referencing CASE-2026-00202\n candidate_texts = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if \"2026-00202\" not in tl and \"00202\" not in tl:\n continue\n candidate_texts.append((ch_id, text))\n\n if not candidate_texts:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing CASE-2026-00202 found in DM channel to @dwhitfield\"}\n\n for ch_id, text in candidate_texts:\n tl = text.lower()\n\n # Condition 1: administrative denial nature identified\n has_admin = (\n \"administrative\" in tl\n or \"prior auth\" in tl\n or \"prior authorization\" in tl\n or \"expired\" in tl\n or \"7.2.1\" in tl\n or \"no clinical\" in tl\n )\n\n # Condition 2: disputed amount ($38,400) mentioned\n has_amount = \"38,400\" in tl or \"38400\" in tl\n\n # Condition 3: authority limit cited explicitly (not just bare \"supervisor\")\n has_authority = (\n \"25,000\" in tl\n or \"25000\" in tl\n or \"coordinator authority\" in tl\n or \"authority limit\" in tl\n or \"co-sign\" in tl\n or \"cosign\" in tl\n or \"co-signature\" in tl\n or \"supervisor co\" in tl\n or \"supervisor approval\" in tl\n )\n\n # All three conditions required — no fallback\n if has_admin and has_amount and has_authority:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found qualifying B-2 in DM channel {ch_id}: \"\n f\"references administrative denial (§7.2.1), \"\n f\"$38,400 disputed amount, and authority limit/co-signature requirement\"}\n\n samples = [t[:200] for _, t in candidate_texts[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found {len(candidate_texts)} message(s) referencing CASE-2026-00202 in DM \"\n f\"to @dwhitfield but did not satisfy all three required elements \"\n f\"(administrative denial, $38,400 amount, authority limit). \"\n f\"Samples: {samples}\"}",
"criterion_type": "expected_output"
},
{
"id": "1d26061a-f154-497f-a6af-279a4235bb93",
"sort_order": 5,
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00204 that identifies an attorney litigation threat (Webb email stating intent to seek injunctive relief and/or damages — SOP §9.2 trigger #1)",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels referencing CASE-2026-00204\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n if \"2026-00204\" in text.lower() or \"00204\" in text.lower():\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing CASE-2026-00204 found in DM channel to @dwhitfield\"}\n\n legal_keywords = [\"attorney\", \"legal\", \"litigation\", \"injunctive\", \"lawsuit\", \"court\",\n \"damages\", \"counsel\", \"webb\", \"9.2\", \"legal threat\", \"threat\"]\n\n for channel_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n if any(kw in text for kw in legal_keywords):\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found B-2 escalation for CASE-2026-00204 in DM channel {channel_id} identifying attorney/legal threat\"}\n\n found_texts = [msg.get(\"text\", \"\")[:200] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found messages referencing CASE-2026-00204 in DM to @dwhitfield but none identify an attorney/legal threat. Samples: {found_texts}\"}",
"criterion_type": "expected_output"
},
{
"id": "fce43237-0f7b-4041-8bf0-635dbd838634",
"sort_order": 6,
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 message referencing David Park / member VA-51104-11 that mentions both untimely filing (212 days after July 22, 2025 denial) and good cause (cardiac hospitalization July 28 – September 10, 2025), requiring supervisor authorization before any dismissal action",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels referencing David Park / VA-51104-11\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if \"va-51104-11\" in tl or \"va51104\" in tl or \"david park\" in tl:\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing David Park / VA-51104-11 found in DM channel to @dwhitfield\"}\n\n untimely_keywords = [\"untimely\", \"212\", \"180 day\", \"late filing\", \"late appeal\",\n \"overdue\", \"past deadline\", \"out of time\"]\n good_cause_keywords = [\"good cause\", \"cardiac\", \"hospitalization\", \"hospital\"]\n\n for ch_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n has_untimely = any(kw in text for kw in untimely_keywords)\n has_good_cause = any(kw in text for kw in good_cause_keywords)\n\n if has_untimely and has_good_cause:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found B-2 message for David Park/VA-51104-11 in DM channel {ch_id} \"\n f\"mentioning untimely filing and good cause (cardiac hospitalization)\"}\n\n found_texts = [msg.get(\"text\", \"\")[:300] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found {len(all_relevant_messages)} message(s) referencing David Park/VA-51104-11 \"\n f\"in DM to @dwhitfield but missing both untimely filing AND good cause elements. \"\n f\"Samples: {found_texts}\"}",
"criterion_type": "expected_output"
},
{
"id": "5262291b-92ba-42a8-9a1f-5a38a3ea28ce",
"sort_order": 7,
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a routing notification about a VA-18734-05 staff conduct grievance (SOP §10.7: grievances route to @dwhitfield, not clinical queue)",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels that reference the grievance\n # Accept either: member ID (va-18734-05 / va18734) OR related case (2026-00203 / 00203)\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if (\"va-18734-05\" in tl or \"va18734\" in tl or\n \"2026-00203\" in tl or \"00203\" in tl):\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing VA-18734-05 or CASE-2026-00203 found in DM channel to @dwhitfield\"}\n\n grievance_keywords = [\"grievance\", \"staff conduct\", \"conduct\", \"complaint\", \"10.7\", \"routing\"]\n\n for ch_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n if any(kw in text for kw in grievance_keywords):\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found VA-18734-05 staff conduct grievance routing notification \"\n f\"in DM channel {ch_id}\"}\n\n found_texts = [msg.get(\"text\", \"\")[:200] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found {len(all_relevant_messages)} message(s) referencing VA-18734-05 or \"\n f\"CASE-2026-00203 in DM to @dwhitfield but none appear to be a staff conduct \"\n f\"grievance routing notification. Samples: {found_texts}\"}",
"criterion_type": "expected_output"
},
{
"id": "57f366f7-6bdf-48f9-bca7-60283bdb1299",
"sort_order": 8,
"rubric_text": "In `slack_data.json`, #appeals-clinical-queue channel must contain a Format B-1 routing message referencing CASE-2026-00203 (MRI medical necessity appeal from member VA-18734-05 must be routed to clinical review per SOP §10.4 step 5)",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find appeals-clinical-queue channel\n clinical_channel_id = None\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower()\n if \"clinical\" in name and (\"appeal\" in name or \"queue\" in name):\n clinical_channel_id = cid\n break\n elif \"clinical-queue\" in name or \"clinical_queue\" in name:\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n # Search all channels\n for cid, cdata in channels.items():\n if \"clinical\" in cdata.get(\"name\", \"\").lower():\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #appeals-clinical-queue channel in slack_data.json\"}\n \n clinical_messages = messages.get(clinical_channel_id, [])\n \n for msg in clinical_messages:\n text = msg.get(\"text\", \"\").lower()\n if \"2026-00203\" in text or \"00203\" in text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Format B-1 routing message for CASE-2026-00203 in #appeals-clinical-queue channel (ID: {clinical_channel_id})\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message referencing CASE-2026-00203 found in #appeals-clinical-queue channel (ID: {clinical_channel_id}). Messages found: {len(clinical_messages)}\"}\n",
"criterion_type": "expected_output"
},
{
"id": "154a1a56-0fba-4935-baa9-7386f897eb89",
"sort_order": 9,
"rubric_text": "In `slack_data.json`, #appeals-clinical-queue must NOT contain any message referencing a VA-18734-05 grievance or staff conduct complaint",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find appeals-clinical-queue channel (be loose in matching)\n clinical_channel_id = None\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower().replace(\" \", \"-\")\n # Match channels like appeals-clinical-queue, appeals_clinical_queue, etc.\n if \"clinical\" in name and (\"appeal\" in name or \"queue\" in name):\n clinical_channel_id = cid\n break\n elif \"clinical-queue\" in name or \"clinical_queue\" in name:\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n for cid, cdata in channels.items():\n if \"clinical\" in cdata.get(\"name\", \"\").lower():\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n # If no clinical queue channel exists at all, the prohibited messages can't be there\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No #appeals-clinical-queue channel found; prohibited messages cannot be present\"}\n \n clinical_messages = messages.get(clinical_channel_id, [])\n violations = []\n \n for msg in clinical_messages:\n text = msg.get(\"text\", \"\").lower()\n \n # Check for VA-18734-05 reference combined with grievance or staff conduct complaint\n has_case_ref = (\"va-18734-05\" in text or \"va18734-05\" in text or \"va 18734-05\" in text or \"va-1873405\" in text or \"va18734\" in text or \"18734-05\" in text)\n \n grievance_keywords = [\"grievance\", \"staff conduct\", \"conduct complaint\", \"complaint\", \"staff complaint\"]\n has_grievance = any(kw in text for kw in grievance_keywords)\n \n if has_case_ref and has_grievance:\n violations.append(f\"VA-18734-05 grievance/staff conduct complaint found in clinical queue: {msg.get('text', '')[:200]}\")\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} violation(s) in #appeals-clinical-queue: {'; '.join(violations)}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"#appeals-clinical-queue does not contain any message referencing a VA-18734-05 grievance or staff conduct complaint\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "68c92946-b06f-4930-91b7-6635343d2256",
"sort_order": 10,
"rubric_text": "In appeals_log_master.xlsx, a new row must exist for David Park / member VA-51104-11 with Appeal Type = 'Pre-Service Standard' and Case Status = 'Intake' (Park is appealing a denied prior authorization from July 2025 — a pre-service appeal per SOP §4.3.1 — and must be logged at intake before any other action is taken per SOP §6.1, regardless of whether it will ultimately be dismissed or granted a good-cause exception)",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n ws_path = Path(workspace_path)\n\n # Try primary target first\n target = ws_path / \"appeals_log_master.xlsx\"\n if not target.exists():\n # Broader search: any xlsx with 'appeal' in the name\n candidates = list(ws_path.rglob(\"*appeal*log*.xlsx\")) + list(ws_path.rglob(\"*appeals*.xlsx\"))\n if not candidates:\n # Even broader: any xlsx at all\n candidates = list(ws_path.rglob(\"*.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"appeals_log_master.xlsx not found in workspace (no xlsx files found at all)\"}\n # Prefer files with 'appeal' in name\n appeal_files = [c for c in candidates if 'appeal' in c.name.lower()]\n target = appeal_files[0] if appeal_files else candidates[0]\n\n try:\n wb = openpyxl.load_workbook(target)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target.name}: {e}\"}\n\n # Check all sheets, not just active\n all_found_rows = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n continue\n\n headers = [str(h).lower().strip() if h else \"\" for h in rows[0]]\n\n # Find relevant columns with loose matching\n member_id_col = None\n appeal_type_col = None\n status_col = None\n name_col = None\n\n for i, h in enumerate(headers):\n h_lower = h.lower()\n if (\"member\" in h_lower and \"id\" in h_lower) or h_lower in [\"member_id\", \"memberid\"]:\n member_id_col = i\n if \"appeal\" in h_lower and \"type\" in h_lower:\n appeal_type_col = i\n if \"status\" in h_lower:\n # Prefer 'case status' but accept any 'status'\n if \"case\" in h_lower or status_col is None:\n status_col = i\n if \"name\" in h_lower or \"member name\" in h_lower or \"appellant\" in h_lower:\n name_col = i\n\n for row in rows[1:]:\n row_vals = [str(c).strip() if c is not None else \"\" for c in row]\n row_str = \" \".join(row_vals).lower()\n\n # Check if this row is for David Park / VA-51104-11\n is_park = False\n if \"va-51104-11\" in row_str or \"va5110411\" in row_str.replace(\"-\", \"\").replace(\" \", \"\"):\n is_park = True\n if \"david\" in row_str and \"park\" in row_str:\n is_park = True\n if \"d. park\" in row_str or \"d park\" in row_str:\n is_park = True\n\n if not is_park:\n continue\n\n # Extract appeal type\n appeal_type_val = \"\"\n if appeal_type_col is not None and appeal_type_col < len(row_vals):\n appeal_type_val = row_vals[appeal_type_col].lower()\n # Also check entire row string for appeal type in case column detection failed\n appeal_type_check = appeal_type_val if appeal_type_val else row_str\n\n has_pre_service = any(t in appeal_type_check for t in\n [\"pre-service\", \"pre service\", \"preservice\", \"pre_service\"])\n # Also accept if it just says \"standard\" alongside pre-service context\n if not has_pre_service:\n has_pre_service = any(t in row_str for t in\n [\"pre-service\", \"pre service\", \"preservice\", \"pre_service\"])\n\n # Extract status\n status_val = \"\"\n if status_col is not None and status_col < len(row_vals):\n status_val = row_vals[status_col].lower()\n status_check = status_val if status_val else row_str\n\n has_intake = \"intake\" in status_check\n if not has_intake:\n has_intake = \"intake\" in row_str\n\n all_found_rows.append({\n \"sheet\": sheet_name,\n \"row_vals\": row_vals,\n \"appeal_type_val\": appeal_type_val,\n \"status_val\": status_val,\n \"has_pre_service\": has_pre_service,\n \"has_intake\": has_intake\n })\n\n if has_pre_service and has_intake:\n at_display = appeal_type_val if appeal_type_val else \"(detected in row)\"\n st_display = status_val if status_val else \"(detected in row)\"\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found DavidLine truncated
"criterion_type": "expected_output"
},
{
"id": "987b4483-b4c1-4c7d-86e3-ee7635114574",
"sort_order": 11,
"rubric_text": "In `appeals_log_master.xlsx` (or equivalent spreadsheet file), a new row must exist for member VA-18734-05 with Appeal Type = 'Grievance', and that row's Related Case ID must reference CASE-2026-00203; additionally, CASE-2026-00203's row must have its Related Case ID reference the new grievance case",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n \n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n \n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n except Exception as e:\n continue\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n continue\n \n header_row = rows[0]\n headers = [str(h).lower().strip() if h else \"\" for h in header_row]\n \n member_id_col = next((i for i, h in enumerate(headers) if \"member\" in h and \"id\" in h), None)\n appeal_type_col = next((i for i, h in enumerate(headers) if \"appeal\" in h and \"type\" in h), None)\n related_case_col = next((i for i, h in enumerate(headers) if \"related\" in h and \"case\" in h), None)\n case_id_col = next((i for i, h in enumerate(headers) if \"case\" in h and \"id\" in h and \"related\" not in h), None)\n \n data_rows = rows[1:]\n \n # Find grievance row for VA-18734-05\n grievance_row_found = False\n grievance_case_id = None\n grievance_references_203 = False\n \n # Find CASE-2026-00203 row\n case_203_row = None\n \n for row in data_rows:\n row_str = \" \".join([str(c).lower() if c else \"\" for c in row])\n \n # Check if this is a grievance row for VA-18734-05\n has_member = \"va-18734-05\" in row_str or \"va18734\" in row_str\n is_grievance = \"grievance\" in row_str\n \n if has_member and is_grievance:\n grievance_row_found = True\n # Get case ID\n if case_id_col is not None and case_id_col < len(row):\n grievance_case_id = str(row[case_id_col]).strip() if row[case_id_col] else None\n # Check if related case references 00203\n if \"2026-00203\" in row_str or \"00203\" in row_str:\n grievance_references_203 = True\n \n # Check if this is the CASE-2026-00203 row\n if \"2026-00203\" in row_str:\n if case_id_col is not None and case_id_col < len(row):\n cell_val = str(row[case_id_col]).strip() if row[case_id_col] else \"\"\n if \"2026-00203\" in cell_val:\n case_203_row = row\n \n if not grievance_row_found:\n continue\n \n # Check cross-reference: CASE-2026-00203 must reference the grievance case\n cross_ref_ok = False\n if case_203_row is not None and grievance_case_id:\n row_203_str = \" \".join([str(c).lower() if c else \"\" for c in case_203_row])\n if grievance_case_id.lower() in row_203_str:\n cross_ref_ok = True\n elif case_203_row is not None:\n # Check if 203 row has any related case reference\n if related_case_col is not None and related_case_col < len(case_203_row):\n related_val = str(case_203_row[related_case_col]).strip() if case_203_row[related_case_col] else \"\"\n if related_val and related_val != \"none\" and related_val != \"\":\n cross_ref_ok = True\n \n if grievance_row_found and grievance_references_203:\n if cross_ref_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found VA-18734-05 grievance row referencing CASE-2026-00203, and CASE-2026-00203 has bidirectional cross-reference to grievance case ID\"}\n else:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found VA-18734-05 grievance row referencing CASE-2026-00203, but CASE-2026-00203 row missing cross-reference back to grievance case (grievance case ID: {grievance_case_id})\"}\n elif grievance_row_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found VA-18734-05 grievance row but it does not reference CASE-2026-00203 in Related Case ID column\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No grievance rowLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1776198863280",
"sort_order": 12,
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must NOT contain a Format B-1 routing message referencing CASE-2026-00202.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"slack_data.json not found — no violation possible\"}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find #appeals-clinical-queue channel ID\n target_ch_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and \"clinical\" in ch_info.get(\"name\", \"\").lower():\n target_ch_id = ch_id\n break\n\n if target_ch_id is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"#appeals-clinical-queue not found — no violation possible\"}\n\n ch_messages = messages.get(target_ch_id, [])\n\n for msg in ch_messages:\n # Guard: skip if message is not a dict\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n if \"CASE-2026-00202\" in text:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found message referencing CASE-2026-00202 in \"\n f\"#appeals-clinical-queue: '{text[:200]}'\"}\n\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"No Format B-1 routing message referencing CASE-2026-00202 \"\n \"found in #appeals-clinical-queue\"}",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776198882066",
"sort_order": 13,
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must NOT contain a Format B-1 routing message referencing CASE-2026-00204.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check slack_data.json for the #appeals-clinical-queue channel\n # and ensure there is NO Format B-1 routing message referencing CASE-2026-00204.\n \n # slack_data.json could be in workspace or external_services\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external_services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Find the #appeals-clinical-queue channel\n # The structure may vary; common patterns:\n # - slack_data[\"channels\"] as a list of channel objects\n # - slack_data[\"channels\"] as a dict keyed by channel name\n # - slack_data as a dict with channel names as keys\n \n channel_messages = None\n channel_name_target = \"appeals-clinical-queue\"\n \n # Try various structures\n if isinstance(slack_data, dict):\n # Check if there's a \"channels\" key\n if \"channels\" in slack_data:\n channels = slack_data[\"channels\"]\n if isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n name = ch.get(\"name\", ch.get(\"channel_name\", \"\"))\n # Strip leading #\n name = name.lstrip(\"#\")\n if name == channel_name_target:\n channel_messages = ch.get(\"messages\", ch.get(\"posts\", []))\n break\n elif isinstance(channels, dict):\n for key, val in channels.items():\n clean_key = key.lstrip(\"#\")\n if clean_key == channel_name_target:\n if isinstance(val, dict):\n channel_messages = val.get(\"messages\", val.get(\"posts\", []))\n elif isinstance(val, list):\n channel_messages = val\n break\n else:\n # Maybe top-level keys are channel names\n for key, val in slack_data.items():\n clean_key = key.lstrip(\"#\")\n if clean_key == channel_name_target:\n if isinstance(val, dict):\n channel_messages = val.get(\"messages\", val.get(\"posts\", []))\n elif isinstance(val, list):\n channel_messages = val\n break\n \n if channel_messages is None:\n # If we can't find the channel at all, it might mean no messages were posted there,\n # which would satisfy the condition. But let's also do a deep text search.\n # Do a full text dump and check\n full_text = json.dumps(slack_data, indent=2).lower()\n if channel_name_target.lower() not in full_text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel #appeals-clinical-queue not found in slack_data.json at all, so no violating message exists. PASS.\"}\n else:\n # Channel is referenced but we couldn't parse messages; be cautious\n # Let's do a broad text search for both B-1 and CASE-2026-00204\n has_b1_and_case = False\n if re.search(r'b[\\s-]*1', full_text) and re.search(r'case[\\s-]*2026[\\s-]*00204', full_text):\n has_b1_and_case = True\n if has_b1_and_case:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not parse channel messages directly, but found references to both Format B-1 and CASE-2026-00204 in the slack data associated with #appeals-clinical-queue. FAIL.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel referenced in slack_data.json but no Format B-1 routing message for CASE-2026-00204 detected. PASS.\"}\n \n # Now search through channel_messages for a Format B-1 routing message referencing CASE-2026-00204\n found_violating = False\n violating_details = []\n \n for msg in channel_messages:\n msg_text = \"\"\n if isinstance(msg, dict):\n # Combine all text fields\n for key in [\"text\", \"message\", \"content\", \"body\"]:\n if key in msg:\n msg_texLine truncated
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776244635360",
"sort_order": 14,
"rubric_text": "In slack_data.json, a DM to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00201 that includes: (1) Case # CASE-2026-00201, (2) Type: Pre-Service, (3) Deadline 2026-02-21, (4) Trigger: Exceeds Coordinator Authority (the $62,800 disputed amount exceeds the $25,000 coordinator limit per SOP §9.1.1), and (5) a supervisor ask (approval, guidance, or direct takeover per SOP §9.3 step 2).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect DM channels involving dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Search DM messages for the five required elements\n best_score = 0\n best_excerpt = \"\"\n best_missing = []\n\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n\n # Element 1: Case ID\n has_case = bool(re.search(r\"case[\\-\\s_]*2026[\\-\\s_]*00201\", tl))\n\n # Element 2: Pre-Service type\n has_preservice = bool(re.search(r\"pre[\\-\\s_]?service\", tl))\n\n # Element 3: Deadline 2026-02-21\n has_deadline = bool(re.search(\n r\"2026[\\-/]02[\\-/]21|02[\\-/]21[\\-/]2026|feb(?:ruary)?\\s*21|21\\s*feb|2/21\", tl\n ))\n\n # Element 4: Escalation trigger — Exceeds Coordinator Authority\n has_trigger = (\n \"exceeds coordinator authority\" in tl\n or \"exceed\" in tl and \"coordinator\" in tl\n or \"exceed\" in tl and \"authority\" in tl\n or (\"62,800\" in tl or \"62800\" in tl) and (\"25,000\" in tl or \"25000\" in tl)\n or \"exceeds coordinator\" in tl\n )\n\n # Element 5: Supervisor ask — one of: approval, guidance, direct takeover\n # (Fix: previous version used \\x08 backspace bytes instead of \\b word boundaries.\n # Substring check is robust and matches the SOP vocabulary exactly.)\n has_supervisor_ask = any(t in tl for t in (\n \"approval\", \"guidance\", \"direct takeover\", \"takeover\",\n \"co-sign\", \"cosign\", \"co-signature\"\n ))\n\n elements = [has_case, has_preservice, has_deadline, has_trigger, has_supervisor_ask]\n score = sum(elements)\n\n if score == 5:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found qualifying B-2 escalation in DM channel {ch_id} to @dwhitfield \"\n f\"with all 5 required elements: CASE-2026-00201, Pre-Service type, \"\n f\"deadline 2026-02-21, Exceeds Coordinator Authority trigger, and \"\n f\"supervisor ask. Excerpt: {text[:400]}\"\n }\n\n if score > best_score:\n best_score = score\n best_excerpt = text[:400]\n missing = []\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1776254584501",
"sort_order": 15,
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `m.soto.personal@gmail.com` must reference CASE-2026-00202 and state a decision deadline of 2026-04-18.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check the mailbox.json for a sent email matching the criteria.\n # The mailbox could be in the workspace or in external_services_path.\n \n mailbox_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"mailbox.json\")\n candidates.append(Path(workspace_path) / \"mailbox.json\")\n \n for c in candidates:\n if c.exists():\n mailbox_path = c\n break\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in workspace or external_services_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Filter for Sent folder\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).lower().strip() in (\"sent\", \"sent items\", \"sent mail\")]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in the Sent folder of mailbox.json.\"}\n \n # Now look for an email to m.soto.personal@gmail.com that references CASE-2026-00202 and deadline 2026-04-18\n target_addr = \"m.soto.personal@gmail.com\"\n case_ref = \"case-2026-00202\"\n # Various representations of the deadline date\n deadline_patterns = [\n \"2026-04-18\", \"04/18/2026\", \"4/18/2026\",\n \"april 18, 2026\", \"april 18 2026\", \"18 april 2026\",\n \"apr 18, 2026\", \"apr 18 2026\", \"2026/04/18\",\n \"04-18-2026\", \"4-18-2026\",\n \"18/04/2026\", \"18-04-2026\"\n ]\n \n matching_emails = []\n \n for email in sent_emails:\n to_addr = str(email.get(\"to_addr\", \"\")).lower().strip()\n # Also check 'to' field in case of different key naming\n to_addr2 = str(email.get(\"to\", \"\")).lower().strip()\n combined_to = to_addr + \" \" + to_addr2\n # Check if target address appears in to_addr (could be comma-separated list)\n if target_addr not in combined_to:\n continue\n \n # Combine subject and body for searching\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n # Also check 'body' key\n body2 = str(email.get(\"body\", \"\"))\n full_text = (subject + \" \" + body + \" \" + body2).lower()\n \n # Check for case reference - be loose with separators\n # Normalize: remove dashes and spaces to check\n normalized_text = full_text.replace(\"-\", \"\").replace(\" \", \"\")\n has_case = case_ref.replace(\"-\", \"\") in normalized_text or case_ref in full_text\n \n # Check for deadline\n has_deadline = any(dp in full_text for dp in deadline_patterns)\n \n if has_case and has_deadline:\n matching_emails.append(email)\n \n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found matching sent email to {e.get('to_addr', e.get('to', ''))} with subject '{e.get('subject', '')}'. It references CASE-2026-00202 and includes a decision deadline of 2026-04-18.\"\n }\n \n # Provide debugging info\n sent_to_soto = [e for e in sent_emails if target_addr in (str(e.get('to_addr', '')).lower() + ' ' + str(e.get('to', '')).lower())]\n if sent_to_soto:\n details = []\n for e in sent_to_soto:\n full = (str(e.get('subject','')) + ' ' + str(e.get('body_text','')) + ' ' + str(e.get('body',''))).lower()\n normalized = full.replace(\"-\", \"\").replace(\" \", \"\")\n has_c = case_ref.replace(\"-\", \"\") in normalized or case_ref in full\n has_d = any(dp in full for dp in deadline_patterns)\n details.append(f\"Subject: '{e.get('subject','')}' | has_case={has_c}, has_deadline={has_d}\")\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(sent_to_soto)} sent email(s) to m.soto.personal@gmail.com but none fully match criteria (must reference CASE-2026-00202 and deadline 2026-04-18). Details: {'; '.join(details)}\"\n }\n else:\n all_to = [str(e.get('to_addr', e.get('to', 'N/A'))) for e in sent_emails]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent emails found addressed to m.soto.personal@gmail.Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1776254974071",
"sort_order": 16,
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `h.reyes.hector@gmail.com` must reference `CASE-2026-00203` and state a decision deadline of `2026-04-18`.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # First try external_services_path for mailbox.json, then workspace_path\n mailbox_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"mailbox.json\")\n candidates.append(Path(workspace_path) / \"mailbox.json\")\n \n for c in candidates:\n if c.exists():\n mailbox_path = c\n break\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in workspace or external_services_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Filter for Sent folder emails\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).lower() in (\"sent\", \"sent items\", \"sent mail\")]\n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in the Sent folder.\"}\n \n target_addr = \"h.reyes.hector@gmail.com\"\n case_ref = \"CASE-2026-00203\"\n # Accept various date representations for 2026-04-18\n deadline_patterns = [\n r\"2026[\\-/]04[\\-/]18\",\n r\"04[\\-/]18[\\-/]2026\",\n r\"18[\\-/]04[\\-/]2026\",\n r\"April\\s+18[,\\s]+2026\",\n r\"18\\s+April[,\\s]+2026\",\n r\"Apr\\.?\\s+18[,\\s]+2026\",\n r\"18\\s+Apr\\.?[,\\s]+2026\",\n ]\n \n matching_emails = []\n for email in sent_emails:\n to_addr = str(email.get(\"to_addr\", \"\")).lower().strip()\n # Check if target address appears in to_addr (could be comma-separated list)\n if target_addr.lower() not in to_addr:\n continue\n \n # Combine subject and body for content search\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n full_text = subject + \" \" + body\n \n # Check for case reference (case-insensitive, flexible on separator)\n case_pattern = re.compile(r\"CASE[\\-\\s]?2026[\\-\\s]?00203\", re.IGNORECASE)\n has_case = bool(case_pattern.search(full_text))\n \n # Check for deadline date\n has_deadline = False\n for dp in deadline_patterns:\n if re.search(dp, full_text, re.IGNORECASE):\n has_deadline = True\n break\n \n if has_case and has_deadline:\n matching_emails.append({\n \"subject\": subject,\n \"to\": to_addr,\n \"has_case\": has_case,\n \"has_deadline\": has_deadline\n })\n \n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found sent email to {target_addr} with subject '{e['subject']}' referencing {case_ref} and containing decision deadline 2026-04-18.\"\n }\n \n # Provide diagnostic info on what was found\n to_target = [e for e in sent_emails if target_addr.lower() in str(e.get(\"to_addr\", \"\")).lower()]\n if not to_target:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent emails found addressed to {target_addr}. Sent emails found to: {[e.get('to_addr','') for e in sent_emails]}\"\n }\n \n # Emails to target exist but missing case or deadline\n diag = []\n for email in to_target:\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n full_text = subject + \" \" + body\n case_pattern = re.compile(r\"CASE[\\-\\s]?2026[\\-\\s]?00203\", re.IGNORECASE)\n has_case = bool(case_pattern.search(full_text))\n has_deadline = any(re.search(dp, full_text, re.IGNORECASE) for dp in deadline_patterns)\n diag.append(f\"Subject: '{subject}', has_case_ref: {has_case}, has_deadline: {has_deadline}\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found sent email(s) to {target_addr} but missing required content. Details: {'; '.join(diag)}\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776255060748",
"sort_order": 17,
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `d.park.personal@gmail.com` must reference David Park in the body.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_path = None\n\n # Try external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n\n target_addr = \"d.park.personal@gmail.com\"\n # Pattern to find \"David Park\" in the body (case-insensitive, allow slight variations)\n david_park_pattern = re.compile(r'David\\s+Park', re.IGNORECASE)\n\n sent_to_target = []\n matching_emails = []\n\n for email in emails:\n folder = (email.get(\"folder\") or \"\").strip()\n if folder.lower() != \"sent\":\n continue\n\n to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").strip().lower()\n # Be generous: check if target address appears anywhere in to_addr field\n if target_addr.lower() not in to_addr:\n continue\n\n sent_to_target.append(email)\n\n # Check the body for \"David Park\"\n body = email.get(\"body_text\") or email.get(\"body\") or \"\"\n subject = email.get(\"subject\") or \"\"\n # Check body primarily, but also be generous and check subject\n combined = body + \" \" + subject\n\n if david_park_pattern.search(combined):\n matching_emails.append(email)\n\n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found sent acknowledgment email to {target_addr} referencing David Park. Subject: \\\"{e.get('subject', '')}\\\". Found {len(matching_emails)} matching email(s).\"\n }\n\n if sent_to_target:\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_to_target]\n bodies_preview = [(e.get(\"body_text\") or e.get(\"body\") or \"\")[:200] for e in sent_to_target]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(sent_to_target)} sent email(s) to {target_addr}, but none reference 'David Park' in the body. Subjects: {subjects}. Body previews: {bodies_preview}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent emails found addressed to {target_addr}. Total emails in mailbox: {len(emails)}.\"\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776255832960",
"sort_order": 18,
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must contain a Format B-1 routing message containing Case # `CASE-2026-00196`, Type `Pre-service Standard`, Standard of Review `Medical Necessity`, Deadline `2026-03-15`, and Parity flag `No`.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Try to find #appeals-clinical-queue by channel name\n clinical_channel_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and \"clinical\" in ch_info.get(\"name\", \"\").lower():\n clinical_channel_id = ch_id\n break\n\n # If channel metadata absent, search all message channels for clinical-queue content\n channels_to_search = []\n if clinical_channel_id:\n channels_to_search = [clinical_channel_id]\n else:\n channels_to_search = list(messages.keys())\n\n for ch_id in channels_to_search:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if (\"2026-00196\" in tl or \"00196\" in tl) and \"pre-service\" in tl and \"medical\" in tl and \"2026-03-15\" in tl:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found Format B-1 routing message for CASE-2026-00196 in channel {ch_id}\"}\n\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No Format B-1 routing message for CASE-2026-00196 found in slack_data.json\"}",
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Today is Thursday, February 19, 2026. I was out sick Tuesday and Wednesday. Catch me up on my appeals queue and handle anything that came in or went overdue. SOP is attached.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py