Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,18 @@
{
"events": {
"CIG_2026_0341": {
"id": "CIG_2026_0341",
"summary": "IVIG Visit — CIG_2026_0341",
"start": {
"dateTime": "2026-10-05T09:00:00-05:00",
"timeZone": "America/Chicago"
},
"end": {
"dateTime": "2026-10-05T12:30:00-05:00",
"timeZone": "America/Chicago"
},
"description": "Home IVIG infusion for Margaret Thornton. Nurse: Patricia Nolan, RN. Drug: Privigen 40g IV. Attendees: pnolan.rn@homeinfusionkc.com",
"location": "4218 Ridgeview Ct, Kansas City, KS 66112"
}
}
}
@@ -0,0 +1,134 @@
{
"mailbox": {
"email": "billing@careig.com",
"name": "CareIG Billing"
},
"contacts": [
{
"email": "lchoi@careig.com",
"name": "Lisa Choi"
},
{
"email": "rkim@careig.com",
"name": "Rachel Kim"
},
{
"email": "pnolan.rn@homeinfusionkc.com",
"name": "Patricia Nolan"
},
{
"email": "evasquez@kcimunocare.com",
"name": "Dr. Elena Vasquez"
},
{
"email": "m.thornton64@gmail.com",
"name": "Margaret Thornton"
},
{
"email": "intake@careig.com",
"name": "CareIG Intake"
},
{
"email": "pa@careig.com",
"name": "CareIG PA"
},
{
"email": "compliance@careig.com",
"name": "CareIG Compliance"
},
{
"email": "legal@careig.com",
"name": "CareIG Legal"
},
{
"email": "pa.review@sunflowerhealthks.com",
"name": "Sunflower Health PA"
},
{
"email": "claims@sunflowerhealthks.com",
"name": "Sunflower Health Claims"
},
{
"email": "appeals@sunflowerhealthks.com",
"name": "Sunflower Health Appeals"
}
],
"folders": [],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "CareIG — Consent Form Required | Thornton",
"from_addr": "intake@careig.com",
"to_addr": "m.thornton64@gmail.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-09-02T11:45:00-05:00",
"message_id": "<msg001-careig-20260902@careig.com>",
"in_reply_to": null,
"body_text": "Dear Margaret,\n\nThank you for speaking with us today. Dr. Vasquez has referred you to CareIG Specialty Pharmacy for home IVIG infusion therapy.\n\nBefore we can proceed with verifying your insurance and scheduling your first visit, we need a signed Consent for Treatment and Financial Responsibility form. Please find the form attached.\n\nYou may sign and return it by:\n- Replying to this email with the signed form attached\n- Faxing to (816) 555-0200\n\nOnce we verify your insurance, we will also contact you with your estimated out-of-pocket costs before scheduling.\n\nIf you have any questions, please call us at (816) 555-0200.\n\nLisa Choi — CareIG Specialty Pharmacy, Intake\nintake@careig.com | (816) 555-0200",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "PA REQUEST CareIG | Thornton | MemberID: SHC-884729103",
"from_addr": "pa@careig.com",
"to_addr": "pa.review@sunflowerhealthks.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-09-12T09:15:00-05:00",
"message_id": "<msg002-careig-20260912@careig.com>",
"in_reply_to": null,
"body_text": "To Whom It May Concern,\n\nPlease find attached the Prior Authorization request packet for the following patient:\n\nPatient: Margaret Thornton\nDOB: 1961-06-15\nMember ID: SHC-884729103\nGroup: GRP-20145\n\nRequesting Provider: CareIG Specialty Pharmacy\nNPI: 1234567893\nTax ID: 47-3821056\n\nPrescribing Physician: Dr. Elena Vasquez, NPI: 1528074836\n\nRequested Drug: Privigen 40g IV every 4 weeks\nDiagnosis: D83.9\n\nAttached: pa_submission_packet_SunflowerHealth.pdf\n\nPlease direct questions to pa@careig.com or (816) 555-0200.\n\nRachel Kim — CareIG Specialty Pharmacy, Prior Authorization\npa@careig.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "pa_submission_packet_SunflowerHealth.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlL1BhZ2UvUGFyZW50IDIgMCBSL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Db250ZW50cyA0IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L0xlbmd0aCA0OD4+CnN0cmVhbQpCVAovRjEgMTIgVGYKNzIgNzIwIFRkCihQQSBTdWJtaXNzaW9uIFBhY2tldCkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iagoxIDAgb2JqCjw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+CmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+CmVuZG9iagp4cmVmCjAgNQowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAyMDggMDAwMDAgbiAKMDAwMDAwMDI1NyAwMDAwMCBuIAowMDAwMDAwMDE1IDAwMDAwIG4gCjAwMDAwMDAxMDQgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDUvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgozMTQKJUVPRgo="
}
]
},
{
"email_id": "3",
"folder": "INBOX",
"subject": "VISIT ASSIGNMENT CIG_2026_0341 | 2026-10-05",
"from_addr": "intake@careig.com",
"to_addr": "pnolan.rn@homeinfusionkc.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-09-28T15:20:00-05:00",
"message_id": "<msg003-careig-20260928@careig.com>",
"in_reply_to": null,
"body_text": "Dear Patricia,\n\nYou have been assigned to the following IVIG home infusion visit:\n\nCase ID: CIG_2026_0341\nPatient: Margaret Thornton\nAddress: 4218 Ridgeview Ct, Kansas City, KS 66112\nDate: 2026-10-05 at 9:00 AM\nEstimated Duration: 3.5 hours\n\nDrug: Privigen\nOrdered Dose: 40g\nRoute: IV\nLot Number: LG26K0489\nNDC: 44206-0455-10\nInfusion Start Rate: 30 mL/hr\n\nPre-Medication Orders: Diphenhydramine 50mg IV prior to infusion\n\nEmergency Contact: (913) 555-0147\nContraindications: No known drug allergies. No history of hyperprolinemia.\n\nPlease confirm receipt of this assignment by replying to this email.\n\nRachel Kim — CareIG Specialty Pharmacy\nintake@careig.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "4",
"folder": "INBOX",
"subject": "Re: VISIT ASSIGNMENT CIG_2026_0341 | 2026-10-05",
"from_addr": "pnolan.rn@homeinfusionkc.com",
"to_addr": "intake@careig.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-09-28T16:05:00-05:00",
"message_id": "<msg004-pnolan-20260928@homeinfusionkc.com>",
"in_reply_to": "<msg003-careig-20260928@careig.com>",
"body_text": "Hi Rachel,\n\nConfirmed. I have the visit on my schedule for 10/05 at 9:00 AM.\n\nI will pick up the Privigen and supplies from the delivery at the patient address beforehand. Please let me know if anything changes.\n\nPatricia Nolan, RN\n(913) 555-0283",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
}
],
"next_email_id": 5
}
@@ -0,0 +1,464 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "lchoi",
"deleted": false,
"real_name": "Lisa Choi",
"profile": {
"real_name": "Lisa Choi",
"display_name": "lchoi",
"email": "lchoi@careig.com",
"title": "Intake Representative"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "rkim",
"deleted": false,
"real_name": "Rachel Kim",
"profile": {
"real_name": "Rachel Kim",
"display_name": "rkim",
"email": "rkim@careig.com",
"title": "Benefits Verification Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "pnolan",
"deleted": false,
"real_name": "Patricia Nolan",
"profile": {
"real_name": "Patricia Nolan",
"display_name": "pnolan",
"email": "pnolan.rn@homeinfusionkc.com",
"title": "Infusion Nurse"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "dpark",
"deleted": false,
"real_name": "David Park",
"profile": {
"real_name": "David Park",
"display_name": "dpark",
"email": "dpark@careig.com",
"title": "Clinical Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "jmartinez",
"deleted": false,
"real_name": "Julio Martinez",
"profile": {
"real_name": "Julio Martinez",
"display_name": "jmartinez",
"email": "jmartinez@careig.com",
"title": "Billing Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "tnguyen",
"deleted": false,
"real_name": "Tanya Nguyen",
"profile": {
"real_name": "Tanya Nguyen",
"display_name": "tnguyen",
"email": "tnguyen@careig.com",
"title": "AR Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "swright",
"deleted": false,
"real_name": "Sandra Wright",
"profile": {
"real_name": "Sandra Wright",
"display_name": "swright",
"email": "swright@careig.com",
"title": "Director of Revenue Cycle"
},
"is_admin": true,
"is_owner": true,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "intake-team",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-team",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Intake workflow notifications \u2014 BVS assignments, PA submissions, formulary issues, financial hardship flags",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Intake workflow notifications \u2014 BVS assignments, PA submissions, formulary issues, financial hardship flags",
"creator": "U007",
"last_set": 1640995200
}
},
"C002": {
"id": "C002",
"name": "intake-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-alerts",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Intake hold alerts \u2014 missing NPI, unreachable patients, consent issues, PA holds, drug mismatches",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Intake hold alerts \u2014 missing NPI, unreachable patients, consent issues, PA holds, drug mismatches",
"creator": "U007",
"last_set": 1640995200
}
},
"C003": {
"id": "C003",
"name": "intake-mgmt",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-mgmt",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Intake management escalations \u2014 OON pharmacy decisions, formulary unresolved, PA delays",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Intake management escalations \u2014 OON pharmacy decisions, formulary unresolved, PA delays",
"creator": "U007",
"last_set": 1640995200
}
},
"C004": {
"id": "C004",
"name": "clinical-scheduling",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "clinical-scheduling",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Clinical workflow \u2014 PA approvals, scheduling, supply holds, visit completion, unsigned notes",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Clinical workflow \u2014 PA approvals, scheduling, supply holds, visit completion, unsigned notes",
"creator": "U007",
"last_set": 1640995200
}
},
"C005": {
"id": "C005",
"name": "billing",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Billing workflow \u2014 claim submissions, AR follow-ups, claim holds, timely filing risks",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Billing workflow \u2014 claim submissions, AR follow-ups, claim holds, timely filing risks",
"creator": "U007",
"last_set": 1640995200
}
},
"C006": {
"id": "C006",
"name": "billing-appeals",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing-appeals",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Denial and appeal notifications \u2014 PA denials, denial filings, appeal submissions",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Denial and appeal notifications \u2014 PA denials, denial filings, appeal submissions",
"creator": "U007",
"last_set": 1640995200
}
},
"C007": {
"id": "C007",
"name": "billing-mgmt",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing-mgmt",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Billing management escalations \u2014 underpayments, benefit type issues, write-off requests, collections, L2 denials",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Billing management escalations \u2014 underpayments, benefit type issues, write-off requests, collections, L2 denials",
"creator": "U007",
"last_set": 1640995200
}
},
"C008": {
"id": "C008",
"name": "compliance",
"is_channel": false,
"is_group": true,
"is_im": false,
"is_mpim": false,
"is_private": true,
"created": 1640995200,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "compliance",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1640995200,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Compliance alerts \u2014 legal threats, fraud reports (restricted access)",
"creator": "U007",
"last_set": 1640995200
},
"purpose": {
"value": "Compliance alerts \u2014 legal threats, fraud reports (restricted access)",
"creator": "U007",
"last_set": 1640995200
}
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "BVS ASSIGNED CIG_2026_0341 | Payer: Sunflower Health Commercial PPO | Drug: Privigen | Dose: 40g",
"ts": "1788461600.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "PA SUBMITTED CIG_2026_0341 | Payer: Sunflower Health Commercial PPO | Drug: Privigen | Ref: SHC-PA-20260912-7803",
"ts": "1789325100.000000",
"team": "T001"
}
],
"C002": [],
"C003": [],
"C004": [
{
"type": "message",
"user": "U002",
"text": "PA APPROVED CIG_2026_0341 | Drug: Privigen | Dose: 40g | PA#: PA-2026-SHC-41092 | Exp: 2027-03-20 | Drug Source: CareIG",
"ts": "1790021200.000000",
"team": "T001"
},
{
"type": "message",
"user": "U004",
"text": "SCHEDULED CIG_2026_0341 | Date: 2026-10-05 | Nurse: Patricia Nolan, RN",
"ts": "1790710700.000000",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "VISIT COMPLETE CIG_2026_0341 | DOS: 2026-10-05",
"ts": "1791320200.000000",
"team": "T001"
}
],
"C005": [],
"C006": [],
"C007": [],
"C008": []
},
"counters": {
"messageId": 1000000
}
}
@@ -0,0 +1,3 @@
I (Julio Martinez) believe we’re ready to advance Margaret’s case to the next step. Please do that for me by following the SOP. Complete all steps possible and needed, including any file updates, communications, and logging specified by the SOP.
All case files and reference files are in the workspace. And you can use my information if you need it for anything.
@@ -0,0 +1 @@
Today's date is, October, 8, 2026, 09:30 AM CDT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/medical_careig_specialty_pharmacy_08ae3378"
description = "SOP task from CSV row 08ae3378"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,86 @@
[
{
"id": "556a749d-f292-4fb9-abe2-d92e0eb17b29",
"sort_order": 0,
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Header worksheet must contain exactly 2 rows (header + 1 data row) with the correct headers in A1:AF1 and the correct claim data in A2:AF2, including CASE_ID=CIG_2026_0341, DOS=2026-10-05, Patient_Last_Name=Thornton, Patient_First_Name=Margaret, Patient_DOB=1961-06-15, Member_ID=SHC-884729103, Group_Number=GRP-20145, Payer_Name=Sunflower Health Commercial PPO, Payer_Type=Commercial PPO, PA_Number=PA-2026-SHC-41092, Physician_NPI=1528074836, CareIG_NPI=1234567893, CareIG_Tax_ID=47-3821056, CareIG_Taxonomy_Code=3336C0003X, Diagnosis_ICD10=D83.9, Drug_Source=CareIG, Claim_Frequency_Code=1, Total_Billed=6154.80, Submission_Date=2026-10-08, Claim_Status=Submitted, and Partial_Infusion_Note containing administered dose info.",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n \"\"\"Lowercase, strip all punctuation, hyphens, em-dashes, apostrophes, whitespace.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019\\u201c\\u201d'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Header' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Header\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Header']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data[idx] if idx < len(data) else None\n\n # ── Headers A1:AF1 ─────────────────────────────────────────────────────────\n expected_headers = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Payer_Member_ID',\n 'COB_Other_Payer_Paid', 'COB_Other_Payer_EOB_Date', 'AR_Followup_Log',\n ]\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data row A2:AF2 — string fields ───────────────────────────────────────\n str_fields = [\n (0, 'CIG_2026_0341', 'CASE_ID'),\n (2, 'Thornton', 'Patient_Last_Name'),\n (3, 'Margaret', 'Patient_First_Name'),\n (5, 'SHC-884729103', 'Member_ID'),\n (6, 'GRP-20145', 'Group_Number')Line truncated
"criterion_type": "expected_output"
},
{
"id": "cd924150-af76-44ea-8a22-8bafaf26cb9a",
"sort_order": 1,
"rubric_text": "In file `claim_CIG_2026_0341.xlsx`, the Claim Lines worksheet must contain exactly 8 rows (header + 7 data rows) with correct headers in A1:I1 and correct line data for all 7 data lines in any order including HCPCS codes J1459 (71 units, $5,573.50), 99600 (1 unit, $185.00), 99603 (2 units, $290.00), J7030 (2 units, $13.50), J0171 (1 unit, $12.30), A4221 (1 unit, $42.00), A4222 (1 unit, $38.50).",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passed_list = [f'{lbl}' for lbl, p, det in results if p]\n feedback = (f'All {total} checks passed: {\", \".join(passed_list)}'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_CIG_2026_0341.xlsx: {e}'}\n if 'Claim Lines' not in wb.sheetnames:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'Worksheet \"Claim Lines\" not found in claim_CIG_2026_0341.xlsx'}\n\n ws = wb['Claim Lines']\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n\n # ── Headers A1:I1 ──────────────────────────────────────────────────────────\n expected_headers = ['CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier']\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"')\n\n # ── Data rows (matched in any order by HCPCS code) ─────────────────────────\n # Expected: (hcpcs, units, line_total)\n expected_lines = [\n ('J1459', 71, 5573.50),\n ('99600', 1, 185.00),\n ('99603', 2, 290.00),\n ('J7030', 2, 13.50),\n ('J0171', 1, 12.30),\n ('A4221', 1, 42.00),\n ('A4222', 1, 38.50),\n ]\n\n # Build a lookup from HCPCS code (normalized) -> list of data rows\n data_rows = rows[1:] if len(rows) > 1 else []\n hcpcs_col = 3 # Column index for HCPCS_Code\n units_col = 5 # Column index for Units\n total_col = 7 # Column index for Line_Total\n\n hcpcs_to_rows = {}\n for dr in data_rows:\n hcpcs_val = _normalize(_cell_str(dr[hcpcs_col] if hcpcs_col < len(dr) else None))\n if hcpcs_val not in hcpcs_to_rows:\n hcpcs_to_rows[hcpcs_val] = []\n hcpcs_to_rows[hcpcs_val].append(dr)\n\n for (hcpcs, exp_units, exp_total) in expected_lines:\n prefix = f'line_{hcpcs}'\n norm_hcpcs = _normalize(hcpcs)\n\n # Find matching row\n matched_rows = hcpcs_to_rows.get(norm_hcpcs, [])\n found = len(matched_rows) > 0\n chk(f'{prefix}_found', found,\n f'HCPCS code \"{hcpcs}\" not found among data rows. Available: {list(hcpcs_to_rows.keys())}')\n\n if not found:\n # Still add checks as failed so score reflects missing data\n chk(f'{prefix}_Units', False, f'Row for {hcpcs} not found, cannot check units')\n chk(f'{prefix}_LiLine truncated
"criterion_type": "expected_output"
},
{
"id": "4e4620e4-5676-488b-9792-26fe4528035f",
"sort_order": 2,
"rubric_text": "In file `audit_log.xlsx`, the Audit Log worksheet must contain exactly 8 rows (header + 7 data rows), with the 8th data row (row 8) recording the claim submission: Date_Time must be within 24 hours after 2026-10-08 9:30 AM, CASE_ID=CIG_2026_0341, Action=Claim Submitted, File_Updated=claim_CIG_2026_0341.xlsx, Field_Updated=Claim_Status, New_Value=Submitted, Staff_User=Julio Martinez, Result=SUCCESS.",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d %H:%M:%S')\n return str(value).strip()\n\n\ndef _parse_datetime(cell_value):\n \"\"\"Try to parse a cell value into a datetime object.\"\"\"\n if cell_value is None:\n return None\n if isinstance(cell_value, datetime.datetime):\n return cell_value\n if isinstance(cell_value, datetime.date):\n return datetime.datetime(cell_value.year, cell_value.month, cell_value.day)\n s = str(cell_value).strip()\n # Try many common datetime formats\n formats = [\n '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d %I:%M %p', '%Y-%m-%d %I:%M:%S %p',\n '%m/%d/%Y %H:%M:%S', '%m/%d/%Y %H:%M', '%m/%d/%Y %I:%M %p', '%m/%d/%Y %I:%M:%S %p',\n '%d/%m/%Y %H:%M:%S', '%d/%m/%Y %H:%M', '%d/%m/%Y %I:%M %p',\n '%Y/%m/%d %H:%M:%S', '%Y/%m/%d %H:%M',\n '%m-%d-%Y %H:%M:%S', '%m-%d-%Y %H:%M',\n '%Y-%m-%dT%H:%M:%S', '%Y-%m-%dT%H:%M',\n '%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y',\n ]\n for fmt in formats:\n try:\n return datetime.datetime.strptime(s, fmt)\n except ValueError:\n continue\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n # Locate worksheet — prefer exact name, fall back to first sheet with audit/log\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 8 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 8, f'expected 8 rows (header + 7 data), got {len(rows)}')\n\n if len(rows) < 8:\n # Can't check row 8 details if not enough rows; still report partial\n return score_and_return()\n\n header_row = rows[0]\n # The 8th row overall = row index 7 = 7th data row\n data_row8 = rows[7]\n\n # ── Headers (informational, lenient) ───────────────────────────────────────\n expected_headers = ['Date_Time', 'CASE_ID', 'Action', 'File_Updated',\n 'Field_Updated', 'New_Value', 'Staff_User', 'Result']\n headers_ok = True\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n if _normalize(actual_h) != _normalize(exp_h):\n headers_ok = False\n chk('headers', headers_ok, f'Header row: {[_cell_str(header_row[i]) if i < len(header_row) else \"\" for i in range(len(expected_headers))]}')\n\n # ── Row 8 field checks ─────────────────────────────────────────────────────\n def c(idx):\n return data_row8[idx] if idx < len(data_row8) elseLine truncated
"criterion_type": "expected_output"
},
{
"id": "59c8eabd-b7f0-415d-a697-6d8c7e64adff",
"sort_order": 3,
"rubric_text": "In `mailbox.json` (external services), a sent email must exist with Subject containing 'CLAIM SUBMISSION', CareIG NPI 1234567893, patient Thornton, MemberID SHC-884729103, DOS 2026-10-05, sent From billing@careig.com To claims@sunflowerhealthks.com, with attachment claim_CIG_2026_0341.xlsx, and body containing patient details (Thornton, Margaret; DOB 1961-06-15; Member ID SHC-884729103; Group GRP-20145; DOS 2026-10-05; PA-2026-SHC-41092; NPI 1234567893; Tax ID 47-3821056; Total Billed $6,154.80) and signed by Julio Martinez.",
"verifier_code": "import re\nimport json\nimport datetime\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'mailbox.json not found at {mailbox_path}'}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n sent_emails = [e for e in emails\n if e.get('folder', '').lower() in ('sent', 'sent items')]\n\n # Find the claim email — any sent email whose subject contains 'claim submission'\n claim_email = next(\n (e for e in sent_emails\n if 'claimsubmission' in _normalize(e.get('subject', ''))),\n None\n )\n\n if claim_email is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'No claim submission email found in sent folder. '\n f'Sent subjects: {[e.get(\"subject\",\"\") for e in sent_emails]}'}\n\n # ── Subject ────────────────────────────────────────────────────────────────\n expected_subject = ('CLAIM SUBMISSION CareIG NPI: 1234567893 | Thornton | '\n 'MemberID: SHC-884729103 | DOS: 2026-10-05')\n actual_subject = claim_email.get('subject', '')\n chk('subject',\n _normalize(actual_subject) == _normalize(expected_subject),\n f'expected \"{expected_subject}\", got \"{actual_subject}\"')\n\n # ── From / To ──────────────────────────────────────────────────────────────\n chk('from_addr',\n _normalize(claim_email.get('from_addr', '')) == _normalize('billing@careig.com'),\n f'expected billing@careig.com, got \"{claim_email.get(\"from_addr\",\"\")}\"')\n chk('to_addr',\n _normalize(claim_email.get('to_addr', '')) == _normalize('claims@sunflowerhealthks.com'),\n f'expected claims@sunflowerhealthks.com, got \"{claim_email.get(\"to_addr\",\"\")}\"')\n\n # ── Attachment ─────────────────────────────────────────────────────────────\n attachments = claim_email.get('attachments', [])\n att_names = []\n if isinstance(attachments, list):\n att_names = [a.get('filename', '') if isinstance(a, dict) else str(a)\n for a in attachments]\n has_att = any(_normalize('claim_CIG_2026_0341.xlsx') in _normalize(n)\n for n in att_names)\n chk('attachment',\n has_att,\n f'expected claim_CIG_2026_0341.xlsx, got {att_names}')\n\n # ── Body — full template line checks ──────────────────────────────────────\n body = claim_email.get('body_text', '') or ''\n nbody = _normalize(body)\n\n template_lines = [\n 'To Whom It May Concern',\n 'Please find attached the claim for the following patient and date of service',\n 'Patient: Thornton, Margaret',\n 'DOB: 1961-06-15',\n 'Member ID: SHC-884729103',\n 'Group: GRP-20145',\n 'Date of Service: 2026-10-05',\n 'PA Number: PA-2026-SHC-41092',\n 'Rendering Provider: CareIG Specialty Pharmacy',\n 'NPI: 1234567893',\n 'Tax ID: 47-3821056',\n 'Taxonomy: 3336C0003X',\n 'Referring Physician: Dr. Elena Vasquez, NPI: 1528074836',\n 'Diagnosis:Line truncated
"criterion_type": "expected_output"
},
{
"id": "8051d798-11b8-4f3c-b809-b76770f40632",
"sort_order": 4,
"rubric_text": "In `slack_data.json` (external services), channel C005 must contain a message with text 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80' (not a BILLING HOLD message).",
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n \"\"\"Normalize a string for loose comparison: lowercase, strip whitespace and punctuation.\"\"\"\n if value is None:\n return ''\n s = str(value).lower()\n # Remove whitespace, hyphens, dashes, quotes, punctuation, dollar signs\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\|$]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n passes = [f'{lbl}: {det}' for lbl, p, det in results if p]\n if not failures:\n feedback = f'All {total} checks passed. ' + ' | '.join(passes)\n else:\n feedback = f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures)\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n c005_messages = data.get('messages', {}).get('C005', [])\n\n # The rubric says the text should be:\n # 'CLAIM SUBMITTED CIG_2026_0341 | DOS: 2026-10-05 | Payer: Sunflower Health Commercial PPO | Billed: 6,154.80'\n # We accept with or without a dollar sign before the amount.\n # We'll check for key fragments rather than exact match for leniency.\n\n # Key fragments that must all be present in a single message\n required_fragments = [\n 'claimsubmitted',\n 'cig_2026_0341',\n 'dos',\n '20261005',\n 'sunflowerhealthcommercialppo',\n '615480', # 6,154.80 normalized\n ]\n\n # Also accept alternate claim id normalizations\n alt_claim_ids = ['cig20260341']\n\n matched_msg = None\n for m in c005_messages:\n norm_text = _normalize(m.get('text', ''))\n # Check all required fragments\n has_claim_id = 'cig_2026_0341' in norm_text or 'cig20260341' in norm_text\n all_others = all(frag in norm_text for frag in required_fragments if frag not in ('cig_2026_0341',))\n if has_claim_id and all_others:\n matched_msg = m\n break\n\n # Also do a looser check: look for the key parts with the pipe-separated format\n if matched_msg is None:\n for m in c005_messages:\n text = m.get('text', '')\n text_upper = text.upper()\n if ('CLAIM SUBMITTED' in text_upper and\n 'CIG_2026_0341' in text_upper and\n '2026-10-05' in text and\n 'SUNFLOWER' in text_upper and\n ('6,154.80' in text or '6154.80' in text)):\n matched_msg = m\n break\n\n # Check: No BILLING HOLD message present in C005\n billing_hold_present = any(\n 'billing hold' in m.get('text', '').lower() or\n 'billinghold' in _normalize(m.get('text', ''))\n for m in c005_messages\n )\n chk('c005_no_billing_hold',\n not billing_hold_present,\n 'A BILLING HOLD message was found in C005' if billing_hold_present else 'No BILLING HOLD message found in C005 (good)')\n\n # Check: Expected CLAIM SUBMITTED message exists\n msg_texts = [m.get('text', '') for m in c005_messages]\n chk('c005_claim_submitted_message',\n matched_msg is not None,\n f'Found matching CLAIM SUBMITTED message: {matched_msg.get(\"text\", \"\")!r}' if matched_msg\n else f'Expected CLAIM SUBMITTED message with CIG_2026_0341, DOS 2026-10-05, Sunflower Health Commercial PPO, Billed 6,154.80 not found in C005. Messages found: {msg_texts}')\n\n return score_and_return()\n",
"criterion_type": "expected_output"
},
{
"id": "e9993229-323e-401c-8d4a-ee8ce5e1554a",
"sort_order": 5,
"rubric_text": "In file `claim_blank.xlsx`, the template file must remain unchanged with Claim Header worksheet containing exactly 1 row (header only) with headers A1:AF1, and Claim Lines worksheet containing exactly 1 row (header only) with headers A1:I1.",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'claim_blank.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'claim_blank.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open claim_blank.xlsx: {e}'}\n\n # ── Claim Header sheet ─────────────────────────────────────────────────────\n expected_header_cols = [\n 'CASE_ID', 'DOS', 'Patient_Last_Name', 'Patient_First_Name', 'Patient_DOB',\n 'Member_ID', 'Group_Number', 'Payer_Name', 'Payer_Type', 'PA_Number',\n 'Physician_NPI', 'CareIG_NPI', 'CareIG_Tax_ID', 'CareIG_Taxonomy_Code',\n 'Diagnosis_ICD10', 'Drug_Source', 'Billing_Model', 'Drug_Modifier',\n 'Claim_Frequency_Code', 'Total_Billed', 'Submission_Date', 'Claim_Status',\n 'Partial_Infusion_Note', 'ERA_Received_Date', 'Paid_Amount', 'Payment_Posted_Date',\n 'Patient_Responsibility', 'COB_Other_Payer_Name', 'COB_Other_Payer_Member_ID',\n 'COB_Other_Payer_Paid', 'COB_Other_Payer_EOB_Date', 'AR_Followup_Log',\n ]\n\n chk('claim_header_sheet_exists', 'Claim Header' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_h = wb['Claim Header'] if 'Claim Header' in wb.sheetnames else None\n if ws_h is not None:\n rows_h = [r for r in ws_h.iter_rows(values_only=True) if any(c is not None for c in r)]\n chk('claim_header_row_count', len(rows_h) == 1,\n f'expected 1 row (header only), got {len(rows_h)}')\n hrow = rows_h[0] if rows_h else ()\n for i, exp in enumerate(expected_header_cols):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'claim_header_col_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Claim Lines sheet ──────────────────────────────────────────────────────\n expected_lines_cols = [\n 'CASE_ID', 'DOS', 'Line_Number', 'HCPCS_Code', 'Description',\n 'Units', 'Billed_Rate', 'Line_Total', 'Modifier',\n ]\n\n chk('claim_lines_sheet_exists', 'Claim Lines' in wb.sheetnames,\n f'sheet not found, available: {wb.sheetnames}')\n\n ws_l = wb['Claim Lines'] if 'Claim Lines' in wb.sheetnames else None\n if ws_l is not None:\n rows_l = [r for r in ws_l.iter_rows(values_only=True) if any(c is not None for c in r)]\n chk('claim_lines_row_count', len(rows_l) == 1,\n f'expected 1 row (header only), got {len(rows_l)}')\n lrow = rows_l[0] if rows_l else ()\n for i, exp in enumerate(expected_lines_cols):\n actual = _cell_str(lrow[i]) if i < len(lrow) else ''\n chk(f'claim_lines_col_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n return score_and_return()\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "af850a02-2c83-4f4b-b248-d9026ae57faa",
"sort_order": 6,
"rubric_text": "In `calendar_data.json` (external services), there must be exactly 1 event and it must have id 'CIG_2026_0341'.",
"verifier_code": "import re\nimport json\nfrom pathlib import Path\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check calendar_data.json'}\n\n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'calendar_data.json not found at {cal_path}'}\n\n try:\n with open(cal_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse calendar_data.json: {e}'}\n\n events = data.get('events', {})\n event_ids = list(events.keys())\n\n # ── Exactly 1 event ────────────────────────────────────────────────────────\n chk('event_count', len(events) == 1,\n f'expected 1 event, got {len(events)}: {event_ids}')\n\n # ── Event id = CIG_2026_0341 ───────────────────────────────────────────────\n # Check both the dict key and the id field inside the event object\n expected_id = 'CIG_2026_0341'\n found_id = any(\n _normalize(k) == _normalize(expected_id) or\n _normalize(str(v.get('id', ''))) == _normalize(expected_id)\n for k, v in events.items()\n )\n chk('event_id',\n found_id,\n f'expected event id \"{expected_id}\", got event ids: {event_ids}')\n\n return score_and_return()\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "2256d643-3276-468d-a6d8-8ef0b3207ca5",
"sort_order": 7,
"rubric_text": "In `mailbox.json` (external services), the inbox must contain exactly 4 emails, no emails in Drafts, no emails in Trash, and no more than 1 email in Sent.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse mailbox.json: {e}'}\n \n emails = data.get('emails', [])\n \n inbox_emails = [e for e in emails if e.get('folder', '').upper() in ['INBOX']]\n draft_emails = [e for e in emails if e.get('folder', '').lower() in ['drafts', 'draft']]\n trash_emails = [e for e in emails if e.get('folder', '').lower() in ['trash', 'deleted']]\n sent_emails = [e for e in emails if e.get('folder', '').lower() in ['sent', 'sent items']]\n \n errors = []\n \n if len(inbox_emails) != 4:\n errors.append(f'Inbox has {len(inbox_emails)} emails, expected exactly 4')\n \n if len(draft_emails) != 0:\n errors.append(f'Drafts has {len(draft_emails)} emails, expected 0')\n \n if len(trash_emails) != 0:\n errors.append(f'Trash has {len(trash_emails)} emails, expected 0')\n \n if len(sent_emails) > 1:\n errors.append(f'Sent has {len(sent_emails)} emails, expected no more than 1')\n \n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json folder counts incorrect: ' + '; '.join(errors)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'mailbox.json folder counts verified: inbox={len(inbox_emails)}, drafts={len(draft_emails)}, trash={len(trash_emails)}, sent={len(sent_emails)}'}\n",
"criterion_type": "expected_output"
},
{
"id": "47d9923f-1c23-4706-a08d-1a467e9ca9ec",
"sort_order": 8,
"rubric_text": "In `slack_data.json` (external services), channel message counts must be: C001 no more than 2 messages, C002 no messages, C003 no messages, C004 no more than 3 messages, C005 no more than 1 message, C006 no messages, C007 no messages, C008 no messages.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'external_services_path is None; cannot check slack_data.json'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': f'slack_data.json not found at {slack_path}'}\n\n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n\n # (channel_id, max_allowed)\n channel_limits = [\n ('C001', 2),\n ('C002', 0),\n ('C003', 0),\n ('C004', 3),\n ('C005', 1),\n ('C006', 0),\n ('C007', 0),\n ('C008', 0),\n ]\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n for channel_id, max_allowed in channel_limits:\n count = len(messages.get(channel_id, []))\n passed = count <= max_allowed\n detail = f'has {count} messages, max allowed is {max_allowed}' if not passed else f'{count} messages'\n chk(f'channel_{channel_id}', passed, detail)\n\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4)\n failures = [f'{lbl} ({det})' for lbl, p, det in results if not p]\n feedback = (f'All {total} channel count checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Violations: ' + ' | '.join(failures))\n\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1774579377910",
"sort_order": 9,
"rubric_text": "The file `claim_CIG_2026_0341.xlsx` exists in the workspace and has exactly 2 worksheets named \"Claim Header\" and \"Claim Lines\".",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"claim_CIG_2026_0341.xlsx\"\n \n # Check file existence\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' does not exist in the workspace. Files found: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, read_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'claim_CIG_2026_0341.xlsx': {e}\"}\n \n sheet_names = wb.sheetnames\n wb.close()\n \n # Check exactly 2 worksheets\n if len(sheet_names) != 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 worksheets but found {len(sheet_names)}: {sheet_names}\"}\n \n # Check worksheet names (case-insensitive and whitespace-tolerant)\n expected_names = {\"claim header\", \"claim lines\"}\n actual_names = {name.strip().lower() for name in sheet_names}\n \n if actual_names != expected_names:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected worksheets named 'Claim Header' and 'Claim Lines', but found: {sheet_names}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'claim_CIG_2026_0341.xlsx' exists with exactly 2 worksheets: {sheet_names}\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1774579558077",
"sort_order": 10,
"rubric_text": "In `audit_log.xlsx`, the \"Audit Log\" worksheet must preserve all 6 pre-existing data rows (rows 2–7) unchanged and exactly:\n- Row 2: 2026-09-02, Referral Received, intake.xlsx, All required fields, Populated from referral, Lisa Choi, SUCCESS\n- Row 3: 2026-09-02, BVS Assigned, intake.xlsx, BVS_Assigned, Rachel Kim, Lisa Choi, SUCCESS\n- Row 4: 2026-09-08, BV Complete, benefits.xlsx, BV_Complete, YES, Rachel Kim, SUCCESS\n- Row 5: 2026-09-12, PA Submitted, auth.xlsx, PA_Status, Pending, Rachel Kim, SUCCESS\n- Row 6: 2026-09-20, PA Approved — Clinical Handoff, auth.xlsx, PA_Status, Approved, Rachel Kim, SUCCESS\n- Row 7: 2026-10-05, Visit Note Signed, visit_note.xlsx, Nurse_Signature, Patricia Nolan RN, Patricia Nolan, SUCCESS\n\nAll rows must have CASE_ID = CIG_2026_0341 and Result = SUCCESS.",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n\ndef _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n\ndef _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n\ndef _flexible_match(actual_str, expected_str):\n \"\"\"More lenient matching: normalize both sides and compare.\n Also tries checking if one contains the other after normalization.\"\"\"\n na = _normalize(actual_str)\n ne = _normalize(expected_str)\n if na == ne:\n return True\n # Also try: if one contains the other (for slight variations)\n if na and ne and (na in ne or ne in na):\n return True\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n file_path = Path(workspace_path) / 'audit_log.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = None\n if 'Audit Log' in wb.sheetnames:\n ws = wb['Audit Log']\n else:\n for name in wb.sheetnames:\n if 'audit' in name.lower() or 'log' in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb[wb.sheetnames[0]]\n\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # Need at least 7 rows to check existing data (1 header + 6 data)\n if len(rows) < 7:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'audit_log.xlsx has only {len(rows)} non-empty rows; expected at least 7 (header + 6 data rows)'}\n\n # ── Row 1: headers ─────────────────────────────────────────────────────────\n expected_headers = ['Date_Time', 'CASE_ID', 'Action', 'File_Updated',\n 'Field_Updated', 'New_Value', 'Staff_User', 'Result']\n hrow = rows[0]\n for i, exp in enumerate(expected_headers):\n actual = _cell_str(hrow[i]) if i < len(hrow) else ''\n chk(f'header_{exp}',\n _normalize(actual) == _normalize(exp),\n f'col {i+1}: expected \"{exp}\", got \"{actual}\"')\n\n # ── Rows 2–7: existing data ────────────────────────────────────────────────\n # The rubric says:\n # Row 2: 2026-09-02, Referral Received, intake.xlsx, All required fields, Populated from referral, Lisa Choi, SUCCESS\n # Row 3: 2026-09-02, BVS Assigned, intake.xlsx, BVS_Assigned, Rachel Kim, Lisa Choi, SUCCESS\n # Row 4: 2026-09-08, BV Complete, benefits.xlsx, BV_Complete, YES, Rachel Kim, SUCCESS\n # Row 5: 2026-09-12, PA Submitted, auth.xlsx, PA_Status, Pending, Rachel Kim, SUCCESS\n # Row 6: 2026-09-20, PA Approved — Clinical Handoff, auth.xlsx, PA_Status, Approved, Rachel Kim, SUCCESS\n # Row 7: 2026-10-05, Visit Note Signed, visit_note.xlsx, Nurse_Signature, Patricia Nolan RN, Patricia Nolan, SUCCESS\n # All rows have CASE_ID = CIG_2026_0341 and Result = SUCCESS\n #\n # The rubric uses short filenames like \"intake.xlsx\" but actual files may include case ID suffix.\n # We match lenientLine truncated
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1774579634882",
"sort_order": 11,
"rubric_text": "In file `visit_note_CIG_2026_0341.xlsx`, the worksheet must be unchanged with exactly 2 rows (header + 1 data row). Headers A1:Z1 must match the 26 expected column names.",
"verifier_code": "import re\nimport datetime\nfrom pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n\n def _normalize(value):\n if value is None:\n return ''\n s = str(value).lower()\n s = re.sub(r\"[\\s\\-\\u2013\\u2014\\u2018\\u2019'\\\".,;:!?/\\\\]\", '', s)\n return s\n\n def _cell_str(value):\n if value is None:\n return ''\n if isinstance(value, (datetime.datetime, datetime.date)):\n return value.strftime('%Y-%m-%d')\n return str(value).strip()\n\n def _to_float(value):\n try:\n return float(re.sub(r'[,$\\s]', '', _cell_str(value)))\n except (ValueError, TypeError):\n return None\n\n def _date_matches(cell_value, year, month, day):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n return cell_value.year == year and cell_value.month == month and cell_value.day == day\n s = str(cell_value).strip()\n for fmt in ('%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', '%m-%d-%Y'):\n try:\n d = datetime.datetime.strptime(s.split()[0], fmt)\n return d.year == year and d.month == month and d.day == day\n except ValueError:\n continue\n return False\n\n def _time_matches(cell_value, hour, minute):\n if cell_value is None:\n return False\n if isinstance(cell_value, (datetime.time, datetime.datetime)):\n return cell_value.hour == hour and cell_value.minute == minute\n s = str(cell_value).strip()\n for fmt in ('%I:%M %p', '%H:%M:%S', '%H:%M', '%I:%M%p'):\n try:\n t = datetime.datetime.strptime(s, fmt)\n return t.hour == hour and t.minute == minute\n except ValueError:\n continue\n return False\n\n results = []\n\n def chk(label, passed, detail=''):\n results.append((label, bool(passed), detail))\n\n def score_and_return():\n total = len(results)\n n_passed = sum(1 for _, p, _ in results if p)\n score = round(n_passed / total, 4) if total else 0.0\n failures = [f'{lbl}: {det}' for lbl, p, det in results if not p]\n feedback = (f'All {total} checks passed.'\n if not failures\n else f'{n_passed}/{total} checks passed. Failures: ' + ' | '.join(failures))\n return {'pass': score == 1.0, 'score': score, 'feedback': feedback}\n\n # ── File / workbook ────────────────────────────────────────────────────────\n file_path = Path(workspace_path) / 'visit_note_CIG_2026_0341.xlsx'\n if not file_path.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'visit_note_CIG_2026_0341.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'Failed to open visit_note_CIG_2026_0341.xlsx: {e}'}\n\n ws = wb[wb.sheetnames[0]]\n rows = [r for r in ws.iter_rows(values_only=True) if any(c is not None for c in r)]\n\n # ── Row count: exactly 2 ───────────────────────────────────────────────────\n chk('row_count', len(rows) == 2, f'expected 2 rows, got {len(rows)}')\n\n header_row = rows[0] if len(rows) > 0 else ()\n data_row = rows[1] if len(rows) > 1 else ()\n\n def d(idx):\n return data_row[idx] if idx < len(data_row) else None\n\n # ── Headers A1:Z1 ──────────────────────────────────────────────────────────\n expected_headers = [\n 'CASE_ID', 'Visit_Date', 'Visit_Start_Time', 'Visit_End_Time',\n 'Visit_Duration_Minutes', 'Drug_Name_Trade', 'Drug_Name_Generic',\n 'Drug_NDC', 'Drug_Lot_Number', 'Drug_Expiration_Date',\n 'Dose_Ordered_Grams', 'Dose_Administered_Grams', 'Route',\n 'Infusion_Start_Rate_mL_hr', 'Infusion_Rate_Adjustments',\n 'Premeds_Administered', 'NS_500mL_Bags_Used', 'NS_250mL_Bags_Used',\n 'Pump_Used', 'Vital_Signs_Pre', 'Vital_Signs_Mid', 'Vital_Signs_Post',\n 'Patient_Response', 'Education_Provided', 'Nurse_Signature',\n 'Dextrose_500mL_Bags_Used',\n ]\n for i, exp_h in enumerate(expected_headers):\n actual_h = _cell_str(header_row[i]) if i < len(header_row) else ''\n chk(f'header_{exp_h}',\n _normalize(actual_h) == _normalize(exp_h),\n f'col {i+1}: expected \"{exp_h}\", got \"{actual_h}\"Line truncated
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,6 @@
[Current datetime: 2026-10-08, 09:30 AM CDT]
[Current user: Julio Martinez]
I believe we’re ready to advance Margaret’s case to the next step. Please do that for me by following the SOP. Complete all steps possible and needed, including any file updates, communications, and logging specified by the SOP.
All case files and reference files are in the workspace. And you can use my information if you need it for anything.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py