Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "maria.santos@pathfinderbilling.com",
|
||||
"name": "Maria Santos"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "david.park@pathfinderbilling.com",
|
||||
"name": "David Park"
|
||||
},
|
||||
{
|
||||
"email": "jason.blake@pathfinderbilling.com",
|
||||
"name": "Jason Blake"
|
||||
},
|
||||
{
|
||||
"email": "rachel.kim@pathfinderbilling.com",
|
||||
"name": "Rachel Kim"
|
||||
},
|
||||
{
|
||||
"email": "providerrelations@uhcprovider.com",
|
||||
"name": "UnitedHealthcare Provider Relations"
|
||||
},
|
||||
{
|
||||
"email": "providerservices@cigna.com",
|
||||
"name": "Cigna Provider Services"
|
||||
},
|
||||
{
|
||||
"email": "provideroperations@bcbstx.com",
|
||||
"name": "BlueCross TX Provider Operations"
|
||||
},
|
||||
{
|
||||
"email": "providerservices@aetna.com",
|
||||
"name": "Aetna Provider Services"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "rcm.team@pathfinderbilling.com",
|
||||
"name": "PBC RCM Team",
|
||||
"members": [
|
||||
"maria.santos@pathfinderbilling.com",
|
||||
"jason.blake@pathfinderbilling.com",
|
||||
"david.park@pathfinderbilling.com",
|
||||
"rachel.kim@pathfinderbilling.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "Escalations"
|
||||
},
|
||||
{
|
||||
"name": "Payer Correspondence"
|
||||
},
|
||||
{
|
||||
"name": "Archive/Q1-2026"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "ERA_20260417 \u2014 Ready for Processing",
|
||||
"from_addr": "jason.blake@pathfinderbilling.com",
|
||||
"to_addr": "maria.santos@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-24T09:15:00-05:00",
|
||||
"message_id": "<pbc-era-20260424-001@pathfinderbilling.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Maria,\n\nJust a heads-up \u2014 ERA_20260417.xlsx came in from last week and is sitting in the queue for you.\n\nJason",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Ops Check-In \u2014 Week of April 20",
|
||||
"from_addr": "david.park@pathfinderbilling.com",
|
||||
"to_addr": "rcm.team@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-20T09:00:00-05:00",
|
||||
"message_id": "<pbc-ops-20260420-001@pathfinderbilling.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Team,\n\nQuick check-in for the week. A few open items to be aware of:\n\n- We have a handful of ERA batches from recent weeks still moving through the queue. Please make sure your assigned items are actioned before EOD Friday.\n- Jason, any update on the BlueCross TX resubmission from March? Let me know if you need me to follow up on our end.\n- Nothing else pressing from my side. Come find me if anything needs escalation.\n\nDavid Park\nRCM Manager \u2014 Pathfinder Billing and Coding",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "Payer Correspondence",
|
||||
"subject": "UnitedHealthcare Provider Portal \u2014 Unplanned Service Interruption Notice",
|
||||
"from_addr": "providerrelations@uhcprovider.com",
|
||||
"to_addr": "maria.santos@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-05T06:30:00-06:00",
|
||||
"message_id": "<uhc-notice-20260305-8814@uhcprovider.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear UnitedHealthcare Provider,\n\nWe are writing to notify you of an unplanned service interruption affecting the UnitedHealthcare Provider Portal and EDI submission gateway.\n\nImpacted Services:\n - Electronic claim (837P/837I) submission queue\n - Electronic Remittance Advice (835) retrieval\n - Real-time eligibility verification\n\nOutage Start: March 5, 2026 at 02:00 AM Central Time\nEstimated Restoration: March 7, 2026 (subject to update)\n\nClaims submitted via EDI during this window will be queued and processed upon restoration. Submission timestamps will reflect the actual received date; no timeliness penalties will apply to claims held in queue during the interruption window.\n\nPlease do not attempt duplicate submissions. We will send a follow-up notification once services are fully restored.\n\nIf you have urgent eligibility questions during the outage, please contact Provider Services at 1-800-842-4844.\n\nThank you for your patience.\n\nUnitedHealthcare Provider Relations\nproviderrelations@uhcprovider.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "Sent",
|
||||
"subject": "Escalation Notice: ESC-2026-0007 \u2014 Operational / Payer System Outage",
|
||||
"from_addr": "maria.santos@pathfinderbilling.com",
|
||||
"to_addr": "david.park@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-05T09:45:00-06:00",
|
||||
"message_id": "<pbc-esc-20260305-001@pathfinderbilling.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi David,\n\nI have logged ESC-2026-0007 in the Escalation Log.\n\nEscalation Details:\n Escalation ID: ESC-2026-0007\n Category: Operational\n Reason: Payer System Outage\n Payer: UnitedHealthcare (UNHC1)\n Date: 03/05/2026\n Impact: UHC's EDI submission gateway has been down since approximately\n 02:00 AM CT. We have 3 claims currently queued for 837P\n submission to UHC that cannot be transmitted.\n\nRequesting direction: should I hold the submission queue and wait for restoration, or attempt an alternate submission pathway?\n\nBest,\nMaria Santos\nResolution Auditor \u2014 Pathfinder Billing and Coding",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "5",
|
||||
"folder": "Escalations",
|
||||
"subject": "Re: Escalation Notice: ESC-2026-0007 \u2014 Operational / Payer System Outage",
|
||||
"from_addr": "david.park@pathfinderbilling.com",
|
||||
"to_addr": "maria.santos@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-05T11:00:00-06:00",
|
||||
"message_id": "<pbc-esc-20260305-002@pathfinderbilling.com>",
|
||||
"in_reply_to": "<pbc-esc-20260305-001@pathfinderbilling.com>",
|
||||
"body_text": "Maria,\n\nReceived. Hold the submission queue \u2014 do not attempt an alternate pathway. UHC has confirmed this is a widespread outage affecting all providers; secondary submission attempts risk duplicates when the main gateway restores.\n\nI'll monitor UHC communications. Once the portal is back up and we confirm queued submissions have processed, I'll update ESC-2026-0007 to Resolved and notify the team.\n\nDavid",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "6",
|
||||
"folder": "Archive/Q1-2026",
|
||||
"subject": "ESC-2026-0007 Resolved \u2014 UHC Portal Restored, Queued Submissions Cleared",
|
||||
"from_addr": "david.park@pathfinderbilling.com",
|
||||
"to_addr": "rcm.team@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-10T14:00:00-05:00",
|
||||
"message_id": "<pbc-esc-20260310-001@pathfinderbilling.com>",
|
||||
"in_reply_to": "<pbc-esc-20260305-002@pathfinderbilling.com>",
|
||||
"body_text": "Team,\n\nUnitedHealthcare confirmed portal restoration as of March 9 at 11:00 PM CT. All 3 queued 837P submissions processed successfully. EDI acknowledgments (999/TA1) received for all claims.\n\nI've updated ESC-2026-0007 to Resolved in the Escalation Log.\n\nNo further action needed.\n\nDavid",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "7",
|
||||
"folder": "Sent",
|
||||
"subject": "Escalation Notice: ESC-2026-0003 \u2014 Financial / High Value Claim Dispute",
|
||||
"from_addr": "maria.santos@pathfinderbilling.com",
|
||||
"to_addr": "david.park@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-01-22T10:30:00-06:00",
|
||||
"message_id": "<pbc-esc-20260122-001@pathfinderbilling.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi David,\n\nI have logged ESC-2026-0003 in the Escalation Log.\n\nUrgent Escalation: 01/22/2026\nClaim Value: $7,200\nPayer: Aetna (AETN1)\nIssue: Denied for CO-45; payer processed claim at a reduced allowed amount that does not align with the contracted rate.\nAction Taken: Reviewed claim register and fee schedule; contracted rate confirmed.\nRequest: Awaiting your direction before submitting the formal appeal to Aetna.\n\nBest,\nMaria Santos\nResolution Auditor \u2014 Pathfinder Billing and Coding",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "8",
|
||||
"folder": "Escalations",
|
||||
"subject": "Re: Escalation Notice: ESC-2026-0003 \u2014 Financial / High Value Claim Dispute",
|
||||
"from_addr": "david.park@pathfinderbilling.com",
|
||||
"to_addr": "maria.santos@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-01-29T15:30:00-06:00",
|
||||
"message_id": "<pbc-esc-20260129-001@pathfinderbilling.com>",
|
||||
"in_reply_to": "<pbc-esc-20260122-001@pathfinderbilling.com>",
|
||||
"body_text": "Maria,\n\nGood news \u2014 I escalated this directly to our Aetna Provider Relations contact. Aetna's billing team reviewed and confirmed the reduction was a system error on their end (incorrect fee schedule version applied during adjudication). They have reprocessed the claim at the correct contracted rate.\n\nPayment correction should post within the next few business days. I've updated ESC-2026-0003 to Resolved in the Escalation Log.\n\nGood catch on the rate mismatch.\n\nDavid",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "9",
|
||||
"folder": "Payer Correspondence",
|
||||
"subject": "Electronic Claim Receipt Acknowledgment \u2014 CLM-2026-0267",
|
||||
"from_addr": "provideroperations@bcbstx.com",
|
||||
"to_addr": "maria.santos@pathfinderbilling.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-03-10T10:00:00-05:00",
|
||||
"message_id": "<bcbstx-ack-20260310-44721@bcbstx.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Dear Pathfinder Billing and Coding,\n\nThis is an automated acknowledgment confirming receipt of the following electronic claim submission:\n\n Payer Claim Reference: BCBSTX-2026-044721\n Your Claim ID: CLM-2026-0267\n Original Claim ID: CLM-2026-0234\n Claim Type: 837P Professional (Corrected \u2014 Frequency Code 7)\n Rendering Provider: Dr. Sarah Chen, NPI 1234567890\n Date of Service: 01/15/2026\n Received Date: 03/10/2026\n Current Status: Received \u2014 Pending Adjudication\n\nExpected processing time is 15\u201320 business days from the received date. An Explanation of Benefits (EOB) or Electronic Remittance Advice (ERA) will be issued upon adjudication.\n\nIf you have questions regarding this submission, please reference payer claim number BCBSTX-2026-044721 when contacting Provider Operations at 1-800-521-2227 (Monday\u2013Friday, 7:00 AM\u20137:00 PM CT).\n\nThank you,\nBlueCross BlueShield of Texas \u2014 Provider Operations\nprovideroperations@bcbstx.com",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 10
|
||||
}
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "maria.santos",
|
||||
"deleted": false,
|
||||
"real_name": "Maria Santos",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Maria Santos",
|
||||
"real_name_normalized": "Maria Santos",
|
||||
"display_name": "maria",
|
||||
"display_name_normalized": "maria",
|
||||
"email": "maria.santos@pathfinderbilling.com",
|
||||
"title": "Resolution Auditor"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "david.park",
|
||||
"deleted": false,
|
||||
"real_name": "David Park",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "David Park",
|
||||
"real_name_normalized": "David Park",
|
||||
"display_name": "david",
|
||||
"display_name_normalized": "david",
|
||||
"email": "david.park@pathfinderbilling.com",
|
||||
"title": "RCM Manager"
|
||||
},
|
||||
"is_admin": true,
|
||||
"is_owner": true,
|
||||
"is_primary_owner": true,
|
||||
"is_bot": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "jason.blake",
|
||||
"deleted": false,
|
||||
"real_name": "Jason Blake",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Jason Blake",
|
||||
"real_name_normalized": "Jason Blake",
|
||||
"display_name": "jason",
|
||||
"display_name_normalized": "jason",
|
||||
"email": "jason.blake@pathfinderbilling.com",
|
||||
"title": "Resolution Auditor"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_email_confirmed": true
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "rachel.kim",
|
||||
"deleted": false,
|
||||
"real_name": "Rachel Kim",
|
||||
"tz": "America/Chicago",
|
||||
"tz_label": "Central Daylight Time",
|
||||
"tz_offset": -18000,
|
||||
"profile": {
|
||||
"real_name": "Rachel Kim",
|
||||
"real_name_normalized": "Rachel Kim",
|
||||
"display_name": "rachel",
|
||||
"display_name_normalized": "rachel",
|
||||
"email": "rachel.kim@pathfinderbilling.com",
|
||||
"title": "Billing Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_primary_owner": false,
|
||||
"is_bot": false,
|
||||
"is_email_confirmed": true
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "general",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1735689600,
|
||||
"is_archived": false,
|
||||
"is_general": true,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "general",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1767247200,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Company-wide updates",
|
||||
"creator": "U002",
|
||||
"last_set": 1767247200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "General announcements for the Pathfinder Billing and Coding team",
|
||||
"creator": "U002",
|
||||
"last_set": 1767247200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "rcm-team",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1735689600,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "rcm-team",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1767247200,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "RCM operations, denials, and escalations",
|
||||
"creator": "U002",
|
||||
"last_set": 1767247200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Day-to-day coordination for the Revenue Cycle Management team",
|
||||
"creator": "U002",
|
||||
"last_set": 1767247200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Happy new year, everyone. Let\u2019s make it a productive one. I sent around some Q1 priority notes via email \u2014 please take a look before Wednesday.",
|
||||
"ts": "1767625200.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Happy new year!",
|
||||
"ts": "1767625560.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Happy new year, all.",
|
||||
"ts": "1767625920.000300",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Heads up \u2014 flagging a rate discrepancy on an Aetna claim for David\u2019s awareness. Sent him an email and logged it.",
|
||||
"ts": "1769099400.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Update on that Aetna item from a few weeks ago \u2014 resolved. Fee schedule error on their end, claim reprocessed at the right rate.",
|
||||
"ts": "1769722500.000200",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Anyone else getting EDI rejections on UHC submissions this morning? Everything I\u2019m trying to push is bouncing.",
|
||||
"ts": "1772722800.000300",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Yeah, got a notice from them. Something\u2019s down on their end. I\u2019m on it.",
|
||||
"ts": "1772723100.000400",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Got it. Holding my queue.",
|
||||
"ts": "1772723280.000500",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "FYI \u2014 UHC is back up as of this morning. Submissions from last week went through.",
|
||||
"ts": "1773169200.000600",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Good, resuming now.",
|
||||
"ts": "1773169380.000700",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "David \u2014 the BCBSTX resubmission from March is still showing pending adjudication on their portal. No ERA received yet.",
|
||||
"ts": "1776798000.000800",
|
||||
"team": "T001"
|
||||
}
|
||||
]
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R /F3 4 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Oblique /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 9 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 9 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 9 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260408202251-04'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260408202251-04'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Count 2 /Kids [ 5 0 R 6 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 2159
|
||||
>>
|
||||
stream
|
||||
Gatm<>Bed\&:Vs/R&<)h%lh"FK>(2*/Su+KY^?5n(Y':oel](.8NCF5r/N$A)j6R@Rj:_HkVD%CF4$p%R.tNthn]f\h$U<'/?fD]emLLm"05XnXKI"Y=li8j%mH,6;22-R_Z;pUZNG\bo];uNW"-AWis6\OO#&?9)L*#72R-M*!/BMGhf0gq5\F203KIU7k$*tPO9b7[//+0.);N>p4D$9VER)ukYkej15G?]UKE+6spG#4fdi]*(IDP1*FE<DuQLLa@"(f(GDioV0pMI%/@lpQeiQqkM.=Y`2=;F#35]G^)cpi+co`YBkVL*i>FNef6]n-+h?u=dDr=\FD.q82f^33b>I!l+"lh2<XnqR-i,.D<cNC0:+l76%6V/Al5WFVo?F_K9e:uZU\_m)c>GW)0FL9'Rr'3IrjZc/q6e6M\]&cpT%4B8Po1#,se;%S'kVeps$\Pj`U%^G+=SN4qTL>gko?&>Ja\aLF>qT@aA/WW7+F.3)AXKPJ4)_JZ9akbsj"bgg(L0)2.c?dDS?I/"#rj+jPnB?eUN+_V);;nKNm?iNA6EW'Ch%hIm6ns\-,,K:pc4X$,Go%33P_q!o3Eg;\MaGYA(V"Id$b9E3j2Q'q2tf*`YGI.\\@E0i*SZ!P.87cp\LGA-)m-jHk)3"uca,=mbSNenj<?S,,2FpbT-K)hm@A63^dktGGR<jf@VHYH`[qm1M!9@P4\JNVLtIaRq>ETb0McAe01'VF"!6u[Il<X83^:/*LW`O&/XKla7?m7i-Z1c>dI!JW16uK4-X]-t^[G5n+8!n=r=s'n]Sc6@JI=PY]Y5@RYZ*Ba'U_H(ZX0@RTBsIb8]u'La]uTFiLl@erDJB&8nJN))%O"^gC'a_W51b/Yt_#nb4XsP@@I?E6VciB3Uuq0($^r;Q!\EbNdYFYDGR=IcT:[:UqSY*e)C3$flFYdn^`h3Vsh*KK&p";@%G5fA\`n9[Q48A))`58"i:a_a'tVD70YcI.<ZESdC<hW3)+4A+OhHk)4'@n_hC+,2,Mu@EKS+=Up1X#NXr.oGJ\qt/jNfE<bLG.*e,ikI2C<6Nj!A)D=rDI/ibeS(i[[LQ)2ZP/09&]ZS(V.GB%/ik8Jc0TTuj2M\S.NQD0c[FtBqFF!Qg*`$;FCf]ET_LV-6(ba,=0)B\&7m[+C!UC2X/f/uK_"3&-2QZ!uZfgnX1<_0+Z$>pjTot&4H_afbMhnfJ%.KTWgpBaLa<$^5sV)W>LH_$[uN@*]#(6^&;lY^<m]?.Q_Xm81B&h@,4<&?(O`?FM<7DQbaAROd!hI*S:E97`;V+4[MMqoF1@5M<l)>*f,HT^%bB)`rBndNNjGE3ms%r6i@b*G;@mi50qRAUK3<7RFfr],.uKsmB&R<JBE/)(DnA*C&'$j//4'Tc;f@.Z1\$=ds*<1o'b17$:Oi_,HS'Z$gdAsT>`OK/gt4))S[=C7fFr9g7lijo7Ea&cu^a-Rf]ipd5W2F3'L'O[tl$?-c'AJl;h!>?AfCKO\J+%5`D"2BSSabh)^fT[]<KD;,2LjSr&8(^C$"'+p($7S/i=MjPDh>u9+Dj4rm6+*ZIB_)K!m+9=XiVd;tqCl&ILTS^hheO9/$dF#6,HkcXbVJ6?itQ1:j-V79*-1jl&$h,4G@SbtS@m_nHPXdo(7dLf2Mjt@ctVlh=O(^/dC9i:X5W]ZogZ1^m>o_K#.++pDA8GCQ@+j84)>,Eh*n`iZ<M5qLV2[p9LDACgDq>LL9[3+4Wn5+:f\k4J!VhqAZh83+u>"'MjON4lpQ^fk!'6N[pn\n*;<lo+ue`(//$`nTCG:>*%Lg:&LNH?9mgc#G98_t/..A+7K^L194[JLVD+iea>K\C7i>%=6S>NU[T3.8/8#k>EJ]baF\%7ujUu-soh;f[[0XY]Xl\6%Km;r*2=6KY!-$Zs)]:Z;2t+"o4/gAq0r6@9Xl8WYQ[/-+$OXHmK9SMQ)Q@`lLg:Xu?P'N&8k2nDFQ^/7*2:&@>#-S1Bg71I_R;$q^/r$3)#6mZHgN\5W&1f9'kL'_&epbV<8e2c*M2K68/Vqq#^6Dl(srdS;8$R'+^Kdr8%=UpL"R5g`=Un(1h8;26EnE4kXUL[37Xp.DO/<&n[d^hN&,9bl>mnc1*-M0;L#O"7sm\cF;Y?Hc=rB43Y&=l"d;re>Q~>endstream
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 2006
|
||||
>>
|
||||
stream
|
||||
GauHK?!#cM&:N_CbY(a*#8Bi1c5Qm!S3imRHhtV&>$=QlUg7V713%eln#S(nI)Kk]3Q9!GBm44PcCI%lO?.W?r\H0_c=Z_2I7Rd7ZNZ&rTLZ==\sOLc4U<tsLEd^r]3(#-=<4gT7,sq9Z\$%Hn)74CKnWm\+e-p=/<a$OX/V1a\QIol.d&9Q&&M0fk>ilaTungTP+^GAF/;=)N$bkR@EZk5(Rq7YBs('B8a(g7K+lemf]P#p_Ss.)?I1^,ffdT/UVih93st29>UegG4Jo[1KoWFLFht)pgD=k,j@Ua"Z(!G]KLMslrEX!:8k"SrAS/$CdO\`1FDG\AEn=KAO;;&Aj07r;Po96i3i(@f,&gOb=m9[.UfFt$"Q/NI&]Y\>-)mUmF-Rsi9E$KI>ipp')VC>9%D@$/2:*D]WN>`=2n)u?%thkPV!ei,*iEs2W`.OqkA`,dS4L(%G!$@a8e5jB!HO!JU:54j(FB%*:7pr$B"HQ*-0*DflXRLApP'@PmWJe\N.utsUV0EVSJQP$MuJuCVe(]I0i'c/!(mp?em$pMG*Z"rn4R:-WPhL^;I:qWVt^SLIS[@V+r+YnQoRfVU6T'G2&crm_`egSAQfbF2-&*p-ZU[+U!%gX9.P!giq&^C_>c"0eC9>Z8^!$GE#F66k`^C-/N,O0Q%@AdZk\eSeU4>9'6r6$-o?)bp=+]YCpq&Ua@,"(ZDOh_g)?-g<0YSa*>!X]G&<=*`k$UPgrI!T^WuJsGoPn:/!4T,8qaj:J'G"_.(a"e^`nQoM-;H\"Z`HTK$)U)kZ"S46H)3@<[uEijX==X`MGH8Cu&R9-;`^!._Pja/WhWXR-2<f[49I^BnKhtJUI0Y)N^HW%(9WLA##5Fn'e.m1Urer\'C1KRp>FE@m)Arr?;oZ`cJ97ct;SqW-Sl?dS@@$WtYdU_f>(#_0ZLiD-V!FBTqJA\5klVO,[uk/$=^K._!;C6m1J%Ek;X=71]'oieet<OUU#:+_@r5bXGkP+08!Y:#LuWWSFN6L.rWF(>-KSebI5!`9@`+(R1WPb<BM,6>e(p_5Yt>&%+NFO9+L\fYSHab2b(V`]<UO=o#!&3"OIo1:4<3;Uc&Xdug*+6Km*9h4.S!8>N4ds-:5Ynt'g^ZC89\dYYnnUd,k\$TQ@b>>pS25Yn<lcm<k.KH5,_VleMIO?*5hN/J&-jBO=(epLC"LU3jTs-HGSD;u8mGI^;]WS16MG>%``<j$/&pI7;]PMTO\I"EN^VeY,u2kA$TQ>Pr`SX-g'U]3jMI&2SjHG,c)6.cLaK91\&'_VcA$@7><f]tb#[BT%pTi\VB/)%sa2Ru#B?(BBse]l?!?/82PHd,%9]aajcb'C+r@bC,\YN+If2"`q0>]%J2!/(EH\H1^5VS>4:9`(@ENr)$CKB_dB]?0`bVnIY*LV6am*X;8I*KK")FY,Y^k!AYV<hskDT3rcAHU3jMlg+McpP5qm-jo7-WV1!$7.FXR_C+K.O:*Z2[G0+!(]B9359i2OPNhJZF>B?@I+i3bC8gVW)WBV$E3O4&<)lGZbIVgYi?9[2YNHZcfNFp(6<_"'C+Ef8,=>t;fF1]Go6jP74:uI8hHA=FT'6E>Bsc)Q$sEF"120MG;HPS*DTB<!_qGH7SCQi:_#X"E;D!1kO:nhTT9s<MM6hOd_BTiRfO,QE"+EHU";02QcQ<Y(?]$0hMXWIRTq%u5*8UL171qaT_FO6E8(X,]G\"?Khs5a`3o6JHSj_J0ARY-2MDlC1$q=TQ=K$$cNS\HYGp`;PX.O8WD^juFqLda?g^3_C#fj-4eoRFLVO/cCns2>m2c0hKBX$)fYGocqQ[e5`hrj[]fH+mF1Q$d(`*AZU>XrK"M2D10!9&F!=IR?V;dd@$Y,N&9+8BK9OMPE4$TDVre9#nZHTAn^ib);IIQP<b>->-RamJ=A2QN+!Go0+-pQ^2VD8H^nUS'\`]_PK9ZH+ra:BILaA*4(djZU[!5IW$-hNMmi#2RS'hRilJ%fQJR(>R:~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 12
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000112 00000 n
|
||||
0000000219 00000 n
|
||||
0000000331 00000 n
|
||||
0000000446 00000 n
|
||||
0000000640 00000 n
|
||||
0000000834 00000 n
|
||||
0000000902 00000 n
|
||||
0000001182 00000 n
|
||||
0000001247 00000 n
|
||||
0000003498 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<4448da1d4c55928077ecd9ad677dbc11><4448da1d4c55928077ecd9ad677dbc11>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 8 0 R
|
||||
/Root 7 0 R
|
||||
/Size 12
|
||||
>>
|
||||
startxref
|
||||
5596
|
||||
%%EOF
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260408202250-04'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260408202250-04'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Count 2 /Kids [ 4 0 R 5 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1634
|
||||
>>
|
||||
stream
|
||||
Gatm:9ip(/']&@2hP,q8LDWV^Af$VP-?B6aM,1%-DM5,KC)reK*GEd`O.Re(G+@J-L4MVZH%*@!8pXhY#_$I_*Gtn\!kA"rJMIM^5V9m*?k1ZDm%6)64^AS'VOZ%G:dTScJmgEL4$FLI_d(B>m+WBZd0=rRE*EHW#[%3b"\Y7pfZ_2+5m2h@GULL6Rcg?8<Z<LI#gtR5i[<(06gu+J%kp+JK=X8*k>3+;Zo58gJ`HdNg<</-"i$nX[12$b[BaerjYhl_QA-U0Ab$;M+t5pg(Y*]t''8oN)i1dNK:$npQ)]OXCVWsH'-)c=bC\V*iGm*5EsTL1MeD)ZQ$k=V@!SYBO1^$9dl!LoCYY_B\:h$fO-$R6_.If1(irR?8,cH?I@$Y`7BIUec+V[D8>fA?=%\OUQG.ZCg7G6#lpe1)7)c$$4;cme1<)C?#=b(ZMs04Z=Viaa?cBY8`F^]c87ZZcj&cI#:EgXR&O,p2+RF4i/W/U%F>tpBT#SB/V?or74jT!;#'D#gmFe[P[2:'+/Ku)'(`#72n]63N_eLP,!@Au'P`GLLNUH@(-<2C;Y2+H&[S8gNe2H,!0s`+ej]!<U*blm=_`)9Qf=5sGd(M4"RbG_ZJ[a9H<Ot0+CiW[1[uOntj8&@E7J)[d53q;Wk,Z"q`V*&;*R*+jgi*imOCjp%qku*mq&e,Bqn<sR?\ZDaEp.f$NI9I'mri9LA\Tjm-\>tOq%X1L4muBZdk2B:2rU]V8k-%I3Q.Z-*X0*q*^gK#LNT&&0d)Z\5XMa7"b$d9GNj_sAn!>IOb=p1Y`,UlY.G>kJI&#R6j>2O_/fXFG`^o^XfSuhLd]'jXPDk.Yq0Dhruil]qthq`,0M_$6PU5B+Wo&N.Lel9_,.Ju@*H1p]la7Hc1jU!SSBjMO(QutR0KG>I!&Tr;a!kHH)Ai6$Oi[&*`;qh8mOsW-0WQ(!Z%(QbW7j0=FYQb?l@XE5tKg0m==JdW-e#2NC#4F/LTh^&Zp6Sb].Z<3O[u1W(fM8+r*LO:3u/>h\5\E!18(tZVK-l^(l(KNVf[^EX;(QJG76iSU8Ms+h1PJ:K6!++M,E`P$u+O]nDahe@mo2-4[lbPq@^Ts/PbPGkiJWrD_I,Y?+FR"'.PE)]3K]EfrfI;iM2EMch^Ep88>>7*"tJEXGrVdWcA&Br1Pp[&`XO*;tY3]>iPLi3UX=E=UI1i-2HM2ct$<)Nm(2SASSoi5Jou\FhH!`fhPlcdHUt.A_c/,Cf4K?3#V7jEI?6\p,fGn1&TA4N_+8J)]5L>Pouc>KKYId!)Q\2O$AYVRqJkl%oQ1s,MB55u&K#^=cABeal56HGM\k"01g^bEi3Pj/GB?(]Jj'iWn$&nrGSC2ig\TgQk9(4iOWTXgRQC6<ZS'gSZP>CTOUK-&n2VXjjDRlD_$LNR_`G)kF@T7N48>@\XAOD6Ys//Zsi.`RGI,GI<d@D:J:KaHa;^WmBVPdiM&:rtaF.1;0*8k-1A9<jt#9h9M]IC3FONlZ>QUS^+*L1i<thLRGe>R]6<S]@1SkjVZ&2gEA^WV^p1rB=uSc!9QEAOi&_!LT5BZqKmiH$YDES#SCJLiu8h(hfmJsiW#=9lAD5aIlc&V2t&dOO84N/7:,e3&9r0f009(\('~>endstream
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1479
|
||||
>>
|
||||
stream
|
||||
Gatm:gMZ"A&:N^llB,3Qa6a8Co9M@H`/6h"nZ6?!]QEKeFQ?)lgP4hY\%7!=^r.Hd'*=PAfBq(e.MFN+7J(5U"5Lh6Gt3Be&?gXEL'-rIB(>*JLpSNm$Tu(mj<ugl9Y?<f[6SM1XUY\"V($X#k:hq!U[SVV[VpSFp_Df':<jI+27g-g>Y8>8>gA8'bhk0V0tc;iN2Q/1/'a"ZjsbcM2.tpiS-nq-I(t[M+.E5nlc=$R2_)2(*8d%g)*FY/RJL*_^3cR,M[gBclkH]+$s&4SX7jOlrAk#(LT0`!J/Q9N_3BIY>K!':?oV*.F5.NtdMINa">9-o+QNOepi[E[cPNolR/M83>VE;@q.b2>ppjVY5-*/KM<c?6g(.4Z?(rQJ4'L/9P)Ss:_ODqeV@*EF0.W@+N_65XqOB:=8tbEaSaqG7phRcYUB]$qTs[iD#-r]DM;Z]R!,q1Wa*Cg0TheA"1^!BbK5jcOl6,BQ]W,'@AYG)?B%)o7.*;m;=sY+90hbW(1<X^W#/QI#U_;7TcoO29[a;m[B?Yn9[t?,Oc;!^"i,?Z:c#Lhr>IY""+-Upa9me%@'p&gUPWQ<RVD3VH.kc2ccqPrc?6'pKr/KTD!,]S&k'N\9.*E`6aqC;c\#YuK0YL-=>`unoT=uH9.;P*Ya_DTK;G)B1\D\u+YgsAls0+\*)(gC58WJj0n178L(o3.*/)$La?-aH6':F;;VMC+-H%9Fe&fht\W_SHd;/Su/QlPL0@C_g?:ZG9"pZ^N@p9S@"ZA5YX)0gt-?u%)cUkl*GQ\*!AJnCZ5Z-7i^\5)N@)E=16q`^(FJ\tJ%c,+`)cO?tohmK^^gs7->%/BgfG+AI^k\F&XB4P(,I`e7j4?_Io&TiSC;O#-CP`GdcCb,[_NWhp9err@CDH"g/qBJ9ML-:YQ4XdO/*X'NDe#1O6d3j*,&dT<J'?db['@RJniB2*[%-2,PPbr<5XH9ht5dLkk$qGEgO&crsVK7h9F&%"3'*3kXn\W`5?l\c6n2HrrcPHPth0^qaIYHl5SqjWq[ru-i@:06\JIoQ!Xb=?Q'lFaDD&M6>EuNLT^59AdV5idl[aqfUTk%WU'6U'KR3$,`bt.1-Nf(>QqTcU$h$/Y?T(TVJ1:/J=\>)/boter;a&6\WYJ?%YCb9=\)i3qTOfZFm!3H>d_gATPE4/ge8s'Xnm?XdgP(Vl'Vnk#5[-^UQ`OJEE,Y7rOIkg09EIR#u*dkoiI9J!^#M7R8(rZ5LP$u8'LVr/r/UO7e;U%&ZcO'm.hM;*)G,-folKNTuZfkSR%auTXA788/Sq>9O]5@-VY-%]]o_X#XG+n\dT!]$J<=c'KMX@IsR?gFHVdNC<'i#=>0%S9:5^%#^c#A\$%hJn.QK5Uj%%+5,?e2HTgS6CNHq.p-+\Ha*>4&=[bIHhQ]U.e4jk)N5ES:MS,KMS/RCERhC$_<9:1;`u'?d\oT)'*5+K4\4(*XUo&$cdf7K~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 11
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000514 00000 n
|
||||
0000000708 00000 n
|
||||
0000000776 00000 n
|
||||
0000001056 00000 n
|
||||
0000001121 00000 n
|
||||
0000002846 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<a265ed952a75bbdcf89b799dc13e75cc><a265ed952a75bbdcf89b799dc13e75cc>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 7 0 R
|
||||
/Root 6 0 R
|
||||
/Size 11
|
||||
>>
|
||||
startxref
|
||||
4417
|
||||
%%EOF
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Hey, it's Maria. If you'll notice from my email I have a batch that I need completed today. Can you use the SOP and process the ERA for me? You can go ahead and do workflows even if they're technically for other roles.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 24, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/medical_pathfinder_billing_and_coding_c01deb6e"
|
||||
description = "SOP task from CSV row c01deb6e"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
[
|
||||
{
|
||||
"id": "a017242d-7557-4fc5-b933-13e3a0a45e9b",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file mailbox.json (in `external_data/final`), There must be a sent email to david.park@pathfinderbilling.com with exact subject \"Pattern Recurrence Alert: Dr. Sarah Chen — CO-16 — 5 Instances\" and exact body (no partial match accepted): \"Pattern Recurrence Report — 04/24/2026\n\nThis is Maria Santos, Resolution Auditor, reporting a pattern recurrence for Dr. Sarah Chen, NPI: 1234567890.\n\nDenial Code: CO-16 — RARC: N382 — 5 instances identified.\n\nAffected Claims:\n\n1. Claim ID: CLM-2026-0234 — Patient ID: PAT-1008 — DOS: 01/15/2026 — CPT: 99214 — Status: Resubmitted\n\n2. Claim ID: CLM-2026-0356 — Patient ID: PAT-1012 — DOS: 01/28/2026 — CPT: 99213 — Status: Resubmitted\n\n3. Claim ID: CLM-2026-0489 — Patient ID: PAT-1019 — DOS: 02/10/2026 — CPT: 99214 — Status: Resubmitted\n\n4. Claim ID: CLM-2026-0612 — Patient ID: PAT-1027 — DOS: 02/22/2026 — CPT: 99213 — Status: Resubmitted\n\n5. Claim ID: CLM-2026-0892 — Patient ID: PAT-1042 — DOS: 03/15/2026 — CPT: 99214 — Status: Resubmitted\n\nAssessment: This pattern suggests a systemic submission error. Requesting management review to determine whether a corrective action plan or direct clinic intervention is warranted.\n\nEscalation Log Entry ID: ESC-2026-0008. Escalation Category: Operational. Escalation Reason: Pattern Recurrence.\" where \"ESC-2026-0008\" could be \"ESC-2026-0009\"",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external services path.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = []\n if isinstance(mailbox_data, list):\n emails = mailbox_data\n elif isinstance(mailbox_data, dict):\n if \"emails\" in mailbox_data:\n emails = mailbox_data[\"emails\"]\n elif \"sent\" in mailbox_data:\n emails = mailbox_data[\"sent\"]\n else:\n for key, val in mailbox_data.items():\n if isinstance(val, list) and len(val) > 0:\n emails = val\n break\n\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n\n def normalize_dashes(s):\n s = str(s)\n s = s.replace('\\u2013', '\\u2014')\n s = s.replace(' -- ', ' \\u2014 ')\n s = s.replace('--', '\\u2014')\n s = s.replace(' - ', ' \\u2014 ')\n return s\n\n def normalize(s):\n s = normalize_dashes(str(s))\n return re.sub(r'\\s+', ' ', s).strip().lower()\n\n expected_subject = \"Pattern Recurrence Alert: Dr. Sarah Chen \\u2014 CO-16 \\u2014 5 Instances\"\n\n # Required content checks — all must be present in body\n required_claim_ids = [\n \"CLM-2026-0234\",\n \"CLM-2026-0356\",\n \"CLM-2026-0489\",\n \"CLM-2026-0612\",\n \"CLM-2026-0892\",\n ]\n required_phrases = [\n \"Maria Santos\",\n \"Dr. Sarah Chen\",\n \"NPI: 1234567890\",\n \"CO-16\",\n \"N382\",\n \"Pattern Recurrence\",\n \"Operational\",\n ]\n esc_pattern = re.compile(r'ESC-2026-00\\d{2}')\n\n to_addr = \"david.park@pathfinderbilling.com\"\n from_addr = \"maria.santos@pathfinderbilling.com\"\n\n best_feedback = f\"No email found to {to_addr}.\"\n best_score = 0.0\n\n for email in emails:\n if not isinstance(email, dict):\n continue\n\n # Check recipient\n e_to = None\n for field in [\"to_addr\", \"to\", \"recipient\", \"to_address\", \"to_email\", \"recipients\"]:\n if field in email and email[field]:\n e_to = email[field]\n break\n if e_to is None:\n continue\n e_to_str = \" \".join(str(x) for x in e_to) if isinstance(e_to, list) else str(e_to)\n if to_addr not in e_to_str.lower():\n continue\n\n # Check sender (allow missing/unknown sender for sent-folder emails)\n e_from = \"\"\n for field in [\"from_addr\", \"from\", \"sender\", \"from_address\", \"from_email\"]:\n if field in email and email[field]:\n e_from = str(email[field]).strip().lower()\n break\n e_folder = str(email.get(\"folder\", \"\")).strip().lower()\n if e_from and from_addr not in e_from and e_folder != \"sent\":\n continue\n\n # Check subject (collapse whitespace + normalize dashes, case-insensitive)\n e_subject = str(email.get(\"subject\", \"\"))\n subject_ok = normalize(e_subject) == normalize(expected_subject)\n\n # Get body\n e_body = \"\"\n for field in [\"body_text\", \"body\", \"content\", \"text\", \"message\", \"html\", \"body_html\"]:\n if field in email and email[field]:\n e_body = str(email[field])\n break\n\n body_norm = normalize(e_body)\n\n # Check all required claim IDs\n missing_claims = [c for c in required_claim_ids if c.lower() not in body_norm]\n\n # Check all required phrases\n missing_phrases = [p for p in required_phrases if p.lower() not in body_norm]\n\n # Check ESC ID present\n esc_ok = bool(esc_pattern.search(e_body))\n\n all_claims_ok = len(missing_claims) == 0\n all_phrases_ok = len(missing_phrases) == 0\n\n if subject_ok and all_claims_ok and all_phrases_ok and esc_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found matching Pattern Recurrence email to david.park with correct subject and all required content.\"}\n\n # Partial scoring for feedback\n total_checks = len(required_claim_ids) + len(required_phrases) + 2 # +subject, +esc\n passed_checks = (\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705367514",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `mailbox.json`, there must be exactly three sent emails in total.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox.json)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'emails' key is missing or not a list in mailbox.json.\"}\n\n sent_emails = [e for e in emails if isinstance(e, dict) and str(e.get(\"folder\", \"\")).strip().lower() == \"sent\"]\n count = len(sent_emails)\n\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_emails]\n subjects_str = \"; \".join(subjects)\n\n if count == 3:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 3 sent emails as expected. Subjects: {subjects_str}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 3 sent emails, but found {count}. Sent email subjects: {subjects_str}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705423565",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be a table with exactly 40 total rows (including the header row). No leniency, must be exactly 40.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Claim_Register.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'PBC_Claim_Register.xlsx': {e}\"}\n \n target = 40\n \n # Check all sheets for a table with exactly 40 rows\n results = []\n for sheet_name in wb.sheetnames:\n sws = wb[sheet_name]\n # Count rows that have at least one non-empty cell\n non_empty_rows = 0\n for row in sws.iter_rows(min_row=1, max_row=sws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n max_row = sws.max_row\n results.append((sheet_name, non_empty_rows, max_row))\n \n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has exactly {non_empty_rows} non-empty rows (including header), which meets the requirement of exactly 40 total rows.\"\n }\n if max_row == target and non_empty_rows != target:\n # Also accept if max_row is exactly 40 (some cells might appear empty but row exists)\n if non_empty_rows == target:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx sheet '{sheet_name}' has max_row={max_row} matching exactly 40 total rows.\"\n }\n \n # If no sheet matched exactly, provide detailed failure feedback\n details = \"; \".join([f\"Sheet '{s}': {nr} non-empty rows, max_row={mr}\" for s, nr, mr in results])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Claim_Register.xlsx does not have exactly 40 total rows (including header) on any sheet. Found: {details}. Expected exactly 40 rows.\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705456779",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0845\", \"PAT-1021\", \"Jennifer Walsh\", \"03/10/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99213\", \"J06.9\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$180.00\", \"$142.00\", \"$38.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0845\",\n \"PAT-1021\",\n \"Jennifer Walsh\",\n \"03/10/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99213\",\n \"J06.9\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Paid - Full\",\n \"$180.00\",\n \"$142.00\",\n \"$38.00\",\n \"$0.00\",\n \"Reconciled\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, stripping $ and commas.\"\"\"\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual_val)\n\n norm_actual = normalize(actual_val)\n if norm_actual is None:\n return False\n\n norm_expected = expected_val.strip()\n\n # Direct case-insensitive match\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Strip $ for comparison\n if norm_actual.replace('$', '').strip().lower() == norm_expected.replace('$', '').strip().lower():\n return True\n\n # Try numeric comparison with tolerance\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0 and abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n if exp_num == 0 and abs(act_num) < 0.01:\n return True\n\n # Date flexibility: 03/10/2026 vs 2026-03-10 etc.\n if norm_expected == \"03/10/2026\":\n date_patterns = [\"03/10/2026\", \"3/10/2026\", \"2026-03-10\", \"2026/03/10\", \"03-10-2026\"]\n if norm_actual in date_patterns:\n return True\n # openpyxl might return a datetime\n import datetime\n if isinstance(actual_val, datetime.datetime) or isinstance(actual_val, datetime.date):\n try:\n if actual_val.month == 3 and actual_val.day == 10 and actual_val.year == 2026:\n return True\n except:\n pass\n\n # \"1\" could also be stored as integer 1\n if norm_expected == \"1\" and norm_actual in [\"1\", \"1.0\"]:\n return True\n\n return False\n\n num_expected = len(expected)\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n cells = [cell.value for cell in row]\n # We need to find the starting column. The row might have extra columns.\n # Try each possible starting position.\n for start_col in range(max(1, len(cells) - num_expected + 1)):\n subset = cells[start_col:start_col + num_expected]\n if len(subset) < num_expected:\n # Pad with None\n subset = subset + [None] * (num_expected - len(subset))\n\n match_count = 0\n mismatches = []\n for i, (exp_v, act_v) in enumerate(zip(expected, subset)):\n if values_match(exp_v, act_v):\n match_count += 1\n else:\n mismatches.append((i, exp_v, normalize(act_v)))\n\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row_idx\n best_mismatches = mismatches\n\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705468789",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-0878\", \"PAT-1035\", \"David Kim\", \"03/12/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"99214\", \"M54.5\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$285.00\", \"$228.00\", \"$57.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected values for a row (columns A through W = 23 columns)\n expected = [\n \"CLM-2026-0878\", # A\n \"PAT-1035\", # B\n \"David Kim\", # C\n \"03/12/2026\", # D\n \"Dr. James Miller\",# E\n \"2345678901\", # F\n None, # G - BLANK\n \"BlueCross TX\", # H\n \"BCTX1\", # I\n \"99214\", # J\n \"M54.5\", # K\n None, # L - BLANK\n \"1\", # M\n None, # N - BLANK\n \"Paid - Full\", # O\n \"$285.00\", # P\n \"$228.00\", # Q\n \"$57.00\", # R\n \"$0.00\", # S\n \"Reconciled\", # T\n None, # U - BLANK\n None, # V - BLANK\n None, # W - BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def money_val(s):\n \"\"\"Extract numeric value from money string like $285.00\"\"\"\n if s is None:\n return None\n cleaned = re.sub(r'[^\\d.\\-]', '', str(s))\n try:\n return float(cleaned)\n except:\n return None\n\n def vals_match(actual, expected_val):\n a = normalize(actual)\n e = normalize(expected_val)\n # Both blank\n if a is None and e is None:\n return True\n # One blank, other not\n if a is None or e is None:\n return False\n # Direct case-insensitive match\n if a.lower() == e.lower():\n return True\n # Try numeric / money comparison with tolerance\n ma = money_val(a)\n me = money_val(e)\n if ma is not None and me is not None:\n if me == 0:\n return abs(ma) < 0.01\n return abs(ma - me) / max(abs(me), 0.001) <= 0.05\n # Date flexibility: 03/12/2026 vs 2026-03-12 etc.\n # Try removing leading zeros and comparing\n a_stripped = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_stripped = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_stripped.lower() == e_stripped.lower():\n return True\n # Date: check if it's a datetime object rendered differently\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n try:\n formatted = actual.strftime('%m/%d/%Y')\n if formatted == e:\n return True\n except:\n pass\n return False\n\n # Search all rows\n found = False\n best_match_count = 0\n best_row_num = None\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [row[i].value if i < len(row) else None for i in range(23)]\n # Quick check: does column A match the claim ID?\n if normalize(cells[0]) is None:\n continue\n if normalize(cells[0]).upper() != \"CLM-2026-0878\":\n continue\n # Detailed check\n match_count = 0\n mismatches = []\n for idx in range(23):\n if vals_match(cells[idx] if idx < len(cells) else None, expected[idx]):\n match_count += 1\n else:\n mismatches.append((idx, normalize(cells[idx] if idx < len(cells) else None), normalize(expected[idx])))\n if match_count > best_match_count:\n best_match_count = match_count\n best_row_num = row[0].row\n best_mismatches = mismatches\n if match_count == 23:\n found = True\n break\n\n wb.close()\n\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching row for CLM-2026-0878 with all 23 expected column values.\"}\n\n if best_row_num is not None:\n mismatch_details = \"; \".join([f\"Col {m[0]+1}: got '{m[1]}' expected '{m[2]}'\" for m in best_mismatches])\n return {\"pass\": False, \"score\": best_match_count / 23.0, \"feedback\": f\"Found row for CLM-2026-0878 at row {best_row_num} but {len(best_mismatches)} column(s) differ: {mismatch_details}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row with Claim ID 'CLM-2026-0878' found in PBC_Claim_Register.xlsx.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705484020",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$285.00\", \"$0.00\", \"$285.00\", \"$0.00\", BLANK, BLANK, \"CO-16\", \"N382\".",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n target_files = list(Path(workspace_path).glob(\"**/PBC_Claim_Register.xlsx\"))\n if not target_files:\n # Also try case-insensitive\n target_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n target_files = [f for f in target_files if 'claim_register' in f.name.lower()]\n if not target_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n target_file = target_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n # Expected values in order. None means BLANK.\n expected = [\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$285.00\",\n \"$0.00\",\n \"$285.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-16\",\n \"N382\"\n ]\n\n # Column index 14 (0-based) is the status column where \"Submitted\" is also acceptable\n status_col_idx = 14\n\n def normalize(val):\n \"\"\"Normalize a cell value to a stripped string, or None if blank.\"\"\"\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def values_match(actual, expected_val, col_idx=None):\n \"\"\"Loosely compare an actual cell value against an expected value.\"\"\"\n actual_n = normalize(actual)\n if expected_val is None:\n # Expect blank\n return actual_n is None\n if actual_n is None:\n return False\n exp_str = str(expected_val).strip()\n\n # Direct case-insensitive match\n if actual_n.lower() == exp_str.lower():\n return True\n\n # Special handling for status column: accept \"Submitted\" as well\n if col_idx == status_col_idx:\n if actual_n.lower() in [\"resubmitted\", \"submitted\"]:\n return True\n\n # Try removing $ and commas for currency comparison\n def strip_currency(s):\n return s.replace('$', '').replace(',', '').strip()\n act_clean = strip_currency(actual_n)\n exp_clean = strip_currency(exp_str)\n if act_clean == exp_clean:\n return True\n\n # Try numeric comparison with tolerance\n try:\n act_num = float(act_clean)\n exp_num = float(exp_clean)\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n except (ValueError, TypeError):\n pass\n\n # Date flexibility: 03/15/2026 vs 2026-03-15 etc.\n date_formats = ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%Y/%m/%d']\n exp_date = None\n act_date = None\n for fmt in date_formats:\n try:\n exp_date = datetime.datetime.strptime(exp_str, fmt).date()\n break\n except ValueError:\n continue\n if exp_date:\n for fmt in date_formats:\n try:\n act_date = datetime.datetime.strptime(actual_n, fmt).date()\n break\n except ValueError:\n continue\n if act_date is None and isinstance(actual, datetime.datetime):\n act_date = actual.date()\n if act_date is None and isinstance(actual, datetime.date):\n act_date = actual\n if act_date is not None and act_date == exp_date:\n return True\n\n # Check if expected is \"1\" and actual could be 1 or 1.0\n if exp_str == \"1\":\n try:\n if float(actual_n) == 1.0:\n return True\n except (ValueError, TypeError):\n pass\n\n return False\n\n num_expected = len(expected)\n best_match_count = 0\n best_row_idx = None\n best_mismatches = []\n\n # Check all sheets\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n if len(cells) < num_expected:\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705498691",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values: \"CLM-2026-1104\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29881\", \"M23.211\", BLANK, \"1\", BLANK, \"Paid - Full\", \"$2,200.00\", \"$1,850.00\", \"$350.00\", \"$0.00\", \"Reconciled\", BLANK, BLANK, BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n fp = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n # Expected row values in column order\n expected = [\n \"CLM-2026-1104\", # col 1\n \"PAT-1058\", # col 2\n \"Linda Park\", # col 3\n \"03/20/2026\", # col 4\n \"Dr. James Miller\",# col 5\n \"2345678901\", # col 6\n None, # col 7 BLANK\n \"BlueCross TX\", # col 8\n \"BCTX1\", # col 9\n \"29881\", # col 10\n \"M23.211\", # col 11\n None, # col 12 BLANK\n \"1\", # col 13\n None, # col 14 BLANK\n \"Paid - Full\", # col 15\n \"$2,200.00\", # col 16\n \"$1,850.00\", # col 17\n \"$350.00\", # col 18\n \"$0.00\", # col 19\n \"Reconciled\", # col 20\n None, # col 21 BLANK\n None, # col 22 BLANK\n None, # col 23 BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string like $2,200.00 or 2200\"\"\"\n if s is None:\n return None\n s = str(s).strip()\n # Remove dollar sign, commas, spaces\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def values_match(actual, expected_val):\n # If expected is blank\n if expected_val is None:\n return is_blank(actual)\n \n act = normalize(actual)\n exp = normalize(expected_val)\n \n if act is None:\n return False\n \n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n \n # Handle date: expected 03/20/2026\n if '/' in exp:\n # Try to match dates loosely\n # actual could be a datetime object\n import datetime\n if isinstance(actual, datetime.datetime) or isinstance(actual, datetime.date):\n dt = actual\n formatted = dt.strftime('%m/%d/%Y')\n if formatted == exp:\n return True\n # Also try without leading zeros\n formatted2 = f\"{dt.month}/{dt.day}/{dt.year}\"\n if formatted2 == exp:\n return True\n # String comparison already done above\n return False\n \n # Numeric comparison with tolerance\n exp_num = parse_number(exp)\n act_num = parse_number(act)\n if exp_num is not None and act_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 0.001) < 0.05:\n return True\n \n # Try comparing as plain numbers (e.g., \"1\" vs 1.0)\n try:\n if float(act) == float(exp):\n return True\n except (ValueError, TypeError):\n pass\n \n return False\n\n # Search all rows for a matching row\n num_expected = len(expected)\n best_match_count = 0\n best_mismatches = []\n\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n # Find the row that starts with CLM-2026-1104\n first_val = normalize(cells[0]) if len(cells) > 0 else None\n if first_val is None or 'CLM-2026-1104' not in first_val.upper().replace(' ', ''):\n # Also check if it might be in the row somewhere\n found_clm = False\n for c in cells:\n cn = normalize(c)\n if cn and 'CLM-2026-1104' in cn.upper().replace(' ', ''):\n found_clm = True\n break\n if not found_clm:\n continue\n\n # We found a candidate row. Check all expected columns.\n match_count = 0\n mismatches = []\n for i, exp_val in enumerate(expected):\n if i < len(cells):\n actual_val = cells[i]\n else:\n actual_val = None\n \n if values_match(actual_val, exp_val):\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705525107",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the following exact values [No partial matches acceptable, must match every value]: \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", BLANK, \"1\", BLANK, \"Resubmitted\" [\"Submitted\" is also acceptable], \"$1,850.00\", \"$0.00\", \"$1,850.00\", \"$0.00\", BLANK, BLANK, \"CO-97\", \"N520\".",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*.xlsx\")) + list(workspace.glob(\"*claim_register*.xlsx\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx and no alternative claim register file found in {workspace}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # Expected values in order across columns\n expected = [\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"Resubmitted\", # \"Submitted\" also acceptable\n \"$1,850.00\",\n \"$0.00\",\n \"$1,850.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n \"CO-97\",\n \"N520\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def is_blank(val):\n \"\"\"Check if a value is effectively blank/empty.\"\"\"\n if val is None:\n return True\n s = str(val).strip()\n if s == '' or s.lower() == 'none' or s == '0' or s == '0.0' or s == '$0.00':\n return True\n return False\n\n def values_match(actual_val, expected_val, col_index):\n act = normalize(actual_val)\n exp = normalize(expected_val)\n\n # Both blank\n if exp is None:\n return is_blank(actual_val)\n\n if act is None:\n return False\n\n # Direct case-insensitive match\n if act.lower() == exp.lower():\n return True\n\n # Special handling for column 14 (index 14): accept both Resubmitted and Submitted\n if col_index == 14:\n if act.lower() in ('resubmitted', 'submitted'):\n return True\n\n # Try numeric comparison with tolerance\n act_num = parse_number(act)\n exp_num = parse_number(exp)\n if act_num is not None and exp_num is not None:\n if exp_num == 0:\n return abs(act_num) < 0.01\n if abs(act_num - exp_num) / max(abs(exp_num), 1e-9) < 0.05:\n return True\n\n # Currency: strip $ and commas and compare\n act_clean = re.sub(r'[\\$,\\s]', '', act)\n exp_clean = re.sub(r'[\\$,\\s]', '', exp)\n if act_clean == exp_clean:\n return True\n\n # Date flexibility\n date_patterns_act = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', act)\n date_patterns_exp = re.findall(r'(\\d{1,4})[/\\-](\\d{1,2})[/\\-](\\d{1,4})', exp)\n if date_patterns_act and date_patterns_exp:\n def norm_date(parts):\n a, b, c = parts\n if len(a) == 4:\n return (b, c, a)\n else:\n return (a, b, c)\n if norm_date(date_patterns_act[0]) == norm_date(date_patterns_exp[0]):\n return True\n\n # Handle datetime objects from openpyxl\n if isinstance(actual_val, (datetime.datetime, datetime.date)):\n formatted = actual_val.strftime(\"%m/%d/%Y\")\n if formatted == exp:\n return True\n formatted3 = f\"{actual_val.month}/{actual_val.day}/{actual_val.year}\"\n if formatted3 == exp:\n return True\n formatted4 = f\"{actual_val.month:02d}/{actual_val.day:02d}/{actual_val.year}\"\n if formatted4 == exp:\n return True\n\n # Try comparing just digits for things like NPI numbers\n act_digits = re.sub(r'\\D', '', act)\n exp_digits = re.sub(r'\\D', '', exp)\n if act_digits and exp_digits and act_digits == exp_digits:\n return True\n\n return False\n\n # Search all sheets\n best_match_count = 0\n best_mismatches = []\n best_row_vals = []\n\n for ws_naLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705535872",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable, must match every value]: \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"Dr. Amanda Foster\", \"3456789012\", BLANK, \"BlueCross TX\", \"BCTX1\", \"27447\", \"M17.11\", BLANK, \"1\", BLANK, \"In Recovery\", \"$7,500.00\", \"$4,200.00\", \"$3,300.00\", \"$0.00\", \"Held - Contractual Variance\", BLANK, \"CO-45\", \"N130\".",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob in case of slight name differences\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"*Claim_Register*\")) + list(workspace.glob(\"*claim_register*\")) + list(workspace.glob(\"*ClaimRegister*\")) + list(workspace.glob(\"*claim*register*\"))\n file_path = workspace / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n # Try any xlsx candidate\n for c in candidates:\n if c.suffix == '.xlsx':\n file_path = c\n break\n if not file_path.exists():\n # Try any xlsx at all\n all_xlsx = list(workspace.glob(\"*.xlsx\"))\n for f in all_xlsx:\n if 'claim' in f.name.lower():\n file_path = f\n break\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Claim_Register.xlsx in {workspace}. Files present: {[f.name for f in workspace.iterdir()]}\"}\n\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n # Expected values in column order. None means BLANK.\n expected = [\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"Dr. Amanda Foster\",\n \"3456789012\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"27447\",\n \"M17.11\",\n None, # BLANK\n \"1\",\n None, # BLANK\n \"In Recovery\",\n \"$7,500.00\",\n \"$4,200.00\",\n \"$3,300.00\",\n \"$0.00\",\n \"Held - Contractual Variance\",\n None, # BLANK\n \"CO-45\",\n \"N130\"\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n \"\"\"Try to extract a numeric value from a string, removing $, commas, etc.\"\"\"\n if s is None:\n return None\n try:\n cleaned = re.sub(r'[\\$,\\s]', '', str(s))\n return float(cleaned)\n except (ValueError, TypeError):\n return None\n\n def match_cell(expected_val, actual_val):\n \"\"\"Check if actual_val matches expected_val with reasonable flexibility.\"\"\"\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n # Both blank\n if e is None and a is None:\n return True\n # One blank, other not\n if e is None or a is None:\n return False\n\n # Direct case-insensitive match\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison with tolerance\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(a_num - e_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(a_num - e_num) / abs(a_num) <= 0.05:\n return True\n\n # Date flexibility: compare 03/08/2026 with various date formats\n date_formats = [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%m/%d/%y\"]\n e_date = None\n a_date = None\n for fmt in date_formats:\n try:\n e_date = datetime.datetime.strptime(e, fmt).date()\n break\n except ValueError:\n pass\n # actual might be a datetime object already\n if isinstance(actual_val, datetime.datetime):\n a_date = actual_val.date()\n elif isinstance(actual_val, datetime.date) and not isinstance(actual_val, datetime.datetime):\n a_date = actual_val\n else:\n for fmt in date_formats:\n try:\n a_date = datetime.datetime.strptime(a, fmt).date()\n break\n except ValueError:\n pass\n if e_date is not None and a_date is not None:\n if e_date == a_date:\n return True\n\n\n return False\n\n # Search all rows for a matching row\n found = False\n best_match_count = 0\n best_row_idx = None\n best_row_vals = None\n num_expected = len(expected)\n\n for row_idx, row in enumerate(ws.iter_rows(min_row=1, values_only=False), start=1):\n row_vals = [cell.value for cell in row]\n # We need at least num_expected columns; pad if shorter\n while len(row_vals) < num_expected:\n row_vals.append(None)\n\n # First, find the row by the claim ID CLM-2026-0967Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705546419",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values [No partial matches acceptable]: \"CLM-2026-1106\" (or \"CLM-2026-1107\"), \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"Dr. Sarah Chen\", \"1234567890\", \"1122334455\", \"BlueCross TX\", \"BCTX1\", \"99214\", \"E11.65\", BLANK, \"7\", \"CLM-2026-0892\", \"Submitted\", \"$285.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob in case of path variations\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Claim_Register.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Claim_Register.xlsx not found in workspace.\"}\n\n file_path = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_Claim_Register.xlsx: {e}\"}\n\n ws = wb.active\n\n # The rubric says CLM-2026-1106 or CLM-2026-1107 for column 1\n expected_values = [\n [\"CLM-2026-1106\", \"CLM-2026-1107\"], # accept either claim ID\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"Dr. Sarah Chen\",\n \"1234567890\",\n \"1122334455\",\n \"BlueCross TX\",\n \"BCTX1\",\n \"99214\",\n \"E11.65\",\n None, # BLANK\n \"7\",\n \"CLM-2026-0892\",\n \"Submitted\",\n \"$285.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n return val is None or str(val).strip() == '' or str(val).strip().lower() == 'none'\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip().replace('$', '').replace(',', '').strip()\n try:\n return float(s)\n except ValueError:\n return None\n\n def single_value_match(expected_str, actual):\n \"\"\"Check if a single expected string matches the actual value.\"\"\"\n if expected_str is None:\n return is_blank(actual)\n\n norm_actual = normalize(actual)\n norm_expected = normalize(expected_str)\n\n if norm_actual is None:\n return False\n\n # Direct string match (case-insensitive)\n if norm_actual.lower() == norm_expected.lower():\n return True\n\n # Whitespace-insensitive\n if norm_expected.lower().replace(' ', '') == norm_actual.lower().replace(' ', ''):\n return True\n\n # Try numeric comparison for currency values\n exp_num = parse_number(norm_expected)\n act_num = parse_number(norm_actual)\n if exp_num is not None and act_num is not None:\n if exp_num == 0 and act_num == 0:\n return True\n if exp_num != 0:\n if abs(act_num - exp_num) / abs(exp_num) <= 0.05:\n return True\n # Also check exact equality for non-zero\n if exp_num == act_num:\n return True\n\n # Date matching - be very flexible\n if '/' in norm_expected:\n # Parse the expected date\n try:\n exp_parts = norm_expected.split('/')\n exp_month = int(exp_parts[0])\n exp_day = int(exp_parts[1])\n exp_year = int(exp_parts[2])\n except:\n exp_month = exp_day = exp_year = None\n\n if exp_month is not None:\n # If actual is a datetime object\n if hasattr(actual, 'year') and hasattr(actual, 'month') and hasattr(actual, 'day'):\n if actual.month == exp_month and actual.day == exp_day and actual.year == exp_year:\n return True\n\n # Try parsing the string form\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y', '%m/%d/%y', '%Y/%m/%d']:\n try:\n dt = datetime.datetime.strptime(norm_actual, fmt)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Try iso format\n try:\n dt = datetime.datetime.fromisoformat(norm_actual)\n if dt.month == exp_month and dt.day == exp_day and dt.year == exp_year:\n return True\n except:\n pass\n\n # Number as string comparison (e.g., \"7\" vs 7)\n try:\n ef = float(norm_expected)\n af = float(norm_actual)\n if ef == af:\n return True\n except:\n pass\n\n # Also try int comparison (\"7\" vs \"7.0\")\n try:\n ei = intLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705558604",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be one row in the table with the exact values: \"CLM-2026-1107\" (or \"CLM-2026-1106\"), \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"Dr. James Miller\", \"2345678901\", BLANK, \"BlueCross TX\", \"BCTX1\", \"29876\", \"M65.862\", \"LT\", \"7\", \"CLM-2026-1105\", \"Submitted\", \"$1,850.00\", \"$0.00\", \"$0.00\", \"$0.00\", BLANK, BLANK, BLANK, BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file using glob to be lenient\n file_path = None\n for f in Path(workspace_path).glob(\"**/*Claim_Register*\"):\n if f.suffix == '.xlsx':\n file_path = f\n break\n if file_path is None:\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # Acceptable claim IDs for the first column per the rubric\n acceptable_claim_ids = [\"CLM-2026-1107\", \"CLM-2026-1106\"]\n\n expected = [\n None, # placeholder for claim ID - checked separately\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"Dr. James Miller\",\n \"2345678901\",\n None, # BLANK\n \"BlueCross TX\",\n \"BCTX1\",\n \"29876\",\n \"M65.862\",\n \"LT\",\n \"7\",\n \"CLM-2026-1105\",\n \"Submitted\",\n \"$1,850.00\",\n \"$0.00\",\n \"$0.00\",\n \"$0.00\",\n None, # BLANK\n None, # BLANK\n None, # BLANK\n None, # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def parse_number(s):\n if s is None:\n return None\n s = str(s).strip()\n cleaned = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(cleaned)\n except ValueError:\n return None\n\n def values_match(expected_val, actual_val):\n e = normalize(expected_val)\n a = normalize(actual_val)\n\n if e is None and a is None:\n return True\n if e is None or a is None:\n return False\n\n if e.lower() == a.lower():\n return True\n\n # Try numeric comparison\n e_num = parse_number(e)\n a_num = parse_number(a)\n if e_num is not None and a_num is not None:\n if e_num == 0 and a_num == 0:\n return True\n if e_num != 0:\n if abs(e_num - a_num) / abs(e_num) <= 0.05:\n return True\n elif a_num != 0:\n if abs(e_num - a_num) / abs(a_num) <= 0.05:\n return True\n\n # Date comparison\n if '/' in e:\n try:\n expected_date = datetime.datetime.strptime(e, '%m/%d/%Y').date()\n if hasattr(actual_val, 'date'):\n if actual_val.date() == expected_date:\n return True\n elif hasattr(actual_val, 'year'):\n if actual_val == expected_date:\n return True\n for fmt in ['%m/%d/%Y', '%Y-%m-%d', '%m-%d-%Y', '%d/%m/%Y']:\n try:\n if datetime.datetime.strptime(a, fmt).date() == expected_date:\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n\n # NPI / numeric string cleaning\n e_clean = re.sub(r'[^\\w.]', '', e)\n a_clean = re.sub(r'[^\\w.]', '', a)\n if e_clean.lower() == a_clean.lower():\n return True\n\n return False\n\n def is_acceptable_claim_id(val):\n n = normalize(val)\n if n is None:\n return False\n for cid in acceptable_claim_ids:\n if n.upper() == cid.upper():\n return True\n return False\n\n best_mismatches = None\n best_sheet = None\n best_claim_id = None\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n cells = [cell.value for cell in row]\n\n # Find a cell that matches one of the acceptable claim IDs\n start_idx = None\n matched_claim_id = None\n for i, c in enumerate(cells):\n if is_acceptable_claim_id(c):\n start_idx = i\n matched_claim_id = normalize(c)\n break\n if start_idx is None:\n continue\n\n # Pad cells if needed\n if start_idx + len(expected) > len(cells):\n cells = cells + [None] * (start_idx + len(expected) - len(cells))\n\n all_match = True\n mismatches = []\n for j, exp_val in enumerate(expLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705566070",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, there must be no duplicate rows or `Claim_ID`s.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n\n rows = list(ws.iter_rows(min_row=1, values_only=True))\n if len(rows) < 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File has 0 or 1 data rows, so no duplicates possible.\"}\n\n header = [str(c).strip().lower() if c is not None else \"\" for c in rows[0]]\n\n # Find Claim_ID column\n claim_id_col = None\n for i, h in enumerate(header):\n if h in (\"claim_id\", \"claimid\", \"claim id\"):\n claim_id_col = i\n break\n\n # Check for duplicate Claim_IDs\n duplicate_claim_ids = []\n if claim_id_col is not None:\n seen_ids = {}\n for row_idx, row in enumerate(rows[1:], start=2):\n val = row[claim_id_col]\n if val is None:\n continue\n val_str = str(val).strip()\n if val_str == \"\":\n continue\n if val_str in seen_ids:\n duplicate_claim_ids.append((val_str, seen_ids[val_str], row_idx))\n else:\n seen_ids[val_str] = row_idx\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find a 'Claim_ID' column in header: {header}\"}\n\n # Check for fully duplicate rows\n def row_key(row):\n return tuple(str(c).strip() if c is not None else \"\" for c in row)\n\n seen_rows = {}\n duplicate_rows = []\n for row_idx, row in enumerate(rows[1:], start=2):\n rk = row_key(row)\n # Skip completely empty rows\n if all(v == \"\" for v in rk):\n continue\n if rk in seen_rows:\n duplicate_rows.append((row_idx, seen_rows[rk]))\n else:\n seen_rows[rk] = row_idx\n\n issues = []\n if duplicate_claim_ids:\n for cid, first_row, dup_row in duplicate_claim_ids:\n issues.append(f\"Duplicate Claim_ID '{cid}' found in rows {first_row} and {dup_row}\")\n if duplicate_rows:\n for dup_row, first_row in duplicate_rows:\n issues.append(f\"Duplicate row found: rows {first_row} and {dup_row} are identical\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicates found in PBC_Claim_Register.xlsx: \" + \"; \".join(issues)\n }\n\n total_data_rows = sum(1 for row in rows[1:] if not all((str(c).strip() if c is not None else \"\") == \"\" for c in row))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Claim_IDs found in PBC_Claim_Register.xlsx. {total_data_rows} unique data rows checked.\"\n }\n\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Claim_Register.xlsx: {str(e)}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705648199",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be a table with 11 total rows (including the header row).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (rows that have at least one non-None cell)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Sheet '{sheet_name}' in PBC_Denial_Worklist.xlsx has exactly 11 non-empty rows (including header), as required.\"\n }\n \n # If no sheet matched exactly, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with 11 total rows (including header) in PBC_Denial_Worklist.xlsx. Found: {'; '.join(details)}\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705668317",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0028\" (where 28 can be 29 or 30), \"CLM-2026-0892\", \"PAT-1042\", \"Michael Torres\", \"03/15/2026\", \"BlueCross TX\", \"CO-16\", \"N382\", \"Technical/Administrative\", \"Correct and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and BLANK (empty cell) in consecutive columns.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob to be lenient\n workspace = Path(workspace_path)\n candidates = list(workspace.glob(\"**/PBC_Denial_Worklist*.xlsx\"))\n if not candidates:\n candidates = list(workspace.glob(\"**/*.xlsx\"))\n \n file_path = None\n for c in candidates:\n if \"denial\" in c.name.lower() or \"worklist\" in c.name.lower() or \"pbc\" in c.name.lower():\n file_path = c\n break\n if file_path is None and candidates:\n file_path = candidates[0]\n if file_path is None:\n file_path = workspace / \"PBC_Denial_Worklist.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found. Searched in {workspace}. No matching xlsx files found.\"}\n\n # The rubric says DEN-2026-0028 where 28 can be 29 or 30.\n # We'll also accept a broader range (28-31) to be lenient.\n # All other values are fixed.\n expected_values = [\n \"DEN-2026-XXXX\", # special handling\n \"CLM-2026-0892\",\n \"PAT-1042\",\n \"Michael Torres\",\n \"03/15/2026\",\n \"BlueCross TX\",\n \"CO-16\",\n \"N382\",\n \"Technical/Administrative\",\n \"Correct and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n def normalize(val):\n if val is None:\n return None\n s = str(val).strip()\n if s == '' or s.lower() == 'none':\n return None\n return s\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() == 'none'\n\n def den_id_matches(actual_val):\n \"\"\"Check if actual matches DEN-2026-00XX where XX is 28-31 (lenient range)\"\"\"\n if actual_val is None:\n return False\n a = str(actual_val).strip().upper()\n # Accept DEN-2026-0028 through DEN-2026-0031\n pattern = r'^DEN-2026-00(2[89]|3[01])$'\n return bool(re.match(pattern, a))\n\n def values_match(actual, expected):\n if expected is None:\n return is_blank(actual)\n if actual is None:\n return False\n a = normalize(actual)\n e = normalize(expected)\n if a is None:\n return False\n # Case-insensitive comparison\n if a.lower() == e.lower():\n return True\n # Try date variants: strip leading zeros\n a_clean = re.sub(r'\\b0+(\\d)', r'\\1', a)\n e_clean = re.sub(r'\\b0+(\\d)', r'\\1', e)\n if a_clean.lower() == e_clean.lower():\n return True\n # Also handle date formats like 2026-03-15 vs 03/15/2026\n # Try converting common date patterns\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\"),\n ]\n def to_canonical_date(s):\n for pat, fn in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fn(m)\n return None\n \n a_date = to_canonical_date(a)\n e_date = to_canonical_date(e)\n if a_date and e_date and a_date == e_date:\n return True\n \n # Handle BCBS TX variants\n if e.lower() == \"bluecross tx\":\n variants = [\"bluecross tx\", \"bcbs tx\", \"bcbstx\", \"blue cross tx\", \"bluecrosstx\", \"bcbs-tx\"]\n if a.lower().replace(\" \", \"\") in [v.replace(\" \", \"\") for v in variants]:\n return True\n \n return False\n\n rows_to_check = list(range(1, ws.max_row + 1)) if ws.max_row and ws.max_row <= 50 else list(range(1, 51))\n # Prioritize rows 9-11 as stated in rubric, but check all rows for leniency\n priority_rows = [9, 10, 11]\n all_rows = priority_rows + [r for r in rows_to_check if r not in priority_rows]\n\n num_expected = len(expected_values)\n best_match_count = 0\n best_row = None\n best_mismatches = []\n\n for row_num in all_rows:\n row_cells = []\n for col in range(1, max(ws.max_column + 1 if ws.max_column else 1, 20)):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n for start_col_idx in range(len(row_cells) - num_expected + 1):\n segment = row_cells[start_col_idx:start_col_idx + num_expected]\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705689978",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the exact values: \"DEN-2026-0029\" (where 29 can be 28 or 30), \"CLM-2026-1105\", \"PAT-1058\", \"Linda Park\", \"03/20/2026\", \"BlueCross TX\", \"CO-97\", \"N520\", \"Technical/Administrative\", \"Add Modifier and Resubmit\", \"Resubmitted\", \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob\n fp = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n fp = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n \n if not fp.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {fp}\"}\n\n try:\n wb = openpyxl.load_workbook(fp, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n ws = wb.active\n\n # The rubric says DEN-2026-0029 where 29 can be 28 or 30\n # So we accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n # We also accept variants like DEN-2026-0035 or other denial IDs with flexible matching\n expected_values = [\n \"DEN-2026-FLEX\", # special handling\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"Linda Park\",\n \"03/20/2026\",\n \"BlueCross TX\",\n \"CO-97\",\n \"N520\",\n \"Technical/Administrative\",\n \"Add Modifier and Resubmit\",\n \"Resubmitted\",\n \"Maria Santos\",\n \"04/24/2026\",\n None # BLANK\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n try:\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n except Exception:\n pass\n s = str(val).strip()\n return s\n\n def match_den_id(actual_str):\n \"\"\"Match DEN-2026-0028, DEN-2026-0029, or DEN-2026-0030 (flexible)\"\"\"\n norm = actual_str.strip().upper()\n # Accept DEN-2026-0028, DEN-2026-0029, DEN-2026-0030\n if re.match(r'^DEN-2026-00(28|29|30)$', norm):\n return True\n # Also accept any DEN-2026-XXXX pattern as a looser fallback\n # But primary match is the three specific ones\n return False\n\n def match_date(actual_str, expected_str):\n \"\"\"Flexible date matching\"\"\"\n date_pat = re.compile(r'^(\\d{1,2})/(\\d{1,2})/(\\d{4})$')\n m_a = date_pat.match(actual_str)\n m_e = date_pat.match(expected_str)\n if m_a and m_e:\n return (int(m_a.group(1)) == int(m_e.group(1)) and\n int(m_a.group(2)) == int(m_e.group(2)) and\n m_a.group(3) == m_e.group(3))\n return False\n\n def match(actual, expected, idx):\n norm_actual = normalize(actual)\n \n # Special case: DEN ID (first column)\n if idx == 0:\n return match_den_id(norm_actual)\n \n # Special case: BLANK (last column)\n if expected is None:\n return norm_actual == \"\" or norm_actual.lower() == \"none\"\n \n norm_expected = str(expected).strip()\n \n # Case-insensitive comparison\n if norm_actual.lower() == norm_expected.lower():\n return True\n \n # Date matching\n if match_date(norm_actual, norm_expected):\n return True\n \n # Contains check as fallback for things like \"BlueCross TX\" vs \"BCBS TX\" etc.\n if norm_expected.lower() in norm_actual.lower() or norm_actual.lower() in norm_expected.lower():\n return True\n \n # Also try matching with flexible whitespace/case\n if re.sub(r'\\s+', ' ', norm_actual.lower()) == re.sub(r'\\s+', ' ', norm_expected.lower()):\n return True\n \n return False\n\n for row_num in [9, 10, 11]:\n row_cells = []\n for col in range(1, ws.max_column + 2): # +2 for safety to check blank\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n # Try to match expected_values against consecutive columns starting from various offsets\n for start_col in range(min(5, len(row_cells))):\n all_match = True\n mismatches = []\n for i in range(len(expected_values)):\n col_idx = start_col + i\n if col_idx >= len(row_cells):\n actual_val = None\n else:\n actual_val = row_cells[col_idx]\n if not match(actual_val, expected_values[i], i):\n all_match = False\n mismatches.append((i, expected_values[i], normalize(actual_val)))\n if all_match:\n vals = [normalize(row_cells[start_col + i]) if (start_col + i) < len(row_cells) else \"\" for i in range(len(expected_values))]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {row_num} contains all expected values starting at column {start_col+1}. Values: {vals}\"}\n\n # If no exact matLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705707748",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, Row 9, 10, or 11 must contain the following exact values: \"DEN-2026-0030\" (where 30 can be 29 or 38), \"CLM-2026-0967\", \"PAT-1063\", \"Robert Daniels\", \"03/08/2026\", \"BlueCross TX\", \"CO-45\", \"N130\", \"Contractual/Payer Error\", \"Appeal - Contractual Rate Dispute\", \"Escalated\" [\"Appealed\" also acceptable], \"Maria Santos\", \"04/24/2026\", and the last cell must be BLANK.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n filepath = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n filepath = candidates[0]\n else:\n fp = Path(workspace_path) / \"PBC_Denial_Worklist.xlsx\"\n if fp.exists():\n filepath = fp\n\n if filepath is None or not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(filepath)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n # The DEN ID can be DEN-2026-0029, DEN-2026-0030, DEN-2026-0038, DEN-2026-0039\n valid_den_ids = [\"den-2026-0029\", \"den-2026-0030\", \"den-2026-0038\", \"den-2026-0039\"]\n\n # Expected values for columns after DEN ID (indices 1..12)\n # Index 0: DEN ID (special)\n # Index 1: CLM-2026-0967\n # Index 2: PAT-1063\n # Index 3: Robert Daniels\n # Index 4: 03/08/2026\n # Index 5: BlueCross TX\n # Index 6: CO-45\n # Index 7: N130\n # Index 8: Contractual/Payer Error\n # Index 9: Appeal - Contractual Rate Dispute\n # Index 10: Escalated or Appealed\n # Index 11: Maria Santos\n # Index 12: 04/24/2026\n # Index 13: BLANK\n\n expected_values = [\n None, # placeholder for DEN ID\n \"CLM-2026-0967\",\n \"PAT-1063\",\n \"Robert Daniels\",\n \"03/08/2026\",\n \"BlueCross TX\",\n \"CO-45\",\n \"N130\",\n \"Contractual/Payer Error\",\n \"Appeal - Contractual Rate Dispute\",\n \"Escalated\", # also accept \"Appealed\"\n \"Maria Santos\",\n \"04/24/2026\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n return s\n\n def norm_date(s):\n \"\"\"Normalize date string to MM/DD/YYYY format.\"\"\"\n date_patterns = [\n (r'(\\d{1,2})/(\\d{1,2})/(\\d{4})', lambda m: f\"{int(m.group(1)):02d}/{int(m.group(2)):02d}/{m.group(3)}\"),\n (r'(\\d{4})-(\\d{1,2})-(\\d{1,2})', lambda m: f\"{int(m.group(2)):02d}/{int(m.group(3)):02d}/{m.group(1)}\")\n ]\n for pat, fmt in date_patterns:\n m = re.match(pat, s.strip())\n if m:\n return fmt(m)\n return s.strip()\n\n def cell_to_normalized_string(actual):\n \"\"\"Convert a cell value to a normalized comparable string.\"\"\"\n if actual is None:\n return \"\"\n if isinstance(actual, (datetime.datetime, datetime.date)):\n d = actual if isinstance(actual, datetime.date) else actual.date()\n return f\"{d.month:02d}/{d.day:02d}/{d.year}\"\n return str(actual).strip()\n\n def values_match(actual, expected, col_index):\n a = cell_to_normalized_string(actual).lower()\n e = expected.lower().strip()\n # Special handling for column 10 (Escalated / Appealed)\n if col_index == 10:\n return a in [\"escalated\", \"appealed\"]\n if a == e:\n return True\n # Try date normalization\n if norm_date(a) == norm_date(e):\n return True\n # Loose matching: strip punctuation differences\n a_clean = re.sub(r'[\\s\\-/]+', '', a)\n e_clean = re.sub(r'[\\s\\-/]+', '', e)\n if a_clean == e_clean:\n return True\n return False\n\n def is_blank(val):\n return val is None or str(val).strip() == \"\"\n\n rows_to_check = list(range(2, ws.max_row + 1)) # Check all data rows, but prioritize 9-11\n priority_rows = [9, 10, 11]\n other_rows = [r for r in rows_to_check if r not in priority_rows]\n ordered_rows = priority_rows + other_rows\n\n best_score = 0\n best_details = []\n best_row = None\n\n for row_num in ordered_rows:\n if row_num > ws.max_row:\n continue\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n\n if len(row_cells) < len(expected_values):\n continue\n\n all_match = True\n match_details = []\n matched_count = 0\n\n # Check column 1 (DEN ID)\n col0_val = cell_to_normalized_string(row_cells[0]).lower()\n if col0_val in valid_den_ids:\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' matches DEN ID pattern\")\n matched_count += 1\n else:\n all_match = False\n match_details.append(f\"Col 1: '{normalize(row_cells[0])}' not in valid DEN IDs {valid_den_ids}\")\n\n # Check remaininLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705711177",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In `PBC_Denial_Worklist.xlsx`, there must be no duplicate rows or Denial IDs.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Denial_Worklist.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if f.name.lower() == \"pbc_denial_worklist.xlsx\":\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Denial_Worklist.xlsx in {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n rows = []\n header_row = None\n denial_id_col = None\n \n for i, row in enumerate(ws.iter_rows(values_only=True)):\n # Skip completely empty rows\n if all(cell is None or str(cell).strip() == '' for cell in row):\n continue\n if header_row is None:\n header_row = tuple(str(c).strip() if c is not None else '' for c in row)\n # Find the Denial ID column (be flexible with naming)\n for col_idx, h in enumerate(header_row):\n h_lower = h.lower().strip()\n if 'denial' in h_lower and 'id' in h_lower:\n denial_id_col = col_idx\n break\n elif h_lower in ('denialid', 'denial_id', 'denial id'):\n denial_id_col = col_idx\n break\n continue\n # Normalize cell values for comparison\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else ''\n for cell in row\n )\n rows.append((i + 1, normalized, row)) # 1-indexed row number\n \n if not rows:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PBC_Denial_Worklist.xlsx has no data rows (only header or empty), so no duplicates.\"}\n \n issues = []\n \n # Check for duplicate full rows\n seen_rows = {}\n duplicate_rows = []\n for row_num, normalized, raw in rows:\n if normalized in seen_rows:\n duplicate_rows.append((row_num, seen_rows[normalized]))\n else:\n seen_rows[normalized] = row_num\n \n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {dup_row} is a duplicate of row {orig_row}\"\n for dup_row, orig_row in duplicate_rows\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate row(s): {dup_details}\")\n \n # Check for duplicate Denial IDs\n if denial_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for row_num, normalized, raw in rows:\n denial_id = normalized[denial_id_col] if denial_id_col < len(normalized) else ''\n if denial_id and denial_id in seen_ids:\n duplicate_ids.append((row_num, denial_id, seen_ids[denial_id]))\n elif denial_id:\n seen_ids[denial_id] = row_num\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Row {dup_row} has Denial ID '{did}' which duplicates row {orig_row}\"\n for dup_row, did, orig_row in duplicate_ids\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Denial ID(s): {dup_id_details}\")\n else:\n # If we can't find a Denial ID column, note it but don't fail on that alone\n pass\n \n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx has duplicates. \" + \" | \".join(issues)\n }\n \n denial_id_info = f\" Denial ID column found at index {denial_id_col} ('{header_row[denial_id_col]}').\" if denial_id_col is not None else \" No Denial ID column detected (checked row uniqueness only).\"\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PBC_Denial_Worklist.xlsx contains {len(rows)} data rows with no duplicate rows or Denial IDs.{denial_id_info}\"\n }\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PBC_Denial_Worklist.xlsx: {str(e)}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705797706",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be a table with 5 total rows (including the header row).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n \n # Check all sheets for a table with 5 rows (1 header + 4 data)\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows: a row is non-empty if at least one cell has a value\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 5:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 5 total rows (including header) in sheet '{sheet_name}' of PBC_Escalation_Log.xlsx.\"\n }\n \n # If no sheet had exactly 5 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected a table with exactly 5 total rows (including header) in PBC_Escalation_Log.xlsx. Found: {'; '.join(details)}\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705829052",
|
||||
"sort_order": 18,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the following exact values: \"ESC-2026-0008\" (or \"ESC-2026-0009\"), \"04/24/2026\", \"CLM-2026-0234, CLM-2026-0356, CLM-2026-0489, CLM-2026-0612, CLM-2026-0892\" (in any order), \"Operational\", \"Pattern Recurrence\", \"Maria Santos\", \"David Park\", \"Open\", and a BLANK cell.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Search for the file with glob to be lenient\n xlsx_files = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if not xlsx_files:\n # Try case-insensitive\n xlsx_files = [f for f in Path(workspace_path).glob(\"**/*.xlsx\") if \"escalation\" in f.name.lower()]\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_Escalation_Log.xlsx not found in workspace.\"}\n\n file_path = xlsx_files[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open workbook: {e}\"}\n\n expected_claims = {\"CLM-2026-0234\", \"CLM-2026-0356\", \"CLM-2026-0489\", \"CLM-2026-0612\", \"CLM-2026-0892\"}\n # Accept ESC-2026-0008 or ESC-2026-0009 (or nearby numbers to be generous)\n valid_esc_ids = {\"ESC-2026-0008\", \"ESC-2026-0009\", \"ESC-2026-0010\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_row(row_num):\n row_vals = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_vals.append(cell_val)\n\n row_strs = [normalize(v) for v in row_vals]\n feedback_parts = []\n score = 0\n total_checks = 8 # 8 required fields\n\n # Check ESC ID\n found_esc = False\n for s in row_strs:\n for esc_id in valid_esc_ids:\n if esc_id in s:\n found_esc = True\n break\n if found_esc:\n break\n if found_esc:\n score += 1\n feedback_parts.append(\"Found valid ESC ID\")\n else:\n feedback_parts.append(f\"Missing ESC ID (expected one of {valid_esc_ids})\")\n\n # Check date 04/24/2026 - be flexible with date formats\n found_date = False\n for idx, s in enumerate(row_strs):\n if not s:\n continue\n if \"04/24/2026\" in s or \"4/24/2026\" in s or \"2026-04-24\" in s:\n found_date = True\n break\n raw_val = row_vals[idx]\n if hasattr(raw_val, 'strftime'):\n try:\n formatted = raw_val.strftime(\"%m/%d/%Y\")\n if formatted == \"04/24/2026\":\n found_date = True\n break\n except:\n pass\n if hasattr(raw_val, 'year'):\n try:\n if raw_val.year == 2026 and raw_val.month == 4 and raw_val.day == 24:\n found_date = True\n break\n except:\n pass\n if found_date:\n score += 1\n feedback_parts.append(\"Found date 04/24/2026\")\n else:\n feedback_parts.append(\"Missing date 04/24/2026\")\n\n # Check claims - they can be in any order, possibly in one cell or across cells\n found_claims = set()\n for s in row_strs:\n for claim in expected_claims:\n if claim in s:\n found_claims.add(claim)\n if found_claims == expected_claims:\n score += 1\n feedback_parts.append(\"Found all 5 claims\")\n else:\n missing = expected_claims - found_claims\n feedback_parts.append(f\"Missing claims: {missing}\")\n\n # Check Operational\n found_operational = any(\"operational\" in s.lower() for s in row_strs)\n if found_operational:\n score += 1\n feedback_parts.append(\"Found 'Operational'\")\n else:\n feedback_parts.append(\"Missing 'Operational'\")\n\n # Check Pattern Recurrence\n found_pattern = any(\"pattern recurrence\" in s.lower() for s in row_strs)\n if found_pattern:\n score += 1\n feedback_parts.append(\"Found 'Pattern Recurrence'\")\n else:\n feedback_parts.append(\"Missing 'Pattern Recurrence'\")\n\n # Check Maria Santos\n found_maria = any(\"maria santos\" in s.lower() for s in row_strs)\n if found_maria:\n score += 1\n feedback_parts.append(\"Found 'Maria Santos'\")\n else:\n feedback_parts.append(\"Missing 'Maria Santos'\")\n\n # Check David Park\n found_david = any(\"david park\" in s.lower() for s in row_strs)\n if found_david:\n score += 1\n feedback_parts.append(\"Found 'David Park'\")\n else:\n feedback_parts.append(\"Missing 'David Park'\")\n\n # Check Open\n found_open = any(s.Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705849430",
|
||||
"sort_order": 19,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, Row 4 or 5 must contain the exact values: \"ESC-2026-0009\" (or \"ESC-2026-0008\"), \"04/24/2026\", \"CLM-2026-0967\", \"Financial\", \"Payer Dispute\", \"Maria Santos\", \"David Park\", \"Open\", and BLANK (empty cell).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for leniency\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Try exact path\n fp = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n if fp.exists():\n file_path = fp\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # The rubric says ESC-2026-0009 or ESC-2026-0008, but we'll accept a range of ESC IDs\n valid_esc_ids = [\"esc-2026-0008\", \"esc-2026-0009\", \"esc-2026-0010\", \"esc-2026-0011\"]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def check_date(val):\n \"\"\"Check if the value represents 04/24/2026\"\"\"\n if isinstance(val, datetime):\n return val.month == 4 and val.day == 24 and val.year == 2026\n s = normalize(val).lower()\n date_patterns = [\"04/24/2026\", \"4/24/2026\", \"2026-04-24\", \"04-24-2026\", \"april 24, 2026\", \"apr 24, 2026\"]\n return s in date_patterns\n\n def check_blank(val):\n return val is None or str(val).strip() == \"\"\n\n # Search rows 4-10 for leniency (rubric says 4 or 5, but be generous)\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n \n row_strs = [normalize(c).lower() for c in row_cells]\n \n # Check if key identifiers are present in the row\n has_esc = any(any(esc_id in s for esc_id in valid_esc_ids) for s in row_strs)\n has_clm = any(\"clm-2026-0967\" in s for s in row_strs)\n has_financial = any(\"financial\" in s for s in row_strs)\n has_payer = any(\"payer dispute\" in s for s in row_strs)\n has_maria = any(\"maria santos\" in s or \"maria\" in s for s in row_strs)\n has_david = any(\"david park\" in s or \"david\" in s for s in row_strs)\n has_open = any(s == \"open\" for s in row_strs)\n \n # Check date\n has_date = any(check_date(c) for c in row_cells)\n \n checks = {\n \"ESC ID\": has_esc,\n \"CLM-2026-0967\": has_clm,\n \"Financial\": has_financial,\n \"Payer Dispute\": has_payer,\n \"Maria Santos\": has_maria,\n \"David Park\": has_david,\n \"Open\": has_open,\n \"Date 04/24/2026\": has_date\n }\n \n if all(checks.values()):\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row {row_num} in PBC_Escalation_Log.xlsx contains all expected escalation values. Row values: {row_strs}\"\n }\n \n # Gather debug info\n debug = []\n for row_num in range(4, 11):\n row_cells = []\n for col in range(1, ws.max_column + 1):\n row_cells.append(ws.cell(row=row_num, column=col).value)\n row_strs = [normalize(c) for c in row_cells]\n if any(s.strip() for s in row_strs): # Only show non-empty rows\n debug.append(f\"Row {row_num}: {row_strs}\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No row (4-10) in PBC_Escalation_Log.xlsx contains the expected escalation entry. Expected ESC-2026-0008/0009, 04/24/2026, CLM-2026-0967, Financial, Payer Dispute, Maria Santos, David Park, Open. Found rows: {'; '.join(debug)}\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705853789",
|
||||
"sort_order": 20,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be no duplicate rows or Escalation IDs",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with glob for flexibility\n file_path = None\n candidates = list(Path(workspace_path).glob(\"**/PBC_Escalation_Log.xlsx\"))\n if candidates:\n file_path = candidates[0]\n else:\n # Also try case-insensitive\n for f in Path(workspace_path).glob(\"**/*.xlsx\"):\n if \"escalation\" in f.name.lower() and \"log\" in f.name.lower():\n file_path = f\n break\n \n if file_path is None or not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: PBC_Escalation_Log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {file_path}: {e}\"}\n\n issues = []\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = tuple(\n str(cell).strip().lower() if cell is not None else \"\"\n for cell in row\n )\n rows.append(normalized)\n\n if not rows:\n continue\n\n header = rows[0]\n data_rows = rows[1:] if len(rows) > 1 else []\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(cell != \"\" for cell in r)]\n\n # Check for duplicate full rows\n seen_rows = set()\n duplicate_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n duplicate_rows.append((i + 2, r))\n else:\n seen_rows.add(r)\n\n if duplicate_rows:\n dup_details = \"; \".join(\n f\"Row {idx}: {row}\" for idx, row in duplicate_rows[:5]\n )\n issues.append(f\"Found {len(duplicate_rows)} duplicate data row(s) in sheet '{sheet_name}'. Examples: {dup_details}\")\n\n # Check for duplicate Escalation IDs\n # Find the column index for Escalation ID (flexible matching)\n esc_id_col = None\n for col_idx, col_name in enumerate(header):\n if col_name and (\"escalation\" in col_name and \"id\" in col_name):\n esc_id_col = col_idx\n break\n \n if esc_id_col is not None:\n seen_ids = {}\n duplicate_ids = []\n for i, r in enumerate(data_rows):\n if esc_id_col < len(r):\n esc_id = r[esc_id_col]\n if esc_id and esc_id != \"\":\n if esc_id in seen_ids:\n duplicate_ids.append((i + 2, esc_id, seen_ids[esc_id]))\n else:\n seen_ids[esc_id] = i + 2\n \n if duplicate_ids:\n dup_id_details = \"; \".join(\n f\"Escalation ID '{eid}' in rows {first_row} and {dup_row}\" \n for dup_row, eid, first_row in duplicate_ids[:5]\n )\n issues.append(f\"Found {len(duplicate_ids)} duplicate Escalation ID(s) in sheet '{sheet_name}'. Examples: {dup_id_details}\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Duplicate issues found in PBC_Escalation_Log.xlsx: \" + \" | \".join(issues)\n }\n\n total_sheets = len(wb.sheetnames)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No duplicate rows or Escalation IDs found in PBC_Escalation_Log.xlsx across {total_sheets} sheet(s).\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705926737",
|
||||
"sort_order": 21,
|
||||
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be a table with 11 total rows (including the header row).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for the file with case-insensitive matching\n target = None\n ws_path = Path(workspace_path)\n \n # Try exact name first\n exact = ws_path / \"PBC_RA_Queue.xlsx\"\n if exact.exists():\n target = exact\n else:\n # Try case-insensitive glob\n for f in ws_path.glob(\"*.xlsx\"):\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target = f\n break\n \n if target is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File 'PBC_RA_Queue.xlsx' not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open '{target.name}': {e}\"}\n \n # Check all sheets for a table with 11 rows\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count non-empty rows (a row is non-empty if at least one cell has a value)\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n \n if non_empty_rows == 11:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found table with 11 total rows (including header) in sheet '{sheet_name}' of '{target.name}'.\"\n }\n \n # If no sheet had exactly 11 rows, report what was found\n details = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n non_empty_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n non_empty_rows += 1\n details.append(f\"Sheet '{sheet_name}': {non_empty_rows} non-empty rows\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected 11 total rows (including header) in 'PBC_RA_Queue.xlsx', but found: {'; '.join(details)}.\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705946912",
|
||||
"sort_order": 22,
|
||||
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0031\" (where 31 can be 32 or 33), \"04/24/2026\", \"CLM-2026-0892\", \"PAT-1042\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Find the file with case-insensitive search\n file_path = None\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n file_path = c\n break\n if file_path is None:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n for f in xlsx_files:\n if \"ra_queue\" in f.name.lower() or \"raqueue\" in f.name.lower():\n file_path = f\n break\n if file_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in {workspace_path}. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n # The rubric says the RA ID can be RA-2026-0031, 0032, or 0033\n valid_ra_ids = [\"ra-2026-0031\", \"ra-2026-0032\", \"ra-2026-0033\"]\n\n # Core required values that MUST be present:\n # RA ID (flexible from list above)\n # Date: 04/24/2026\n # Claim: CLM-2026-0892\n # Patient: PAT-1042\n # Task: ERA Processing\n # Priority: [Any value is acceptable] - we won't check this\n # Assigned: Maria Santos\n #\n # Status: \"Resolved\" or \"Open\" or BLANK\n # Resolution Date: \"04/24/2026\" or BLANK\n\n core_required = [\n \"04/24/2026\", # date\n \"CLM-2026-0892\", # claim id\n \"PAT-1042\", # patient id\n \"ERA Processing\", # task type\n \"Maria Santos\", # assigned to\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime.datetime, datetime.date)):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def values_match(cell_val, expected_val):\n n_cell = normalize(cell_val).lower()\n n_exp = expected_val.strip().lower()\n if n_cell == n_exp:\n return True\n # Handle date variations\n if n_exp == \"04/24/2026\":\n if n_cell in [\"4/24/2026\", \"04/24/2026\", \"2026-04-24\", \"04-24-2026\", \"4-24-2026\"]:\n return True\n # Handle minor whitespace/case differences\n if n_cell.replace(\" \", \"\") == n_exp.replace(\" \", \"\"):\n return True\n # Partial matching for things like \"era processing\"\n if n_exp in n_cell or n_cell in n_exp:\n if len(n_exp) > 3 and len(n_cell) > 3:\n return True\n return False\n\n def is_blank(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == \"\" or s.lower() == \"none\"\n\n # Check rows 9, 10, 11 first, then expand search\n priority_rows = [9, 10, 11]\n check_rows = list(range(2, min(ws.max_row + 1, 50)))\n all_rows = priority_rows + [r for r in check_rows if r not in priority_rows]\n\n best_match_count = 0\n best_row_info = \"\"\n\n for row_num in all_rows:\n row_values = []\n for col in range(1, ws.max_column + 1):\n row_values.append(ws.cell(row=row_num, column=col).value)\n\n # Check RA ID\n ra_id_found = False\n for cv in row_values:\n nv = normalize(cv).lower().strip()\n if nv in valid_ra_ids:\n ra_id_found = True\n break\n\n # Check core required values\n core_matched = []\n for exp in core_required:\n found = False\n for cv in row_values:\n if values_match(cv, exp):\n found = True\n break\n core_matched.append(found)\n\n total_core = sum(core_matched) + (1 if ra_id_found else 0)\n if total_core > best_match_count:\n best_match_count = total_core\n best_row_info = f\"Row {row_num}: {[normalize(v) for v in row_values]}\"\n\n # Need RA ID + all core required values (6 total)\n if ra_id_found and all(core_matched):\n # Check status - rubric says \"Resolved\" and \"04/24/2026\" for last two,\n # but also says they could be \"Open\", BLANK instead\n has_resolved = False\n has_open = False\n status_val = \"unknown\"\n\n for cv in row_values:\n nv = normalize(cv).lower()\n if nv == \"resolved\":\n has_resolved = True\n status_val = \"Resolved\"\n if nv == \"open\":\n has_open = True\n status_val = \"Open\"\n\n #Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705963295",
|
||||
"sort_order": 23,
|
||||
"rubric_text": "In `PBC_RA_Queue.xlsx`, row 9, 10, or 11 must contain the exact values: \"RA-2026-0032\" (where 32 can be 31 or 33), \"04/24/2026\", \"CLM-2026-1105\", \"PAT-1058\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Resolved\", \"04/24/2026\". (where the last two cells could be \"Open\", BLANK instead)",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\n\ndef normalize(val):\n \"\"\"Normalize a cell value to a comparable string.\"\"\"\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n\ndef values_match(actual, expected):\n \"\"\"Loosely compare two values.\"\"\"\n a = normalize(actual)\n e = expected.strip()\n # Direct match (case-insensitive)\n if a.lower() == e.lower():\n return True\n # Try date parsing: expected is MM/DD/YYYY\n if '/' in e:\n try:\n exp_date = datetime.strptime(e, \"%m/%d/%Y\")\n # Try parsing actual in various formats\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\"]:\n try:\n act_date = datetime.strptime(a, fmt)\n if act_date.date() == exp_date.date():\n return True\n except ValueError:\n pass\n except ValueError:\n pass\n return False\n\n\ndef ra_id_match(actual):\n \"\"\"Check if a cell value matches RA-2026-00{31,32,33}.\"\"\"\n a = normalize(actual).upper()\n return a in [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n\ndef verify(workspace_path, external_services_path=None):\n ws_path = Path(workspace_path)\n # Try case-insensitive search for file\n target_file = None\n for f in ws_path.iterdir():\n if f.name.lower() == \"pbc_ra_queue.xlsx\":\n target_file = f\n break\n\n if target_file is None:\n # Also try glob\n candidates = list(ws_path.glob(\"*RA_Queue*\")) + list(ws_path.glob(\"*ra_queue*\")) + list(ws_path.glob(\"*Ra_Queue*\")) + list(ws_path.glob(\"*Queue*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n # The core required values (excluding RA ID which has special matching,\n # excluding priority which accepts any value,\n # and excluding status/completion date which are flexible)\n core_expected = [\n \"04/24/2026\",\n \"CLM-2026-1105\",\n \"PAT-1058\",\n \"ERA Processing\",\n \"Maria Santos\",\n ]\n\n # Search rows 8-12 to be generous (rubric says 9, 10, or 11)\n search_rows = list(range(8, 13))\n\n for row_num in search_rows:\n row_cells = []\n for col in range(1, ws.max_column + 1):\n cell_val = ws.cell(row=row_num, column=col).value\n row_cells.append(cell_val)\n\n # Check if RA ID matches\n ra_found = False\n for cell in row_cells:\n if ra_id_match(cell):\n ra_found = True\n break\n\n if not ra_found:\n continue\n\n # Check if all core expected values can be found in this row\n all_core_found = True\n matched_details = []\n matched_details.append(\" RA ID found (one of RA-2026-0031/32/33)\")\n\n for exp in core_expected:\n found = False\n for i, cell in enumerate(row_cells):\n if values_match(cell, exp):\n found = True\n matched_details.append(f\" Expected '{exp}' -> found '{normalize(cell)}' in column {i+1}\")\n break\n if not found:\n all_core_found = False\n matched_details.append(f\" Expected '{exp}' -> NOT FOUND\")\n break\n\n if all_core_found:\n # Status can be \"Resolved\", \"Open\", or blank - check if present\n status_info = \"\"\n for i, cell in enumerate(row_cells):\n n = normalize(cell).lower()\n if n in [\"resolved\", \"open\"]:\n status_info = f\"Status='{normalize(cell)}' in column {i+1}\"\n break\n if not status_info:\n status_info = \"Status is blank or not found (acceptable)\"\n\n detail = \"\\n\".join(matched_details)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All expected values found in row {row_num} of PBC_RA_Queue.xlsx:\\n{detail}\\n {status_info}\\nPriority: any value accepted.\"\n }\n\n # If not found, provide diagnostic info\n diag = []\n for row_num in search_rows:\n row_vals = []\n for col in range(1, ws.max_column + 1Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705979579",
|
||||
"sort_order": 24,
|
||||
"rubric_text": "In `PBC_RA_Queue.xlsx`, Row 9, 10, or 11 must contain the exact values: \"RA-2026-0033\" (where 33 can be 32 or 31), \"04/24/2026\", \"CLM-2026-0967\", \"PAT-1063\", \"ERA Processing\", [Any priority value is acceptable], \"Maria Santos\", \"Open\", and a BLANK cell (in the appropriate columns).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n if not file_path.exists():\n # Try case-insensitive glob\n candidates = list(Path(workspace_path).glob(\"PBC_RA_Queue*\")) + list(Path(workspace_path).glob(\"PBC_Ra_Queue*\")) + list(Path(workspace_path).glob(\"pbc_ra_queue*\"))\n if candidates:\n file_path = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File PBC_RA_Queue.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n acceptable_ra_ids = [\"RA-2026-0031\", \"RA-2026-0032\", \"RA-2026-0033\"]\n\n # Expected fixed values (excluding RA ID, Date, and Priority which are handled separately)\n # The rubric says: RA ID, Date, CLM-2026-0967, PAT-1063, ERA Processing, [Any priority], Maria Santos, Open, BLANK\n expected_fixed = {\n \"CLM\": \"CLM-2026-0967\",\n \"PAT\": \"PAT-1063\",\n \"TYPE\": \"ERA Processing\",\n \"ASSIGNEE\": \"Maria Santos\",\n \"STATUS\": \"Open\"\n }\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def clean(s):\n return s.lower().replace(\" \", \"\").replace(\"-\", \"\").replace(\"/\", \"\").replace(\"_\", \"\")\n\n def match_date(actual):\n a_clean = normalize(actual)\n # Try multiple date formats\n try:\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%B %d, %Y\", \"%m/%d/%y\"]:\n try:\n parsed = datetime.strptime(a_clean, fmt)\n if parsed.month == 4 and parsed.day == 24 and parsed.year == 2026:\n return True\n except:\n pass\n except:\n pass\n # Fallback: clean comparison\n if clean(a_clean) == clean(\"04/24/2026\"):\n return True\n if clean(a_clean) == \"04242026\":\n return True\n return False\n\n def match_ra_id(actual):\n a = normalize(actual)\n if not a:\n return False\n for rid in acceptable_ra_ids:\n if clean(a) == clean(rid):\n return True\n if a.upper().replace(\"-\", \"\").replace(\" \", \"\") == rid.upper().replace(\"-\", \"\"):\n return True\n # Regex: RA-2026-003[1-3] with flexible separators\n pattern = r'(?i)ra[\\-_ ]?2026[\\-_ ]?0{0,2}3[123]'\n if re.search(pattern, a):\n return True\n return False\n\n def fuzzy_match(expected, actual):\n e_norm = normalize(expected)\n a_norm = normalize(actual)\n if not a_norm and not e_norm:\n return True\n if clean(e_norm) == clean(a_norm):\n return True\n if e_norm.lower().strip() == a_norm.lower().strip():\n return True\n return False\n\n def check_row(row_values):\n \"\"\"Check if a row contains all expected values. Returns (True, msg) or (False, msg).\"\"\"\n # Strategy 1: Ordered match\n # Expected column order: A=RA_ID, B=Date, C=CLM, D=PAT, E=Type, F=Priority, G=Assignee, H=Status, I=blank(notes)\n if len(row_values) >= 8:\n ordered_ok = True\n issues = []\n # Column A (index 0): RA ID\n if not match_ra_id(row_values[0]):\n ordered_ok = False\n issues.append(f\"Col A: expected RA-2026-003[1-3], got '{normalize(row_values[0])}'\")\n # Column B (index 1): Date\n if ordered_ok and not match_date(row_values[1]):\n ordered_ok = False\n issues.append(f\"Col B: expected 04/24/2026, got '{normalize(row_values[1])}'\")\n # Column C (index 2): CLM-2026-0967\n if ordered_ok and not fuzzy_match(\"CLM-2026-0967\", row_values[2]):\n ordered_ok = False\n issues.append(f\"Col C: expected CLM-2026-0967, got '{normalize(row_values[2])}'\")\n # Column D (index 3): PAT-1063\n if ordered_ok and not fuzzy_match(\"PAT-1063\", row_values[3]):\n ordered_ok = False\n issues.append(f\"Col D: expected PAT-1063, got '{normalize(row_values[3])}'\")\n # Column E (index 4): ERA Processing\n if ordered_ok and not fuzzy_match(\"ERA Processing\", row_values[4]):\n ordered_ok = False\n issues.append(f\"Col E: expected ERALine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775705982031",
|
||||
"sort_order": 25,
|
||||
"rubric_text": "In `PBC_RA_Queue.xlsx`, there must be no duplicate rows or Queue Entry IDs.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n target = Path(workspace_path) / \"PBC_RA_Queue.xlsx\"\n # Also try case-insensitive search\n if not target.exists():\n candidates = list(Path(workspace_path).glob(\"*\"))\n for c in candidates:\n if c.name.lower() == \"pbc_ra_queue.xlsx\":\n target = c\n break\n\n if not target.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File PBC_RA_Queue.xlsx not found in workspace. Files present: {[f.name for f in Path(workspace_path).iterdir()]}\"}\n\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open PBC_RA_Queue.xlsx: {e}\"}\n\n ws = wb.active\n\n rows = []\n for row in ws.iter_rows(min_row=1, values_only=True):\n normalized = []\n for cell in row:\n if isinstance(cell, str):\n normalized.append(cell.strip().lower())\n elif cell is None:\n normalized.append(None)\n else:\n normalized.append(cell)\n rows.append(tuple(normalized))\n\n if len(rows) <= 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(rows)} row(s); no duplicates possible.\"}\n\n header = rows[0]\n data_rows = rows[1:]\n\n # Filter out completely empty rows\n data_rows = [r for r in data_rows if any(c is not None for c in r)]\n\n issues = []\n\n # Check 1: Duplicate entire rows\n seen_rows = set()\n dup_rows = []\n for i, r in enumerate(data_rows):\n if r in seen_rows:\n dup_rows.append((i + 2, r)) # +2 for 1-indexed + header\n else:\n seen_rows.add(r)\n\n if dup_rows:\n dup_info = \"; \".join([f\"Row {idx}\" for idx, vals in dup_rows[:5]])\n issues.append(f\"Found {len(dup_rows)} duplicate row(s). Examples: {dup_info}\")\n\n # Check 2: Duplicate Queue Entry IDs\n # Try to find a column that looks like a Queue Entry ID\n qe_col_idx = None\n if header:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue entry id' in h_lower or 'queueentryid' in h_lower or 'queue_entry_id' in h_lower or h_lower == 'id' or 'entry id' in h_lower:\n qe_col_idx = idx\n break\n # If not found, also check for columns starting with 'queue'\n if qe_col_idx is None:\n for idx, h in enumerate(header):\n if h is not None and isinstance(h, str):\n h_lower = h.lower().strip()\n if 'queue' in h_lower and 'id' in h_lower:\n qe_col_idx = idx\n break\n\n if qe_col_idx is not None:\n seen_ids = {}\n dup_ids = []\n for i, r in enumerate(data_rows):\n if qe_col_idx < len(r):\n val = r[qe_col_idx]\n if val is not None:\n val_key = str(val).strip().lower()\n if val_key in seen_ids:\n dup_ids.append((i + 2, val))\n else:\n seen_ids[val_key] = i + 2\n if dup_ids:\n dup_id_info = \"; \".join([f\"Row {idx}: ID={val}\" for idx, val in dup_ids[:5]])\n issues.append(f\"Found {len(dup_ids)} duplicate Queue Entry ID(s). Examples: {dup_id_info}\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_RA_Queue.xlsx has duplicates: \" + \" | \".join(issues)}\n else:\n id_note = f\" Queue Entry ID column found at index {qe_col_idx}, all unique.\" if qe_col_idx is not None else \" No Queue Entry ID column identified; only checked full row uniqueness.\"\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_RA_Queue.xlsx has {len(data_rows)} data rows with no duplicate rows or Queue Entry IDs.{id_note}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775706048803",
|
||||
"sort_order": 26,
|
||||
"rubric_text": "In `slack_data.json` (in `external_data/final`), there must be exactly 3 total messages in #general and exactly 8 total messages in #rcm-team.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n # Try multiple possible locations for slack_data.json\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n \n # Also check workspace-based paths\n wp = Path(workspace_path)\n candidates.append(wp / \"external_data\" / \"final\" / \"slack_data.json\")\n candidates.append(wp / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find slack_data.json. Checked: {[str(c) for c in candidates]}\"}\n \n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n \n messages = data.get(\"messages\", {})\n \n # Find the general channel and rcm-team channel\n # First, map channel names to IDs from channels info\n channels = data.get(\"channels\", {})\n general_id = None\n rcm_id = None\n for ch_id, ch_info in channels.items():\n name = ch_info.get(\"name\", \"\").lower()\n if name == \"general\":\n general_id = ch_id\n elif name == \"rcm-team\":\n rcm_id = ch_id\n \n # Count messages\n general_msgs = []\n rcm_msgs = []\n \n # Try by channel ID first, then by channel name as key\n for key, msg_list in messages.items():\n if key == general_id or key.lower() == \"general\":\n general_msgs = msg_list\n elif key == rcm_id or key.lower() == \"rcm-team\":\n rcm_msgs = msg_list\n \n general_count = len(general_msgs)\n rcm_count = len(rcm_msgs)\n \n feedback_parts = []\n passed = True\n score = 0.0\n \n if general_count == 3:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#general has {general_count} messages (expected 3). FAIL.\")\n passed = False\n \n if rcm_count == 8:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). PASS.\")\n score += 0.5\n else:\n feedback_parts.append(f\"#rcm-team has {rcm_count} messages (expected 8). FAIL.\")\n passed = False\n \n return {\"pass\": passed, \"score\": score, \"feedback\": \" | \".join(feedback_parts)}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Hey, it's Maria. If you'll notice from my email I have a batch that I need completed today, April 24, 2026. Can you use the SOP and process the ERA for me? You can go ahead and do workflows even if they're technically for other roles.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user