Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,204 @@
{
"mailbox": {
"email": "d.okafor@careig.com",
"name": "Diane Okafor"
},
"contacts": [
{
"email": "payments@silverpath-claims.com",
"name": "SilverPath Claims \u2014 Payments"
},
{
"email": "billing@silverpath-claims.com",
"name": "SilverPath Claims \u2014 Billing"
},
{
"email": "appeals@silverpath-claims.com",
"name": "SilverPath Claims \u2014 Appeals"
},
{
"email": "claims@bluecrest-health.com",
"name": "BlueCrest Health \u2014 Claims"
},
{
"email": "appeals@bluecrest-health.com",
"name": "BlueCrest Health \u2014 Appeals"
},
{
"email": "billing@careig.com",
"name": "CareIG Billing Inbox"
},
{
"email": "d.okafor@careig.com",
"name": "Diane Okafor"
},
{
"email": "r.walsh@careig.com",
"name": "Raymond Walsh"
},
{
"email": "j.alcott@careig.com",
"name": "Jennifer Alcott"
},
{
"email": "s.obi@careig.com",
"name": "Sandra Obi"
},
{
"email": "m.holloway@emailhost.com",
"name": "Margaret Holloway"
}
],
"folders": [
{
"name": "Cases/Active"
},
{
"name": "Cases/Closed"
},
{
"name": "Payer-Correspondence"
}
],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "Electronic Remittance Advice \u2014 SilverPath Medicare Advantage \u2014 CareIG NPI: 1234567893",
"from_addr": "payments@silverpath-claims.com",
"to_addr": "billing@careig.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-04-08T07:42:00Z",
"message_id": "<era-spa-20260408-0047@silverpath-claims.com>",
"in_reply_to": null,
"body_text": "CareIG Specialty Pharmacy,\n\nPlease find attached your Electronic Remittance Advice (ERA) for the remittance date of 04/08/2026.\n\nEFT Reference: SPA-EFT-20260408-0047\nProvider NPI: 1234567893\n\nThis remittance contains 1 claim line for review. Please see the attached PDF for full detail including CARC codes.\n\nQuestions: providerrelations@silverpath-claims.com\nAppeal submissions: appeals@silverpath-claims.com\n\nSilverPath Medicare Advantage\nProvider Relations",
"body_html": null,
"is_read": false,
"is_important": true,
"attachments": [
{
"filename": "era_02142026_SilverPath.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjMKJcTl8uXrp/Og0MTGCjMgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyMTMyID4+CnN0cmVhbQp4AeVa227cNhB911fQjpNItlcmRYkSe0nTuEnbNGmbdnMp6j4Z7UOBFEjz/0DPjKjRhdr17tq7fQgMWFzxPnPOzHCoD+qV+qA0/irvVFMW6t8/1Vv1j7q4/GjU9Ueu0urjNbXKi7L93RZcqXPdqOv36skSlVprkyyvVUWjYTytrNaq9KVavlcXy2WhjFr+pX5X6dHxvZNMGa3S+w8eZsoalV6lmarxyE7Pzu8vjvMs8fTyAZqh9v7ZxYJ6cEN9laFItQbPArWZ+kMtn6unS2yGtjNde3LD2vsVG62LvPTxqtOzTC3/xhRJO8UdSKyfleVkbdLKqezllKmFybHV40z5vFLpPbzQuVUphAFt8NbxAjUkqCKHBCFP1CQs0LYtiQulsu91yi8w7pkMeM6vMBC1bgcuqcv9LMGvBQ2KaUlL4wXIADJmTmtFzwvqgwF1NxlGTsbrQOf1ilsHugSg60Xorcm99oUqrQ14C3JMsTLWXIv0WQwnLYb74Ugjti7CSK7VSHrUj7SGNckWrJnMCIC3GGhnTMCVYwhJEwZOAHPWMFQNMUIrKNig8lYpvXpIi9yr633EsscwkIbOwS3gpAnAKXPH0OIOwDljBXvFoOjQIqMFQTsvyIkCVCxV0HVXRQVUtVpHL2kjb8DagdbBzSld12mdTE0vNG/LvPYwICS3BDZmO01tZN8CNti+wZRN7VuB1Qx1RvatsBA7REdawKPk/3nlIDdIus5UibcQNP4fNfzwGe0q/WwkGpIDGdJZw5y0VvZmw9xLi+x1461T0zUn6ecbIXtreYFAQ3m5qs690UW0ABIaxKOCeAI89mFqa2tz65uSlhDsbYeZw8igrsAPU8UiSPW5NueLc3CpM1cwMqzmeQQwODZxzT0CvCvhsZ3/33bvG/gzbee2f36++GK0dbYLuxlToxsgzfI+O3YmHH1saMO3QToMntbg6Ajq5EBM47vJWwcClPsvwX6VPvoKYOcACE9Y48cwDXjAbPBbNOInGWFq9XV4wkzw7yfhCRtC3bpml/w6YeNL3b4JzeC+udvT8IQF5t+QNz/JfFP7Z+GJfmMK7m6hm8pBOpUnYUz4hlV3QN84PmOxFs3qkdb45W102lOGZnR1rEeo0ZPzhckuK3KkMOsc75BuOFiq2E3CBZL22JWS5ec2kZP99hH1RtvvglvGsx0FFaSa77Ok8/Wapn2OcLkgv/3D1F+/gGdvuwJFpiYX/jJTLvh8qklSqrE0XRceSExBUOK1/iilF12jk5+6EuEzbCQZRwtwZ9wdSBX5DLz9Xbs0UyACKAo/VdGhPJopKoOIsprOj7jnXD+7vJiYtGR3IpkSttubejDT3izaBP0R+AtWP8wYobSDZYs5BKmM4xMJOCWA/Zl74RQRoV/Ge0W9waUXYoDaE93dekEHzNhKO+UmduTuI4Bekg5AsQ5HCjgKPg6HsINC/IolAzoiBFpJlmTHg3m/hNpWuUPIfIB9c7Q8if7qCscXM91/G/boy3HYMwx8b79x71xuG9McYOO9tH0TIs1Y4eRnmSVSIE/Nb8iFo5CwT58Y4tNfuBK866xsxKT4tCWHtTAyH+bbuXBSR+FghzVT+iq3SNiQQFYc1rYKBBZVtXqkvQQCC2Nibf7KYoSbFQlTnMXapECLtSl6Ea8qBTk7x22Wwbf353pKx5D7h79mu/s6mFsZTQpvujVQ8MeL6SKFwYR0jofZkZnfhgnlhRRkXCkMPTyGKZNUIp9TaSSrkIE2moLWBYHCL5TkDoD7lWZx63xllOsxxiM6qi1i4Kl215+Lk53zpBPLaApEYx6nBVrBxCc9mzkUbRTJIi87TpGZwtV5qeu5fUJhXSw+T5xbRC4UQS9MLTF0G7bA74n1k0LgywCRghuB1KAQYCK8k6o54kySZ9I2FAYWV6oEvTKDcEcoPoPnwCoZJi5MuaPWcCfZjZ5z3EFCZV1KaYeEgjFVlfuKMFXXE+TefTQ1F1UYg3OQr8sRxNrjdjrHnR0TCkWBEALp5RjIe8go9EEEc6ewTRQzCmWkMOVOwsd/tu8xAOWNZGXlzRx3NnU6MogwR8aPXeEMc4KnkmHiwoA5RDMEL3fjdQb0n2NOe3ewMhl7M3NmvI7zyMaCOZF6D+V1TAO/Z8AcrGDC3Tnm7Op1kPKwFsyJ9wn17tfrFF4ycOJ1JFyj7Nc4/JEofBOvI40FpHPMEa8zF64NQCeDSEEoJAyXVc0w55PyOaXNTdMAUX6aUzyUz3EAjq54BWPTfKc+B1GhKxCXRjDev8+xLr7WEuYIECUuClxK+jOQ4DguSC+pmmOO+JyWpNJ2k4JwJ5A0SdfFa5+U1yk88pXgTqTgg3md0uWFA3ewgv15HeN97hpwJ94nALRfr1MW8uGAeB1cE7ThihzHJTKSQsyLAdYPdtYRtygFofun7neqBglMpxZQ8AS5h/I7dZXXFEsNILaPs47PcUM9nCS+atguU1eCiiHtHQ9121NpFFvzqakquw/P+kx7jU9o6rzhjBIOdLg6xRvYIxwjAkGT9FW4gxDuvZMTRlfVPSVvfxolYt90l3/h3NNf1AnLxRKs4xf7pkGc2F/L3bwXFe/FZ4lcy02PTzN7mAbIEwPAqYlxZu4WF1xwCnmDq7QFNDeh140me/NviAJYsHmtMVl0ab9wiOtGNzRpgfzjCo9x+7sJW9MFIr6JoYkn2z6MVbFNkxeVQ0wQbZ2SdF8AF/Sh0jt8ooQL46v03m+4i0YKGkoZ6371x0obn485wwNTMfpqCCFLZRzs7tz6rlK6IaflPEQyHPej6esMJoY+CsX/l1juYy4BuuPVbozUyLzgFjav8aHozIJGgfl/1lWHsgplbmRzdHJlYW0KZW5kb2JqCjEgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9Db250ZW50cyAzIDAgUiAvTWVkaWFCb3ggWzAgMCA1OTUuMiA4NDEuOTJdCj4+CmVuZG9iago0IDAgb2JqCjw8IC9Qcm9jU2V0IFsgL1BERiAvVGV4dCBdIC9Db2xvclNwYWNlIDw8IC9DczEgNSAwIFIgPj4gL0ZvbnQgPDwgL1RUMiA3IDAgUgovVFQ0IDkgMCBSIC9UVDYgMTEgMCBSID4+ID4+CmVuZG9iagoxMiAwIG9iago8PCAvTiAzIC9BbHRlcm5hdGUgL0RldmljZVJHQiAvTGVuZ3RoIDI2MTIgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCngBnZZ3VFPZFofPvTe90BIiICX0GnoJINI7SBUEUYlJgFAChoQmdkQFRhQRKVZkVMABR4ciY0UUC4OCYtcJ8hBQxsFRREXl3YxrCe+tNfPemv3HWd/Z57fX2Wfvfde6AFD8ggTCdFgBgDShWBTu68FcEhPLxPcCGBABDlgBwOFmZgRH+EQC1Py9PZmZqEjGs/buLoBku9ssv1Amc9b/f5EiN0MkBgAKRdU2PH4mF+UClFOzxRky/wTK9JUpMoYxMhahCaKsIuPEr2z2p+Yru8mYlybkoRpZzhm8NJ6Mu1DemiXho4wEoVyYJeBno3wHZb1USZoA5fco09P4nEwAMBSZX8znJqFsiTJFFBnuifICAAiUxDm8cg6L+TlongB4pmfkigSJSWKmEdeYaeXoyGb68bNT+WIxK5TDTeGIeEzP9LQMjjAXgK9vlkUBJVltmWiR7a0c7e1Z1uZo+b/Z3x5+U/09yHr7VfEm7M+eQYyeWd9s7KwvvRYA9iRamx2zvpVVALRtBkDl4axP7yAA8gUAtN6c8x6GbF6SxOIMJwuL7OxscwGfay4r6Df7n4Jvyr+GOfeZy+77VjumFz+BI0kVM2VF5aanpktEzMwMDpfPZP33EP/jwDlpzcnDLJyfwBfxhehVUeiUCYSJaLuFPIFYkC5kCoR/1eF/GDYnBxl+nWsUaHVfAH2FOVC4SQfIbz0AQyMDJG4/egJ961sQMQrIvrxorZGvc48yev7n+h8LXIpu4UxBIlPm9gyPZHIloiwZo9+EbMECEpAHdKAKNIEuMAIsYA0cgDNwA94gAISASBADlgMuSAJpQASyQT7YAApBMdgBdoNqcADUgXrQBE6CNnAGXARXwA1wCwyAR0AKhsFLMAHegWkIgvAQFaJBqpAWpA+ZQtYQG1oIeUNBUDgUA8VDiZAQkkD50CaoGCqDqqFDUD30I3Qaughdg/qgB9AgNAb9AX2EEZgC02EN2AC2gNmwOxwIR8LL4ER4FZwHF8Db4Uq4Fj4Ot8IX4RvwACyFX8KTCEDICAPRRlgIG/FEQpBYJAERIWuRIqQCqUWakA6kG7mNSJFx5AMGh6FhmBgWxhnjh1mM4WJWYdZiSjDVmGOYVkwX5jZmEDOB+YKlYtWxplgnrD92CTYRm40txFZgj2BbsJexA9hh7DscDsfAGeIccH64GFwybjWuBLcP14y7gOvDDeEm8Xi8Kt4U74IPwXPwYnwhvgp/HH8e348fxr8nkAlaBGuCDyGWICRsJFQQGgjnCP2EEcI0UYGoT3QihhB5xFxiKbGO2EG8SRwmTpMUSYYkF1IkKZm0gVRJaiJdJj0mvSGTyTpkR3IYWUBeT64knyBfJQ+SP1CUKCYUT0ocRULZTjlKuUB5QHlDpVINqG7UWKqYup1aT71EfUp9L0eTM5fzl+PJrZOrkWuV65d7JU+U15d3l18unydfIX9K/qb8uAJRwUDBU4GjsFahRuG0wj2FSUWaopViiGKaYolig+I1xVElvJKBkrcST6lA6bDSJaUhGkLTpXnSuLRNtDraZdowHUc3pPvTk+nF9B/ovfQJZSVlW+Uo5RzlGuWzylLine truncated
}
]
},
{
"email_id": "2",
"folder": "Payer-Correspondence",
"subject": "Explanation of Benefits \u2014 BlueCrest Commercial PPO \u2014 CareIG NPI: 1234567893",
"from_addr": "claims@bluecrest-health.com",
"to_addr": "billing@careig.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-01-30T09:15:00Z",
"message_id": "<eob-bcr-20260130-4401@bluecrest-health.com>",
"in_reply_to": null,
"body_text": "CareIG Specialty Pharmacy,\n\nPlease find attached your Explanation of Benefits for the claim referenced below.\n\nClaim Reference: CLM-REF-20260113-4401\nPatient: Patel, Sunita | Member ID: BCR-10047291\nDOS: 01/09/2026\nTotal Paid: $4,368.00\n\nBlueCrest Commercial PPO\nProvider Claims",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "eob_01092026_BlueCrest_Patel.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgW0VPQiBzdHViXQpCbHVlQ3Jlc3QgQ29tbWVyY2lhbCBQUE8KRXhwbGFuYXRpb24gb2YgQmVuZWZpdHMKRU9CIERhdGU6IDAxLzMwLzIwMjYKUHJvdmlkZXI6IENhcmVJRyBTcGVjaWFsdHkgUGhhcm1hY3kgTlBJIDEyMzQ1Njc4OTMKClBBVElFTlQ6IFBhdGVsLCBTdW5pdGEgfCBNZW1iZXI6IEJDUi0xMDA0NzI5MQpDbGFpbSBSZWY6IENMTS1SRUYtMjAyNjAxMTMtNDQwMSB8IERPUzogMDEvMDkvMjAyNgoKSjE1NjEgIDcwIHVuaXRzICAkNCw5MDAuMDAgIEFMTE9XRUQgJDMsOTIwLjAwICBQQUlEICQzLDEzNi4wMAo5OTYwMCAgIDEgdW5pdCAgICAkMTg1LjAwICAgQUxMT1dFRCAgJDE0OC4wMCAgIFBBSUQgICQxMTguNDAKOTk2MDMgICAxIHVuaXQgICAgJDEzMC4wMCAgIEFMTE9XRUQgICQxMDQuMDAgICBQQUlEICAgJDgzLjIwCko3MDMwICAgMyB1bml0cyAgICAkOTAuMDAgICBBTExPV0VEICAgJDcyLjAwICAgUEFJRCAgICQ1Ny42MApBNDIyMSAgIDEgdW5pdCAgICAkMTU1LjAwICAgQUxMT1dFRCAgJDEyNC4wMCAgIFBBSUQgICAkOTkuMjAKClRPVEFMIEJJTExFRDogJDUsNDYwLjAwIHwgQ09OVFJBQ1RVQUwgQURKOiAkMSwwOTIuMDAgfCBQQUlEOiAkNCwzNjguMDAKUGF0aWVudCBSZXNwb25zaWJpbGl0eTogJDAuMDAgKGRlZHVjdGlibGUgbWV0KQo="
}
]
},
{
"email_id": "3",
"folder": "Payer-Correspondence",
"subject": "Appeal Decision \u2014 Level 1 \u2014 Okonkwo, Chidi \u2014 DOS 11/18/2025",
"from_addr": "appeals@bluecrest-health.com",
"to_addr": "billing@careig.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-01-15T13:55:00Z",
"message_id": "<appeal-dec-bcr-20260115-7703@bluecrest-health.com>",
"in_reply_to": "<appeal-l1-careig-20251208-7703@careig.com>",
"body_text": "CareIG Specialty Pharmacy,\n\nWe have completed our review of your Level 1 appeal for the claim referenced below.\n\nPatient: Okonkwo, Chidi | Member ID: BCR-20081147\nDOS: 11/18/2025 | Claim: CLM-REF-20251121-7703 | CARC: CO-50\n\nDECISION: Upheld in favor of provider. Payment of $4,736.00 will be issued within 10 business days.\n\nPlease see the attached decision notice for full detail.\n\nBlueCrest Commercial PPO\nProvider Appeals Department",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "appeal_decision_L1_11182025_BlueCrest_Okonkwo.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgW0FwcGVhbCBEZWNpc2lvbiBzdHViXQpCbHVlQ3Jlc3QgQ29tbWVyY2lhbCBQUE8KQXBwZWFsIERlY2lzaW9uIE5vdGljZQpEZWNpc2lvbiBEYXRlOiAwMS8xNS8yMDI2ClByb3ZpZGVyOiBDYXJlSUcgU3BlY2lhbHR5IFBoYXJtYWN5IE5QSSAxMjM0NTY3ODkzCgpSRTogTGV2ZWwgMSBBcHBlYWwKUGF0aWVudDogT2tvbmt3bywgQ2hpZGkgfCBNZW1iZXI6IEJDUi0yMDA4MTE0NwpET1M6IDExLzE4LzIwMjUgfCBDbGFpbSBSZWY6IENMTS1SRUYtMjAyNTExMjEtNzcwMyB8IENBUkM6IENPLTUwCgpERUNJU0lPTjogVVBIRUxEIElOIEZBVk9SIE9GIFBST1ZJREVSClRoZSBzZXJ2aWNlcyByZW5kZXJlZCBvbiAxMS8xOC8yMDI1IGFyZSBtZWRpY2FsbHkgbmVjZXNzYXJ5IGFuZApjb3ZlcmVkIHVuZGVyIHRoZSBtZW1iZXIncyBwbGFuLiBQYXltZW50IHdpbGwgYmUgaXNzdWVkIHdpdGhpbgoxMCBidXNpbmVzcyBkYXlzLgpQYXltZW50IEFtb3VudDogJDQsNzM2LjAwCg=="
}
]
},
{
"email_id": "4",
"folder": "Sent",
"subject": "[CLAIM SUBMISSION] CareIG NPI: 1234567893 | Holloway | MemberID: SPA-00291847 | DOS: 02/14/2026",
"from_addr": "d.okafor@careig.com",
"to_addr": "billing@silverpath-claims.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-02-18T14:22:00Z",
"message_id": "<claim-careig-20260218-9931@careig.com>",
"in_reply_to": null,
"body_text": "SilverPath Medicare Advantage \u2014 Provider Billing,\n\nPlease find attached the claim for the patient and date of service referenced below.\n\nPatient: Holloway, Margaret\nMember ID: SPA-00291847\nDOS: 02/14/2026\nTotal Billed: $6,140.00\nPA Number: SPA-PA-2026-3302\n\nDiane Okafor\nBilling Specialist | CareIG Specialty Pharmacy\nd.okafor@careig.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "claim_02142026_Holloway.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgW0NsYWltIHN0dWJdCkNhcmVJRyBTcGVjaWFsdHkgUGhhcm1hY3kgfCBOUEk6IDEyMzQ1Njc4OTMgfCBUYXggSUQ6IDQ3LTM4MjEwNTYKQ0xBSU0gU1VCTUlTU0lPTgpQYXRpZW50OiBIb2xsb3dheSwgTWFyZ2FyZXQgfCBNZW1iZXI6IFNQQS0wMDI5MTg0NwpET1M6IDAyLzE0LzIwMjYgfCBJQ0QtMTA6IEQ4My45IHwgUEEjOiBTUEEtUEEtMjAyNi0zMzAyClBoeXNpY2lhbiBOUEk6IDE0ODcyMDM5NTYKCkoxNTYwICA4MCB1bml0cyAgJDQsODAwLjAwCjk5NjAwICAgMSB1bml0ICAgICQxODUuMDAKOTk2MDMgICAyIHVuaXRzICAgJDI2MC4wMApKNzAzMCAgIDMgdW5pdHMgICAgJDkwLjAwCkowMTcxICAgMSB1bml0ICAgICAkMzUuMDAKQTQyMjEgICAxIHVuaXQgICAgJDc3MC4wMApUT1RBTCBCSUxMRUQ6ICQ2LDE0MC4wMApCZW5lZml0OiBNZWRpY2FsIHwgQ2xhaW0gRnJlcXVlbmN5IENvZGU6IDEK"
}
]
},
{
"email_id": "5",
"folder": "Sent",
"subject": "[APPEAL L1] CareIG | Okonkwo | MemberID: BCR-20081147 | DOS: 11/18/2025 | Claim: CLM-REF-20251121-7703 | CARC: CO-50",
"from_addr": "d.okafor@careig.com",
"to_addr": "appeals@bluecrest-health.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2025-12-08T11:25:00Z",
"message_id": "<appeal-l1-careig-20251208-7703@careig.com>",
"in_reply_to": null,
"body_text": "BlueCrest Commercial PPO \u2014 Provider Appeals Department,\n\nCareIG Specialty Pharmacy submits this Level 1 appeal for the claim referenced below.\n\nPatient: Okonkwo, Chidi\nMember ID: BCR-20081147\nDOS: 11/18/2025\nClaim Reference: CLM-REF-20251121-7703\nCARC: CO-50 \u2014 Non-covered service\nBilled: $5,920.00\n\nPlease find the full appeal letter and supporting documents attached.\n\nDiane Okafor\nBilling Specialist | CareIG Specialty Pharmacy\nd.okafor@careig.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "appeal_L1_11182025.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjQgW0FwcGVhbCBQYWNrZXQgc3R1Yl0KQ2FyZUlHIFNwZWNpYWx0eSBQaGFybWFjeSB8IE5QSTogMTIzNDU2Nzg5MyB8IFRheCBJRDogNDctMzgyMTA1NgpBUFBFQUwgTEVWRUw6IDEKUGF0aWVudDogT2tvbmt3bywgQ2hpZGkgfCBNZW1iZXI6IEJDUi0yMDA4MTE0NwpET1M6IDExLzE4LzIwMjUgfCBDbGFpbTogQ0xNLVJFRi0yMDI1MTEyMS03NzAzIHwgQ0FSQzogQ08tNTAKQmlsbGVkOiAkNSw5MjAuMDAKCkJhc2lzOiBQQSAjQkNSLVBBLTIwMjUtNjYxNCBhcHByb3ZlZCBPY3RhZ2FtIDUlIDMwZywgZWZmZWN0aXZlCjEwLzIzLzIwMjUgdGhyb3VnaCAwNC8yMy8yMDI2LiBTZXJ2aWNlIGNvdmVyZWQgdW5kZXIgbWVkaWNhbApiZW5lZml0IHBlciB2ZXJpZmllZCBiZW5lZml0cy4gQXR0YWNoZWQ6IGNsYWltLCBQQSBhcHByb3ZhbCwKTE1OLCB2aXNpdCBub3RlLCBFUkEuCg=="
}
]
},
{
"email_id": "6",
"folder": "Sent",
"subject": "[APPEAL L1] CareIG | Holloway | MemberID: BCR-00184523 | DOS: 12/09/2025 | Claim: CLM-REF-20251215-6612 | CARC: PI-204",
"from_addr": "d.okafor@careig.com",
"to_addr": "appeals@bluecrest-health.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-23T11:25:00Z",
"message_id": "<appeal-l1-careig-20260323-6612@careig.com>",
"in_reply_to": null,
"body_text": "BlueCrest Commercial PPO \u2014 Provider Appeals Department,\n\nCareIG Specialty Pharmacy submits this Level 1 appeal for the claim referenced below.\n\nPatient: Holloway, Margaret\nMember ID: BCR-00184523\nDOS: 12/09/2025\nClaim Reference: CLM-REF-20251215-6612\nCARC: PI-204 \u2014 Not medically necessary\nBilled: $5,840.00\n\nPlease find the full appeal letter and supporting documents attached.\n\nDiane Okafor\nBilling Specialist | CareIG Specialty Pharmacy\nd.okafor@careig.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": [
{
"filename": "appeal_L1_12092025.pdf",
"content_type": "application/pdf",
"content_base64": "JVBERi0xLjMKJcTl8uXrp/Og0MTGCjMgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCA0MTEzID4+CnN0cmVhbQp4AdVca2/kthX9rl+hscfbGdsji3pSefmxfq13162TSdKi7qdFgyJICqT5/0DPJQ9F6mFLM9sCLYK1RhJ5Sd7HuQ9S+S1+in+Lz97+ruJPv8ep+e/3T3iUJllh7+2PKlVJk8Wffo2vtniZpkUebz/FpTaNcMnTNC7yOtr+Gp9tt1ms4u1P8V/j1eJgjd+rw3Wcxaul+X20jvN49eYP5vK8Ms+e1+ZiGxyb3yfrCM1s91PTdmOIJOat7X1mfqfmbziQWscbjGofCblo1Xb7W7x9iG+2ZuWvLtMvTqV1kTR5M1zhKlvH2593JafyPMnqbIQcljWT3JgQlCqapKhH5glJ5ORxh9Uhey1LC8NMKwjL8M06amUHOc7mX1EZ5cBFlVmWZHkdLLiwGrJa+AWLKqYit0rlcd3k8b/+Hv8Y/9OrZ6J0kSuQSZM8awqldJQmpdZZXuYx1LbGXYW/WdJU8a9xWaikKNz9L+E9lPuXoLncRv+If+paQwIL4L9pm8itJeAilpBXFS0BixdLCNc518w2ImJMbfsp8qwU6lmjY2tn5CKke3C4PHpjtR4snZZRZE3ZjKEasWU/RlHUSZ2WKhhIBwYNHRDjWh64H4vl0TqaHpLo4cep6jpJm7IMxsGCoh67JsDIkzOsybAWA0G7U/L8DnghRFXZUJqe30tYC5iwC8vt+qPxYXSTKFWCfukW4FkuUCkcf16J7cqv4xO5Ru098FEew17lAqgzjc4sMNq/rqvpGa+IkIAvaZqvI7lwHPYvinKOJg3E2jSA1TKrA75xLTKzmfA2yiUFe0gaPUIZFgBtcmvM7aq4HDKHq9KuUc1W8DOydvKuy6wGT/dQbFUUOqlgpV5z9uRA3zRzrZO6TLO4SamSXk3y2qik/fu8zt3yvzBu80vzstUkc/e8Vq0ine63VJ3WSaMA1k1Kw9ORRbyvPk/WQHXgfTa60q+dDLuCpgi//saKnyouVtDKMJqNJiqtykRlOlgYodwpcWQc1muBUg+bwBcHTT2fMDWtUWsQaNoUDi4cNAEUAAuQM3TaBlIeo0TPT0axAoGUvLTaDwciNyqzlkHDIa8RH2wQDpAKEELuKtsDqicd2V839tY0jZyBQRvRhN1Js9sPmhpQ+Upzbu0y5rsaw7a+CSllBFsJ77o2tDpfHlxcHl29vUZYNBOnvIhVkcFh5yNkncJMh5x2wlnXHasiz5K6KgLKTtjx6suFDdXAULD1a/N3cWbDujefHQmoStUJgq5AzxyQXVxN84hBRsCjqtIJLHuE3gy4eC1mUZUuklQbqTogor0evF2eLy6RdUyJdDhdXWVJqfTIdGfIdEDOGCyi1hdh4MXEK1y5iQh7EcpGwSl2wAUO8VDro0NiIU3yG2vRR+dHfN4alYfIFycxcPRV2SCqr8BzDN83pWl2D+lBhDXihoAcFT0Mn2cjuGF3VrqZjZBKhxkFQv4go3hprEl55JIfG7D3/rmobm4ttN3d3xBU3yVWIscPFd+9NybM1/ruxj1/+BC4MZuaS84zMUNvek2qk7KAu9j4uY2wZCQJi7pJGBKdxP5rU65SVeIG25SL923KhXuXb0X29yDZMit6ZT1UllccYeZSLg+Omm6G7omui7y9sLLgu64hRHPqA6NzKQBwWO2myCqnADYegkF+PGP4xcvZo5E1neAf7Xz+ZC+cpNUGtmCk+tTeGj/pYqE94zcElEWaCyL3GbiD0RlWmIqQVziGJ+UgdTrm2r647CxVojeEEoJUuIjTxwWOTGIBkY55t6Pv72NlWWYS0ykBdDcx59DeHH+7D2aVKDJkVTZGcYZLC9W6H6hUVZVUGgY7nOticf724nK5T5iiUSWpEAOFZEdw4CVcCSfc566ReFkVA4kf2HBUdY1sTtXMZst9zhQZHIXWiIgwmjMz6+1hZudLFiYW175CcblfhaIsElWLDPxAYNaOJYpRoDDMqvIhs04YUxO7CFo2bKYbf17ThBCDiXnwDoW64I74RrBgExJ4680K3fGwTZCmHMroWlAsSlKk/PEGC+rKY3X8AAzbOfLSaZHkKex0SHCGVb3mnrVqUCVEnW6E9Ld33+0x10YphEFpOUZxx8n2Fb3JqqRoBJ2HfFgulpfny/PLYMYoh8ImTX1xpGzvoVmpHJVNwYAhWTfj6ezWqEJ/xihBl0mFEnRImhWBo6NzJCtTujAIC1WGeqSkP5hsL8qEVs8kN6q3xgbrNB8A1snh8/pkd7SKOUrULaeWZZLnFfQDQ/Wso7idXsEgjygaVJcKGMeQnpPeK6lmaBx96ZUqT5pUjGNIerkAokxxezDXEsXAvC5GKc6Q3oBelaIslCLbGZnhdo/5VRnC4qYZZeb8+Y1qF0QEucN2B7yEk7o/Y7WKwJyuYSRtPfd71mVZwiKGd6NHxovs37gw0HQZVIbpTLqRIyjMxv0QPOoK64KF+3XtEECE6jcaQNS1MxIfyC+vrcNisM7VlPB3jBjBuxt6SrpPXsgemJmLI63Dk7u7YP2zKnADc8EGRqWxgbHxk2b5EyK+ZCj/9mLhsvBZnnYAgHVZyT4JEBDDdLFqhxjdS9DAni6yl0n99xLjBhXVfmJcFcVd9UBJPdzd3NwhEv9b5LZp5+6X+QVW0FBdS93GDhdhS3pERUdy3d6G4zDXjfK6REXM57q8b3NduW+z3djdjeS7Ufpa/j4ZZ6tUtfXkNsM8IBNpIIQYBn2HBBVeuk1pLvc2hGQPR449ulRpizYwJbFuTv1ur5gbiplUeQ1EHi5xB3UfhWSj+SptejUhGOtOZfEw6ObWGcGnRSaTr5JF3fJ2pyze7ju5sriJ4AnT3X73FsIe7AVhn0CY7cbBORwbLqwo+ZC1eLZsmxgiRCfKEOF6kEVQL0Bs2lUMgEtlCH0yU9EDz53dMxo8n/bWIwR1mtRKSoSW4AuWPSd77YO5ypo8KbDrbWh3sREKcoojDKfCDes/ZnPD6KEC1oY77NggzBNkIu065NCMd3dXcHcYZlZFZxAh4QxIjdxJqrKe6SPo9xKPQgfdK+moUmGLA34ooOzrm++D6AJu+cnqX/ehi1JY7BGVQ9OPlq1S88GyJczBQ17Mw2j1SDqzYpYQPFV/N6XBloa47CEMrD4A96Yi3JB2P35RDcSait8ZEocKLWmENDSCLBdGQyOA0E5d1m9Q4SPfeePdUQnlmEfvKMkGhw7cxq3Xv9vkQiYAWSS3nCVEhFs+NZgSrRq+OzHvUGtBCxEwLvYR3s+aYoRDZaN4Xek6yRQcXThPV7DbK3eqkfmWysjfr9xR/MzsqUbqW9dSlgq46mjvkj4RM7rbgNgLR1KPnfaQurNr2eqb0tzQsPuaq+XIj0LkHRL3pv1/nK3kGpunsuMYyMRx7cNspvnYUuVNCTchLsgrkCP4mVKw4Qm21JyrhCnZY5RviBKEABOQRW4XnXjyA1EUthc48BePxfwYGOdU9uOXL6F1k0tFVPl5dpZv905wujFHDS/ViJQVdlFRAeD9L/4+S7SWjSFpy98SKM/xTH0FtqzD6UCyjlMC6upDwhSKni0aTS14FI3ypklQx0GZU/mB3HGX/bYPTCmnkiKBkGTQ4SBjBhy95o1MMcdsTYzQLoDhU4AxCL9K5Lslzlb+Bybbj77KBgpQGr0aMkJKnXtMt2oKHDaQKvIIA3Zkbn++tURCelxuV7OwfsBdrTIEV3LYYmS6sPGZ4hpVXS07JrWm6g6C22FoIQgSgk0sJ53loURuuLAg894GeUQgVllob9ycYEv2w7vWCGcZej8EVTgpiWKIZdIAJ0tWzGQ0xJBMzp7snImhqcwWL5GAyEo42z/vMDEPh6rAdCrkNkZmXeyZk6yGPrmvYnAvwM2c6ttd6uoC055SiGFqUJZIcRARj832cTbB0fBElbVG7XiUNlD43QJcB7evruwVksFdxYASOSfu7JtZGhLinsIxrk5yVSNqlPPj4Sp9LLO44q7g4pw/LLxQLaf3Pbz0gS01jpB0R/LOMJoSUGcVvTwlS7MGJyupWQOLrZBLgGUbmymiKIAbSBB/7YtZXHxN++hQcbqnX65r91cPOgycXyAY1Z8Sy80bk1z60/Pei3/gOTe7OmoKMqFWbTZWh6hQMHW+iVYLyyPUpvBoFl86cpGsKThupeWARwMdgHa1p/y9drXM6Ww+Y767oR7S986oyIGwj2IKBnLgqn/Uq7CcwOKwRGta4aLtWzaC42+ZZnlin9jOyPMAjRIdoFFjjNS+cE98Z/scTgIt4Rzav8Be/Lbje8OOVvAOeB72smobUra9LIUCuT462AFso0LeR6tWlO1r+yQcAXUDvEUsjOXgnMRs9gfWjWPN2GuSUHfA8xVq+1PWPW1dhUB+50jw07fgzy6EB3FwlSORM+rpqXv11AiCxemBI3LheUBwdZo/g3il0NiWgDsGJTvU5xTjeugncXFdYq/A0B6g33eQK6YPD46/PO36bmHVyaqCVXmrlhYtLCQUNmC5cvUNQ8HqzXYvLuimwpEiE7WBC92Zrj5MC3PAVjmoh0hYHOmQ4AztCMFroB1yZBxxhTv75PF1vDYFLoqagIFykaAKF9i0XISDPsRiE8wveMfufNjZKWCo2BIx3U6tcFgK46VbBYd0ZQAOx4iOdxyOFxLjpLtUGMt218VpAmlkhH6tzpTgGBGzCS98yLl0+3GZJN0W2g2xLpXRucAwZS6MpEnl2Ogs299alvX5aLqRgez2FXu8zLPIyZQzI8/e2UmQjKnHmlhDZtb5YCAsYUary043ioFzoIhgsEKE/TgqxpkGow62Ykcx9NA2cilRoulgK4LQ/FpOrcuQwi9czKyiYBcFzy4eudJbNjp9vLi+5+97vgN+oun3P8DAhc6PfPydlUYHnWLine truncated
}
]
}
],
"next_email_id": 7
}
@@ -0,0 +1,714 @@
{
"users": {
"U001": {
"id": "U001",
"team_id": "T001",
"name": "carlos.mendez",
"deleted": false,
"real_name": "Carlos Mendez",
"profile": {
"real_name": "Carlos Mendez",
"display_name": "carlosm",
"email": "c.mendez@careig.com",
"title": "Patient Representative"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U002": {
"id": "U002",
"team_id": "T001",
"name": "jennifer.alcott",
"deleted": false,
"real_name": "Jennifer Alcott",
"profile": {
"real_name": "Jennifer Alcott",
"display_name": "jennifera",
"email": "j.alcott@careig.com",
"title": "Benefits Verification Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U003": {
"id": "U003",
"team_id": "T001",
"name": "marcus.reid",
"deleted": false,
"real_name": "Marcus Reid",
"profile": {
"real_name": "Marcus Reid",
"display_name": "marcusr",
"email": "m.reid@careig.com",
"title": "Benefits Verification Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U004": {
"id": "U004",
"team_id": "T001",
"name": "patricia.nguyen",
"deleted": false,
"real_name": "Patricia Nguyen",
"profile": {
"real_name": "Patricia Nguyen",
"display_name": "patrician",
"email": "p.nguyen@careig.com",
"title": "Infusion Nurse (RN)"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U005": {
"id": "U005",
"team_id": "T001",
"name": "diane.park",
"deleted": false,
"real_name": "Diane Park",
"profile": {
"real_name": "Diane Park",
"display_name": "dianep",
"email": "d.park@careig.com",
"title": "Infusion Nurse (RN)"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U006": {
"id": "U006",
"team_id": "T001",
"name": "diane.okafor",
"deleted": false,
"real_name": "Diane Okafor",
"profile": {
"real_name": "Diane Okafor",
"display_name": "dianeo",
"email": "d.okafor@careig.com",
"title": "Billing Specialist"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U007": {
"id": "U007",
"team_id": "T001",
"name": "raymond.walsh",
"deleted": false,
"real_name": "Raymond Walsh",
"profile": {
"real_name": "Raymond Walsh",
"display_name": "raymondw",
"email": "r.walsh@careig.com",
"title": "Director of Revenue Cycle Management"
},
"is_admin": true,
"is_owner": true,
"is_bot": false
},
"U008": {
"id": "U008",
"team_id": "T001",
"name": "intake.supervisor",
"deleted": false,
"real_name": "Sandra Obi",
"profile": {
"real_name": "Sandra Obi",
"display_name": "sandrao",
"email": "s.obi@careig.com",
"title": "Intake Supervisor"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
},
"U009": {
"id": "U009",
"team_id": "T001",
"name": "rachel.kim",
"deleted": false,
"real_name": "Rachel Kim",
"profile": {
"real_name": "Rachel Kim",
"display_name": "rachelk",
"email": "r.kim@careig.com",
"title": "Clinical Coordinator"
},
"is_admin": false,
"is_owner": false,
"is_bot": false
}
},
"channels": {
"C001": {
"id": "C001",
"name": "intake-team",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-team",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Intake coordination and BVS assignment",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Patient intake workflow: referral receipt, consent, BVS assignment, PA submission updates",
"creator": "U007",
"last_set": 1735689600
}
},
"C002": {
"id": "C002",
"name": "intake-alerts",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-alerts",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Intake holds and patient contact issues",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Automated and manual alerts for intake holds: missing NPI, unreachable patients, consent violations, formulary issues",
"creator": "U007",
"last_set": 1735689600
}
},
"C003": {
"id": "C003",
"name": "intake-mgmt",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "intake-mgmt",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Escalations requiring supervisor action",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Supervisor-level intake escalations: OON pharmacy decisions, formulary unresolved, PA delays",
"creator": "U007",
"last_set": 1735689600
}
},
"C004": {
"id": "C004",
"name": "clinical-scheduling",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "clinical-scheduling",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "PA approvals, nurse scheduling, visit documentation",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Clinical coordination: PA approvals handed to clinical team, nurse scheduling, visit completion",
"creator": "U007",
"last_set": 1735689600
}
},
"C005": {
"id": "C005",
"name": "billing",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Claim submission and AR monitoring",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Billing workflow: claim submissions, billing holds, AR follow-up, timely filing alerts",
"creator": "U007",
"last_set": 1735689600
}
},
"C006": {
"id": "C006",
"name": "billing-appeals",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing-appeals",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Denial management and appeal tracking",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Denial alerts, appeal filings, and appeal follow-up for all active cases",
"creator": "U007",
"last_set": 1735689600
}
},
"C007": {
"id": "C007",
"name": "billing-mgmt",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "billing-mgmt",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Escalations: underpayments, write-offs, collections",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Director-level billing escalations: underpayments, write-off requests, collections referrals, L2 denials",
"creator": "U007",
"last_set": 1735689600
}
},
"C008": {
"id": "C008",
"name": "compliance",
"is_channel": true,
"is_group": false,
"is_im": false,
"is_mpim": false,
"is_private": false,
"created": 1735689600,
"is_archived": false,
"is_general": false,
"unlinked": 0,
"name_normalized": "compliance",
"is_shared": false,
"is_org_shared": false,
"is_ext_shared": false,
"is_pending_ext_shared": false,
"pending_shared": [],
"pending_connected_team_ids": [],
"context_team_id": "T001",
"updated": 1735689600,
"creator": "U007",
"shared_team_ids": [
"T001"
],
"is_member": true,
"topic": {
"value": "Legal threats and compliance reports \u2014 STOP and post here",
"creator": "U007",
"last_set": 1735689600
},
"purpose": {
"value": "Compliance and legal escalation channel. Do not discuss case details elsewhere once flagged here.",
"creator": "U007",
"last_set": 1735689600
}
}
},
"messages": {
"C001": [
{
"type": "message",
"user": "U001",
"text": "[BVS ASSIGNED] Okonkwo_DOB_03281965 | Payer: BlueCrest Commercial PPO | Drug: Octagam 5% | Dose: 30g",
"ts": "1760440200.000000",
"team": "T001"
},
{
"type": "message",
"user": "U003",
"text": "[PA SUBMITTED] Okonkwo_DOB_03281965 | Payer: BlueCrest Commercial PPO | Drug: Octagam 5% | Ref: PA-REF-2025-05512",
"ts": "1760606100.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "[BVS ASSIGNED] Patel_DOB_07221958 | Payer: BlueCrest Commercial PPO | Drug: Gamunex-C 10% | Dose: 35g",
"ts": "1765190100.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA SUBMITTED] Patel_DOB_07221958 | Payer: BlueCrest Commercial PPO | Drug: Gamunex-C 10% | Ref: PA-REF-2025-07741",
"ts": "1765359900.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "[BVS ASSIGNED] Holloway_DOB_03141961 | Payer: SilverPath Medicare Advantage | Drug: Octagam 10% | Dose: 40g",
"ts": "1768814520.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA SUBMITTED] Holloway_DOB_03141961 | Payer: SilverPath Medicare Advantage | Drug: Octagam 10% | Ref: PA-REF-2026-00814",
"ts": "1768992900.000000",
"team": "T001"
},
{
"type": "message",
"user": "U001",
"text": "[BVS ASSIGNED] Brennan_DOB_11051970 | Payer: SilverPath Medicare Advantage | Drug: Privigen 10% | Dose: 25g",
"ts": "1772528700.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA SUBMITTED] Brennan_DOB_11051970 | Payer: SilverPath Medicare Advantage | Drug: Privigen 10% | Ref: PA-REF-2026-01122",
"ts": "1772702100.000000",
"team": "T001"
}
],
"C002": [
{
"type": "message",
"user": "U001",
"text": "[INTAKE HOLD] Nguyen_DOB_05091955 \u2014 Physician NPI missing. Template 1 sent to physician.",
"ts": "1756981200.000000",
"team": "T001",
"reply_count": 1,
"latest_reply": "1757061900.000000",
"reply_users": [
"U008"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U008",
"text": "NPI received and updated \u2014 hold cleared. Proceeding with BVS assignment.",
"ts": "1757061900.000000",
"team": "T001",
"thread_ts": "1756981200.000000",
"parent_user_id": "U001"
}
],
"C003": [
{
"type": "message",
"user": "U008",
"text": "Brennan_DOB_11051970 \u2014 PA pending follow-up call placed with SilverPath today. Rep confirmed decision expected by 03/10/2026. auth.xlsx PA_Call_Log updated.",
"ts": "1772800800.000000",
"team": "T001"
}
],
"C004": [
{
"type": "message",
"user": "U003",
"text": "[PA APPROVED] Okonkwo_DOB_03281965 | Drug: Octagam 5% | Dose: 30g | PA#: BCR-PA-2025-6614 | Exp: 04/23/2026 | Drug Source: CareIG",
"ts": "1761217200.000000",
"team": "T001"
},
{
"type": "message",
"user": "U009",
"text": "[SCHEDULED] Okonkwo_DOB_03281965 | Date: 11/18/2025 | Nurse: Diane Park, RN",
"ts": "1761661800.000000",
"team": "T001"
},
{
"type": "message",
"user": "U005",
"text": "[VISIT COMPLETE] Okonkwo_DOB_03281965 | DOS: 11/18/2025",
"ts": "1763468700.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA APPROVED] Patel_DOB_07221958 | Drug: Gamunex-C 10% | Dose: 35g | PA#: BCR-PA-2025-8801 | Exp: 06/17/2026 | Drug Source: CareIG",
"ts": "1765361100.000000",
"team": "T001"
},
{
"type": "message",
"user": "U009",
"text": "[SCHEDULED] Patel_DOB_07221958 | Date: 01/09/2026 | Nurse: Diane Park, RN",
"ts": "1765796400.000000",
"team": "T001"
},
{
"type": "message",
"user": "U005",
"text": "[VISIT COMPLETE] Patel_DOB_07221958 | DOS: 01/09/2026",
"ts": "1767963300.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA APPROVED] Holloway_DOB_03141961 | Drug: Octagam 10% | Dose: 40g | PA#: SPA-PA-2026-3302 | Exp: 07/28/2026 | Drug Source: CareIG",
"ts": "1769594520.000000",
"team": "T001"
},
{
"type": "message",
"user": "U009",
"text": "[SCHEDULED] Holloway_DOB_03141961 | Date: 02/14/2026 | Nurse: Patricia Nguyen, RN",
"ts": "1770286500.000000",
"team": "T001"
},
{
"type": "message",
"user": "U004",
"text": "[VISIT COMPLETE] Holloway_DOB_03141961 | DOS: 02/14/2026",
"ts": "1771070400.000000",
"team": "T001"
},
{
"type": "message",
"user": "U002",
"text": "[PA APPROVED] Brennan_DOB_11051970 | Drug: Privigen 10% | Dose: 25g | PA#: SPA-PA-2026-4471 | Exp: 09/12/2026 | Drug Source: CareIG",
"ts": "1773311400.000000",
"team": "T001"
},
{
"type": "message",
"user": "U009",
"text": "[SCHEDULED] Brennan_DOB_11051970 | Date: 04/14/2026 | Nurse: Patricia Nguyen, RN",
"ts": "1773842400.000000",
"team": "T001"
}
],
"C005": [
{
"type": "message",
"user": "U006",
"text": "[CLAIM SUBMITTED] Okonkwo_DOB_03281965 | DOS: 11/18/2025 | Payer: BlueCrest Commercial PPO | Billed: $5,920.00",
"ts": "1763732700.000000",
"team": "T001"
},
{
"type": "message",
"user": "U006",
"text": "[DENIAL] Okonkwo_DOB_03281965 | CARC: CO-50 \u2014 posting to #billing-appeals now.",
"ts": "1764925800.000000",
"team": "T001"
},
{
"type": "message",
"user": "U006",
"text": "[CLAIM SUBMITTED] Patel_DOB_07221958 | DOS: 01/09/2026 | Payer: BlueCrest Commercial PPO | Billed: $5,460.00",
"ts": "1768302300.000000",
"team": "T001"
},
{
"type": "message",
"user": "U006",
"text": "[CLAIM SUBMITTED] Holloway_DOB_03141961 | DOS: 02/14/2026 | Payer: SilverPath Medicare Advantage | Billed: $6,140.00",
"ts": "1771424520.000000",
"team": "T001"
}
],
"C006": [
{
"type": "message",
"user": "U006",
"text": "[DENIAL] Okonkwo_DOB_03281965 | DOS: 11/18/2025 | Payer: BlueCrest Commercial PPO | CARC: CO-50",
"ts": "1764926100.000000",
"team": "T001"
},
{
"type": "message",
"user": "U006",
"text": "[APPEAL FILED L1] Okonkwo_DOB_03281965 | DOS 11/18/2025 | BlueCrest Commercial PPO | CO-50",
"ts": "1765193400.000000",
"team": "T001",
"reply_count": 1,
"latest_reply": "1768486200.000000",
"reply_users": [
"U006"
],
"reply_users_count": 1
},
{
"type": "message",
"user": "U006",
"text": "Appeal approved \u2014 payment received 01/20/2026. Case closed.",
"ts": "1768486200.000000",
"team": "T001",
"thread_ts": "1765193400.000000",
"parent_user_id": "U006"
},
{
"type": "message",
"user": "U006",
"text": "[DENIAL] Holloway_DOB_03141961 | DOS: 12/09/2025 | Payer: BlueCrest Commercial PPO | CARC: PI-204",
"ts": "1765792800.000000",
"team": "T001"
},
{
"type": "message",
"user": "U006",
"text": "[APPEAL FILED L1] Holloway_DOB_03141961 | DOS 12/09/2025 | BlueCrest Commercial PPO | PI-204",
"ts": "1774265400.000000",
"team": "T001"
}
],
"C007": [
{
"type": "message",
"user": "U007",
"text": "Okonkwo_DOB_03281965 \u2014 appeal payment $4,736.00 confirmed received 01/20/2026. claim.xlsx updated. Case cleared.",
"ts": "1774861200.000000",
"team": "T001"
}
],
"C008": [
{
"type": "message",
"user": "U007",
"text": "Q1 2026 billing audit documentation submitted to Director for review.",
"ts": "1759309200.000000",
"team": "T001"
}
]
},
"counters": {
"messageId": 2000000
}
}
@@ -0,0 +1 @@
It's the start of your morning shift as billing specialist - work through your inbox and the active case files, and take care of anything that needs actioning today in line with the CareIG SOP. All relevant files are available in your workspace.
@@ -0,0 +1 @@
Today's date is, Wednesday, April, 8, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/medical_careig_specialty_pharmacy_3c62ee67"
description = "SOP task from CSV row 3c62ee67"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,128 @@
[
{
"id": "292496ce-fecf-4a85-a8ac-244d4df88aae",
"sort_order": 0,
"rubric_text": "In file `claim.xlsx`, the row where `DOS` = `02/14/2026` must have `Claim_Status` = `Denied`, `ERA_Received_Date` = `04/08/2026` or 8 April 2026, `Payment_Posted_Date` blank/null, `Benefit_Tier_Billed` = `Medical`, `Paid_Amount` = `0` and `Claim_Frequency_Code` = `1`.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find claim.xlsx - be flexible with naming\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n # Try glob for any xlsx with 'claim' in the name\n candidates = list(Path(workspace_path).glob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n # Also try recursively\n candidates = list(Path(workspace_path).rglob('*laim*.xlsx'))\n if candidates:\n claim_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n\n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n headers_upper = [h.upper().replace(' ', '_') for h in headers]\n\n # Find columns flexibly\n dos_col = None\n status_col = None\n era_received_col = None\n payment_posted_col = None\n tier_col = None\n freq_col = None\n paid_amount_col = None\n\n for i, h in enumerate(headers_upper):\n if dos_col is None and 'DOS' in h:\n dos_col = i\n if status_col is None and ('CLAIM_STATUS' in h or ('CLAIM' in h and 'STATUS' in h) or h == 'STATUS' or h == 'CLAIM_STATUS'):\n status_col = i\n if era_received_col is None and ('ERA' in h and 'RECEIVED' in h):\n era_received_col = i\n if payment_posted_col is None and ('PAYMENT_POSTED' in h or ('PAYMENT' in h and 'POSTED' in h)):\n payment_posted_col = i\n if tier_col is None and ('BENEFIT_TIER' in h or ('TIER' in h and 'BILLED' in h) or ('BENEFIT' in h and 'TIER' not in h)):\n tier_col = i\n if tier_col is None and 'BENEFIT' in h:\n tier_col = i\n if freq_col is None and ('FREQUENCY' in h or 'FREQ' in h):\n freq_col = i\n if paid_amount_col is None and ('PAID_AMOUNT' in h or ('PAID' in h and 'AMOUNT' in h) or h == 'PAID_AMOUNT'):\n paid_amount_col = i\n\n if dos_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'DOS column not found. Headers: {headers}'}\n\n # Find the target row with DOS = 02/14/2026\n target_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n dos_val = row[dos_col]\n if dos_val is None:\n continue\n dos_str = str(dos_val).strip()\n match = False\n if '02/14/2026' in dos_str or '2/14/2026' in dos_str or '2026-02-14' in dos_str:\n match = True\n if isinstance(dos_val, datetime.datetime):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n elif isinstance(dos_val, datetime.date):\n if dos_val.year == 2026 and dos_val.month == 2 and dos_val.day == 14:\n match = True\n if match:\n target_row = row\n break\n\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=02/14/2026 found in claim.xlsx'}\n\n issues = []\n details = []\n\n # Check Claim_Status = Denied\n if status_col is not None:\n status_val = str(target_row[status_col]).strip().lower() if target_row[status_col] is not None else ''\n if 'denied' in status_val or 'deny' in status_val or 'denial' in status_val:\n details.append(f'Claim_Status={target_row[status_col]}')\n else:\n issues.append(f'Claim_Status expected Denied, got: {target_row[status_col]}')\n else:\n issues.append(f'Claim_Status column not found. Headers: {headers}')\n\n # Check ERA_Received_Date = 04/08/2026 or 8 April 2026\n if era_received_col is not None:\n era_val = target_row[era_received_col]\n era_match = False\n if era_val is not None:\n if isinstance(era_val, (datetime.datetime, datetime.date)):\n if era_val.year == 2026 and era_val.month == 4 and era_val.day == 8:\n era_match = True\n else:\n era_str = str(era_val).strip()\n if '04/08/2026' in era_str or '4/8/2026' in era_str or '2026-04-08' in era_str or '8 April 2026' in era_str or 'April 8' in era_str or '08-Apr' in era_str or '8-Apr' in era_str or '04-08-2026' in era_str:\n era_match = True\n if era_match:\n details.append(f'ERA_Received_Date={era_val}')\n else:\n issues.append(f'ERA_Received_Date expected 04/08/2026 (8 April 2026), got: {era_val}')\n else:\n issues.append(f'ERA_Received_Date column not found. Headers: {headers}')\n\n # Check Payment_Posted_Date blank/null\n if payment_posted_col is not None:\n payment_val = target_Line truncated
"criterion_type": "expected_output"
},
{
"id": "826aeb1f-eea5-4a46-b9d1-3bb715ba3013",
"sort_order": 1,
"rubric_text": "In file `claim.xlsx`, total row count must be exactly 5 (1 header + 4 data rows).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n claim_path = Path(workspace_path) / 'claim.xlsx'\n if not claim_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'claim.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(claim_path)\n ws = wb.active\n \n row_count = sum(1 for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row))\n \n if row_count == 5:\n return {'pass': True, 'score': 1.0, 'feedback': f'claim.xlsx has exactly 5 rows (1 header + 4 data rows)'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'claim.xlsx has {row_count} non-empty rows, expected 5 (1 header + 4 data rows)'}\n",
"criterion_type": "expected_output"
},
{
"id": "f7996998-b356-45fc-a368-c4ad88757797",
"sort_order": 2,
"rubric_text": "A file named `appeal_L1_02142026.pdf` must exist in the workspace.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n\n all_files = [f for f in ws.rglob('*') if f.is_file()]\n\n # --- Pass 1: exact stem match, PDF only ---\n target_stem = 'appeal_l1_02142026'\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n if f.stem.lower() == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found appeal_L1_02142026.pdf at {f.relative_to(ws)}'\n }\n\n # --- Pass 2: normalised separators (hyphens → underscores), PDF only ---\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n normalised = f.stem.lower().replace('-', '_').replace(' ', '_')\n if normalised == target_stem:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found matching PDF with alternate separators: '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Pass 3: component match, PDF only ---\n date_variants = ['02142026', '02_14_2026', '02-14-2026']\n for f in all_files:\n if f.suffix.lower() != '.pdf':\n continue\n n = f.name.lower()\n if 'appeal' in n and 'l1' in n and any(d in n for d in date_variants):\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'Found appeal letter PDF (component match): '\n f'{f.name} (at {f.relative_to(ws)})'\n )\n }\n\n # --- Diagnosis: did the model produce the file in a non-PDF format? ---\n non_pdf_matches = [\n f for f in all_files\n if f.suffix.lower() != '.pdf'\n and 'appeal' in f.name.lower()\n and 'l1' in f.name.lower()\n and any(d in f.name.lower() for d in date_variants)\n ]\n if non_pdf_matches:\n names = [f.name for f in non_pdf_matches]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'Appeal letter found but not in PDF format: {names}. '\n f'The criterion requires a .pdf file. '\n f'Rename or re-export the file as appeal_L1_02142026.pdf.'\n )\n }\n\n all_names = [f.name for f in all_files]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No appeal_L1_02142026.pdf found in workspace. '\n f'Expected a .pdf file whose name contains \"appeal\", \"l1\", '\n f'and date \"02142026\". '\n f'Files present: {all_names}'\n )\n }",
"criterion_type": "expected_output"
},
{
"id": "f7446601-8f7d-4712-9097-7620699d893c",
"sort_order": 3,
"rubric_text": "In `mailbox.json` (external services), exactly 1 email must exist in the `Sent` folder with `to_addr` = `appeals@silverpath-claims.com`, with subject containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `SPA-00291847`, `DOS:`, `02/14/2026`, `Claim:`, `CLM-REF-20260218-9931`, `CARC:`, `CO-97`, and attachments containing exactly `appeal_L1_02142026.pdf`, `claim_02142026_Holloway_SilverPath.pdf`, `pa_approval_SilverPath_Holloway.pdf`, `lmn_DrOsei_Holloway.pdf`, `era_02142026_SilverPath.pdf`, and filename containing `visit_note`.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find sent emails to the appeals address (case-insensitive)\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@silverpath-claims.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No sent email to appeals@silverpath-claims.com found'}\n\n if len(target_emails) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 sent email to appeals@silverpath-claims.com, found {len(target_emails)}'}\n\n email = target_emails[0]\n\n # Check subject for required substrings\n subject = str(email.get('subject', ''))\n required_substrings = [\n 'APPEAL L1',\n 'CareIG',\n 'Holloway',\n 'MemberID:',\n 'SPA-00291847',\n 'DOS:',\n '02/14/2026',\n 'Claim:',\n 'CLM-REF-20260218-9931',\n 'CARC:',\n 'CO-97'\n ]\n\n # Case-sensitive IDs that must match exactly\n exact_match_required = {'SPA-00291847', 'CLM-REF-20260218-9931', 'CO-97', '02/14/2026'}\n\n missing = []\n for s in required_substrings:\n if s in subject:\n continue\n # For non-exact fields, try case-insensitive\n if s not in exact_match_required and s.upper() in subject.upper():\n continue\n missing.append(s)\n\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Email subject missing required substrings: {missing}. Subject was: {subject}'}\n\n # Check attachments\n attachments = email.get('attachments', [])\n attach_names = []\n for a in attachments:\n if isinstance(a, str):\n attach_names.append(a)\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', str(a)))\n attach_names.append(name)\n else:\n attach_names.append(str(a))\n\n attach_names_lower = [n.lower() for n in attach_names]\n\n # Required exact attachment filenames\n required_exact_attachments = [\n 'appeal_L1_02142026.pdf',\n 'claim_02142026_Holloway_SilverPath.pdf',\n 'pa_approval_SilverPath_Holloway.pdf',\n 'lmn_DrOsei_Holloway.pdf',\n 'era_02142026_SilverPath.pdf'\n ]\n\n missing_attachments = []\n for req in required_exact_attachments:\n # Check case-insensitive\n found = any(req.lower() in n for n in attach_names_lower)\n if not found:\n missing_attachments.append(req)\n\n if missing_attachments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required attachments: {missing_attachments}. Found attachments: {attach_names}'}\n\n # Check for an attachment whose filename contains 'visit_note' (case-insensitive)\n found_visit_note = any('visit_note' in n.lower() for n in attach_names)\n if not found_visit_note:\n return {'pass': False, 'score': 0.0, 'feedback': f'No attachment with filename containing \"visit_note\" found. Found attachments: {attach_names}'}\n\n # Total expected: 5 exact + 1 visit_note = 6\n # Be lenient: as long as all required ones are present, pass even if there are extras\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Sent email to appeals@silverpath-claims.com found with correct subject substrings and all required attachments. Subject: {subject}. Attachments: {attach_names}'\n }\n",
"criterion_type": "expected_output"
},
{
"id": "74e31c33-9667-455a-8fd5-dd989a7231eb",
"sort_order": 4,
"rubric_text": "In `mailbox.json` (external services), no email in any folder must have `to_addr` containing `billing@silverpath-claims.com` with a date of `2026-04-01` or later.",
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n \n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n cutoff = datetime(2026, 4, 1)\n \n violations = []\n for e in emails:\n to_addr = str(e.get('to_addr', ''))\n if 'billing@silverpath-claims.com' in to_addr.lower():\n date_str = str(e.get('date', ''))\n try:\n email_date = datetime.fromisoformat(date_str[:10])\n if email_date >= cutoff:\n violations.append(f\"Subject: {e.get('subject', 'N/A')}, Date: {date_str}\")\n except Exception:\n pass\n \n if violations:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found emails to billing@silverpath-claims.com on/after 2026-04-01: {violations}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'No emails sent to billing@silverpath-claims.com on or after 2026-04-01 (no corrected claim submitted to payer)'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "bce26050-60a1-4108-8bf3-836752efd837",
"sort_order": 5,
"rubric_text": "File `appeals_log.xlsx` must contain exactly 4 rows (1 header + 3 data rows), with exactly 1 row having `DOS` = `02/14/2026`, and that row must have `CARC_Code` = `CO-97`, `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, `Appeal_Status` = `Submitted`, `Payer` = `SilverPath Medicare Advantage`.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) != 4:\n return {'pass': False, 'score': 0.0, 'feedback': f'appeals_log.xlsx has {len(rows)} rows, expected 4 (1 header + 3 data rows)'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n carc_col = col_idx('CARC')\n level_col = col_idx('LEVEL')\n subdate_col = col_idx('SUBMITTED_DATE') or col_idx('SUBMIT')\n status_col = col_idx('STATUS')\n payer_col = col_idx('PAYER')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '02/14/2026' in dos_val or '2026-02-14' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) != 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 1 row with DOS=02/14/2026 in appeals_log.xlsx, found {len(target_rows)}'}\n \n row = target_rows[0]\n issues = []\n \n if carc_col is not None:\n carc_val = str(row[carc_col]).strip() if row[carc_col] is not None else ''\n if 'CO-97' not in carc_val.upper() and 'CO97' not in carc_val.upper():\n issues.append(f'CARC_Code expected CO-97, got: {carc_val}')\n else:\n issues.append('CARC_Code column not found')\n \n if level_col is not None:\n level_val = str(row[level_col]).strip().upper() if row[level_col] is not None else ''\n if 'L1' not in level_val:\n issues.append(f'Appeal_Level expected L1, got: {row[level_col]}')\n else:\n issues.append('Appeal_Level column not found')\n \n if subdate_col is not None:\n date_val = str(row[subdate_col]).strip() if row[subdate_col] is not None else ''\n if '04/08/2026' not in date_val and '2026-04-08' not in date_val:\n issues.append(f'Appeal_Submitted_Date expected 04/08/2026, got: {date_val}')\n else:\n issues.append('Appeal_Submitted_Date column not found')\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if payer_col is not None:\n payer_val = str(row[payer_col]).strip() if row[payer_col] is not None else ''\n if 'SILVERPATH' not in payer_val.upper():\n issues.append(f'Payer expected SilverPath Medicare Advantage, got: {payer_val}')\n else:\n issues.append('Payer column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=02/14/2026 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx has 4 rows, DOS=02/14/2026 row has CARC_Code=CO-97, Appeal_Level=L1, Appeal_Submitted_Date=04/08/2026, Appeal_Status=Submitted, Payer=SilverPath Medicare Advantage'}\n",
"criterion_type": "expected_output"
},
{
"id": "3e721c8d-53cd-47f9-a9a0-19515432c546",
"sort_order": 6,
"rubric_text": "In `appeals_log.xlsx`, the row where `DOS` = `12/09/2025` must have `Appeal_Status` = `Submitted` and `Followup_Log` that is non-blank and contains a date matching `04/08/2026` or `8 April 2026`.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n log_path = Path(workspace_path) / 'appeals_log.xlsx'\n if not log_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx not found in workspace'}\n \n wb = openpyxl.load_workbook(log_path)\n ws = wb.active\n \n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n \n if len(rows) < 2:\n return {'pass': False, 'score': 0.0, 'feedback': 'appeals_log.xlsx has no data rows'}\n \n headers = [str(h).strip().upper().replace(' ', '_') if h is not None else '' for h in rows[0]]\n \n def col_idx(substr):\n return next((i for i, h in enumerate(headers) if substr in h), None)\n \n dos_col = col_idx('DOS')\n status_col = col_idx('STATUS')\n followup_col = col_idx('FOLLOWUP') or col_idx('FOLLOW_UP') or col_idx('FOLLOW')\n \n target_rows = []\n for row in rows[1:]:\n if dos_col is not None:\n dos_val = str(row[dos_col]).strip() if row[dos_col] is not None else ''\n if '12/09/2025' in dos_val or '2025-12-09' in dos_val:\n target_rows.append(row)\n \n if len(target_rows) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No row with DOS=12/09/2025 found in appeals_log.xlsx'}\n \n row = target_rows[0]\n issues = []\n \n if status_col is not None:\n status_val = str(row[status_col]).strip().lower() if row[status_col] is not None else ''\n if 'submitted' not in status_val:\n issues.append(f'Appeal_Status expected Submitted, got: {row[status_col]}')\n else:\n issues.append('Appeal_Status column not found')\n \n if followup_col is not None:\n followup_val = str(row[followup_col]).strip() if row[followup_col] is not None else ''\n if followup_val in ('', 'None', 'nan', 'NaN', 'NULL'):\n issues.append('Followup_Log is blank')\n else:\n has_date = '04/08/2026' in followup_val or '2026-04-08' in followup_val or '8 april 2026' in followup_val.lower() or 'april 8, 2026' in followup_val.lower()\n if not has_date:\n issues.append(f'Followup_Log does not contain date 04/08/2026 or 8 April 2026. Value: {followup_val}')\n else:\n issues.append('Followup_Log column not found')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues in appeals_log.xlsx DOS=12/09/2025 row: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': 'appeals_log.xlsx DOS=12/09/2025 row has Appeal_Status=Submitted and Followup_Log containing 04/08/2026'}\n",
"criterion_type": "expected_output"
},
{
"id": "fc648229-7c0f-4abc-9ba6-d8d05942a39a",
"sort_order": 7,
"rubric_text": "In `mailbox.json` (external services), exactly 3 emails must exist in the `Sent` folder with `to_addr` = `appeals@bluecrest-health.com`. The 2 of the emails must have subjects containing `APPEAL L1`, `CareIG`, `Holloway`, `MemberID:`, `BCR-00184523`, `DOS:`, `12/09/2025`, `Claim:`, `CLM-REF-20251215-6612`, `CARC:`, `PI-204`, email attachment name = `appeal_L1_12092025.pdf`, email body containing `BlueCrest Commercial PPO — Provider Appeals Department`, `Member ID: BCR-00184523`, `DOS: 12/09/2025`, `Claim Reference: CLM-REF-20251215-6612`, `Billed: $5,840.00`",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external_services_path'}\n\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n\n # Find all sent emails to appeals@bluecrest-health.com\n target_emails = [\n e for e in emails\n if str(e.get('folder', '')).lower() == 'sent'\n and 'appeals@bluecrest-health.com' in str(e.get('to_addr', '')).lower()\n ]\n\n if len(target_emails) != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails to appeals@bluecrest-health.com, found {len(target_emails)}'}\n\n # Required substrings in the subject (case-insensitive, whitespace-flexible)\n required_subject_substrings = [\n 'APPEAL L1', 'CareIG', 'Holloway', 'MemberID:',\n 'BCR-00184523', 'DOS:', '12/09/2025', 'Claim:',\n 'CLM-REF-20251215-6612', 'CARC:', 'PI-204'\n ]\n\n # Required body substrings\n required_body_substrings = [\n 'BlueCrest Commercial PPO',\n 'Provider Appeals Department',\n 'Member ID: BCR-00184523',\n 'DOS: 12/09/2025',\n 'Claim Reference: CLM-REF-20251215-6612',\n 'Billed: $5,840.00'\n ]\n\n required_attachment = 'appeal_L1_12092025.pdf'\n\n def normalize(text):\n \"\"\"Normalize whitespace for flexible matching.\"\"\"\n return re.sub(r'\\s+', ' ', str(text)).strip()\n\n def check_subject(subject):\n subj_norm = normalize(subject).lower()\n # Also try without spaces around colons\n subj_no_space = re.sub(r'\\s*:\\s*', ':', subj_norm)\n missing = []\n for s in required_subject_substrings:\n s_lower = s.lower()\n s_no_space = re.sub(r'\\s*:\\s*', ':', s_lower)\n # Try multiple matching strategies\n if s_lower in subj_norm:\n continue\n if s_no_space in subj_no_space:\n continue\n # Try without space: \"MemberID:\" might appear as \"Member ID:\" or \"MemberID:\"\n if s == 'MemberID:':\n if 'memberid:' in subj_norm or 'member id:' in subj_norm or 'memberid :' in subj_norm:\n continue\n missing.append(s)\n return missing\n\n def check_body(body):\n body_norm = normalize(body)\n missing = []\n for s in required_body_substrings:\n # Case-insensitive check with normalized whitespace\n s_norm = normalize(s)\n if s_norm.lower() in body_norm.lower():\n continue\n # Try with em-dash variants\n s_variants = [\n s_norm,\n s_norm.replace('\\u2014', '-'),\n s_norm.replace('\\u2014', '--'),\n s_norm.replace('-', '\\u2014'),\n ]\n found = False\n for variant in s_variants:\n if variant.lower() in body_norm.lower():\n found = True\n break\n if not found:\n missing.append(s)\n return missing\n\n def get_attachment_names(attachments):\n names = []\n for a in (attachments or []):\n if isinstance(a, str):\n names.append(a.strip())\n elif isinstance(a, dict):\n name = a.get('filename', a.get('name', a.get('file_name', '')))\n names.append(str(name).strip())\n else:\n names.append(str(a).strip())\n return names\n\n def check_attachment(email_obj):\n att_names = get_attachment_names(email_obj.get('attachments', []))\n for name in att_names:\n if name.lower() == required_attachment.lower():\n return True\n return False\n\n # Find emails that match all criteria\n fully_matching = []\n partial_details = []\n\n for email in target_emails:\n subject = str(email.get('subject', ''))\n body = str(email.get('body_text', email.get('body', '')))\n\n subj_missing = check_subject(subject)\n body_missing = check_body(body)\n has_attachment = check_attachment(email)\n\n if not subj_missing and not body_missing and has_attachment:\n fully_matching.append(email)\n else:\n partial_details.append({\n 'subject': subject[:100],\n 'subj_missing': subj_missing,\n 'body_missing': body_missing,\n 'has_attachment': has_attachment\n })\nLine truncated
"criterion_type": "expected_output"
},
{
"id": "52595972-13bc-49f5-b35b-4674e117b8ff",
"sort_order": 8,
"rubric_text": "In `slack_data.json` (external services), channel `billing-appeals` must contain a message with text containing all of: `APPEAL FILED L1`, `Holloway_DOB_03141961`, `02/14/2026`, `SilverPath`, `CO-97`.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check Slack'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external_services_path'}\n \n with open(slack_path, 'r') as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n messages = data.get('messages', {})\n \n # Find #billing-appeals channel (be flexible with naming)\n billing_appeals_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if ch_name == 'billingappeals':\n billing_appeals_id = ch_id\n break\n \n if billing_appeals_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Channel #billing-appeals not found in slack_data.json. Available channels: {[ch.get(\"name\") for ch in channels.values()]}'}\n \n channel_messages = messages.get(billing_appeals_id, [])\n \n # Required content pieces to find in a message\n # Be loose: case-insensitive where appropriate, flexible date formats, flexible separators\n required_checks = {\n 'APPEAL FILED L1': lambda t: 'appeal' in t.lower() and ('filed' in t.lower() or 'file' in t.lower()) and ('l1' in t.lower() or 'level 1' in t.lower() or 'level1' in t.lower()),\n 'Holloway_DOB_03141961': lambda t: 'holloway' in t.lower() and ('03141961' in t.replace('/', '').replace('-', '').replace(' ', '') or '03/14/1961' in t or '03-14-1961' in t or '3/14/1961' in t),\n '02/14/2026': lambda t: any(d in t for d in ['02/14/2026', '2/14/2026', '02-14-2026', '2-14-2026', '2026-02-14', '02142026']),\n 'SilverPath': lambda t: 'silverpath' in t.lower().replace(' ', '').replace('-', '').replace('_', ''),\n 'CO-97': lambda t: 'co-97' in t.lower() or 'co 97' in t.lower() or 'co97' in t.lower(),\n }\n \n best_match_count = 0\n best_match_text = ''\n matched_keys_best = []\n \n for msg in channel_messages:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n \n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n \n if len(matched_keys) == len(required_checks):\n return {'pass': True, 'score': 1.0, 'feedback': f'Found matching message in #billing-appeals containing all required elements: {text[:300]}'}\n \n # Also check all messages across all channels in case channel name matching failed\n for ch_id, ch_msgs in messages.items():\n if ch_id == billing_appeals_id:\n continue\n for msg in ch_msgs:\n text = str(msg.get('text', ''))\n matched_keys = []\n for key, check_fn in required_checks.items():\n if check_fn(text):\n matched_keys.append(key)\n if len(matched_keys) > best_match_count:\n best_match_count = len(matched_keys)\n best_match_text = text\n matched_keys_best = matched_keys\n # Note: don't pass here since it's wrong channel, but track for feedback\n \n missing = [k for k in required_checks if k not in matched_keys_best]\n return {\n 'pass': False,\n 'score': best_match_count / len(required_checks) * 0.5, # partial credit\n 'feedback': f'No message in #billing-appeals found containing all required elements. '\n f'Best match had {best_match_count}/{len(required_checks)} elements. '\n f'Missing: {missing}. '\n f'Found {len(channel_messages)} messages in #billing-appeals. '\n f'Best matching text: \"{best_match_text[:300]}\"'\n }\n",
"criterion_type": "expected_output"
},
{
"id": "29097e98-d53d-4af6-ac8a-8df8f3ec442e",
"sort_order": 9,
"rubric_text": "In `audit_log.xlsx`, total row count must be exactly 37 (1 header + 36 data rows).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find audit_log.xlsx - be flexible with location\n audit_path = Path(workspace_path) / 'audit_log.xlsx'\n if not audit_path.exists():\n # Try glob search\n candidates = list(Path(workspace_path).rglob('audit_log.xlsx'))\n if candidates:\n audit_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'audit_log.xlsx not found in workspace'}\n\n try:\n wb = openpyxl.load_workbook(audit_path)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open audit_log.xlsx: {e}'}\n\n ws = wb.active\n\n # Collect all non-empty rows\n rows = [row for row in ws.iter_rows(values_only=True) if any(cell is not None for cell in row)]\n\n total_row_count = len(rows)\n\n # Rubric says exactly 37 rows (1 header + 36 data rows)\n if total_row_count != 37:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'audit_log.xlsx has {total_row_count} non-empty rows, expected exactly 37 (1 header + 36 data rows).'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'audit_log.xlsx has exactly {total_row_count} non-empty rows (1 header + 36 data rows), as required.'\n }\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775206985975",
"sort_order": 10,
"rubric_text": "File `DENIAL_CO97_02142026.xlsx` must exist and contain exactly 2 rows (1 header + 1 data row) with the following values in the data row: `CASE_ID` = `Holloway_DOB_03141961`, `CARC_Code` = `CO-97`, `CARC_Description` = `Benefit not assigned`, `DOS` = `02/14/2026`, `Payer` = `SilverPath Medicare Advantage`, `Denied_Code` containing `J1560`, `99600`, `99603`, `J7030`, `J0171`, `A4221`, and `Denied_Amount` = `6140.00` (numerical or string). It must also have `Appeal_Level` = `L1`, `Appeal_Submitted_Date` = `04/08/2026`, and `Appeal_Status` = `Submitted`.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file with flexible matching\n target_file = None\n exact_path = Path(workspace_path) / \"DENIAL_CO97_02142026.xlsx\"\n if exact_path.exists():\n target_file = exact_path\n else:\n # Try glob for similar filenames\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*02142026*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*denial*co*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*DENIAL*CO*97*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*denial*co*97*.xlsx\"))\n if not candidates:\n # Even more lenient: any xlsx with denial and 97\n candidates = list(Path(workspace_path).glob(\"**/*enial*97*.xlsx\"))\n if not candidates:\n # Try any xlsx with denial in name\n candidates = list(Path(workspace_path).glob(\"**/*enial*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"**/*ENIAL*.xlsx\"))\n if candidates:\n target_file = candidates[0]\n\n if target_file is None or not target_file.exists():\n # List all xlsx files to help debug\n all_xlsx = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File DENIAL_CO97_02142026.xlsx not found in workspace. Found xlsx files: {[str(f.name) for f in all_xlsx]}\"}\n\n try:\n wb = openpyxl.load_workbook(str(target_file))\n ws = wb.active\n # If active sheet is empty, try other sheets\n if ws.max_row < 2:\n for sheet_name in wb.sheetnames:\n candidate = wb[sheet_name]\n if candidate.max_row >= 2:\n ws = candidate\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target_file.name}: {e}\"}\n\n rows = list(ws.iter_rows(values_only=False))\n if len(rows) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 rows (1 header + 1 data), but found {len(rows)} rows.\"}\n\n # Be lenient: ignore completely empty rows beyond row 2\n non_empty = [r for r in rows if any(c.value is not None for c in r)]\n if len(non_empty) < 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected at least 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n if len(non_empty) > 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 non-empty rows (1 header + 1 data), but found {len(non_empty)} non-empty rows.\"}\n\n # Build header map from first non-empty row\n header_row = non_empty[0]\n headers = {}\n for idx, cell in enumerate(header_row):\n if cell.value is not None:\n headers[str(cell.value).strip()] = idx\n\n # Helper to find a column by flexible name matching\n def find_col(possible_names):\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if h_lower == pn_lower:\n return idx\n # Also try substring matching\n for pn in possible_names:\n pn_lower = pn.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n for h, idx in headers.items():\n h_lower = h.lower().replace(\" \", \"\").replace(\"_\", \"\").replace(\"-\", \"\")\n if pn_lower in h_lower or h_lower in pn_lower:\n return idx\n return None\n\n data_row = non_empty[1]\n\n def cell_val(row, col_idx):\n if col_idx is None:\n return None\n if col_idx < len(row):\n return row[col_idx].value\n return None\n\n def normalize_str(v):\n if v is None:\n return \"\"\n return str(v).strip()\n\n def normalize_date(v):\n \"\"\"Try to normalize a date value to MM/DD/YYYY for comparison.\"\"\"\n if v is None:\n return \"\"\n if isinstance(v, (datetime.datetime, datetime.date)):\n return v.strftime(\"%m/%d/%Y\")\n s = str(v).strip()\n for fmt in [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%m/%d/%y\", \"%Y/%m/%d\", \"%d/%m/%Y\"]:\n try:\n dt = datetime.datetime.strptime(s, fmt)\n return dt.strftime(\"Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775268135625",
"sort_order": 11,
"rubric_text": "File `era_02142026_SilverPath.pdf` must exist in the workspace.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"era_02142026_SilverPath.pdf\"\n if target_file.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' exists at {target_file}.\"}\n else:\n # Also do a case-insensitive search as a generous fallback\n found = [f for f in Path(workspace_path).rglob(\"*\") if f.name.lower() == \"era_02142026_silverpath.pdf\"]\n if found:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"File found (case-insensitive match): {found[0]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'era_02142026_SilverPath.pdf' was NOT found in the workspace at '{workspace_path}'. Files present: {[f.name for f in Path(workspace_path).rglob('*') if f.is_file()]}\"}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1775268234332",
"sort_order": 12,
"rubric_text": "File `statement_02142026.pdf` must not exist in the workspace.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"statement_02142026.pdf\"\n if target_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'statement_02142026.pdf' still exists in the workspace at '{target_file}'. It should have been removed.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'statement_02142026.pdf' does not exist in the workspace, as expected.\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775268384702",
"sort_order": 13,
"rubric_text": "In `mailbox.json`, an email with the following details must not exist: `to_addr` = `m.holloway@emailhost.com`, subject containing `CareIG Statement of Account` and `02/14/2026` (or the equivalent date in any format).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check mailbox.json for the absence of a specific email.\n # The mailbox.json could be in the workspace or in external_services_path.\n \n mailbox_path = None\n \n # Check external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json found at all, the email trivially doesn't exist -> PASS\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so the prohibited email does not exist. PASS.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n \n # Define date patterns that all represent February 14, 2026\n date_patterns = [\n r'02/14/2026',\n r'02-14-2026',\n r'2/14/2026',\n r'2-14-2026',\n r'14/02/2026',\n r'14-02-2026',\n r'2026-02-14',\n r'2026/02/14',\n r'Feb(?:ruary)?\\s*14,?\\s*2026',\n r'14\\s*Feb(?:ruary)?,?\\s*2026',\n ]\n \n def subject_contains_date(subject):\n for pat in date_patterns:\n if re.search(pat, subject, re.IGNORECASE):\n return True\n return False\n \n violating_emails = []\n \n for email in emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n subject = email.get(\"subject\", \"\")\n \n # Check to_addr matches\n if to_addr != \"m.holloway@emailhost.com\":\n continue\n \n # Check subject contains \"CareIG Statement of Account\" (case-insensitive)\n if \"careig statement of account\" not in subject.lower():\n continue\n \n # Check subject contains the date 02/14/2026 in any format\n if subject_contains_date(subject):\n violating_emails.append(email)\n \n if len(violating_emails) > 0:\n details = []\n for e in violating_emails:\n details.append(f\" - email_id={e.get('email_id','?')}, to={e.get('to_addr','?')}, subject='{e.get('subject','?')}', folder={e.get('folder','?')}\")\n detail_str = \"\\n\".join(details)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) that should NOT exist (to=m.holloway@emailhost.com, subject containing 'CareIG Statement of Account' and date 02/14/2026):\\n{detail_str}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No email found in mailbox.json with to_addr='m.holloway@emailhost.com' and subject containing both 'CareIG Statement of Account' and the date 02/14/2026.\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775268649908",
"sort_order": 14,
"rubric_text": "The file `appeal_L1_02142026.pdf` must contain the following:\n- Date: `04/08/2026`\n- Patient name: `Margaret` `Holloway`\n- Member ID: `SPA-00291847`\n- Date of Service (DOS): `02/14/2026`\n- Claim #: `CLM-REF-20260218-9931`\n- CARC: `CO-97`\n- Billed amount: `$6,140.00`\n- The letter body must contain: `#SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026` and `D83.9`.\n- Contracted rate: '$4,800.00'",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n pdf_file = None\n\n # Search for the file - try exact name and case-insensitive\n target_name = \"appeal_L1_02142026.pdf\"\n candidates = list(workspace.rglob(\"*.pdf\"))\n for c in candidates:\n if c.name.lower() == target_name.lower():\n pdf_file = c\n break\n\n # Also try broader matching in case of slight name variations\n if pdf_file is None:\n for c in candidates:\n if \"appeal\" in c.name.lower() and \"02142026\" in c.name:\n pdf_file = c\n break\n\n if pdf_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File '{target_name}' not found in workspace. PDF files found: {[str(c.relative_to(workspace)) for c in candidates]}\"}\n\n # Extract text from PDF\n text = \"\"\n try:\n import pdfplumber\n with pdfplumber.open(str(pdf_file)) as pdf:\n for page in pdf.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e1:\n try:\n import PyPDF2\n with open(str(pdf_file), 'rb') as f:\n reader = PyPDF2.PdfReader(f)\n for page in reader.pages:\n page_text = page.extract_text()\n if page_text:\n text += page_text + \"\\n\"\n except Exception as e2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read PDF. pdfplumber error: {e1}; PyPDF2 error: {e2}\"}\n\n if not text.strip():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PDF file exists but no text could be extracted.\"}\n\n # Normalize text for searching\n text_lower = text.lower()\n # Also create a version with collapsed whitespace for matching longer strings\n text_collapsed = re.sub(r'\\s+', ' ', text).strip()\n text_collapsed_lower = text_collapsed.lower()\n\n checks = []\n total = 0\n passed = 0\n\n # Helper function for flexible matching\n def check_present(label, patterns, required=True):\n nonlocal total, passed\n total += 1\n for pattern in patterns:\n if pattern.lower() in text_collapsed_lower:\n passed += 1\n checks.append(f\"PASS: {label} - found '{pattern}'\")\n return True\n checks.append(f\"FAIL: {label} - none of {patterns} found in PDF text\")\n return False\n\n # 1. Date = 04/08/2026\n check_present(\"Date 04/08/2026\", [\n \"04/08/2026\", \"04-08-2026\", \"april 8, 2026\", \"april 08, 2026\",\n \"4/8/2026\", \"8 april 2026\", \"08 april 2026\"\n ])\n\n # 2. Margaret\n check_present(\"Patient first name 'Margaret'\", [\"margaret\"])\n\n # 3. Holloway\n check_present(\"Patient last name 'Holloway'\", [\"holloway\"])\n\n # 4. Member ID SPA-00291847\n check_present(\"Member ID SPA-00291847\", [\n \"spa-00291847\", \"spa00291847\", \"spa 00291847\"\n ])\n\n # 5. DOS 02/14/2026\n check_present(\"DOS 02/14/2026\", [\n \"02/14/2026\", \"02-14-2026\", \"2/14/2026\", \"february 14, 2026\",\n \"february 14 2026\", \"14 february 2026\"\n ])\n\n # 6. Claim # CLM-REF-20260218-9931\n check_present(\"Claim # CLM-REF-20260218-9931\", [\n \"clm-ref-20260218-9931\", \"clm ref 20260218 9931\"\n ])\n\n # 7. CARC CO-97\n check_present(\"CARC CO-97\", [\n \"co-97\", \"co 97\", \"co97\"\n ])\n\n # 8. Billed $6,140.00\n check_present(\"Billed $6,140.00\", [\n \"$6,140.00\", \"$6140.00\", \"6,140.00\", \"6140.00\", \"$ 6,140.00\", \"$6,140\"\n ])\n\n # 9. Letter body: #SPA-PA-2026-3302 was approved for Octagam 10% 40g, effective 01/28/2026 through 07/28/2026\n # Check for the key components since exact whitespace/formatting may vary\n spa_pa_found = False\n spa_patterns = [\"spa-pa-2026-3302\"]\n for p in spa_patterns:\n if p in text_collapsed_lower:\n spa_pa_found = True\n break\n\n octagam_found = False\n octagam_patterns = [\"octagam 10% 40g\", \"octagam 10% 40 g\", \"octagam10% 40g\", \"octagam 10%, 40g\", \"octagam 10% 40g\"]\n for p in octagam_patterns:\n if p in text_collapsed_lower:\n octagam_found = True\n break\n # Also try regex for more flexible matching\n if not octagam_found:\n if re.search(r'octagam\\s*10\\s*%\\s*,?\\s*40\\s*g', text_collapsed_lower):\n octagam_found = True\n\n dates_found = False\n start_date_found = any(d in text_collapsed for d in [\"01/28/2026\", \"1/28/2026\", \"01-28-2026\"]) or \\\n any(d in text_collapsed_lower for d in [\"january 28, 2026\", \"january 28 2026Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775269539963",
"sort_order": 15,
"rubric_text": "In `audit_log.xlsx`, rows 36 and 37 should have: `CASE_ID` = `Holloway_DOB_03141961`, the date in `Datetime` = `04/08/2026`, `Action` = `Denial Received — CO-97` (case-insensitive) for row 36, and `Action` = `Appeal Submitted` (case-insensitive) for row 37.",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"audit_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening audit_log.xlsx: {e}\"}\n\n # Find column indices by header row (row 1)\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip().upper()] = col_idx\n\n # Try to locate relevant columns\n case_id_col = None\n datetime_col = None\n action_col = None\n\n for key, idx in headers.items():\n if 'CASE' in key and 'ID' in key:\n case_id_col = idx\n if key == 'DATETIME': # FIX: was ('DATE' in key or 'TIME' in key or key == 'DATETIME')\n datetime_col = idx # 'DATE' in key matches FILE_UPDATED and FIELD_UPDATED,\n if key == 'ACTION': # overwriting datetime_col with the wrong column index.\n action_col = idx # Exact match eliminates the substring collision.\n\n if case_id_col is None or datetime_col is None or action_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find required columns. Headers found: {headers}\"}\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def check_case_id(val):\n normed = normalize(val).lower()\n return normed == \"holloway_dob_03141961\"\n\n def check_date(val):\n if val is None:\n return False\n import datetime as dt\n if isinstance(val, (dt.datetime, dt.date)):\n d = val.date() if isinstance(val, dt.datetime) else val\n return d.month == 4 and d.day == 8 and d.year == 2026\n s = normalize(val)\n patterns = [\n r'0?4[/\\-]0?8[/\\-]2026',\n r'2026[/\\-]0?4[/\\-]0?8',\n r'April\\s+8,?\\s+2026',\n r'8\\s+April\\s+2026',\n r'04082026',\n ]\n for p in patterns:\n if re.search(p, s, re.IGNORECASE):\n return True\n return False\n\n def check_action_denial(val):\n normed = normalize(val).lower()\n normed_dashes = re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\-]+', '-', normed)\n if 'denial received' in normed_dashes and ('co-97' in normed_dashes or 'co 97' in normed_dashes or 'co97' in normed_dashes):\n return True\n return False\n\n def check_action_appeal(val):\n normed = normalize(val).lower()\n return 'appeal submitted' in normed\n\n # Check row 36\n row36_case = ws.cell(row=36, column=case_id_col).value\n row36_date = ws.cell(row=36, column=datetime_col).value\n row36_action = ws.cell(row=36, column=action_col).value\n\n # Check row 37\n row37_case = ws.cell(row=37, column=case_id_col).value\n row37_date = ws.cell(row=37, column=datetime_col).value\n row37_action = ws.cell(row=37, column=action_col).value\n\n issues = []\n passes = []\n\n if check_case_id(row36_case):\n passes.append(f\"Row 36 CASE_ID: '{row36_case}' matches\")\n else:\n issues.append(f\"Row 36 CASE_ID: expected 'Holloway_DOB_03141961', got '{row36_case}'\")\n\n if check_date(row36_date):\n passes.append(f\"Row 36 Datetime: '{row36_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 36 Datetime: expected date 04/08/2026, got '{row36_date}'\")\n\n if check_action_denial(row36_action):\n passes.append(f\"Row 36 Action: '{row36_action}' matches 'Denial Received — CO-97'\")\n else:\n issues.append(f\"Row 36 Action: expected 'Denial Received — CO-97', got '{row36_action}'\")\n\n if check_case_id(row37_case):\n passes.append(f\"Row 37 CASE_ID: '{row37_case}' matches\")\n else:\n issues.append(f\"Row 37 CASE_ID: expected 'Holloway_DOB_03141961', got '{row37_case}'\")\n\n if check_date(row37_date):\n passes.append(f\"Row 37 Datetime: '{row37_date}' matches 04/08/2026\")\n else:\n issues.append(f\"Row 37 Datetime: expected date 04/08/2026, got '{row37_date}'\")\n\n if check_action_appeal(row37_action):\n passes.append(f\"Row 37 Action: '{row37_action}' matches 'Appeal Submitted'\")\n else:\n issues.append(f\"Row 37 Action: expected 'Appeal Submitted', got '{row37_action}'\")\n\n total_checks = 6\n passed_checks = total_checks - len(issues)\n score = passed_checks / total_checks\n\n if len(issues) == 0:\n return {\"pLine truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775271368334",
"sort_order": 16,
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must contain 2 messages with text containing all of: `APPEAL`, `L1`, `Holloway_DOB_03141961`, `12/09/2025`, `BlueCrest Commercial PPO`, `PI-204` and with exactly one message containing `FOLLOW-UP`",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_path}\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n if not isinstance(slack_data, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json top-level is not a dict.\"}\n\n # Collect all messages from billing-appeals channel using multiple possible structures\n all_channel_messages = []\n\n channels_map = slack_data.get(\"channels\", {})\n messages_map = slack_data.get(\"messages\", {})\n\n # Standard format: channels dict keyed by channel ID, messages dict keyed by channel ID\n if isinstance(channels_map, dict) and isinstance(messages_map, dict):\n for ch_id, ch_info in channels_map.items():\n ch_name = \"\"\n if isinstance(ch_info, dict):\n ch_name = ch_info.get(\"name\", \"\") or ch_info.get(\"channel_name\", \"\") or \"\"\n elif isinstance(ch_info, str):\n ch_name = ch_info\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n msgs = messages_map.get(ch_id, [])\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n # Also check if messages_map is keyed by channel name directly\n if not all_channel_messages:\n for key, msgs in messages_map.items():\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(msgs, list):\n all_channel_messages.extend(msgs)\n break\n\n # Fallback: channels is a dict with channel names as keys containing messages\n if not all_channel_messages and isinstance(channels_map, dict):\n for ch_name, ch_data in channels_map.items():\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n all_channel_messages.extend(ch_data[\"messages\"])\n elif isinstance(ch_data, list):\n all_channel_messages.extend(ch_data)\n break\n\n # Fallback: channels is a list of channel objects\n if not all_channel_messages and isinstance(channels_map, list):\n for ch in channels_map:\n if isinstance(ch, dict):\n ch_name = ch.get(\"name\", \"\") or ch.get(\"channel_name\", \"\")\n if \"billing-appeals\" in ch_name.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n all_channel_messages.extend(ch.get(\"messages\", []))\n break\n\n # Fallback: top-level keys are channel names\n if not all_channel_messages:\n for key in slack_data:\n if key in (\"channels\", \"messages\", \"users\", \"counters\"):\n continue\n if \"billing-appeals\" in key.lower().replace(\" \", \"-\").replace(\"_\", \"-\"):\n val = slack_data[key]\n if isinstance(val, dict) and \"messages\" in val:\n all_channel_messages.extend(val[\"messages\"])\n elif isinstance(val, list):\n all_channel_messages.extend(val)\n break\n\n if not all_channel_messages:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find any messages in #billing-appeals channel in slack_data.json. Top-level keys: {list(slack_data.keys())}\"\n }\n\n def get_text(msg):\n if isinstance(msg, dict):\n return msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"content\", \"\") or msg.get(\"body\", \"\") or \"\"\n elif isinstance(msg, str):\n return msg\n return \"\"\n\n def term_in_text(term, text):\n \"\"\"Check if a required term appears in text, with flexible matching.\"\"\"\n text_upper = text.upper()\n term_upper = term.upper()\n if term_upper in text_upper:\n return True\n # Flexible matching for specific terms\n if term == \"Holloway_DOB_03141961\":\n pattern = re.compile(r\"holloway[\\s_\\-]*dob[\\s_\\-]*0?3[\\s_\\-/]*14[\\s_\\-/]*1961\", re.IGNORECASE)\Line truncated
"criterion_type": "expected_output"
},
{
"id": "rubric_1775274130585",
"sort_order": 17,
"rubric_text": "In `slack_data.json` (external services), the channel `billing-appeals` must have a message containing: `[DENIAL]`, `Holloway_DOB_03141961`, and `CO-97`.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (Slack data)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json.\"}\n\n slack_file = Path(external_services_path) / \"slack_data.json\"\n if not slack_file.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {slack_file}\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n\n # According to the documented Slack data format:\n # \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } }\n # \"messages\": { \"C001\": [ { \"text\": \"...\", ... } ] }\n\n # Step 1: Find the channel ID for billing-appeals\n channels = slack_data.get(\"channels\", {})\n messages_by_channel = slack_data.get(\"messages\", {})\n\n billing_appeals_channel_id = None\n\n if isinstance(channels, dict):\n for cid, cinfo in channels.items():\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\":\n billing_appeals_channel_id = cid\n break\n # Also check loosely\n if \"billing\" in cname and \"appeal\" in cname:\n billing_appeals_channel_id = cid\n break\n elif isinstance(channels, list):\n for cinfo in channels:\n if isinstance(cinfo, dict):\n cname = (cinfo.get(\"name\", \"\") or \"\").lower().replace(\"#\", \"\").strip()\n if cname == \"billing-appeals\" or (\"billing\" in cname and \"appeal\" in cname):\n billing_appeals_channel_id = cinfo.get(\"id\", cname)\n break\n\n # Collect message texts from the identified channel\n message_texts = []\n\n if billing_appeals_channel_id and isinstance(messages_by_channel, dict):\n channel_msgs = messages_by_channel.get(billing_appeals_channel_id, [])\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for key in [\"text\", \"message\", \"content\", \"body\"]:\n if key in msg and isinstance(msg[key], str):\n message_texts.append(msg[key])\n elif isinstance(msg, str):\n message_texts.append(msg)\n\n # Also try direct key lookup by name in messages dict (fallback)\n if not message_texts and isinstance(messages_by_channel, dict):\n for key in messages_by_channel:\n key_lower = key.lower().replace(\"#\", \"\").strip()\n if key_lower == \"billing-appeals\" or (\"billing\" in key_lower and \"appeal\" in key_lower):\n channel_msgs = messages_by_channel[key]\n if isinstance(channel_msgs, list):\n for msg in channel_msgs:\n if isinstance(msg, dict):\n for mkey in [\"text\", \"message\", \"content\", \"body\"]:\n if mkey in msg and isinstance(msg[mkey], str):\n message_texts.append(msg[mkey])\n elif isinstance(msg, str):\n message_texts.append(msg)\n break\n\n # Fallback: recursively search the entire structure for billing-appeals messages\n if not message_texts:\n def find_channel_messages(data):\n results = []\n if isinstance(data, dict):\n for key, value in data.items():\n key_lower = key.lower().replace('#', '').replace(' ', '-')\n if 'billing' in key_lower and 'appeal' in key_lower:\n results.extend(_extract_texts(value))\n else:\n results.extend(find_channel_messages(value))\n elif isinstance(data, list):\n for item in data:\n if isinstance(item, dict):\n name = (item.get('name', '') or item.get('channel_name', '') or '').lower().replace('#', '').replace(' ', '-')\n if 'billing' in name and 'appeal' in name:\n results.extend(_extract_texts(item.get('messages', item)))\n else:\n results.extend(find_channel_messages(item))\n return results\n\n def _extract_texts(obj):\n texts = []\n if isinstance(obj, lLine truncated
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,2 @@
Date: Wednesday, 8 April 2026 - Morning shift
It's the start of your morning shift as billing specialist - work through your inbox and the active case files, and take care of anything that needs actioning today in line with the CareIG SOP. All relevant files are available in your workspace.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py