Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-24 14:16:58 -07:00
commit 25c9eda5ca
1800 files changed
+323575

No files matched your search

@@ -0,0 +1,7 @@
FROM handbook_base
COPY initial_external_services/ /data/
COPY initial_external_services/ /initial_data/
COPY initial_workspace/ /workdir/
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
@@ -0,0 +1,237 @@
{
"mailbox": {
"email": "sanjeeta.kaur@pathfinderbilling.com",
"name": "Sanjeeta Kaur"
},
"contacts": [
{
"email": "priya.nair@pathfinderbilling.com",
"name": "Priya Nair"
},
{
"email": "dominic.reyes@pathfinderbilling.com",
"name": "Dominic Reyes"
},
{
"email": "m.okafor@pathfinderbilling.com",
"name": "Marcus Okafor"
},
{
"email": "claire.hudson@pathfinderbilling.com",
"name": "Claire Hudson"
},
{
"email": "edi.submissions@bcbsil.com",
"name": "BCBS EDI Operations"
},
{
"email": "provider.services@cignagroup.com",
"name": "Cigna Provider Services"
},
{
"email": "clearinghouse@availity.net",
"name": "Availity Clearinghouse"
},
{
"email": "billing@northsideinternal.com",
"name": "Northside Internal Medicine"
},
{
"email": "billing@riversideclinic.com",
"name": "Riverside Family Clinic"
},
{
"email": "billing@sunrisederm.com",
"name": "Sunrise Dermatology"
}
],
"groups": [
{
"email": "rcm-team@pathfinderbilling.com",
"name": "RCM Team",
"members": [
"sanjeeta.kaur@pathfinderbilling.com",
"priya.nair@pathfinderbilling.com",
"dominic.reyes@pathfinderbilling.com",
"m.okafor@pathfinderbilling.com",
"claire.hudson@pathfinderbilling.com"
]
}
],
"folders": [
{
"name": "ERA \u2014 Pending Posting"
},
{
"name": "Clearinghouse Alerts"
},
{
"name": "Provider Correspondence"
},
{
"name": "Archive"
}
],
"emails": [
{
"email_id": "1",
"folder": "INBOX",
"subject": "835 ERA File \u2014 EFT-BCBS-20260315-00047312 \u2014 Northside Internal Medicine",
"from_addr": "edi.submissions@bcbsil.com",
"to_addr": "sanjeeta.kaur@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-16T10:24:00Z",
"message_id": "<bcbs-era-20260315-00047312@bcbsil.com>",
"in_reply_to": null,
"body_text": "Pathfinder Billing and Coding \u2014 Attention: Billing Specialist\n\nPlease find attached the 835 Electronic Remittance Advice for the following payment:\n\n EFT Reference: EFT-BCBS-20260315-00047312\n Payment Date: 15 March 2026\n Payee: Northside Internal Medicine\n Payer: Blue Cross Blue Shield of Illinois\n\nThis file contains claim payment detail and provider-level balance adjustments. Please post in accordance with your internal payment posting procedures.\n\nFor queries relating to this remittance, please contact BCBS Provider Services on 1-800-972-8088 (option 3) or reply to this address.\n\nBlue Cross Blue Shield of Illinois\nEDI Operations\n300 East Randolph Street, Chicago, IL 60601",
"body_html": null,
"is_read": false,
"is_important": true,
"attachments": [
{
"filename": "ERA_20260315_BCBS.xlsx",
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"content_base64": "UEsDBBQAAAAIADphilxGx01IlQAAAM0AAAAQAAAAZG9jUHJvcHMvYXBwLnhtbE3PTQvCMAwG4L9SdreZih6kDkQ9ip68zy51hbYpbYT67+0EP255ecgboi6JIia2mEXxLuRtMzLHDUDWI/o+y8qhiqHke64x3YGMsRoPpB8eA8OibdeAhTEMOMzit7Dp1C5GZ3XPlkJ3sjpRJsPiWDQ6sScfq9wcChDneiU+ixNLOZcrBf+LU8sVU57mym/8ZAW/B7oXUEsDBBQAAAAIADphilw+69VY+QAAACsCAAARAAAAZG9jUHJvcHMvY29yZS54bWzNklFLwzAQx7+K5L29pN0GC11fFJ8UBAuKbyG5bcGmDclJu29vWrdO0Q8g5CV3//zud5BKe6n7gE+h9xjIYrwZXdtFqf2OHYm8BIj6iE7FPCW61Nz3wSlK13AAr/S7OiAUnG/AISmjSMEEzPxCZHVltNQBFfXhjDd6wfuP0M4wowFbdNhRBJELYPU00Z/GtoIrYIIRBhe/CmgW4lz9Ezt3gJ2TY7RLahiGfCjnXNpBwOvjw/O8bma7SKrTmF5FK+nkcccuk1/K27vmntUFLzYZX2V824i1TGfF3ybXH35XYdcbu7f/wljwRmwk38p1+c34IlhX8Otf1J9QSwMEFAAAAAgAOmGKXArmYDUpAwAAuQ4AABMAAAB4bC90aGVtZS90aGVtZTEueG1szVddb9sgFP0F+w+I99UfsZM4alI16aI9bJq0bNozsbHNirEFZF3//TB2bPzVVmsq1S+By7mXw7nAJdc3fzMK/mAuSM7W0LmyIcAszCPCkjX8+WP/cQmBkIhFiOYMr+EjFvBm8+EarWSKMwyUOxMrtIaplMXKskSozEhc5QVmaizOeYak6vLEijh6UGEzarm2PbcyRBis/flL/PM4JiG+y8NThpmsgnBMkVTURUoKAQFDmeJ4SDGWAm7OJD9RXHqI0hBSfgg18wE2unfKH8GT445y8AfRNbT1B63NtdUAqBzi9vqrcTUgunefi+dW8Ya4XjwNQGGoVjGc29svne1djTVAVXMYe2f7ttfFG/FnA3yw3W79oIOftXhvgF/ac+/W7eC9Fu8P+W9vd7t5B++3+PlQm0Uw97p4DUopYfejijdKNpA4p5+fh7coy9g5lT+TU/soQ79zvlcAnVy1PRmQjwWOUahwO0TJkZNyArTCaGokFOMjVi98RtibztWGt8xFawmyrgLf9PHUCsSE0oN8pPiL0MRETkm0V0bd0U6N4EWqmvV0HVzCkW4DnstfRKaHFBVqGkfPkIg6dCJAkQuVNzgZW0tzyr7mUWV1nPMZVA5ItnZ1Ls52JaSsrPNFe2Cb8LqXCJOAr4O+nIQxWZfEbITEYvYyEo59KRbBCIul8xQLy8iKOjQAlRXE9ypGQISI4qjMU+V/zu7FMz0lZnfZ7sjyAu9ime6QMLZbl4SxDVMU4b75wrkOgvFUu6M0Fsu3yLU1vBso6/bAgzpzM1+FCVGxhrG61FQzK1Q8wRIIEE3UQyWUtdD/c7MUXMg7JNIKpoeq9WdEYg4oycoiZqSBspab4y7s90susN+fclY/yTiOcSgnLG1XjVVBRkdfCS47+UmRPqTRAzjSE/+OlFD+wikFjIiQjZoR4cbmblXsXVf1URx57enHDC1SVFcU8zKv4Lrd0DHWoZn2V2WNSXhM9peous879S7NiQKymLzF3q7IG6xm46z80bsuWNpPV4nXFwSD2nKc2myc2lTtuOCDwJhuPqGbO5nNV1aD/q61jHel7vX+wJ0tm39QSwMEFAAAAAgAOmGKXGRCd4gKFAAA7/gAABgAAAB4bC93b3Jrc2hlZXRzL3NoZWV0MS54bWyl3WtT20qawPGvomK2tnaqNrH63n02SRUYc0nAUMCcqX21pWAB3mMsVhbJZD79yDcOtuW2/N/zIgf8qFst9SOp+ydhffpZlH9MnvK8Sv7xPBpPPh88VdXLb53O5P4pf84mH4uXfFxHHoryOavqX8vHzuSlzLPBrNDzqCPT1Haes+H44Mun2WfX5ZdPxWs1Go7z6zKZvD4/Z+Wvo3xU/Px8IA6WH9wMH5+q6QedL59essf8Nq/+9nJd1r913moZDJ/z8WRYjJMyf/h8cCh+6ys1LTBb4vdh/nPy7udkuinfi+KP6S/ng88H6cG06nGe/Lp9GQ2nKzMHSVW8XOQPVTcfjaY12oMku6+GP/LresHPB9+LqiqepwvUDa2yqv7ooSz+mY9na81Heb1s3ZyXjYXnlcxrPRKuLv1/szZPf37bpmnD3v+8bP3JbOfWO+t7Nsm7xejvw0H19PnAHySD/CF7HVV/fib0R63ePr8pfp7lix1Zb9v966Ru0Nsn0xXfF6PJ7N/k57wCLT56u1x0Wel0Y3+N8nktz8Px7KPn7B9/VrIsLlqUlYuycq2sTHeXVYuyan29endZvSir18v63WXNoqxZL9uizXZR1oI2u0VZv15W7S4bln2UghWLtx7e6OIWmyyWfSzWO7lNw8WylwXpZrHsZ7He0f6jE+m7/0SLypYdL2e915kfL7Oj8jirsi+fyuJnUs6Kzo+yj67hQEsms5rqxkyXPVx8UJ9sPh8Mx9MT4W1V1tFhXXX15a7MxpNsfiKpz3qfOlW93mmoc7+o4GhegdpSgVcm+fe/eCnkfyVneTaqnpJuVuZJd5QNn5Pr7Fd91qw6h4Mfw/t8tfZOvTVvmyT32SQZ3aTzcZWX90/Z+LFuRTGuyrpP+sXHpk2T0U2TqbSpEuZ/jrpHt5G2q33arqJtP3kdz/oiGyWnZfH6kpwfN7VbRdt9dh1prN6nsTra2Lpz8zLpZ895Uxt1tI1Ho9e6d8piMklmP94+DfPRINJus0+7TYt2N+9ZE211PboQaaSRdp9G2haNPBwMynwyaWqpjbZUpWnSyyZVcpONB8Xo5Smpo/V55D+T7tPwPnsskvOLxKY2FZHNcftsjtu5OfnWXHHRbekXZfU0GQ7yZHZoTw+Oy3wwvK8XizTe79N4H298WQxe5+fI+jzcuAW+1Zkk0tywT3PDrn09PenWO6l6KgZNrQ3R1h52z95O6b3pKLMsxsP7pD43DSbJ7ILxkJeRTZles/e4PqWtNmbbjl8U/3/sebHf5TR+PV029/C5eK3/9x//9tfGRr+/pk7nKj++COs+dX40NW+vS6OIXxu7T/n9H53eyV3Sf33+vt6Li8bFr4p14Q/TC+KH5a79UA9utFNCxvbx4hrpPsY2YbXI8krld2/04jplt231xeH5ZXJ9+N+Xvf5dcty7O6xPfssM715cd25/7ya3vdNp+LZxlyxW4N467C3WjcSOI7FeJHYSiZ1GYmeR2Hkk9jUS+xaJXURil5FYfxnzK7HV7l9c8KWJpswiAeaXbbet/2dj0eYL/o6i11k1nB7P8YFkt2UtW65/xzuKT899SfFQD9DLzWH0IpviNdzk40FeDsePSX0t+1FfSutx2/V5Q0Unu3bl9V29LwZNbTjdUfRoOBrlg/iZ8WxHHYejUfFzVyXnO3tjuKOGr7uaMfjfeAXfWlRwk2eTekSxZW9e7OrRw5tuQ7HLlpl4k09etrS93+pouq2y6nXS0PrVo3ivEbGYj2n91tP45Yd6Su0bD+N40evurOSHVKSm8fiNj6a/9vq3vX5y1njsxtc8u0o2r7W3q6jSxjof1sfoiwM1XjgEuT7qWRyl78vNxx7Sp29jj8Vh2LCQWFvovGEhsV7T14aF3Noy33b1+9UH3bQpFxt1vzsMNler1lbb37Ha2MxI7DU1EvP5TYjmtVKNeR0vOstrpT6kUurGvF7MytJt3nB1e9e7SXqNiR1f9Syxm1fb21U0ntjxwtPEblrp6Uq5Rc7KjcTeXEhYs57YDQtpuZ7YmwsZv57YOzp+a2Kv1/0usRu2ciOx46uNJvZe02axmDdvnW7MBt53V3eHF83j6pXya+Pq7bHjSKwXiZ1EYqebsbpD0438aVhKObmeQE1LyY0MalhKyI0c2t7mi0jssnl71jOlsYLVjAh7T91k2nrqJtP41O365ur38+P6FHXR+713kRwdXhz2u723ydv1xVF04rasvmniFokdR2K9SOwkEjuNxM4isfNI7Gsk9i0Su4jELiOx/jIWmbjJBadIvbvzRXysWnfuTf6Ql/n4Pt8yfWtRxZYBdrdF0eN8cl8OX6by13SF3FVDfKrV21H8qhw+DqfYGZm+nrSt4/iq6fg4bVt8NmXaMmXbUUc91amzYIFiDfOl1eTZ7z7QArv0tjlSr3v1t+sZVE1H/q4xgXbU8ferxtxZlNo2Ybj6kZcvCwq8ye+L+tdfjRm0svbF5fzd6OjHSr7saOp8HClsY540rWg2NRFro5/TpkXF+iXurGmpD++G2KvdutctMql2bOdN73jRqRuyu+jUnTU0dqqKd+rb8dwt80Hds8NsNBWVk7zRdHa1IToI7q2WXrvIbI+dbsamYwqz3nkNS314v9hq7+11z1Dq+Jhwel7dPiJcLb12wd4eO47EepHYSSR2Gomdbcame9CszyQiVXyNxL5FYheR2GUk1m+Orfa02X+sZ9uP9Wx8rHd3c9i/PezenV/160HdXVL/fn5RD/2Wg73b3nKs15g5NjLU2x47jsR6kdhJJHYaiZ1FYueR2NdI7FskdhGJXUZi/WUsNtTbS0Fk/A7xXVG9jXW6U1pt7OT3d4oXV7EtJ629JrIyfgN43rbdbnwk/Ub73s/7Vlu41y1fGb/nu2hhfW79c7w12drKsNHKlXPX6tMte93NVfG7uf28SqZ3H9vdJlXpRju33iZVe93FVXs9FbXrMSIVf0QqbXGDVO018FXxu7wbTxHtan+756DS6MMiar/HoOLPQe3xDJeKPwzVaue3HOCsFmr5RNJqoZb3Q1YLtTzNrhZqef5bLdTylLT6RFnLE8RqoZaH62qhlgfJaqGWeblaiGSEJhmhSULine truncated
}
]
},
{
"email_id": "2",
"folder": "INBOX",
"subject": "999 Acknowledgement \u2014 Batch 20260312-NIM \u2014 Accepted",
"from_addr": "clearinghouse@availity.net",
"to_addr": "sanjeeta.kaur@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-12T14:08:00Z",
"message_id": "<availity-ack-20260312-NIM-001@availity.net>",
"in_reply_to": null,
"body_text": "Availity Clearinghouse \u2014 Transaction Acknowledgement\n\nBatch ID: 20260312-NIM\nSubmitted by: Sanjeeta Kaur, Pathfinder Billing and Coding\nSubmission date: 12 March 2026 13:47 CT\nStatus: ACCEPTED \u2014 No errors detected\n\nAll 837P files in this batch have passed format and payer validation. Claims are now queued for payer adjudication.\n\nPlease update your claim register to Acknowledged for the following NPI: 2345678901.\n\nAvaility EDI Operations",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "3",
"folder": "INBOX",
"subject": "RE: Portal Login Issue \u2014 Northside Internal Medicine",
"from_addr": "billing@northsideinternal.com",
"to_addr": "sanjeeta.kaur@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-13T11:55:00Z",
"message_id": "<northside-portal-re-20260313@northsideinternal.com>",
"in_reply_to": "<pbc-portal-20260312-001@pathfinderbilling.com>",
"body_text": "Hi Sanjeeta,\n\nThank you for looking into this. The login issue was on our end \u2014 our office manager reset the password this morning and access has been restored.\n\nCould you also let us know the status of the claims we submitted in late February? We haven't had an update in a while.\n\nThanks,\nDeirdre Walsh\nBilling Office Manager\nNorthside Internal Medicine",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "4",
"folder": "Sent",
"subject": "RE: Portal Login Issue \u2014 Northside Internal Medicine",
"from_addr": "sanjeeta.kaur@pathfinderbilling.com",
"to_addr": "billing@northsideinternal.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-12T16:45:00Z",
"message_id": "<pbc-portal-20260312-001@pathfinderbilling.com>",
"in_reply_to": null,
"body_text": "Hi Deirdre,\n\nThank you for reaching out. I have logged a support ticket for the portal access issue and our team is investigating. I will follow up as soon as we have a resolution.\n\nRegarding your recent submissions, I will pull a current status report and send it across shortly.\n\nBest regards,\nSanjeeta Kaur\nBilling Specialist\nPathfinder Billing and Coding\nsanjeeta.kaur@pathfinderbilling.com",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "5",
"folder": "Sent",
"subject": "837P Batch Submission \u2014 Northside Internal Medicine \u2014 12 Mar 2026",
"from_addr": "sanjeeta.kaur@pathfinderbilling.com",
"to_addr": "clearinghouse@availity.net",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-12T13:47:00Z",
"message_id": "<pbc-batch-20260312-NIM@pathfinderbilling.com>",
"in_reply_to": null,
"body_text": "Availity EDI Operations,\n\nPlease find attached the 837P batch file for the following provider for the 4:00 PM CT submission window:\n\n Provider: Northside Internal Medicine\n NPI: 2345678901\n Batch ID: 20260312-NIM\n Claim count: 2\n\nPlease confirm receipt via 999 acknowledgement.\n\nSanjeeta Kaur\nBilling Specialist\nPathfinder Billing and Coding",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "6",
"folder": "INBOX",
"subject": "Weekly Team Standup Notes \u2014 13 Mar 2026",
"from_addr": "priya.nair@pathfinderbilling.com",
"to_addr": "rcm-team@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-13T17:02:00Z",
"message_id": "<pbc-standup-20260313@pathfinderbilling.com>",
"in_reply_to": null,
"body_text": "Hi team,\n\nNotes from today's standup below.\n\nCLAIMS REGISTER HIGHLIGHTS\n- CLM-1032 remains on hold pending QA \u2014 Claire to complete review by EOD Monday.\n- CLM-1020 escalated to RCM Manager this morning. Dominic handling.\n- CLM-1034 provider query sent 09 Mar \u2014 response window closes Tuesday. Monitor closely.\n\nERA POSTING\n- Anticipating a BCBS ERA for the Northside batch submitted last month. Sanjeeta to monitor inbox and post on receipt.\n\nOTHER\n- Reminder: daily posting log must be uploaded with RCM Manager sign-off before 4:00 PM CT.\n- CLM-1012 patient balance outstanding \u2014 PFR reviewing write-off eligibility.\n\nSee everyone Monday.\nPriya",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "7",
"folder": "INBOX",
"subject": "Action Required \u2014 CLM-1024 CO-16 Additional Information Request",
"from_addr": "provider.services@cignagroup.com",
"to_addr": "sanjeeta.kaur@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-11T09:31:00Z",
"message_id": "<cigna-addl-info-clm1024-20260311@cignagroup.com>",
"in_reply_to": null,
"body_text": "Dear Provider Representative,\n\nWe are writing regarding the following claim, which requires additional information before processing can continue:\n\n Claim Reference: CLM-1024\n Patient Control: PC-1024-0201\n Date of Service: 01 February 2026\n CPT Code: 99214\n Payer: UnitedHealth\n Denial Code: CO-16\n RARC: M51\n\nOur records indicate that the referring provider NPI is missing from the submitted claim. Please resubmit with the complete referring provider information using Claim Frequency Code 7.\n\nIf you believe this claim was submitted correctly, please contact Cigna Provider Services at 1-800-88-CIGNA.\n\nCigna Provider Services\nClaims Resolution Department",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "8",
"folder": "Sent",
"subject": "RE: Action Required \u2014 CLM-1024 CO-16 Additional Information Request",
"from_addr": "sanjeeta.kaur@pathfinderbilling.com",
"to_addr": "m.okafor@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-11T11:15:00Z",
"message_id": "<pbc-clm1024-cmc-20260311@pathfinderbilling.com>",
"in_reply_to": "<cigna-addl-info-clm1024-20260311@cignagroup.com>",
"body_text": "Hi Marcus,\n\nRouting CLM-1024 to you \u2014 UnitedHealth is requesting the referring provider NPI which is missing from the original submission (CO-16, RARC M51). Can you check the chart and confirm the correct NPI so I can correct and resubmit with Frequency Code 7?\n\nClaim details:\n Claim ID: CLM-1024\n Patient: PT-122, Martin Holloway\n DOS: 01 February 2026\n CPT: 99214\n Payer: UnitedHealth\n\nThanks,\nSanjeeta",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
},
{
"email_id": "9",
"folder": "INBOX",
"subject": "Claim Status Enquiry \u2014 Riverside Family Clinic",
"from_addr": "billing@riversideclinic.com",
"to_addr": "sanjeeta.kaur@pathfinderbilling.com",
"cc_addr": null,
"bcc_addr": null,
"date": "2026-03-16T08:45:00Z",
"message_id": "<riverside-status-20260316@riversideclinic.com>",
"in_reply_to": null,
"body_text": "Hi Sanjeeta,\n\nHope you had a good weekend. We wanted to check in on a couple of open items.\n\nWe submitted a claim for a patient with a left knee arthroscopy in mid-February and have not yet received a payment confirmation from UnitedHealth. Could you let us know where that stands?\n\nWe also received a notice from BCBS about an account adjustment on an older claim but the details were not clear to us. Is there anything outstanding on our BCBS account that we should be aware of?\n\nThanks for your help.\n\nKind regards,\nTom Breen\nRiverside Family Clinic",
"body_html": null,
"is_read": true,
"is_important": false,
"attachments": []
}
],
"next_email_id": 10
}
@@ -0,0 +1 @@
Hi, can you deal with the unread email in my inbox, as per the processes in the SOP (PBC_SOP_RCM024_Rev3_28.pdf). I've already downloaded its attachment. Thanks.
@@ -0,0 +1 @@
Today's date is, March, 16, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
@@ -0,0 +1,23 @@
schema_version = "1.1"
[task]
name = "sop-tasks/medical_pathfinder_billing_and_coding_1d8667fc"
description = "SOP task from CSV row 1d8667fc"
[agent]
timeout_sec = 3600.0
[verifier]
timeout_sec = 3600.0
[environment]
os = "linux"
cpus = 2
memory_mb = 4096
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
[[environment.mcp_servers]]
name = "mcp-proxy"
transport = "streamable-http"
url = "http://localhost:8000/mcp"
@@ -0,0 +1,37 @@
[
{
"id": "495d3ee8-810b-47fd-8706-2838d3b884ca",
"sort_order": 0,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cells for CLM-1008 (Harold Jensen) in row 9 must have: M9=210.00 (Allowed Amount), N9=180.00 (Paid Amount), O9=70.00 (Contractual Adj), P9=30.00 (Patient Rep), Q9='Paid - Full' (Claim Status), R9='Posted' (Payment Posting Status), Y9='20260315_BCBS' (ERA Batch ID)",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find the claim register file - be loose with naming\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n # Try any xlsx file\n for f in xlsx_files:\n if 'claim' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'No claim register xlsx file found in workspace. Files found: ' + str([f.name for f in xlsx_files])}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n # First, try to find the row for CLM-1008 / Harold Jensen dynamically\n target_row = 9 # default from rubric\n for row in range(1, ws.max_row + 1):\n for col in range(1, min(ws.max_column + 1, 10)):\n cell_val = ws.cell(row=row, column=col).value\n if cell_val is not None and 'CLM-1008' in str(cell_val).upper().replace(' ', ''):\n target_row = row\n break\n \n issues = []\n found_values = {}\n \n # Helper to get cell value by column letter and row\n def get_cell(col_letter, row):\n return ws[f'{col_letter}{row}'].value\n \n # Check M: Allowed Amount = 210.00\n m_val = get_cell('M', target_row)\n try:\n m_num = float(m_val) if m_val is not None else None\n if m_num is None or abs(m_num - 210.00) > 210.00 * 0.02:\n issues.append(f'M{target_row} (Allowed Amount) expected 210.00, got {m_val}')\n else:\n found_values['M'] = m_num\n except (TypeError, ValueError):\n issues.append(f'M{target_row} (Allowed Amount) expected 210.00, got {m_val}')\n \n # Check N: Paid Amount = 180.00\n n_val = get_cell('N', target_row)\n try:\n n_num = float(n_val) if n_val is not None else None\n if n_num is None or abs(n_num - 180.00) > 180.00 * 0.02:\n issues.append(f'N{target_row} (Paid Amount) expected 180.00, got {n_val}')\n else:\n found_values['N'] = n_num\n except (TypeError, ValueError):\n issues.append(f'N{target_row} (Paid Amount) expected 180.00, got {n_val}')\n \n # Check O: Contractual Adj = 70.00\n o_val = get_cell('O', target_row)\n try:\n o_num = float(o_val) if o_val is not None else None\n if o_num is None or abs(o_num - 70.00) > 70.00 * 0.02:\n issues.append(f'O{target_row} (Contractual Adj) expected 70.00, got {o_val}')\n else:\n found_values['O'] = o_num\n except (TypeError, ValueError):\n issues.append(f'O{target_row} (Contractual Adj) expected 70.00, got {o_val}')\n \n # Check P: Patient Rep = 30.00\n p_val = get_cell('P', target_row)\n try:\n p_num = float(p_val) if p_val is not None else None\n if p_num is None or abs(p_num - 30.00) > 30.00 * 0.02:\n issues.append(f'P{target_row} (Patient Rep) expected 30.00, got {p_val}')\n else:\n found_values['P'] = p_num\n except (TypeError, ValueError):\n issues.append(f'P{target_row} (Patient Rep) expected 30.00, got {p_val}')\n \n # Check Q: Claim Status = 'Paid - Full'\n q_val = get_cell('Q', target_row)\n q_str = str(q_val).strip() if q_val is not None else ''\n if q_val is None or ('paid' not in q_str.lower() or 'full' not in q_str.lower()):\n issues.append(f'Q{target_row} (Claim Status) expected \"Paid - Full\", got {q_val}')\n else:\n found_values['Q'] = q_str\n \n # Check R: Payment Posting Status = 'Posted'\n r_val = get_cell('R', target_row)\n r_str = str(r_val).strip() if r_val is not None else ''\n if r_val is None or 'posted' not in r_str.lower():\n issues.append(f'R{target_row} (Payment Posting Status) expected \"Posted\", got {r_val}')\n else:\n found_values['R'] = r_str\n \n # Check Y: ERA Batch ID = '20260315_BCBS'\n y_val = get_cell('Y', target_row)\n y_str = str(y_val).strip() if y_val is not None else ''\n if y_val is None or y_str != '20260315_BCBS':\n issues.append(f'Y{target_row} (ERA Batch ID) expected \"20260315_BCBS\", got {y_val}')\n else:\n found_values['Y'] = y_str\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': f'CLM-1008 (Harold Jensen) row {tarLine truncated
"criterion_type": "expected_output"
},
{
"id": "b1182ec4-1939-43e2-8f0e-7a882d3911c4",
"sort_order": 1,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cells for CLM-1033 (Elaine Foster) in row 34 must have: M34=165.00 (Allowed Amount), N34=142.00 (Paid Amount), O34=58.00 (Contractual Adj), P34=20.00 (Patient Rep), Q34='Written Off' (Claim Status), R34='Posted' (Payment Posting Status), Y34='20260315_BCBS' (ERA Batch ID)",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Search for claim register file - be loose with naming\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n # Also try exact name\n if not target_file:\n exact = Path(workspace_path) / 'PBC_Claim_Register.xlsx'\n if exact.exists():\n target_file = exact\n if not target_file:\n # Try any xlsx\n for f in xlsx_files:\n if 'claim' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': f'No claim register xlsx file found in workspace. Files found: {[f.name for f in xlsx_files]}'}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n # Try to find the correct row for CLM-1033 dynamically\n target_row = 34\n # Scan for CLM-1033 in case it's not exactly row 34\n for row in range(1, ws.max_row + 1):\n for col in range(1, min(ws.max_column + 1, 10)):\n cell_val = ws.cell(row=row, column=col).value\n if cell_val is not None and 'CLM-1033' in str(cell_val):\n target_row = row\n break\n \n issues = []\n \n # Check M: Allowed Amount = 165.00\n m_val = ws.cell(row=target_row, column=13).value # M column = 13\n try:\n m_num = float(m_val) if m_val is not None else None\n if m_num is None or abs(m_num - 165.00) > 165.00 * 0.02:\n issues.append(f'M{target_row} (Allowed Amount) expected 165.00, got {m_val}')\n except (TypeError, ValueError):\n issues.append(f'M{target_row} (Allowed Amount) expected 165.00, got {m_val}')\n \n # Check N: Paid Amount = 142.00\n n_val = ws.cell(row=target_row, column=14).value # N column = 14\n try:\n n_num = float(n_val) if n_val is not None else None\n if n_num is None or abs(n_num - 142.00) > 142.00 * 0.02:\n issues.append(f'N{target_row} (Paid Amount) expected 142.00, got {n_val}')\n except (TypeError, ValueError):\n issues.append(f'N{target_row} (Paid Amount) expected 142.00, got {n_val}')\n \n # Check O: Contractual Adj = 58.00\n o_val = ws.cell(row=target_row, column=15).value # O column = 15\n try:\n o_num = float(o_val) if o_val is not None else None\n if o_num is None or abs(o_num - 58.00) > 58.00 * 0.02:\n issues.append(f'O{target_row} (Contractual Adj) expected 58.00, got {o_val}')\n except (TypeError, ValueError):\n issues.append(f'O{target_row} (Contractual Adj) expected 58.00, got {o_val}')\n \n # Check P: Patient Rep = 20.00\n p_val = ws.cell(row=target_row, column=16).value # P column = 16\n try:\n p_num = float(p_val) if p_val is not None else None\n if p_num is None or abs(p_num - 20.00) > 20.00 * 0.02:\n issues.append(f'P{target_row} (Patient Rep) expected 20.00, got {p_val}')\n except (TypeError, ValueError):\n issues.append(f'P{target_row} (Patient Rep) expected 20.00, got {p_val}')\n \n # Check Q: Claim Status = 'Written Off'\n q_val = ws.cell(row=target_row, column=17).value # Q column = 17\n if q_val is None or ('written' not in str(q_val).lower() or 'off' not in str(q_val).lower()):\n issues.append(f'Q{target_row} (Claim Status) expected \"Written Off\", got {q_val}')\n \n # Check R: Payment Posting Status = 'Posted' (rubric says 'Posted')\n r_val = ws.cell(row=target_row, column=18).value # R column = 18\n if r_val is None or 'posted' not in str(r_val).lower():\n issues.append(f'R{target_row} (Payment Posting Status) expected \"Posted\", got {r_val}')\n \n # Check Y: ERA Batch ID = '20260315_BCBS'\n y_val = ws.cell(row=target_row, column=25).value # Y column = 25\n y_str = str(y_val).strip() if y_val is not None else ''\n if y_val is None or y_str != '20260315_BCBS':\n issues.append(f'Y{target_row} (ERA Batch ID) expected \"20260315_BCBS\", got {y_val}')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': f'CLM-1033 (Elaine Foster) row {target_row} issues in {target_file.name}: ' + '; '.join(issues)}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'CLM-1033 (Elaine Foster) row {target_row} verified in {target_file.name}: M=165.00, N=142.00, O=58.00, P=20.00, Q={q_val}, R={r_val}, Y={y_val}'}\n except ExceptLine truncated
"criterion_type": "expected_output"
},
{
"id": "4ff463fc-9192-4f9a-a68b-878c966f5510",
"sort_order": 2,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, cell Z17 (Notes for CLM-1016, Sarah Mitchell) must contain text indicating a clawback of $30.00 applied on 2026-03-15 via batch 20260315_BCBS (e.g., 'Clawback applied 2026-03-15: $30.00 via 20260315_BCBS.')",
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n target_file = None\n for f in xlsx_files:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n xlsx_files_all = list(Path(workspace_path).glob('**/*.xlsx'))\n for f in xlsx_files_all:\n if 'claim' in f.name.lower() and 'register' in f.name.lower():\n target_file = f\n break\n if not target_file:\n return {'pass': False, 'score': 0.0, 'feedback': 'No claim register xlsx file found in workspace'}\n \n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n ws = wb.active\n \n z17 = ws['Z17'].value\n if z17 is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Z17 (Notes for CLM-1016 Sarah Mitchell) is empty, expected clawback note'}\n \n z17_str = str(z17).lower()\n \n issues = []\n if 'clawback' not in z17_str:\n issues.append('missing keyword \"clawback\"')\n if '30' not in z17_str:\n issues.append('missing amount \"30\"')\n if '2026-03-15' not in z17_str and '20260315' not in z17_str:\n issues.append('missing date \"2026-03-15\" or \"20260315\"')\n if '20260315_bcbs' not in z17_str:\n issues.append('missing batch ID \"20260315_BCBS\"')\n \n if issues:\n joiner = '; '\n return {'pass': False, 'score': 0.0, 'feedback': f'Z17 clawback note issues: {joiner.join(issues)}. Found: \"{z17}\"'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Z17 clawback note verified for CLM-1016 (Sarah Mitchell): \"{z17}\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading xlsx file {target_file.name}: {str(e)}'}\n",
"criterion_type": "expected_output"
},
{
"id": "e87a6e28-1fa3-4df9-b6fd-d13b2aeeb509",
"sort_order": 3,
"rubric_text": "In `mailbox.json`, there must be exactly 3 emails in the Sent folder",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None; cannot check mailbox.json'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'mailbox.json not found at {mailbox_path}'}\n \n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n count = len(sent_emails)\n \n if count != 3:\n return {'pass': False, 'score': 0.0, 'feedback': f'Expected exactly 3 sent emails in mailbox.json, found {count}. Sent email subjects: {[e.get(\"subject\", \"\") for e in sent_emails]}'}\n \n return {'pass': True, 'score': 1.0, 'feedback': f'Found exactly 3 sent emails in mailbox.json. Subjects: {[e.get(\"subject\", \"\") for e in sent_emails]}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading mailbox.json: {str(e)}'}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1775866205434",
"sort_order": 4,
"rubric_text": "In file `PBC_Claim_Register.xlsx`, the financial and status fields for CLM-1016 must remain unchanged from their original values: L17=220 (Billed Amount), M17=162 (Allowed Amount), N17=172 (Paid Amount), O17=58 (Contractual Adj), P17=20 (Patient Resp), Q17='Paid - Full' (Claim Status), R17='Reconciled' (Payment Posting Status), S17=empty (CO Code), U17=empty (Recovery Action), V17=empty (Recovery Status), Y17='20260120_BCBS' (ERA Batch ID).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Claim_Register.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening workbook: {e}\"}\n\n row = 17\n issues = []\n details = []\n\n def check_numeric(cell_ref, expected, label):\n val = ws[cell_ref].value\n if val is None:\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got empty/None\")\n return\n try:\n num_val = float(val)\n except (ValueError, TypeError):\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got non-numeric '{val}'\")\n return\n if abs(num_val - expected) > 0.01:\n issues.append(f\"{cell_ref} ({label}): expected {expected}, got {num_val}\")\n else:\n details.append(f\"{cell_ref} ({label}): {num_val} == {expected} OK\")\n\n def check_string(cell_ref, expected, label):\n val = ws[cell_ref].value\n if val is None:\n val_str = \"\"\n else:\n val_str = str(val).strip()\n if val_str.lower() != expected.lower():\n issues.append(f\"{cell_ref} ({label}): expected '{expected}', got '{val_str}'\")\n else:\n details.append(f\"{cell_ref} ({label}): '{val_str}' OK\")\n\n def check_empty(cell_ref, label):\n val = ws[cell_ref].value\n if val is not None and str(val).strip() != \"\":\n issues.append(f\"{cell_ref} ({label}): expected empty, got '{val}'\")\n else:\n details.append(f\"{cell_ref} ({label}): empty OK\")\n\n # First verify row 17 actually contains CLM-1016 (likely in an early column)\n # We'll check common columns A-D for the claim ID\n claim_found = False\n for col_letter in ['A', 'B', 'C', 'D', 'E']:\n cell_val = ws[f\"{col_letter}{row}\"].value\n if cell_val is not None and 'CLM-1016' in str(cell_val).strip().upper():\n claim_found = True\n break\n if not claim_found:\n # Still proceed but note it\n details.append(f\"Note: CLM-1016 identifier not found in columns A-E of row {row}, but checking specified cells anyway.\")\n\n # Numeric checks\n check_numeric(f\"L{row}\", 220, \"Billed Amount\")\n check_numeric(f\"M{row}\", 162, \"Allowed Amount\")\n check_numeric(f\"N{row}\", 172, \"Paid Amount\")\n check_numeric(f\"O{row}\", 58, \"Contractual Adj\")\n check_numeric(f\"P{row}\", 20, \"Patient Resp\")\n\n # String checks\n check_string(f\"Q{row}\", \"Paid - Full\", \"Claim Status\")\n check_string(f\"R{row}\", \"Reconciled\", \"Payment Posting Status\")\n check_string(f\"Y{row}\", \"20260120_BCBS\", \"ERA Batch ID\")\n\n # Empty checks\n check_empty(f\"S{row}\", \"CO Code\")\n check_empty(f\"U{row}\", \"Recovery Action\")\n check_empty(f\"V{row}\", \"Recovery Status\")\n\n wb.close()\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"CLM-1016 fields were modified. Issues: \" + \"; \".join(issues) + \". Verified: \" + \"; \".join(details)\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"All CLM-1016 fields remain unchanged. Details: \" + \"; \".join(details)\n }\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
Hi, it's 16 March 2026. Can you deal with the unread email in my inbox, as per the processes in the SOP (PBC_SOP_RCM024_Rev3_28.pdf). I've already downloaded its attachment. Thanks.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py