Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-1": {
|
||||
"id": "evt-1",
|
||||
"summary": "Weekly Team Huddle",
|
||||
"start": {
|
||||
"dateTime": "2026-04-06T10:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-06T11:00:00-06:00"
|
||||
}
|
||||
},
|
||||
"evt-2": {
|
||||
"id": "evt-2",
|
||||
"summary": "Preparation for Quarterly Board meeting",
|
||||
"start": {
|
||||
"dateTime": "2026-04-07T14:00:00-06:00"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-07T15:00:00-06:00"
|
||||
}
|
||||
},
|
||||
"evt-3": {
|
||||
"id": "evt-3",
|
||||
"summary": "OOO",
|
||||
"start": {
|
||||
"date": "2026-04-06"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-09"
|
||||
},
|
||||
"description": "Out of Office"
|
||||
},
|
||||
"evt-4": {
|
||||
"id": "evt-4",
|
||||
"summary": "D15 to TFL (BGI-31199221)",
|
||||
"start": {
|
||||
"date": "2026-04-15"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-16"
|
||||
}
|
||||
},
|
||||
"evt-5": {
|
||||
"id": "evt-5",
|
||||
"summary": "D15 to TFL (BGI-33991911)",
|
||||
"start": {
|
||||
"date": "2026-04-17"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-18"
|
||||
}
|
||||
},
|
||||
"evt-6": {
|
||||
"id": "evt-6",
|
||||
"summary": "D15 to TFL (BGI-36110292)",
|
||||
"start": {
|
||||
"date": "2026-05-02"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-05-03"
|
||||
}
|
||||
},
|
||||
"evt-7": {
|
||||
"id": "evt-7",
|
||||
"summary": "D15 to TFL (BGI-37918822, BGI-39882091, BGI-39667288)",
|
||||
"start": {
|
||||
"date": "2026-05-14"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-05-15"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "jillm@pathfinder.com",
|
||||
"name": "Jill Mayfield"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "jamies@pathfinder.com",
|
||||
"name": "Jamie Scott"
|
||||
},
|
||||
{
|
||||
"email": "joem@pathfinder.com",
|
||||
"name": "Joe Malfroy"
|
||||
},
|
||||
{
|
||||
"email": "mayj@pathfinder.com",
|
||||
"name": "May Jollier"
|
||||
},
|
||||
{
|
||||
"email": "russellc@pathfinder.com",
|
||||
"name": "Russell Commy"
|
||||
},
|
||||
{
|
||||
"email": "hopeK@email.com",
|
||||
"name": "Kelly Hope"
|
||||
},
|
||||
{
|
||||
"email": "ursula.james@hull.com",
|
||||
"name": "Ursula James"
|
||||
},
|
||||
{
|
||||
"email": "popler.wen@bulliongoldinsurance.com",
|
||||
"name": "Popler Wen"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "rcm-internal@pathfinder.com",
|
||||
"name": "RCM Internal Team",
|
||||
"members": [
|
||||
"jillm@pathfinder.com",
|
||||
"jamies@pathfinder.com",
|
||||
"joem@pathfinder.com",
|
||||
"mayj@pathfinder.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Patient ID - FQ12578",
|
||||
"from_addr": "ursula.james@hull.com",
|
||||
"to_addr": "jillm@pathfinder.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-03T18:45:00-05:00",
|
||||
"message_id": "<msg001@hull.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Jill\n\nThanks for the call earlier today to clarify about Mr Edward Tee's coverage and the upcoming claims to expect from Pathfinder. As spoken, please send through the next 837P batch and let your team know to drop me an email that we can expedite the processing for this particular batch containing Mr Tee's claims.\n\nBest regards,\nUrsula James\nCustomer Sales Manager, Hull Insurance",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Get well soon!",
|
||||
"from_addr": "mayj@pathfinder.com",
|
||||
"to_addr": "jillm@pathfinder.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-06T10:32:00-05:00",
|
||||
"message_id": "<msg002@pathfinder.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hey girl, I heard you are unwell, take care and see you in office soon!",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 3
|
||||
}
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "jamie.scott",
|
||||
"deleted": false,
|
||||
"real_name": "Jamie Scott",
|
||||
"profile": {
|
||||
"real_name": "Jamie Scott",
|
||||
"display_name": "jamies",
|
||||
"email": "jamies@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "joe.malfroy",
|
||||
"deleted": false,
|
||||
"real_name": "Joe Malfroy",
|
||||
"profile": {
|
||||
"real_name": "Joe Malfroy",
|
||||
"display_name": "joem",
|
||||
"email": "joem@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "may.jollier",
|
||||
"deleted": false,
|
||||
"real_name": "May Jollier",
|
||||
"profile": {
|
||||
"real_name": "May Jollier",
|
||||
"display_name": "mayj",
|
||||
"email": "mayj@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "jill.mayfield",
|
||||
"deleted": false,
|
||||
"real_name": "Jill Mayfield",
|
||||
"profile": {
|
||||
"real_name": "Jill Mayfield",
|
||||
"display_name": "jillm",
|
||||
"email": "jillm@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "escalations-rcm-mgr",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "escalations-rcm-mgr",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1765358880,
|
||||
"creator": "U001",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "RCM Manager Escalations",
|
||||
"creator": "U001",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for escalating issues to RCM managers",
|
||||
"creator": "U001",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "ra",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ra",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1775348700,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Resolution Auditors",
|
||||
"creator": "U003",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for Resolution Auditors discussion, escalations and notifications",
|
||||
"creator": "U003",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "billing-specs",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "billing-specs",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Billing Specialists",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for Billing Specialists discussion, escalations and notifications",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "coders",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "coders",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Medical Coders",
|
||||
"creator": "U004",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for Certified Medical Coders (CMC) team discussions & notifications",
|
||||
"creator": "U004",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Majestic's system is down & estimated to be back up earliest by 6pm today. 837P saved.",
|
||||
"ts": "1765384320.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Noted, will email the office mgrs. Thanks Jamie.",
|
||||
"ts": "1765384480.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C002": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Escalation: 04/03/2026. Category: Operational. Payer: Bullion Gold Insurance. Issue: Credit balance in payer account. Action/s Taken: Verifications against Claim Register, duplicate payer payments for 6 claims. Escalation log updated.",
|
||||
"ts": "1775250300.000100",
|
||||
"team": "T001"
|
||||
}
|
||||
],
|
||||
"C003": [],
|
||||
"C004": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,20 @@
|
||||
Hi, it's Jill. I am still on sick leave today, but recalled that there is an overpayment case under me that might be due today or yesterday. Refer to the escalation log for case details.
|
||||
|
||||
Some instructions:
|
||||
|
||||
1) Draft an email to the payer to inform them of the overpayment situation, and give them the details of the claims that caused the problem (IDs & Amounts for each). Mention that any excess received from the overpayment will be either re-classed for yet unpaid claims for the same patient or processed for refund (subject to approval) back to them, and that updates will be provided as the situation evolves.
|
||||
|
||||
2) If any refund is going to be necessary, draft an email to Joe. The email should inform him of:
|
||||
- The fact that the payer (give their name) will require a refund
|
||||
- The problematic claim IDs
|
||||
- The exact amount that was overpaid
|
||||
- The exact amount that can be re-classed to each unpaid claim (provide the claim IDs along with the amount that would be re-classed to each)
|
||||
- The exact amount that is expected to be refunded
|
||||
|
||||
A refund is necessary if there are any unpaid claims by this patient for the same payer EXCEPT for the claims that caused the overpayment situation. The refund amount is the overpaid amount minus the amount that can be re-classed.
|
||||
|
||||
3) Do NOT send any email out yet, leave them in draft and I will review them before sending them out.
|
||||
|
||||
4) Add May into the BCC for emails (1) and (2).
|
||||
|
||||
On top of those instructions, ensure that everything required by the SOP in this situation is done.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 9, 2026, 1:30 PM. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/medical_pathfinder_billing_and_coding_bbf1a560"
|
||||
description = "SOP task from CSV row bbf1a560"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"id": "5a168b3e-f630-4242-8d93-3c57a0b096b4",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json`, there must be no emails in the Sent folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n if sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(sent_emails)} email(s) in Sent folder; expected none\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No emails found in Sent folder\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "186f0d01-4504-4393-b453-11f3e54d335c",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `mailbox.json`, there must be exactly 2 emails in the Drafts folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n draft_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() in [\"drafts\", \"draft\"]]\n if len(draft_emails) != 2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(draft_emails)} draft email(s); expected exactly 2\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found exactly 2 emails in Drafts folder\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "be8ac84d-c515-4bb2-adfb-413166231d6c",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `mailbox.json`, one draft email must be addressed to joem@pathfinder.com with mayj@pathfinder.com as BCC, and contain 'Bullion Gold Insurance', 'refund', claim IDs 'BGI-31199221', 'BGI-33991911', 'BGI-36110292', 'BGI-37918822', 'BGI-39667288', 'BGI-39882091', 'BGI-38399991' associated with $350, 'BGI-40001965' associated with $75, '$850', and '$1,275'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n draft_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() in [\"drafts\", \"draft\"]]\n \n # Find the draft addressed to joem@pathfinder.com\n joe_drafts = []\n for e in draft_emails:\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n if \"joem@pathfinder.com\" in to_addr.lower():\n joe_drafts.append(e)\n \n if not joe_drafts:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No draft email addressed to joem@pathfinder.com found\"}\n \n failures = []\n for email in joe_drafts:\n f_list = []\n # Check BCC\n bcc = (email.get(\"bcc_addr\") or email.get(\"bcc\") or \"\").lower()\n # Also check body/headers combined\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n full_text = body + \" \" + subject\n \n if \"mayj@pathfinder.com\" not in bcc:\n f_list.append(\"BCC does not contain mayj@pathfinder.com\")\n \n required_strings = [\n \"bullion gold insurance\",\n \"refund\",\n \"bgi-31199221\",\n \"bgi-33991911\",\n \"bgi-36110292\",\n \"bgi-37918822\",\n \"bgi-39667288\",\n \"bgi-39882091\",\n \"bgi-38399991\",\n \"bgi-40001965\",\n ]\n for s in required_strings:\n if s not in full_text:\n f_list.append(f\"Missing '{s}' in email body\")\n \n # Check BGI-38399991 associated with $350\n # Check BGI-40001965 associated with $75\n # Check $850 and $1,275 present\n amount_checks = [\"850\", \"1,275\", \"1275\"]\n found_850 = \"850\" in full_text\n found_1275 = \"1,275\" in full_text or \"1275\" in full_text\n if not found_850:\n f_list.append(\"Missing '$850' in email body\")\n if not found_1275:\n f_list.append(\"Missing '$1,275' (or '$1275') in email body\")\n \n # Check $350 in context of BGI-38399991\n if \"bgi-38399991\" in full_text and \"350\" not in full_text:\n f_list.append(\"Missing '$350' associated with BGI-38399991\")\n \n # Check $75 in context of BGI-40001965\n if \"bgi-40001965\" in full_text and \"75\" not in full_text:\n f_list.append(\"Missing '$75' associated with BGI-40001965\")\n \n if not f_list:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Draft email to joem@pathfinder.com contains all required content with correct BCC\"}\n failures.append(f_list)\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Draft email to joem@pathfinder.com failed checks: {failures[0]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "c8a402ed-e9dc-4be5-aa7c-7e182d9a1f3c",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, one draft email must be addressed to popler.wen@bulliongoldinsurance.com with mayj@pathfinder.com as BCC, contain 'overpayment'/'overpaid', 're-classed', 'unpaid claims', 'same patient', 'refund', 'subject to approval', 'updates', and claim ID/amount pairs: BGI-31199221/$350, BGI-33991911/$75, BGI-36110292/$75, BGI-37918822/$75, BGI-39667288/$350, BGI-39882091/$350",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n draft_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() in [\"drafts\", \"draft\"]]\n \n payer_drafts = []\n for e in draft_emails:\n to_addr = (e.get(\"to_addr\") or \"\").lower()\n if \"popler.wen@bulliongoldinsurance.com\" in to_addr:\n payer_drafts.append(e)\n \n if not payer_drafts:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No draft email addressed to popler.wen@bulliongoldinsurance.com found\"}\n \n for email in payer_drafts:\n f_list = []\n bcc = (email.get(\"bcc_addr\") or email.get(\"bcc\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n full_text = body + \" \" + subject\n \n if \"mayj@pathfinder.com\" not in bcc:\n f_list.append(\"BCC does not contain mayj@pathfinder.com\")\n \n # Check for overpayment/overpaid\n if not any(word in full_text for word in [\"overpayment\", \"overpaid\", \"over payment\", \"over paid\"]):\n f_list.append(\"Missing 'overpayment' or 'overpaid'\")\n \n required_phrases = [\"re-classed\", \"unpaid claims\", \"same patient\", \"refund\", \"subject to approval\", \"updates\"]\n # Be flexible with re-classed (also accept reclassed, re classed)\n reclassed_variants = [\"re-classed\", \"reclassed\", \"re classed\"]\n if not any(v in full_text for v in reclassed_variants):\n f_list.append(\"Missing 're-classed' or equivalent\")\n \n for phrase in [\"unpaid claims\", \"same patient\", \"refund\", \"subject to approval\", \"updates\"]:\n if phrase not in full_text:\n f_list.append(f\"Missing '{phrase}'\")\n \n # Check claim ID/amount pairs\n # BGI-31199221 with $350\n # BGI-33991911 with $75\n # BGI-36110292 with $75\n # BGI-37918822 with $75\n # BGI-39667288 with $350\n # BGI-39882091 with $350\n claim_ids = [\"bgi-31199221\", \"bgi-33991911\", \"bgi-36110292\", \"bgi-37918822\", \"bgi-39667288\", \"bgi-39882091\"]\n for cid in claim_ids:\n if cid not in full_text:\n f_list.append(f\"Missing claim ID '{cid.upper()}'\")\n \n # Check amounts present\n if \"350\" not in full_text:\n f_list.append(\"Missing '$350' amount\")\n if \"75\" not in full_text:\n f_list.append(\"Missing '$75' amount\")\n \n if not f_list:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Draft email to popler.wen@bulliongoldinsurance.com contains all required content with correct BCC\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Draft email to payer failed checks: {f_list}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No valid payer draft email found\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "cfa447fa-41b4-4058-b1f5-deeb4e157b27",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `PBC_Claim_Register.xlsx`, cells I40, I41, I46, I47, I48, and I49 must each contain the value 'Unapplied'",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_files = list(Path(workspace_path).glob(\"*Claim_Register*.xlsx\")) + list(Path(workspace_path).glob(\"*claim_register*.xlsx\"))\n if not wb_files:\n wb_files = list(Path(workspace_path).glob(\"PBC_Claim_Register.xlsx\"))\n if not wb_files:\n # Try case-insensitive search\n all_xlsx = list(Path(workspace_path).glob(\"*.xlsx\"))\n wb_files = [f for f in all_xlsx if \"claim\" in f.name.lower() and \"register\" in f.name.lower()]\n if not wb_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Claim_Register.xlsx not found in workspace\"}\n \n wb_path = wb_files[0]\n try:\n wb = openpyxl.load_workbook(wb_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n ws = wb.active\n cells_to_check = [\"I40\", \"I41\", \"I46\", \"I47\", \"I48\", \"I49\"]\n failures = []\n for cell_ref in cells_to_check:\n val = ws[cell_ref].value\n if val is None or str(val).strip().lower() != \"unapplied\":\n failures.append(f\"{cell_ref}='{val}'\")\n \n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cells not set to 'Unapplied': {failures}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"All cells I40, I41, I46, I47, I48, I49 contain 'Unapplied'\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a7957495-62c0-4d52-9783-31d6d8e2f574",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, cells row 3 through 8 must each contain 'Financial' as their Category, and 'Escalated to RCM Manager - 2026-04-09' as their Resolution_Tkg.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find the escalation log file\n wb_files = list(Path(workspace_path).glob(\"*Escalation_Log*.xlsx\")) + list(Path(workspace_path).glob(\"*escalation_log*.xlsx\"))\n if not wb_files:\n all_xlsx = list(Path(workspace_path).glob(\"*.xlsx\"))\n wb_files = [f for f in all_xlsx if \"escalation\" in f.name.lower() and \"log\" in f.name.lower()]\n if not wb_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PBC_Escalation_Log.xlsx not found in workspace\"}\n \n wb_path = wb_files[0]\n try:\n wb = openpyxl.load_workbook(wb_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n ws = wb.active\n \n # Determine column indices for Category and Resolution_Tkg from header row\n header_row = 1\n max_col = ws.max_column or 20\n cat_col = None\n res_col = None\n for col_idx in range(1, max_col + 1):\n val = ws.cell(row=header_row, column=col_idx).value\n if val is not None:\n val_str = str(val).strip().lower()\n if val_str in (\"category\",):\n cat_col = col_idx\n if \"resolution\" in val_str or \"tkg\" in val_str:\n res_col = col_idx\n \n # Fallback: if headers not detected, assume common column positions\n # Try row 2 as header as well if row 1 didn't work\n if cat_col is None or res_col is None:\n for try_header in [2]:\n for col_idx in range(1, max_col + 1):\n val = ws.cell(row=try_header, column=col_idx).value\n if val is not None:\n val_str = str(val).strip().lower()\n if val_str in (\"category\",) and cat_col is None:\n cat_col = col_idx\n if (\"resolution\" in val_str or \"tkg\" in val_str) and res_col is None:\n res_col = col_idx\n \n # If still not found, use defaults (common layout: Category might be col D or E, Resolution_Tkg might be col E or later)\n # We'll do a broader search across all columns for the expected values\n if cat_col is None or res_col is None:\n # Scan all columns to find which ones contain 'Financial' and which contain 'Escalated to RCM Manager'\n for col_idx in range(1, max_col + 1):\n for row_idx in range(3, 9):\n val = ws.cell(row=row_idx, column=col_idx).value\n if val is not None:\n val_str = str(val).strip()\n if val_str.lower() == \"financial\" and cat_col is None:\n cat_col = col_idx\n if \"escalated to rcm manager\" in val_str.lower() and res_col is None:\n res_col = col_idx\n \n failures = []\n \n expected_category = \"Financial\"\n expected_resolution = \"Escalated to RCM Manager - 2026-04-09\"\n \n rows_to_check = [3, 4, 5, 6, 7, 8]\n \n # Check Category column\n if cat_col is not None:\n for row in rows_to_check:\n val = ws.cell(row=row, column=cat_col).value\n val_str = str(val).strip() if val is not None else \"\"\n if val_str.lower() != expected_category.lower():\n failures.append(f\"Row {row} Category (col {cat_col})='{val_str}', expected '{expected_category}'\")\n else:\n failures.append(\"Could not identify Category column in the spreadsheet\")\n \n # Check Resolution_Tkg column\n if res_col is not None:\n for row in rows_to_check:\n val = ws.cell(row=row, column=res_col).value\n val_str = str(val).strip() if val is not None else \"\"\n if val_str != expected_resolution:\n # Be a bit lenient: check case-insensitive and allow minor formatting differences\n if val_str.lower().replace(\" \", \"\") != expected_resolution.lower().replace(\" \", \"\"):\n failures.append(f\"Row {row} Resolution_Tkg (col {res_col})='{val_str}', expected '{expected_resolution}'\")\n else:\n failures.append(\"Could not identify Resolution_Tkg column in the spreadsheet\")\n \n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failures found: {'; '.join(failures)}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"All rows 3-8 contain Category='{expected_category}' and Resolution_Tkg='{expected_resolution}'\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "963d503e-dcb7-4cb6-aa32-a208b87bfa94",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `slack_data.json`, channel C001 must contain a message with text 'Escalation: 2026-04-09. Category: Financial. Payer: Bullion Gold Insurance.'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n messages = data.get(\"messages\", {})\n c001_messages = messages.get(\"C001\", [])\n \n expected = \"Escalation: 2026-04-09. Category: Financial. Payer: Bullion Gold Insurance.\"\n expected_lower = expected.lower().strip()\n \n for msg in c001_messages:\n text = (msg.get(\"text\") or \"\").strip().lower()\n # Normalize whitespace\n import re\n text_norm = re.sub(r'\\s+', ' ', text)\n exp_norm = re.sub(r'\\s+', ' ', expected_lower)\n if exp_norm in text_norm or text_norm == exp_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found expected escalation message in C001\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message matching '{expected}' found in C001. Messages found: {[m.get('text','') for m in c001_messages]}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "530d25ac-62c7-4548-a433-f90b592628dd",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `slack_data.json`, channels C003 and C004 must both contain a message that reads 'Refund case - AA45175 - B8475Q'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n messages = data.get(\"messages\", {})\n expected = \"Refund case - AA45175 - B8475Q\"\n expected_lower = expected.lower().strip()\n \n failures = []\n for channel in [\"C003\", \"C004\"]:\n channel_messages = messages.get(channel, [])\n found = False\n for msg in channel_messages:\n text = (msg.get(\"text\") or \"\").strip().lower()\n text_norm = re.sub(r'\\s+', ' ', text)\n exp_norm = re.sub(r'\\s+', ' ', expected_lower)\n if exp_norm in text_norm:\n found = True\n break\n if not found:\n failures.append(f\"{channel} missing message '{expected}'\")\n \n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Missing refund messages: {failures}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Both C003 and C004 contain message '{expected}'\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "718f72d9-4438-443a-8dd4-3d3bc33e95ae",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `calendar_data.json`, there must be an event titled 'Escalation Attempt - Jill - RCM Manager' scheduled on April 9th, 2026 from 1:30 PM to 3:30 PM",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n \n events = data.get(\"events\", {})\n expected_title = \"Escalation Attempt - Jill - RCM Manager\"\n expected_date = \"2026-04-09\"\n expected_start_time = \"13:30\"\n expected_end_time = \"15:30\"\n \n for event_id, event in events.items():\n summary = (event.get(\"summary\") or \"\").strip()\n if summary.lower() != expected_title.lower():\n continue\n \n start = event.get(\"start\", \"\")\n end = event.get(\"end\", \"\")\n start_str = str(start)\n end_str = str(end)\n \n # Check date is 2026-04-09\n if expected_date not in start_str:\n continue\n \n # Check times - accept various formats (13:30, 1:30 PM, T13:30, etc.)\n start_ok = \"13:30\" in start_str or \"1:30\" in start_str.lower()\n end_ok = \"15:30\" in end_str or \"3:30\" in end_str.lower()\n \n if start_ok and end_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found event '{expected_title}' on 2026-04-09 from 1:30 PM to 3:30 PM\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found event '{expected_title}' on correct date but wrong times: start='{start}', end='{end}'\"}\n \n found_titles = [e.get(\"summary\", \"\") for e in events.values()]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No event titled '{expected_title}' found on 2026-04-09. Events found: {found_titles}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775842232676",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `PBC_Escalation_Log.xlsx`, there must be exactly 7 rows of data (excluding headers).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"PBC_Escalation_Log.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'PBC_Escalation_Log.xlsx' not found in workspace at {file_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path)\n ws = wb.active\n \n # Count data rows (non-empty rows excluding the header)\n # Assume the first row is the header\n data_row_count = 0\n for row_idx, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2):\n # A row is considered a data row if it has at least one non-None, non-empty-string value\n if any(cell is not None and str(cell).strip() != '' for cell in row):\n data_row_count += 1\n \n wb.close()\n \n if data_row_count == 7:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PBC_Escalation_Log.xlsx has exactly 7 data rows (excluding headers) as required. Found {data_row_count} data rows.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PBC_Escalation_Log.xlsx should have exactly 7 data rows (excluding headers), but found {data_row_count} data rows.\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading 'PBC_Escalation_Log.xlsx': {str(e)}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
Assume today is April 9th, 2026 at 1:30 PM.
|
||||
|
||||
Hi, it's Jill. I am still on sick leave today, but recalled that there is an overpayment case under me that might be due today or yesterday. Refer to the escalation log for case details.
|
||||
|
||||
Some instructions:
|
||||
|
||||
1) Draft an email to the payer to inform them of the overpayment situation, and give them the details of the claims that caused the problem (IDs & Amounts for each). Mention that any excess received from the overpayment will be either re-classed for yet unpaid claims for the same patient or processed for refund (subject to approval) back to them, and that updates will be provided as the situation evolves.
|
||||
|
||||
2) If any refund is going to be necessary, draft an email to Joe. The email should inform him of:
|
||||
- The fact that the payer (give their name) will require a refund
|
||||
- The problematic claim IDs
|
||||
- The exact amount that was overpaid
|
||||
- The exact amount that can be re-classed to each unpaid claim (provide the claim IDs along with the amount that would be re-classed to each)
|
||||
- The exact amount that is expected to be refunded
|
||||
|
||||
A refund is necessary if there are any unpaid claims by this patient for the same payer EXCEPT for the claims that caused the overpayment situation. The refund amount is the overpaid amount minus the amount that can be re-classed.
|
||||
|
||||
3) Do NOT send any email out yet, leave them in draft and I will review them before sending them out.
|
||||
|
||||
4) Add May into the BCC for emails (1) and (2).
|
||||
|
||||
On top of those instructions, ensure that everything required by the SOP in this situation is done.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user