Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-1": {
|
||||
"id": "evt-1",
|
||||
"summary": "New Year's Day",
|
||||
"start": {
|
||||
"date": "2026-01-01"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-01-02"
|
||||
},
|
||||
"description": "Public Holiday"
|
||||
},
|
||||
"evt-2": {
|
||||
"id": "evt-2",
|
||||
"summary": "Martin Luther King Jr Day",
|
||||
"start": {
|
||||
"date": "2026-01-19"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-01-20"
|
||||
},
|
||||
"description": "Public Holiday"
|
||||
},
|
||||
"evt-3": {
|
||||
"id": "evt-3",
|
||||
"summary": "Washington's Birthday",
|
||||
"start": {
|
||||
"date": "2026-02-16"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-02-17"
|
||||
},
|
||||
"description": "Public Holiday"
|
||||
},
|
||||
"evt-4": {
|
||||
"id": "evt-4",
|
||||
"summary": "Billing Specs Huddle",
|
||||
"start": {
|
||||
"dateTime": "2026-02-11T10:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-11T11:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
}
|
||||
},
|
||||
"evt-5": {
|
||||
"id": "evt-5",
|
||||
"summary": "835 - B8475Q posting due Feb 17",
|
||||
"start": {
|
||||
"dateTime": "2026-02-17T16:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-17T17:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Posting deadline"
|
||||
},
|
||||
"evt-6": {
|
||||
"id": "evt-6",
|
||||
"summary": "RA Huddle",
|
||||
"start": {
|
||||
"dateTime": "2026-02-17T14:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-17T15:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
}
|
||||
},
|
||||
"evt-7": {
|
||||
"id": "evt-7",
|
||||
"summary": "835 - A4789P posting due Feb 19",
|
||||
"start": {
|
||||
"dateTime": "2026-02-19T16:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-19T17:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Posting deadline"
|
||||
},
|
||||
"evt-8": {
|
||||
"id": "evt-8",
|
||||
"summary": "YELLOW - HOH-7199789 - TFL: Apr 25, 2026",
|
||||
"start": {
|
||||
"dateTime": "2026-02-24T09:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-02-24T10:00:00-06:00",
|
||||
"timeZone": "America/Chicago"
|
||||
}
|
||||
},
|
||||
"evt-9": {
|
||||
"id": "evt-9",
|
||||
"summary": "ORANGE - HOH-7199789 - TFL: Apr 25, 2026",
|
||||
"start": {
|
||||
"dateTime": "2026-03-26T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-26T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
}
|
||||
},
|
||||
"evt-10": {
|
||||
"id": "evt-10",
|
||||
"summary": "RED - HOH-7199789 - TFL: Apr 25, 2026",
|
||||
"start": {
|
||||
"dateTime": "2026-04-10T09:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-10T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "rcm@pathfinder.com",
|
||||
"name": "RCM Team"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "jamies@pathfinder.com",
|
||||
"name": "Jamie Scott"
|
||||
},
|
||||
{
|
||||
"email": "joem@pathfinder.com",
|
||||
"name": "Joe Malfroy"
|
||||
},
|
||||
{
|
||||
"email": "mayj@pathfinder.com",
|
||||
"name": "May Jollier"
|
||||
},
|
||||
{
|
||||
"email": "russellc@pathfinder.com",
|
||||
"name": "Russell Commy"
|
||||
},
|
||||
{
|
||||
"email": "hopeK@email.com",
|
||||
"name": "Kelly Hope"
|
||||
},
|
||||
{
|
||||
"email": "ursula.james@hull.com",
|
||||
"name": "Ursula James"
|
||||
},
|
||||
{
|
||||
"email": "popler.wen@bulliongoldinsurance.com",
|
||||
"name": "Popler Wen"
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"email": "rcm-internal@pathfinder.com",
|
||||
"name": "RCM Internal Team",
|
||||
"members": [
|
||||
"rcm@pathfinder.com",
|
||||
"jamies@pathfinder.com",
|
||||
"joem@pathfinder.com",
|
||||
"mayj@pathfinder.com",
|
||||
"russellc@pathfinder.com"
|
||||
]
|
||||
}
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Farewell Lunch",
|
||||
"from_addr": "joem@pathfinder.com",
|
||||
"to_addr": "rcm-internal@pathfinder.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-02-13T22:25:00Z",
|
||||
"message_id": "<farewell-lunch-20260213@pathfinder.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Team\n\nAs many of you would have known from the last department meeting, Russell's last day with us will be on the 20 February 2026, Friday and we hope to organise a farewell lunch for him on the 18 Feb, 12pm at Favos Mexican. Please let May know if you are able to make it by 17 Feb, 1pm, so that she can make a table reservation.\n\nRussell would still be in office through the week, if any would like to say your goodbyes to him.\n\nWarm regards,\nJoe",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "Sent",
|
||||
"subject": "DPJ-47089945",
|
||||
"from_addr": "rcm@pathfinder.com",
|
||||
"to_addr": "popler.wen@bulliongoldinsurance.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-02-13T20:43:00Z",
|
||||
"message_id": "<dpj-47089945-20260213@pathfinder.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Hi Popler\n\nThanks for the call earlier. As spoken, we received the payment for claim ID DPJ-47089945 and would need your assistance to clarify on the coinsurance specifics for this case.\n\nBest regards,\nJamie Scott\nBilling Specialist\nPathfinder",
|
||||
"body_html": null,
|
||||
"is_read": true,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 3
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "jamies",
|
||||
"deleted": false,
|
||||
"real_name": "Jamie Scott",
|
||||
"profile": {
|
||||
"real_name": "Jamie Scott",
|
||||
"display_name": "jamies",
|
||||
"email": "jamies@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "joem",
|
||||
"deleted": false,
|
||||
"real_name": "Joe Malfroy",
|
||||
"profile": {
|
||||
"real_name": "Joe Malfroy",
|
||||
"display_name": "joem",
|
||||
"email": "joem@pathfinder.com",
|
||||
"title": "RCM Manager"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "mayj",
|
||||
"deleted": false,
|
||||
"real_name": "May Jollier",
|
||||
"profile": {
|
||||
"real_name": "May Jollier",
|
||||
"display_name": "mayj",
|
||||
"email": "mayj@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "russellc",
|
||||
"deleted": false,
|
||||
"real_name": "Russell Commy",
|
||||
"profile": {
|
||||
"real_name": "Russell Commy",
|
||||
"display_name": "russellc",
|
||||
"email": "russellc@pathfinder.com",
|
||||
"title": "Team Member"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "coders",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1733000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "coders",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1733000000,
|
||||
"creator": "U001",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Coders channel",
|
||||
"creator": "U001",
|
||||
"last_set": 1733000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for coders",
|
||||
"creator": "U001",
|
||||
"last_set": 1733000000
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "billing-specs",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1733000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "billing-specs",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1733000000,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Billing specifications",
|
||||
"creator": "U002",
|
||||
"last_set": 1733000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for billing specifications",
|
||||
"creator": "U002",
|
||||
"last_set": 1733000000
|
||||
}
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003",
|
||||
"name": "ra",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1733000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "ra",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1733000000,
|
||||
"creator": "U003",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "RA channel",
|
||||
"creator": "U003",
|
||||
"last_set": 1733000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for RA",
|
||||
"creator": "U003",
|
||||
"last_set": 1733000000
|
||||
}
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004",
|
||||
"name": "escalations-rcm-mgr",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1733000000,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "escalations-rcm-mgr",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1733848080,
|
||||
"creator": "U004",
|
||||
"shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Escalations for RCM Manager",
|
||||
"creator": "U004",
|
||||
"last_set": 1733000000
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for RCM Manager escalations",
|
||||
"creator": "U004",
|
||||
"last_set": 1733000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [],
|
||||
"C002": [],
|
||||
"C003": [],
|
||||
"C004": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Majestic's system is down & estimated to be back up earliest by 6pm today. 837P saved.",
|
||||
"ts": "1765358581.000100",
|
||||
"team": "T001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Noted, will email the office mgrs. Thanks Jamie.",
|
||||
"ts": "1765358880.000200",
|
||||
"team": "T001"
|
||||
}
|
||||
]
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000000
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
It's Joe Malfroy, RCM manager. Noticed a discrepancy with Hull's payments for CPT 11241 despite the recent pass in the weekly audit. Review the attached SOP to trigger the follow-up processes. Follow all the relevant steps. Put me as the staff assigned first for any issues trigger. if any payer correspondence is required, create an email draft with the completed template (do not send the draft email out & if information is not known, retain the placeholders).
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, February, 17, 2026, 9.53am, central time. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/medical_pathfinder_billing_and_coding_073e602b"
|
||||
description = "SOP task from CSV row 073e602b"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
It's Joe Malfroy, RCM manager. Current timestamp is Feb 17, 2026 (9.53am), central time. Noticed a discrepancy with Hull's payments for CPT 11241 despite the recent pass in the weekly audit. Review the attached SOP to trigger the follow-up processes. Follow all the relevant steps. Put me as the staff assigned first for any issues trigger. if any payer correspondence is required, create an email draft with the completed template (do not send the draft email out & if information is not known, retain the placeholders).
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user