Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-1": {
|
||||
"summary": "CLM-2026-0061 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-05-21"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-05-22"
|
||||
},
|
||||
"id": "evt-1"
|
||||
},
|
||||
"evt-2": {
|
||||
"summary": "CLM-2026-0062 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-04-27"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-28"
|
||||
},
|
||||
"id": "evt-2"
|
||||
},
|
||||
"evt-3": {
|
||||
"summary": "CLM-2026-0063 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-01-11"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-01-12"
|
||||
},
|
||||
"id": "evt-3"
|
||||
},
|
||||
"evt-4": {
|
||||
"summary": "CLM-2026-0064 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-02-22"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-02-23"
|
||||
},
|
||||
"id": "evt-4"
|
||||
},
|
||||
"evt-5": {
|
||||
"summary": "CLM-2026-0065 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-06-13"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-06-14"
|
||||
},
|
||||
"id": "evt-5"
|
||||
},
|
||||
"evt-6": {
|
||||
"summary": "CLM-2026-0066 TFL Deadline",
|
||||
"start": {
|
||||
"date": "2026-05-15"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-05-16"
|
||||
},
|
||||
"id": "evt-6"
|
||||
},
|
||||
"evt-7": {
|
||||
"summary": "CLM-2026-0062 Orange Alert",
|
||||
"start": {
|
||||
"date": "2026-03-28"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-03-29"
|
||||
},
|
||||
"id": "evt-7"
|
||||
},
|
||||
"evt-8": {
|
||||
"summary": "CLM-2026-0065 Yellow Alert",
|
||||
"start": {
|
||||
"date": "2026-04-14"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-15"
|
||||
},
|
||||
"id": "evt-8"
|
||||
},
|
||||
"evt-9": {
|
||||
"summary": "Weekly RCM Team Standup",
|
||||
"start": {
|
||||
"dateTime": "2026-03-30T09:30:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-03-30T10:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"id": "evt-9"
|
||||
},
|
||||
"evt-10": {
|
||||
"summary": "New RA Onboarding \u2014 Alex Rivera",
|
||||
"start": {
|
||||
"date": "2026-03-30"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-03-31"
|
||||
},
|
||||
"id": "evt-10"
|
||||
},
|
||||
"evt-11": {
|
||||
"summary": "Monthly Batch Submission Deadline",
|
||||
"start": {
|
||||
"date": "2026-03-31"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-01"
|
||||
},
|
||||
"description": "End-of-month 837P batch cutoff. All pending claims must be queued by 4:00 PM CT.",
|
||||
"id": "evt-11"
|
||||
},
|
||||
"evt-12": {
|
||||
"summary": "Provider Call \u2014 Valley Medical Group",
|
||||
"start": {
|
||||
"dateTime": "2026-04-02T14:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-02T15:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Quarterly provider relations call. Catherine Lawson to lead.",
|
||||
"id": "evt-12"
|
||||
},
|
||||
"evt-13": {
|
||||
"summary": "Aetna Portal Planned Maintenance",
|
||||
"start": {
|
||||
"dateTime": "2026-04-04T02:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-04T06:00:00-05:00",
|
||||
"timeZone": "America/Chicago"
|
||||
},
|
||||
"description": "Aetna payer portal unavailable. No 837P submissions or ERA retrieval during this window.",
|
||||
"id": "evt-13"
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "alex.rivera@pathfinder.med",
|
||||
"name": "Alex Rivera"
|
||||
},
|
||||
"contacts": [
|
||||
{ "email": "catherine.lawson@pathfinder.med", "name": "Catherine Lawson" },
|
||||
{ "email": "sarah.mitchell@pathfinder.med", "name": "Sarah Mitchell" },
|
||||
{ "email": "david.nguyen@pathfinder.med", "name": "David Nguyen" },
|
||||
{ "email": "marcus.thompson@pathfinder.med", "name": "Marcus Thompson" },
|
||||
{ "email": "jessica.alvarez@pathfinder.med", "name": "Jessica Alvarez" },
|
||||
{ "email": "lauren.patel@pathfinder.med", "name": "Lauren Patel" },
|
||||
{ "email": "kevin.obrien@pathfinder.med", "name": "Kevin O'Brien" },
|
||||
{ "email": "natalie.simmons@pathfinder.med", "name": "Natalie Simmons" },
|
||||
{ "email": "brian.caldwell@pathfinder.med", "name": "Brian Caldwell" }
|
||||
],
|
||||
"folders": [],
|
||||
"emails": [],
|
||||
"next_email_id": 1
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001", "team_id": "T001", "name": "alex.rivera", "deleted": false,
|
||||
"real_name": "Alex Rivera",
|
||||
"profile": { "real_name": "Alex Rivera", "display_name": "alexr",
|
||||
"email": "alex.rivera@pathfinder.med", "title": "Resolution Auditor" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002", "team_id": "T001", "name": "natalie.simmons", "deleted": false,
|
||||
"real_name": "Natalie Simmons",
|
||||
"profile": { "real_name": "Natalie Simmons", "display_name": "natalies",
|
||||
"email": "natalie.simmons@pathfinder.med", "title": "Resolution Auditor" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003", "team_id": "T001", "name": "sarah.mitchell", "deleted": false,
|
||||
"real_name": "Sarah Mitchell",
|
||||
"profile": { "real_name": "Sarah Mitchell", "display_name": "sarahm",
|
||||
"email": "sarah.mitchell@pathfinder.med", "title": "Certified Medical Coder" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004", "team_id": "T001", "name": "david.nguyen", "deleted": false,
|
||||
"real_name": "David Nguyen",
|
||||
"profile": { "real_name": "David Nguyen", "display_name": "davidn",
|
||||
"email": "david.nguyen@pathfinder.med", "title": "Certified Medical Coder" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005", "team_id": "T001", "name": "marcus.thompson", "deleted": false,
|
||||
"real_name": "Marcus Thompson",
|
||||
"profile": { "real_name": "Marcus Thompson", "display_name": "marcust",
|
||||
"email": "marcus.thompson@pathfinder.med", "title": "Patient Financial Representative" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U006": {
|
||||
"id": "U006", "team_id": "T001", "name": "jessica.alvarez", "deleted": false,
|
||||
"real_name": "Jessica Alvarez",
|
||||
"profile": { "real_name": "Jessica Alvarez", "display_name": "jessicaa",
|
||||
"email": "jessica.alvarez@pathfinder.med", "title": "Patient Financial Representative" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U007": {
|
||||
"id": "U007", "team_id": "T001", "name": "lauren.patel", "deleted": false,
|
||||
"real_name": "Lauren Patel",
|
||||
"profile": { "real_name": "Lauren Patel", "display_name": "laurenp",
|
||||
"email": "lauren.patel@pathfinder.med", "title": "Billing Specialist" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U008": {
|
||||
"id": "U008", "team_id": "T001", "name": "kevin.obrien", "deleted": false,
|
||||
"real_name": "Kevin O'Brien",
|
||||
"profile": { "real_name": "Kevin O'Brien", "display_name": "kevino",
|
||||
"email": "kevin.obrien@pathfinder.med", "title": "Billing Specialist" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U009": {
|
||||
"id": "U009", "team_id": "T001", "name": "brian.caldwell", "deleted": false,
|
||||
"real_name": "Brian Caldwell",
|
||||
"profile": { "real_name": "Brian Caldwell", "display_name": "brianc",
|
||||
"email": "brian.caldwell@pathfinder.med", "title": "Patient Financial Representative" },
|
||||
"is_admin": false, "is_owner": false, "is_bot": false
|
||||
},
|
||||
"U010": {
|
||||
"id": "U010", "team_id": "T001", "name": "catherine.lawson", "deleted": false,
|
||||
"real_name": "Catherine Lawson",
|
||||
"profile": { "real_name": "Catherine Lawson", "display_name": "catherinel",
|
||||
"email": "catherine.lawson@pathfinder.med", "title": "RCM Manager" },
|
||||
"is_admin": true, "is_owner": true, "is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001", "name": "special-authorization", "is_channel": true,
|
||||
"is_group": false, "is_im": false, "is_mpim": false, "is_private": false,
|
||||
"created": 1609459200, "is_archived": false, "is_general": false,
|
||||
"unlinked": 0, "name_normalized": "special-authorization",
|
||||
"is_shared": false, "is_org_shared": false, "is_ext_shared": false,
|
||||
"is_pending_ext_shared": false, "pending_shared": [],
|
||||
"pending_connected_team_ids": [], "context_team_id": "T001",
|
||||
"updated": 1609459200, "creator": "U010", "shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": { "value": "Special authorization requests and approvals", "creator": "U010", "last_set": 1609459200 },
|
||||
"purpose": { "value": "Channel for logging special authorization actions per SOP Section 2.2", "creator": "U010", "last_set": 1609459200 }
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002", "name": "escalations", "is_channel": true,
|
||||
"is_group": false, "is_im": false, "is_mpim": false, "is_private": false,
|
||||
"created": 1609459200, "is_archived": false, "is_general": false,
|
||||
"unlinked": 0, "name_normalized": "escalations",
|
||||
"is_shared": false, "is_org_shared": false, "is_ext_shared": false,
|
||||
"is_pending_ext_shared": false, "pending_shared": [],
|
||||
"pending_connected_team_ids": [], "context_team_id": "T001",
|
||||
"updated": 1609459200, "creator": "U010", "shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": { "value": "Escalated issues requiring immediate attention", "creator": "U010", "last_set": 1609459200 },
|
||||
"purpose": { "value": "Channel for escalated matters and urgent issues", "creator": "U010", "last_set": 1609459200 }
|
||||
},
|
||||
"C003": {
|
||||
"id": "C003", "name": "billing", "is_channel": true,
|
||||
"is_group": false, "is_im": false, "is_mpim": false, "is_private": false,
|
||||
"created": 1609459200, "is_archived": false, "is_general": false,
|
||||
"unlinked": 0, "name_normalized": "billing",
|
||||
"is_shared": false, "is_org_shared": false, "is_ext_shared": false,
|
||||
"is_pending_ext_shared": false, "pending_shared": [],
|
||||
"pending_connected_team_ids": [], "context_team_id": "T001",
|
||||
"updated": 1609459200, "creator": "U010", "shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": { "value": "Billing operations and discussions", "creator": "U010", "last_set": 1609459200 },
|
||||
"purpose": { "value": "Channel for billing team coordination and issues", "creator": "U010", "last_set": 1609459200 }
|
||||
},
|
||||
"C004": {
|
||||
"id": "C004", "name": "general", "is_channel": true,
|
||||
"is_group": false, "is_im": false, "is_mpim": false, "is_private": false,
|
||||
"created": 1609459200, "is_archived": false, "is_general": true,
|
||||
"unlinked": 0, "name_normalized": "general",
|
||||
"is_shared": false, "is_org_shared": false, "is_ext_shared": false,
|
||||
"is_pending_ext_shared": false, "pending_shared": [],
|
||||
"pending_connected_team_ids": [], "context_team_id": "T001",
|
||||
"updated": 1609459200, "creator": "U010", "shared_team_ids": ["T001"],
|
||||
"is_member": true,
|
||||
"topic": { "value": "Company-wide announcements and updates", "creator": "U010", "last_set": 1609459200 },
|
||||
"purpose": { "value": "General team communication", "creator": "U010", "last_set": 1609459200 }
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [],
|
||||
"C002": [],
|
||||
"C003": [],
|
||||
"C004": []
|
||||
},
|
||||
"counters": { "messageId": 1000000 }
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Hi! My name is Alex Rivera and I've just joined the Pathfinder team as a Resolution Auditor. I'm not sure what to do about the open items in the RA queue so please could you handle them on my behalf, following the processes in the SOP?
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, March, 30, 2026, 11am CT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/medical_pathfinder_billing_and_coding_a25684b9"
|
||||
description = "SOP task from CSV row a25684b9"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
[
|
||||
{
|
||||
"id": "8b2fb62f-2aca-45b9-bda8-fa26cc4f08c9",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `PBC_Claim_Register.xlsx`, row 2 must have Claim_Status equal to 'On Hold'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n # Find Claim_Status column\n header_row = [cell.value for cell in ws[1]]\n try:\n claim_status_col = header_row.index('Claim_Status') + 1\n except ValueError:\n return {'pass': False, 'score': 0.0, 'feedback': f'Claim_Status column not found. Headers: {header_row}'}\n row2_status = ws.cell(row=2, column=claim_status_col).value\n if row2_status is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 Claim_Status is empty/None, expected \"On Hold\"'}\n if str(row2_status).strip() == 'On Hold':\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 Claim_Status is \"On Hold\" as expected'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Row 2 Claim_Status is \"{row2_status}\", expected \"On Hold\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e93679bd-96c4-43a4-bd26-29973129e54e",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In file `PBC_Claim_Register.xlsx`, row 3 must have Claim_Status equal to 'Written Off', Billed_Amount equal to 285, Paid_Amount equal to 285, and Adjustment_Amount equal to 0.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n claim_status_col = get_col('Claim_Status')\n billed_col = get_col('Billed_Amount')\n paid_col = get_col('Paid_Amount')\n adj_col = get_col('Adjustment_Amount')\n missing = [n for n, c in [('Claim_Status', claim_status_col), ('Billed_Amount', billed_col), ('Paid_Amount', paid_col), ('Adjustment_Amount', adj_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row3_status = ws.cell(row=3, column=claim_status_col).value\n row3_billed = ws.cell(row=3, column=billed_col).value\n row3_paid = ws.cell(row=3, column=paid_col).value\n row3_adj = ws.cell(row=3, column=adj_col).value\n errors = []\n if str(row3_status).strip() != 'Written Off':\n errors.append(f'Claim_Status is \"{row3_status}\", expected \"Written Off\"')\n try:\n if abs(float(str(row3_billed).replace(',','')) - 285) > 2.85:\n errors.append(f'Billed_Amount is \"{row3_billed}\", expected 285')\n except:\n errors.append(f'Billed_Amount \"{row3_billed}\" is not numeric')\n try:\n if abs(float(str(row3_paid).replace(',','')) - 285) > 2.85:\n errors.append(f'Paid_Amount is \"{row3_paid}\", expected 285')\n except:\n errors.append(f'Paid_Amount \"{row3_paid}\" is not numeric')\n try:\n if abs(float(str(row3_adj).replace(',','')) - 0) > 2.85:\n errors.append(f'Adjustment_Amount is \"{row3_adj}\", expected 0')\n except:\n errors.append(f'Adjustment_Amount \"{row3_adj}\" is not numeric')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Claim_Status=\"Written Off\", Billed_Amount={row3_billed}, Paid_Amount={row3_paid}, Adjustment_Amount={row3_adj} as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "9df589fb-fede-40cb-8361-20e290f6f684",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 2 must have Recovery_Action equal to 'Route to PFR' and Recovery_Status equal to 'In Progress'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n action_col = get_col('Recovery_Action')\n status_col = get_col('Recovery_Status')\n missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row2_action = ws.cell(row=2, column=action_col).value\n row2_status = ws.cell(row=2, column=status_col).value\n errors = []\n if str(row2_action).strip() != 'Route to PFR':\n errors.append(f'Recovery_Action is \"{row2_action}\", expected \"Route to PFR\"')\n if str(row2_status).strip() != 'In Progress':\n errors.append(f'Recovery_Status is \"{row2_status}\", expected \"In Progress\"')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 has Recovery_Action=\"Route to PFR\" and Recovery_Status=\"In Progress\" as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "98830c8d-31c2-4120-8294-2c4bd73b5000",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 3 must have Recovery_Action equal to 'Write Off' and Recovery_Status equal to 'Written Off'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n action_col = get_col('Recovery_Action')\n status_col = get_col('Recovery_Status')\n missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row3_action = ws.cell(row=3, column=action_col).value\n row3_status = ws.cell(row=3, column=status_col).value\n errors = []\n if str(row3_action).strip() != 'Write Off':\n errors.append(f'Recovery_Action is \"{row3_action}\", expected \"Write Off\"')\n if str(row3_status).strip() != 'Written Off':\n errors.append(f'Recovery_Status is \"{row3_status}\", expected \"Written Off\"')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Recovery_Action=\"Write Off\" and Recovery_Status=\"Written Off\" as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "fd59b3bf-8c01-42a6-beae-7c1ba3b035ae",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `slack_data.json`, the #special-authorization channel must contain exactly one message with text 'Alex Rivera: CLM-2026-0062 special action authorized (or authorised)'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n # Check both external_services_path and workspace_path for slack_data.json\n slack_path = None\n candidates = []\n if external_services_path is not None:\n p = Path(external_services_path) / 'slack_data.json'\n if p.exists():\n candidates.append(p)\n wp = Path(workspace_path) / 'slack_data.json'\n if wp.exists():\n candidates.append(wp)\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in workspace or external_services_path'}\n slack_path = candidates[0]\n with open(slack_path, 'r') as f:\n data = json.load(f)\n\n # Find the #special-authorization channel\n channels = data.get('channels', {})\n special_auth_channel_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '')\n if 'special-authorization' in ch_name.lower() or 'special_authorization' in ch_name.lower() or 'special-authorisation' in ch_name.lower() or 'special_authorisation' in ch_name.lower():\n special_auth_channel_id = ch_id\n break\n\n if special_auth_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'#special-authorization channel not found. Channels: {[v.get(\"name\") for v in channels.values()]}'}\n\n messages = data.get('messages', {})\n channel_msgs = messages.get(special_auth_channel_id, [])\n\n # Accept both \"authorized\" and \"authorised\" spellings\n # Pattern: alex rivera: clm-2026-0062 special action authorized/authorised\n # Be loose: allow extra whitespace, punctuation at end, case insensitive\n target_pattern = re.compile(\n r'alex\\s+rivera\\s*[:\\-]\\s*clm[\\-\\s]2026[\\-\\s]0062\\s+special\\s+action\\s+authori[sz]ed',\n re.IGNORECASE\n )\n\n matching = []\n for msg in channel_msgs:\n msg_text = str(msg.get('text', '')).strip()\n if target_pattern.search(msg_text):\n matching.append(msg_text)\n\n if len(matching) == 0:\n all_texts = [m.get('text', '') for m in channel_msgs]\n return {'pass': False, 'score': 0.0, 'feedback': f'No message matching \"Alex Rivera: CLM-2026-0062 special action authorized/authorised\" found in #special-authorization. Messages found: {all_texts}'}\n\n if len(matching) > 1:\n return {'pass': False, 'score': 0.5, 'feedback': f'Expected exactly 1 matching message but found {len(matching)}: {matching}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': f'Found exactly 1 message in #special-authorization matching the required text: \"{matching[0]}\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {str(e)}'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775144961819",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "There must be no sent emails in the file `mailbox.json`.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Look for mailbox.json in external_services_path first, then workspace_path\n mailbox_path = None\n\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n # No mailbox.json found at all — no sent emails can exist\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json file found. No sent emails present.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except json.JSONDecodeError as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n # Extract the list of emails from various possible structures\n emails = []\n if isinstance(data, dict):\n if \"emails\" in data:\n emails = data[\"emails\"]\n elif \"messages\" in data:\n emails = data[\"messages\"]\n else:\n for key, value in data.items():\n if isinstance(value, list):\n emails = value\n break\n elif isinstance(data, list):\n emails = data\n\n if not isinstance(emails, list):\n emails = []\n\n # Filter to only SENT emails\n # Be generous: check multiple possible field names and values for \"sent\"\n sent_emails = []\n for e in emails:\n if not isinstance(e, dict):\n continue\n folder = e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))).strip().lower() if isinstance(e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))), str) else \"\"\n if folder in (\"sent\", \"outbox\", \"sent mail\", \"sent items\"):\n sent_emails.append(e)\n\n num_sent = len(sent_emails)\n\n if num_sent == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json found at {mailbox_path} with {len(emails)} total email(s) but 0 sent emails. Pass.\"}\n else:\n # Summarize sent emails for feedback\n subjects = []\n for e in sent_emails[:5]:\n if isinstance(e, dict):\n subj = e.get(\"subject\", e.get(\"Subject\", \"(no subject)\"))\n subjects.append(str(subj))\n subjects_str = \", \".join(subjects)\n if num_sent > 5:\n subjects_str += f\", ... and {num_sent - 5} more\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json contains {num_sent} sent email(s). Expected 0 sent emails. Sent email subjects: [{subjects_str}]. File: {mailbox_path}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Hi! Right now it's 11am CT on 30th March 2026. My name is Alex Rivera and I've just joined the Pathfinder team as a Resolution Auditor. I'm not sure what to do about the open items in the RA queue so please could you handle them on my behalf, following the processes in the SOP?
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user