Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,7 @@
|
||||
FROM handbook_base
|
||||
|
||||
COPY initial_external_services/ /data/
|
||||
COPY initial_external_services/ /initial_data/
|
||||
COPY initial_workspace/ /workdir/
|
||||
|
||||
ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"events": {
|
||||
"evt-1": {
|
||||
"id": "evt-1",
|
||||
"summary": "Operations \u2014 Weekly Team Sync",
|
||||
"start": {
|
||||
"dateTime": "2026-04-10T09:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-10T09:30:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Conference Room B / Teams link",
|
||||
"description": "Standing weekly sync. Agenda distributed by organizer day prior.\n\nOrganizer: rachel.osei@vanguardshield.com"
|
||||
},
|
||||
"evt-2": {
|
||||
"id": "evt-2",
|
||||
"summary": "Q2 Planning Kickoff \u2014 All Departments",
|
||||
"start": {
|
||||
"dateTime": "2026-04-11T13:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"end": {
|
||||
"dateTime": "2026-04-11T15:00:00-04:00",
|
||||
"timeZone": "America/New_York"
|
||||
},
|
||||
"location": "Main Boardroom / Teams link",
|
||||
"description": "Please review the Q2 Draft Objectives deck (shared via internal portal) before attending.\n\nOrganizer: strategy@vanguardshield.com"
|
||||
},
|
||||
"evt-3": {
|
||||
"id": "evt-3",
|
||||
"summary": "Supplemental Benefits Enrollment Deadline",
|
||||
"start": {
|
||||
"date": "2026-04-18"
|
||||
},
|
||||
"end": {
|
||||
"date": "2026-04-19"
|
||||
},
|
||||
"description": "Last day to update Q2 supplemental benefit elections in the Benefits Portal.\n\nOrganizer: benefits@vanguardshield.com"
|
||||
}
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"mailbox": {
|
||||
"email": "financial.analyst@vanguardshield.com",
|
||||
"name": "Financial Analyst"
|
||||
},
|
||||
"contacts": [
|
||||
{
|
||||
"email": "it-security@vanguardshield.com",
|
||||
"name": "IT Security Team"
|
||||
},
|
||||
{
|
||||
"email": "ap-notifications@vanguardshield.com",
|
||||
"name": "Accounts Payable Notifications"
|
||||
},
|
||||
{
|
||||
"email": "benefits@vanguardshield.com",
|
||||
"name": "Benefits Department"
|
||||
},
|
||||
{
|
||||
"email": "facilities@northsidepropertyservices.com",
|
||||
"name": "Northside Property Services"
|
||||
},
|
||||
{
|
||||
"email": "ap@vanguardshield.com",
|
||||
"name": "Accounts Payable"
|
||||
}
|
||||
],
|
||||
"folders": [
|
||||
{
|
||||
"name": "Archive"
|
||||
}
|
||||
],
|
||||
"emails": [
|
||||
{
|
||||
"email_id": "1",
|
||||
"folder": "INBOX",
|
||||
"subject": "Reminder: Annual Security Awareness Training Due April 15",
|
||||
"from_addr": "it-security@vanguardshield.com",
|
||||
"to_addr": "financial.analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-09T08:14:00Z",
|
||||
"message_id": "<sec-training-2026-04-09@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "This is a reminder that your annual security awareness training must be completed by April 15, 2026. Please log in to the Learning Portal and complete the \"Information Security Fundamentals\" module. Contact the IT Help Desk with any access issues.",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "2",
|
||||
"folder": "INBOX",
|
||||
"subject": "Expense Report ER-2026-0334 Approved",
|
||||
"from_addr": "ap-notifications@vanguardshield.com",
|
||||
"to_addr": "financial.analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-08T15:47:00Z",
|
||||
"message_id": "<expense-er-2026-0334@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Your expense report ER-2026-0334 (Q1 Travel & Accommodation, submitted March 28) has been approved and is scheduled for reimbursement in the April 15 payroll cycle. Total reimbursement: $1,142.60. Questions? Contact Accounts Payable.",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "3",
|
||||
"folder": "INBOX",
|
||||
"subject": "Q2 Supplemental Benefits Enrollment Closes April 18",
|
||||
"from_addr": "benefits@vanguardshield.com",
|
||||
"to_addr": "financial.analyst@vanguardshield.com",
|
||||
"cc_addr": null,
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-07T09:02:00Z",
|
||||
"message_id": "<benefits-q2-2026@vanguardshield.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Open enrollment for Q2 supplemental benefits (dental, vision, and supplemental life) closes on April 18, 2026. Log in to the Benefits Portal to review your current elections and make any changes. No action is required if you wish to keep your current coverage.",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"email_id": "4",
|
||||
"folder": "INBOX",
|
||||
"subject": "Invoice INV-84201 Received — April Facilities Services",
|
||||
"from_addr": "facilities@northsidepropertyservices.com",
|
||||
"to_addr": "ap@vanguardshield.com",
|
||||
"cc_addr": "financial.analyst@vanguardshield.com",
|
||||
"bcc_addr": null,
|
||||
"date": "2026-04-09T11:23:00Z",
|
||||
"message_id": "<inv-84201@northsidepropertyservices.com>",
|
||||
"in_reply_to": null,
|
||||
"body_text": "Please find attached Invoice INV-84201 for April facilities maintenance services as outlined in Contract FM-2024-009. Total due: $6,800.00. Payment terms: Net 30. Please direct any questions to our billing department.",
|
||||
"body_html": null,
|
||||
"is_read": false,
|
||||
"is_important": false,
|
||||
"attachments": []
|
||||
}
|
||||
],
|
||||
"next_email_id": 5
|
||||
}
|
||||
+1102
File diff suppressed because it is too large
Load Diff
+297
@@ -0,0 +1,297 @@
|
||||
{
|
||||
"users": {
|
||||
"U001": {
|
||||
"id": "U001",
|
||||
"team_id": "T001",
|
||||
"name": "james.okafor",
|
||||
"deleted": false,
|
||||
"real_name": "James Okafor",
|
||||
"profile": {
|
||||
"real_name": "James Okafor",
|
||||
"display_name": "jamesko",
|
||||
"email": "james.okafor@vanguardshield.com",
|
||||
"title": "Treasury Operations Specialist"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U002": {
|
||||
"id": "U002",
|
||||
"team_id": "T001",
|
||||
"name": "karen.stills",
|
||||
"deleted": false,
|
||||
"real_name": "Karen Stills",
|
||||
"profile": {
|
||||
"real_name": "Karen Stills",
|
||||
"display_name": "karens",
|
||||
"email": "karen.stills@vanguardshield.com",
|
||||
"title": "VP, Treasury Operations"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U003": {
|
||||
"id": "U003",
|
||||
"team_id": "T001",
|
||||
"name": "michael.drury",
|
||||
"deleted": false,
|
||||
"real_name": "Michael Drury",
|
||||
"profile": {
|
||||
"real_name": "Michael Drury",
|
||||
"display_name": "miked",
|
||||
"email": "michael.drury@vanguardshield.com",
|
||||
"title": "SVP, Finance & Risk"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U004": {
|
||||
"id": "U004",
|
||||
"team_id": "T001",
|
||||
"name": "brett.connors",
|
||||
"deleted": false,
|
||||
"real_name": "Brett Connors",
|
||||
"profile": {
|
||||
"real_name": "Brett Connors",
|
||||
"display_name": "brettc",
|
||||
"email": "brett.connors@vanguardshield.com",
|
||||
"title": "Senior Manager, Treasury Operations"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
},
|
||||
"U005": {
|
||||
"id": "U005",
|
||||
"team_id": "T001",
|
||||
"name": "priya.mehta",
|
||||
"deleted": false,
|
||||
"real_name": "Priya Mehta",
|
||||
"profile": {
|
||||
"real_name": "Priya Mehta",
|
||||
"display_name": "priyam",
|
||||
"email": "priya.mehta@vanguardshield.com",
|
||||
"title": "AVP, Enterprise Risk"
|
||||
},
|
||||
"is_admin": false,
|
||||
"is_owner": false,
|
||||
"is_bot": false
|
||||
}
|
||||
},
|
||||
"channels": {
|
||||
"C001": {
|
||||
"id": "C001",
|
||||
"name": "treasury-wires",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "treasury-wires",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Wire transfer submission and approval tracking",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for treasury wire transfer approvals and documentation",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
},
|
||||
"C002": {
|
||||
"id": "C002",
|
||||
"name": "compliance-urgent",
|
||||
"is_channel": true,
|
||||
"is_group": false,
|
||||
"is_im": false,
|
||||
"is_mpim": false,
|
||||
"is_private": false,
|
||||
"created": 1609459200,
|
||||
"is_archived": false,
|
||||
"is_general": false,
|
||||
"unlinked": 0,
|
||||
"name_normalized": "compliance-urgent",
|
||||
"is_shared": false,
|
||||
"is_org_shared": false,
|
||||
"is_ext_shared": false,
|
||||
"is_pending_ext_shared": false,
|
||||
"pending_shared": [],
|
||||
"pending_connected_team_ids": [],
|
||||
"context_team_id": "T001",
|
||||
"updated": 1609459200,
|
||||
"creator": "U002",
|
||||
"shared_team_ids": [
|
||||
"T001"
|
||||
],
|
||||
"is_member": true,
|
||||
"topic": {
|
||||
"value": "Urgent compliance alerts and escalations",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
},
|
||||
"purpose": {
|
||||
"value": "Channel for time-sensitive compliance issues requiring immediate attention",
|
||||
"creator": "U002",
|
||||
"last_set": 1609459200
|
||||
}
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"C001": [
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0147. Payee: Meridian Reinsurance Partners Ltd. Amount: $312,500.00. Please review and approve per policy. Jira: <https://vanguardshield.atlassian.net/browse/OPS-2|OPS-2>",
|
||||
"ts": "1772636400.000100",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1772639220.000300",
|
||||
"reply_users": [
|
||||
"U002",
|
||||
"U003"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1772637720.000200",
|
||||
"thread_ts": "1772636400.000100",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U003",
|
||||
"text": "Approved.",
|
||||
"ts": "1772639220.000300",
|
||||
"thread_ts": "1772636400.000100",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0151. Payee: Castlewood Actuarial Services B.V. Amount: $87,200.00. Jira: <https://vanguardshield.atlassian.net/browse/OPS-3|OPS-3>",
|
||||
"ts": "1773241200.000400",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 1,
|
||||
"latest_reply": "1773242520.000500",
|
||||
"reply_users": [
|
||||
"U002"
|
||||
],
|
||||
"reply_users_count": 1
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1773242520.000500",
|
||||
"thread_ts": "1773241200.000400",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0158. Payee: Nordic Claims Recovery AS. Amount: $204,750.00. Jira: <https://vanguardshield.atlassian.net/browse/OPS-4|OPS-4>",
|
||||
"ts": "1773932400.000600",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1773935220.000800",
|
||||
"reply_users": [
|
||||
"U002",
|
||||
"U004"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1773933720.000700",
|
||||
"thread_ts": "1773932400.000600",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U004",
|
||||
"text": "Approved.",
|
||||
"ts": "1773935220.000800",
|
||||
"thread_ts": "1773932400.000600",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U001",
|
||||
"text": "Wire submitted for processing. Wire ID: INTL-2026-0163. Payee: Avondale Risk Consulting Ltd. Amount: $519,000.00. Jira: <https://vanguardshield.atlassian.net/browse/OPS-5|OPS-5>",
|
||||
"ts": "1774623600.000900",
|
||||
"team": "T001",
|
||||
"channel": "C001",
|
||||
"reply_count": 2,
|
||||
"latest_reply": "1774626420.001100",
|
||||
"reply_users": [
|
||||
"U002",
|
||||
"U005"
|
||||
],
|
||||
"reply_users_count": 2
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U002",
|
||||
"text": "Approved.",
|
||||
"ts": "1774624920.001000",
|
||||
"thread_ts": "1774623600.000900",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"user": "U005",
|
||||
"text": "Approved.",
|
||||
"ts": "1774626420.001100",
|
||||
"thread_ts": "1774623600.000900",
|
||||
"parent_user_id": "U001",
|
||||
"team": "T001",
|
||||
"channel": "C001"
|
||||
}
|
||||
],
|
||||
"C002": []
|
||||
},
|
||||
"counters": {
|
||||
"messageId": 1000012
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
You are a junior operations analyst at Vanguard Shield Mutual. Complete the Q1 '26 international wire transfer dual-authorization audit, strictly following SOP. Open the Jira issue and the wire pop workbook identify any wire that lacks dual authorization and follow the escalation procedure for each. Follow the §3.3 exactly, record each outcome in the workbook.
|
||||
@@ -0,0 +1 @@
|
||||
Today's date is, April, 10, 2026, 09:00 AM EDT. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.
|
||||
@@ -0,0 +1,23 @@
|
||||
schema_version = "1.1"
|
||||
|
||||
[task]
|
||||
name = "sop-tasks/insurance_vanguard_shield_mutual_fe30fce5"
|
||||
description = "SOP task from CSV row fe30fce5"
|
||||
|
||||
[agent]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[verifier]
|
||||
timeout_sec = 3600.0
|
||||
|
||||
[environment]
|
||||
os = "linux"
|
||||
cpus = 2
|
||||
memory_mb = 4096
|
||||
env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
|
||||
|
||||
[[environment.mcp_servers]]
|
||||
name = "mcp-proxy"
|
||||
transport = "streamable-http"
|
||||
url = "http://localhost:8000/mcp"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
It is April 10, 2026, 09:00 AM EDT. You are a junior operations analyst at Vanguard Shield Mutual. Complete the Q1 '26 international wire transfer dual-authorization audit, strictly following SOP. Open the Jira issue and the wire pop workbook identify any wire that lacks dual authorization and follow the escalation procedure for each. Follow the §3.3 exactly, record each outcome in the workbook.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in New Issue
Block a user