Add Handbook.md benchmark tasks
This commit is contained in:
commit
25c9eda5ca
1800 files changed
+323575
No files matched your search
@@ -0,0 +1,51 @@
|
||||
[
|
||||
{
|
||||
"id": "53c53e31-7fa0-4136-9885-e8eb58789e27",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In file `ap_ledger.xlsx`, row 13 must contain invoice data for GLFD-0441 with vendor V-00009 (Great Lakes Furniture & Design Co.), invoice date 2026-03-20, received date 2026-03-24, due date 2026-04-19, type Non-PO, amount $7,900.00, GL code 1000-100-6200, and status Open through the 'Status' column",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n # Find Status column from header row\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n row_str = ' '.join(row_data).lower()\n\n has_invoice_id = 'glfd-0441' in row_str\n has_vendor_id = 'v-00009' in row_str\n has_vendor_name = 'great lakes furniture & design co' in row_str\n has_invoice_date = '2026-03-20' in row_str\n has_received_date = '2026-03-24' in row_str\n has_due_date = '2026-04-19' in row_str\n has_non_po = 'non-po' in row_str or 'non po' in row_str\n amount_found = '7,900' in row_str or '7900' in row_str\n has_gl = '1000-100-6200' in row_str\n\n if status_col is not None:\n status_val = ws.cell(row=13, column=status_col).value\n has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n else:\n has_open = 'open' in row_str # fallback if header not found\n\n checks = {\n 'invoice_id GLFD-0441': has_invoice_id,\n 'vendor_id V-00009': has_vendor_id,\n 'vendor_name Great Lakes': has_vendor_name,\n 'invoice_date 2026-03-20': has_invoice_date,\n 'received_date 2026-03-24': has_received_date,\n 'due_date 2026-04-19': has_due_date,\n 'type Non-PO': has_non_po,\n 'amount $7,900.00': amount_found,\n 'gl_code 1000-100-6200': has_gl,\n 'status Open': has_open\n }\n\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 in {fpath.name} sheet '{ws.title}' contains all required fields: {', '.join(passed)}\"}\n except Exception as e:\n continue\n\n # Partial feedback pass\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n row_str = ' '.join(row_data).lower()\n\n if status_col is not None:\n status_val = ws.cell(row=13, column=status_col).value\n has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n else:\n has_open = 'open' in row_str\n\n checks = {\n 'invoice_id GLFD-0441': 'glfd-0441' in row_str,\n 'vendor_id V-00009': 'v-00009' in row_str,\n 'vendor_name Great Lakes': 'great lakes' in row_str,\n 'invoice_date 2026-03-20': '2026-03-20' in row_str,\n 'received_date 2026-03-24': '2026-03-24' in row_str,\n 'due_date 2026-04-19': '2026-04-19' in row_str,\n 'type Non-PO': 'non-po' in row_str or 'non po' in row_str,\n 'amount $7,900.00': '7,900' in row_str or '7900' in row_str,\n 'gl_code 1000-100-6200': '1000-100-6200' in row_str,\n 'status Open': has_open\n }\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n if passed:\n return {\"pass\": False, \"score\": len(passed)/len(checks), \"feedback\": f\"Row 13 in {fpath.name} missing: {', '.join(failed)}. Found: {', '.join(passed)}. Row content: {row_data}\"}\n Line truncated
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "1bcd6b5d-4a16-42e7-9464-4a8a2c676eda",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In mailbox.json (Drafts folder), exactly one draft email must exist addressed to m.chen@meridianpartners.com and r.whitfield@meridianpartners.com with CC to ap@meridianpartners.com, subject 'AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441', and body containing only the following information and no other information: 'A non-PO invoice requires your approval before payment can be processed. Vendor: Great Lakes Furniture & Design Co. (V-00009). Invoice: GLFD-0441, dated 2026-03-20. Amount: $7,900.00. Description: Executive conference table, 10-person, walnut finish. GL Code: 1000-100-6200. Please reply APPROVED or REJECTED with reason. If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\nimport unicodedata\n\ndef normalize(s):\n s = unicodedata.normalize('NFKC', s)\n s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n s = s.replace('\\u2019', \"'\").replace('\\u2018', \"'\")\n s = s.replace('\\u201c', '\"').replace('\\u201d', '\"')\n s = re.sub(r'[();:!?]', '', s)\n s = re.sub(r'[,.](?!\\d)', '', s)\n s = s.lower()\n s = re.sub(r'\\s+', ' ', s).strip()\n return s\n\nEXPECTED_SUBJECT = \"AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441\"\n\nEXPECTED_BODY = (\n \"A non-PO invoice requires your approval before payment can be processed.\\n\"\n \"Vendor: Great Lakes Furniture & Design Co. (V-00009).\\n\"\n \"Invoice: GLFD-0441, dated 2026-03-20\\n\"\n \"Amount: $7,900.00.\\n\"\n \"Description: Executive conference table, 10-person, walnut finish.\\n\"\n \"GL Code: 1000-100-6200.\\n\"\n \"Please reply APPROVED or REJECTED with reason.\\n\"\n \"If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.\"\n)\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get('emails', [])\n drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n\n if not drafts:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in Drafts folder\"}\n\n # Pick best matching draft by subject keywords\n best_draft = drafts[0]\n best_score = -1\n for draft in drafts:\n subject = str(draft.get('subject', '')).lower()\n score = ('glfd-0441' in subject) + ('great lakes' in subject)\n if score > best_score:\n best_score = score\n best_draft = draft\n\n draft = best_draft\n\n to_raw = draft.get('to_addr', '') or draft.get('to', '') or ''\n if isinstance(to_raw, list):\n to_addr = ' '.join([str(x).lower().strip() for x in to_raw])\n else:\n to_addr = str(to_raw).lower()\n\n cc_raw = draft.get('cc_addr', '') or draft.get('cc', '') or ''\n if isinstance(cc_raw, list):\n cc_addr = ' '.join([str(x).lower().strip() for x in cc_raw])\n else:\n cc_addr = str(cc_raw).lower()\n\n subject = str(draft.get('subject', ''))\n body = str(draft.get('body_text', '') or draft.get('body', '') or '')\n\n checks = {\n 'to: m.chen@meridianpartners.com': 'm.chen@meridianpartners.com' in to_addr,\n 'to: r.whitfield@meridianpartners.com': 'r.whitfield@meridianpartners.com' in to_addr,\n 'cc: ap@meridianpartners.com': 'ap@meridianpartners.com' in cc_addr,\n 'subject exact match': normalize(subject) == normalize(EXPECTED_SUBJECT),\n 'body exact match': normalize(body) == normalize(EXPECTED_BODY),\n }\n\n failed = [k for k, v in checks.items() if not v]\n passed = [k for k, v in checks.items() if v]\n score = len(passed) / len(checks)\n\n if not failed:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Draft email verified: all {len(checks)} checks passed.\"}\n\n details = f\"Failed: {', '.join(failed)}. Passed: {', '.join(passed)}.\"\n if 'subject exact match' in failed:\n details += f\" Subject found: '{subject}'.\"\n if 'body exact match' in failed:\n norm_actual = normalize(body)\n norm_expected = normalize(EXPECTED_BODY)\n if norm_actual == norm_expected:\n details += \" Body: normalized content matches — difference is line-break/formatting only.\"\n else:\n actual_words = norm_actual.split()\n expected_words = norm_expected.split()\n divergence = None\n for i, (aw, ew) in enumerate(zip(actual_words, expected_words)):\n if aw != ew:\n start = max(0, i - 3)\n expected_ctx = ' '.join(expected_words[start:i+6])\n actual_ctx = ' '.join(actual_words[start:i+6])\n divergence = f\"First divergence at word {i+1}: expected '...{expected_ctx}...' | got '...{actual_ctx}...'\"\n break\n if divergence is None:\n divergence = f\"Word count differs: expected {len(expected_words)}, got {len(actual_words)}\"\n details += f\" Body diff: {divergLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e64b48a4-131c-4e84-b578-61ab0558ed20",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In slack_data.json, no new Slack messages must have been sent (the agent must not have sent any Slack messages as part of this task)",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport time\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n\n messages = data.get('messages', {})\n\n # Look for any messages that mention Great Lakes, GLFD-0441, or 7900 that might have been sent by the agent\n # The task says 'No new Slack messages are sent'\n # We check if there are any messages related to this invoice task\n task_related_messages = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = str(msg.get('text', '')).lower()\n if (\n 'great lakes' in text\n or 'glfd-0441' in text\n or 'glfd0441' in text\n or ('7,900' in text and 'furniture' in text)\n or ('7900' in text and 'furniture' in text)\n ):\n task_related_messages.append({'channel': channel_id, 'text': msg.get('text', '')[:100]})\n\n if task_related_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(task_related_messages)} Slack message(s) related to Great Lakes/GLFD-0441 that should not have been sent: {task_related_messages}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Slack messages related to Great Lakes Furniture or GLFD-0441 found, as expected\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774390462114",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "No fields are populated after the \"Status\" column in Row 13 of `ap_ledger.xlsx`.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n status_col = None\n for col in range(1, ws.max_column + 1):\n hdr = ws.cell(row=1, column=col).value\n if hdr is not None and str(hdr).strip().lower() == \"status\":\n status_col = col\n break\n\n if status_col is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find 'Status' column header in {fpath.name}\"}\n\n populated_after_status = []\n for col in range(status_col + 1, ws.max_column + 1):\n cell_value = ws.cell(row=13, column=col).value\n if cell_value is None:\n continue\n if cell_value == 0 or cell_value == 0.0:\n continue\n if str(cell_value).strip() == \"\" or str(cell_value).strip() == \"0\":\n continue\n col_header = ws.cell(row=1, column=col).value\n populated_after_status.append((col, col_header, cell_value))\n\n if not populated_after_status:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 has no populated fields after 'Status' (column {status_col})\"}\n else:\n details = \"; \".join([f\"Col {c} ('{h}'): '{v}'\" for c, h, v in populated_after_status])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 13 has {len(populated_after_status)} populated field(s) after 'Status': {details}\"}\n\n except Exception as e:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774486274859",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json`,no more than one email must exist in the Drafts folder, and no emails related to Great Lakes Furniture & Design or invoice GLFD-0441 must exist in the Sent folder.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get('emails', [])\n drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n sent = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n\n failures = []\n\n if len(drafts) != 1:\n failures.append(f\"Expected exactly 1 draft, found {len(drafts)}\")\n\n sent_related = [\n e for e in sent\n if any(kw in str(e.get('subject', '')).lower() or kw in str(e.get('body_text', '')).lower() or kw in str(e.get('body', '')).lower() or kw in str(e.get('to_addr', '')).lower()\n for kw in ['great lakes', 'glfd-0441', 'orders@greatlakesfurniture.com'])\n ]\n\n if sent_related:\n failures.append(f\"Found {len(sent_related)} sent email(s) related to Great Lakes/GLFD-0441\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(failures)}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Exactly 1 draft found and no related emails in Sent folder\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774487311749",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `ap_ledger.xlsx`, the Invoice Register must contain no more than 13 rows of data (header row plus 12 invoice entries, including the new GLFD-0441 entry in row 13).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n target_files = ledger_files if ledger_files else xlsx_files\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n extra_rows = []\n for row in range(14, ws.max_row + 1):\n if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n extra_rows.append(row)\n\n if extra_rows:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Data found beyond row 13 in {fpath.name}: rows {extra_rows}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Invoice Register contains no data beyond row 13\"}\n except Exception:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1774487313895",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `vendor_master.xlsx`, no new vendor rows must have been added (the file must contain no more than 12 rows: 1 header row and 11 vendor rows).",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n vendor_files = [f for f in xlsx_files if 'vendor' in f.name.lower()]\n if not vendor_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No vendor_master.xlsx found in workspace\"}\n\n for fpath in vendor_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n ws = wb.worksheets[0]\n\n extra_rows = []\n for row in range(13, ws.max_row + 1):\n if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n extra_rows.append(row)\n\n if extra_rows:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"New vendor rows found in {fpath.name} beyond row 12: rows {extra_rows}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"vendor_master.xlsx contains no extra vendor rows\"}\n except Exception:\n continue\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read vendor_master.xlsx\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Find the invoice from Great Lakes Furniture & Design that we received today, March 24, 2026. It's an Office Supplies Account. First, if it should be added to the Invoice Register, update the Invoice Register through the "Status" column. Second, if an email should be sent, draft if for me and put it in Drafts. Do NOT send it - I will review it myself before sending. Third, if a Slack message is required, send it.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user