Add Handbook.md benchmark tasks

tasks/insurance_mojave_crest_assurance_company_2a800bf7/environment/Dockerfile

@@ -0,0 +1,7 @@
+FROM handbook_base
+
+COPY initial_external_services/ /data/
+COPY initial_external_services/ /initial_data/
+COPY initial_workspace/ /workdir/
+
+ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"

tasks/insurance_mojave_crest_assurance_company_2a800bf7/environment/initial_workspace/Member_Profile_KPX658320914.pdf

@@ -0,0 +1,74 @@
+%PDF-1.4
+%“Œ‹ž ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (\(anonymous\)) /CreationDate (D:20260414190050+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260414190050+00'00') /Producer (ReportLab PDF Library - \(opensource\)) 
+  /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 974
+>>
+stream
+Gas1^b>R(K']%q&mP8m9A'+2B)P.$7)B'.RJrO`6@)fLNA4-1i?<O^HT[#O%jeJSlma^?dJ;Y4Xm[4&78n!0<J5Acq5^^V-oEn@Bae&lRR8FB$AB9QMS+Hb=KN(4$3(]4iCUs^Bc04s@"VMb;\%1!(VUtBq]l5r3>Z\/V\lPoVn6Loa<k]S>q?2n/!AAEo@^W)b>Wcs/9-lNB_-p=O_3Cc?o8o'Pn\`op"2;b?IF,]HJf^X@!T+E).U?kSD?em.#&'M++O^n.#u%@)82j63/ro[*lL)`[Fhopde=)a(-R1L-)X7X0362OKjf3X^UYO9(R4!'=6tB9K:PSCGSF"X>WSa&W_]\ZORh1'KeftXu>flhf^M:rf?6&@[,Mk,u<K:9mO(QRYaW,f[V;@SW@M`ETYS'_253If'\LbS@o!j'/LsW]aW/"mJ;:MY-X0nb8>ZV=L=iRk(jmpGn=:&"B?tU$p^YgTb=Y]%RfYok:03kb6*eK8L[2ueoq2;>s?Uf&F_7+3d=Kh6OmLa5[1lMM2L$/.e0@u;+V-(`ME5aZhnL`3G\@I5/\'=,q;7geFK&n5F)S;=.j=kXcS;;RIRK6/1`TZsHQ$APTSqXSJqe>->qZ@+Ui%U*P3oHM(E/FUmM%dBE#ml!,r]J4RGb('g1l4MK+4sTpWs3]Nb^G6T3O\\R8.tZ1]GjR3g7d0a;(QE5MQIr=6JS<fl;>$Oe(qP\SQW8iME7eOk7i#-_9U*eCqP<lDpio-fB-e=?(SNtX0YU>O4>+F06'3XkDru?O4AQF53<d7191R3/&4%FD8-j`W!Za7Gc>4N=@Ti`9%]j\bMpctM[IL7[4\<?c&-hliIR>CP@Fj$E#\(F[pu:<_/c4M9e34"Afqh!h#Tr"=!B::EUCOZG&D%Wn^"'L%?+qUE2GWD^;9"_M4ng-;[XLYDq:b]X\Ds'XFdNCCFgL!\U)@RmL49FR;&&d"09N2If~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000061 00000 n 
+0000000102 00000 n 
+0000000209 00000 n 
+0000000321 00000 n 
+0000000514 00000 n 
+0000000582 00000 n 
+0000000862 00000 n 
+0000000921 00000 n 
+trailer
+<<
+/ID 
+[<ec89ca6c7a8f99d720d1224782688641><ec89ca6c7a8f99d720d1224782688641>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+1985
+%%EOF

tasks/insurance_mojave_crest_assurance_company_2a800bf7/instruction.md

@@ -0,0 +1 @@
+Please process the fax to my email this morning. Follow all the procedures outlined in the SOP (Mojave_Crest_Appeals_SOP.pdf).

tasks/insurance_mojave_crest_assurance_company_2a800bf7/system_prompt.md

@@ -0,0 +1 @@
+Today's date is, May, 12, 2026. You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.

tasks/insurance_mojave_crest_assurance_company_2a800bf7/task.toml

@@ -0,0 +1,23 @@
+schema_version = "1.1"
+
+[task]
+name = "sop-tasks/insurance_mojave_crest_assurance_company_2a800bf7"
+description = "SOP task from CSV row 2a800bf7"
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 3600.0
+
+[environment]
+os = "linux"
+cpus = 2
+memory_mb = 4096
+env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
+
+[[environment.mcp_servers]]
+name = "mcp-proxy"
+transport = "streamable-http"
+url = "http://localhost:8000/mcp"
+

tasks/insurance_mojave_crest_assurance_company_2a800bf7/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/insurance_mojave_crest_assurance_company_2a800bf7/tests/task_prompt.txt

@@ -0,0 +1 @@
+Please process the fax to my email this morning. It's May 12, 2026. Follow all the procedures outlined in the SOP (Mojave_Crest_Appeals_SOP.pdf).

tasks/insurance_mojave_crest_assurance_company_2a800bf7/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py