Add Handbook.md benchmark tasks

tasks/hr_crestwood_university_d93daa01/environment/Dockerfile

@@ -0,0 +1,7 @@
+FROM handbook_base
+
+COPY initial_external_services/ /data/
+COPY initial_external_services/ /initial_data/
+COPY initial_workspace/ /workdir/
+
+ENV INPUTDIR=/data OUTPUTDIR=/data WORLDBENCH_TOOL_SETS="syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core"

tasks/hr_crestwood_university_d93daa01/instruction.md

@@ -0,0 +1,7 @@
+Hi, this is Jennifer Alexander and I could use some help scheduling some exit interviews (as per section 12.5 of our SOP, attached as Crestwood_University_HR_SOP.pdf). An intern was working on it earlier, I think, and I'm trying to make sure we haven't missed anyone. 
+
+I've given you access to some emails with some context, as well as the Jira tickets if you need them, as well as Nicole's Calendar. 
+
+All I need you to do is schedule the meetings into Nicole's calendar for everyone who (according to Jira and the SOP) needs to have one. The emails will help here. Nicole sent me some notes yesterday that you'll need to account for (she wrote the SOP, you'll note, so if there's a conflict anywhere go with her email), as well as those the workers gave in their replies. You'll also likely need to refer to employee_roster.xlsx. Anything I've given you is a valid source of info/context: make sure you don't miss any details. 
+
+I think something might have gotten lost in the shuffle with setting everything up, so don't schedule anything today or tomorrow, just so the workers have time to react. Thanks!

tasks/hr_crestwood_university_d93daa01/system_prompt.md

@@ -0,0 +1 @@
+You are an office assistant with access to the user's filesystem (/workdir), email (google_mail_*), slack (slack_*), calendar (google_calendar_*), jira (jira_*), and shopify (shopify_*) tools. The workspace directory /workdir contains files for the task. Do NOT ask the user for more information — use your tools to discover what you need. Begin by listing /workdir and reading relevant files.

tasks/hr_crestwood_university_d93daa01/task.toml

@@ -0,0 +1,23 @@
+schema_version = "1.1"
+
+[task]
+name = "sop-tasks/hr_crestwood_university_d93daa01"
+description = "SOP task from CSV row d93daa01"
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 3600.0
+
+[environment]
+os = "linux"
+cpus = 2
+memory_mb = 4096
+env = { INPUTDIR = "/data", OUTPUTDIR = "/data", WORLDBENCH_TOOL_SETS = "syntara_ds_all google_mail_contacts slack_core google_mail_core google_calendar_core jira_core shopify_core" }
+
+[[environment.mcp_servers]]
+name = "mcp-proxy"
+transport = "streamable-http"
+url = "http://localhost:8000/mcp"
+

tasks/hr_crestwood_university_d93daa01/tests/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()

tasks/hr_crestwood_university_d93daa01/tests/task_prompt.txt

@@ -0,0 +1,7 @@
+Hi, this is Jennifer Alexander and I could use some help scheduling some exit interviews (as per section 12.5 of our SOP, attached as Crestwood_University_HR_SOP.pdf). An intern was working on it earlier, I think, and I'm trying to make sure we haven't missed anyone. 
+
+I've given you access to some emails with some context, as well as the Jira tickets if you need them, as well as Nicole's Calendar. 
+
+All I need you to do is schedule the meetings into Nicole's calendar for everyone who (according to Jira and the SOP) needs to have one. The emails will help here. Nicole sent me some notes yesterday that you'll need to account for (she wrote the SOP, you'll note, so if there's a conflict anywhere go with her email), as well as those the workers gave in their replies. You'll also likely need to refer to employee_roster.xlsx. Anything I've given you is a valid source of info/context: make sure you don't miss any details. 
+
+I think something might have gotten lost in the shuffle with setting everything up, so don't schedule anything today or tomorrow, just so the workers have time to react. Thanks!

tasks/hr_crestwood_university_d93daa01/tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install openpyxl pdfplumber python-docx 2>/dev/null
+python /tests/sop_verifier.py