Add Handbook.md benchmark tasks

agent_harness/README.md

@@ -0,0 +1,4 @@
+# agent-harness
+
+Internal Harbor agent package for the SOP benchmark distribution. See the
+top-level [README](../README.md) for setup and smoke-test instructions.

agent_harness/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "agent-harness"
+version = "0.1.0"
+description = "Harbor agent harness for Syntara tasks"
+requires-python = ">=3.12"
+dependencies = [
+    "harbor",
+    "deepdiff",
+    "litellm",
+    # NOTE: openhands-sdk is NOT a host dep. openhands_agent.py runs the agent
+    # loop in-container via the vendored openhands_runner.py, installed into an
+    # isolated venv baked into the syntara image (see docker/Dockerfile, which
+    # pins the SDK). The host side only does exec/upload/download.
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_harness"]

agent_harness/src/agent_harness/openhands_agent.py

@@ -0,0 +1,254 @@
+"""Harbor agent that runs the OpenHands SDK agent loop *inside* the task
+container.
+
+It execs the vendored Python OpenHands runner (``/app/openhands-runner``)
+in-container, driving the agent loop through the ``BaseEnvironment``
+exec/upload/download contract. That makes it work on any harbor environment —
+including Modal sandboxes, where the host can't reach the in-sandbox MCP port.
+
+This agent targets the **single-container** task shape only (as staged for
+Modal, and its local equivalent). It
+launches the MCP proxy in-container itself; it does not handle the compose
+host-side path.
+
+Container prerequisites (baked into the ``syntara`` base image at build time):
+- the runner at ``/app/openhands-runner/openhands_runner.py``
+- an isolated venv with ``openhands-sdk`` at ``/app/openhands-runner/.venv``
+- the syntara MCP proxy launcher at ``/app/scripts/start.sh``
+
+Usage:
+    harbor run -p .modal_tasks/some_task \
+        --agent-import-path agent_harness.openhands_agent:OpenHandsAgent \
+        -m anthropic/claude-opus-4-8 -e modal --ek registry_secret=aws-ecr-secret
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shlex
+import time
+from pathlib import Path
+
+from harbor.agents.base import BaseAgent
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+MAX_TOOL_CALLS = 200
+
+# In-container locations.
+RUNNER_DIR = "/app/openhands-runner"
+RUNNER_PATH = f"{RUNNER_DIR}/openhands_runner.py"
+RUNNER_PYTHON = f"{RUNNER_DIR}/.venv/bin/python"
+PROXY_START_CMD = "/app/scripts/start.sh --method http --port 8000"
+MCP_URL = "http://localhost:8000/mcp"
+HEALTH_URL = "http://localhost:8000/health"
+REMOTE_LOG_DIR = "/logs/agent"
+REMOTE_CONFIG_PATH = "/tmp/openhands_run_config.json"
+REMOTE_TRAJECTORY_PATH = f"{REMOTE_LOG_DIR}/trajectory.json"
+REMOTE_RUNNER_LOG_PATH = f"{REMOTE_LOG_DIR}/run-openhands.log"
+REMOTE_PROXY_LOG_PATH = f"{REMOTE_LOG_DIR}/mcp-proxy.log"
+
+# Timeouts (seconds).
+PROBE_TIMEOUT = 15
+HEALTH_TIMEOUT = 180
+# Generous ceiling for the runner exec; the effective limit is harbor's [agent]
+# timeout_sec (× --agent-timeout-multiplier).
+AGENT_EXEC_TIMEOUT = 7200
+
+# Host env vars forwarded into the container for the runner. The runner resolves
+# LLM credentials from these per provider (see openhands_runner._resolve_llm_auth).
+FORWARDED_ENV_VARS = (
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_BASE_URL",
+    "OPENAI_API_KEY",
+    "OPENAI_BASE_URL",
+    "OPENROUTER_API_KEY",
+    "GEMINI_API_KEY",
+)
+
+
+def _forwarded_env() -> dict[str, str]:
+    return {k: v for k in FORWARDED_ENV_VARS if (v := os.environ.get(k))}
+
+
+FORWARDED_LLM_KWARGS = (
+    "reasoning_effort",
+    "reasoning_summary",
+    # Opt-in: --ak log_completions=true makes the runner dump each LLM call's
+    # request payload (including the kwargs actually sent, e.g. reasoning_effort)
+    # into the trial's agent/ log dir for verification.
+    "log_completions",
+)
+
+
+class OpenHandsAgent(BaseAgent):
+    """Runs a single OpenHands SDK agent loop inside the task container."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        # Pull the allow-listed LLM kwargs out of harbor's ``--ak`` kwargs;
+        # BaseAgent would otherwise drop them into **kwargs and discard them.
+        # Forwarded to the runner via the run config (see run()).
+        self.llm_kwargs = {
+            k: kwargs.pop(k) for k in FORWARDED_LLM_KWARGS if k in kwargs
+        }
+        super().__init__(*args, **kwargs)
+
+    @staticmethod
+    def name() -> str:
+        return "openhands-agent"
+
+    def version(self) -> str | None:
+        return "0.2.0"
+
+    async def setup(self, environment: BaseEnvironment) -> None:
+        probe = await environment.exec(
+            f"test -f {shlex.quote(RUNNER_PATH)} && test -x {shlex.quote(RUNNER_PYTHON)}",
+            timeout_sec=PROBE_TIMEOUT,
+        )
+        if probe.return_code != 0:
+            raise RuntimeError(
+                f"container is missing the OpenHands runner at {RUNNER_DIR}. "
+                "Rebuild the base image so the runner is staged into it."
+            )
+
+        if not await self._proxy_healthy(environment):
+            self.logger.info("Launching MCP proxy in-container")
+            launch = (
+                f"mkdir -p {REMOTE_LOG_DIR} && "
+                f"nohup {PROXY_START_CMD} </dev/null "
+                f">>{REMOTE_PROXY_LOG_PATH} 2>&1 &"
+            )
+            result = await environment.exec(launch, timeout_sec=PROBE_TIMEOUT)
+            if result.return_code != 0:
+                raise RuntimeError(
+                    f"failed to launch MCP proxy (rc={result.return_code}): "
+                    f"{(result.stderr or '').strip()}"
+                )
+        await self._wait_for_proxy(environment)
+
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        task_dir = Path(environment.environment_dir).parent
+        sp_path = task_dir / "system_prompt.md"
+        system_prompt = sp_path.read_text()
+
+        config = {
+            "instruction": instruction,
+            "systemPrompt": system_prompt,
+            "model": self.model_name,
+            "mcpUrl": MCP_URL,
+            "maxToolCalls": MAX_TOOL_CALLS,
+            "llmKwargs": self.llm_kwargs,
+            # Host-visible log dir, so opt-in completion logs land where the
+            # trial syncs them back (see openhands_runner log_completions).
+            "logDir": REMOTE_LOG_DIR,
+        }
+        config_path = (self.logs_dir / "openhands_run_config.json").resolve()
+        config_path.write_text(json.dumps(config, indent=2))
+        await environment.upload_file(str(config_path), REMOTE_CONFIG_PATH)
+
+        self.logger.info("Running OpenHands runner in-container (model=%s)", self.model_name)
+
+        # The runner writes the trajectory JSON to a dedicated file (argv[2]);
+        # the OpenHands SDK's human-readable transcript + our logs go to stdout/
+        # stderr, which we capture together in the runner log. Keeping the two
+        # streams separate is essential — the SDK prints to stdout, so a stdout
+        # redirect would corrupt the machine-readable trajectory.
+        command = (
+            f"mkdir -p {REMOTE_LOG_DIR} && "
+            f"{shlex.quote(RUNNER_PYTHON)} {shlex.quote(RUNNER_PATH)} "
+            f"{shlex.quote(REMOTE_CONFIG_PATH)} {shlex.quote(REMOTE_TRAJECTORY_PATH)} "
+            f"> {REMOTE_RUNNER_LOG_PATH} 2>&1"
+        )
+        result = await environment.exec(
+            command=command,
+            env=_forwarded_env(),
+            timeout_sec=AGENT_EXEC_TIMEOUT,
+        )
+
+        runner_log = await self._download_quiet(
+            environment, REMOTE_RUNNER_LOG_PATH, "run-openhands.log"
+        )
+        trajectory = await self._download_quiet(
+            environment, REMOTE_TRAJECTORY_PATH, "trajectory.json"
+        )
+
+        if result.return_code != 0:
+            log_tail = "<runner log unavailable>"
+            if runner_log is not None:
+                log_tail = runner_log.read_text(errors="replace")[-2000:]
+            raise RuntimeError(
+                f"OpenHands runner exited {result.return_code}\n"
+                f"--- exec stderr (last 2000) ---\n{(result.stderr or '')[-2000:]}\n"
+                f"--- runner log tail ---\n{log_tail}"
+            )
+
+        if trajectory is None or trajectory.stat().st_size == 0:
+            raise RuntimeError(
+                "OpenHands runner produced no trajectory output; "
+                f"see {self.logs_dir} for the runner log"
+            )
+
+        traj = json.loads(trajectory.read_text())
+        context.n_input_tokens = traj.get("input_tokens")
+        context.n_output_tokens = traj.get("output_tokens")
+        context.n_cache_tokens = traj.get("cache_tokens")
+        context.cost_usd = traj.get("cost_usd")
+        context.metadata = {
+            "agent_id": traj.get("agent_id"),
+            "model": traj.get("model"),
+            "n_tool_calls": traj.get("n_tool_calls"),
+            "stopped_reason": traj.get("stopped_reason"),
+            "final_output_chars": len(traj.get("final_output") or ""),
+            "error_message": traj.get("error_message"),
+        }
+
+        # Surface genuine agent-loop errors as a trial exception so broken runs
+        # aren't averaged into results.
+        if traj.get("stopped_reason") == "error":
+            raise RuntimeError(
+                "OpenHands agent loop ended with an error after "
+                f"{traj.get('n_tool_calls') or 0} tool call(s): "
+                f"{traj.get('error_message') or '<no error message>'}"
+            )
+
+    async def _proxy_healthy(self, environment: BaseEnvironment) -> bool:
+        result = await environment.exec(
+            f"curl -fsS -o /dev/null --max-time 5 {shlex.quote(HEALTH_URL)}",
+            timeout_sec=PROBE_TIMEOUT,
+        )
+        return result.return_code == 0
+
+    async def _wait_for_proxy(self, environment: BaseEnvironment) -> None:
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            if await self._proxy_healthy(environment):
+                self.logger.info("MCP proxy healthy")
+                return
+            await asyncio.sleep(2)
+        tail = await environment.exec(
+            f"tail -c 2000 {REMOTE_PROXY_LOG_PATH} 2>/dev/null || true",
+            timeout_sec=PROBE_TIMEOUT,
+        )
+        raise RuntimeError(
+            f"MCP proxy at {HEALTH_URL} never became healthy within {HEALTH_TIMEOUT}s"
+            f"\n--- proxy log tail ---\n{(tail.stdout or '').strip() or '<no log>'}"
+        )
+
+    async def _download_quiet(
+        self, environment: BaseEnvironment, remote_path: str, filename: str
+    ) -> Path | None:
+        """Download a container file into logs_dir; None if unavailable."""
+        target = self.logs_dir / filename
+        try:
+            await environment.download_file(remote_path, target)
+        except Exception as e:
+            self.logger.warning("could not download %s: %s", remote_path, e)
+            return None
+        return target if target.exists() else None

agent_harness/src/agent_harness/openhands_runner.py

@@ -0,0 +1,359 @@
+"""In-container OpenHands agent runner.
+
+Runs the OpenHands SDK agent loop *inside* the task container, talking to the
+syntara MCP proxy at ``localhost:8000/mcp`` (reachable in-container on every
+harbor environment, including Modal sandboxes where the host can't reach the
+port).
+
+Baked into the ``syntara`` image at build time (this
+single file is staged to ``/app/openhands-runner/openhands_runner.py`` next to an
+isolated venv holding ``openhands-sdk``). The host-side ``OpenHandsAgent`` uploads
+a config JSON, execs this script, and downloads the trajectory it writes.
+
+Contract:
+- argv[1] = path to a config JSON: {instruction, systemPrompt, model, mcpUrl,
+  maxToolCalls}.
+- argv[2] = path to write the result/trajectory JSON to. This is a **dedicated
+  file**, NOT stdout: the OpenHands SDK prints a human-readable transcript to
+  stdout, so the machine-readable result must go to its own file for the host
+  to parse it.
+- The process exits 0 even when the agent loop errored — the error is recorded
+  in the trajectory's ``stopped_reason``/``error_message`` so the host decides
+  whether to fail the trial.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any
+
+MCP_TOOL_TIMEOUT = 300
+WORKSPACE_DIR = "/tmp/openhands_workspace"
+STATE_DIR = "/tmp/openhands_state"
+
+def _log(msg: str) -> None:
+    print(msg, file=sys.stderr, flush=True)
+
+
+def _resolve_llm_kwargs(config: dict[str, Any]) -> tuple[dict[str, Any], str | None]:
+    """Split LLM kwargs into ``(constructor_kwargs, reasoning_effort)``.
+
+    ``reasoning_effort`` is pulled out so it can be applied post-construction. 
+    openhands.sdk otherwise won't accept `max` as a value. 
+    """
+    llm_kwargs = dict(config.get("llmKwargs") or {})
+
+    effort = llm_kwargs.pop("reasoning_effort", None)
+    if effort is not None:
+        effort = str(effort).strip().lower() or None
+
+    return llm_kwargs, effort
+
+
+def _resolve_llm_auth(model_name: str) -> tuple[str, str | None, str | None]:
+    """Map harbor's ``provider/model`` id to a litellm (model, api_key, base_url).
+
+    ``openrouter/anthropic/<model>`` is rewritten to ``anthropic/<model>`` (dots
+    -> dashes) so Anthropic models route through the Anthropic provider, never
+    OpenRouter. To send Anthropic traffic through an LLM proxy, point
+    ``ANTHROPIC_BASE_URL`` at the proxy endpoint and set ``ANTHROPIC_API_KEY`` to
+    the proxy token.
+    """
+    if "/" not in model_name:
+        raise ValueError(
+            f"model must be in 'provider/model' form; got {model_name!r}"
+        )
+    provider, model = model_name.split("/", 1)
+
+    if provider == "openrouter" and model.startswith("anthropic/"):
+        provider = "anthropic"
+        model = model.split("/", 1)[1].replace(".", "-")
+
+    if provider == "anthropic":
+        litellm_model = f"anthropic/{model}"
+        return (
+            litellm_model,
+            os.environ.get("ANTHROPIC_API_KEY"),
+            os.environ.get("ANTHROPIC_BASE_URL"),
+        )
+
+    _provider_key_env = {
+        "openai": "OPENAI_API_KEY",
+        "openrouter": "OPENROUTER_API_KEY",
+        "google": "GEMINI_API_KEY",
+        "gemini": "GEMINI_API_KEY",
+    }
+    api_key = os.environ.get(_provider_key_env.get(provider, ""))
+    base_url = os.environ.get("OPENAI_BASE_URL") if provider == "openai" else None
+    return model_name, api_key, base_url
+
+
+def _usage(*llms: Any) -> dict[str, int | float | None]:
+    """Pull token/cost totals from one or more LLMs' metrics, tolerant of API drift.
+
+    Sums across every LLM passed in (e.g. the agent loop plus the condenser) so
+    the reported cost and cache usage reflect *all* model traffic for the trial,
+    not just the main agent call. Each field stays ``None`` if no LLM exposed it,
+    so a metrics-shape change degrades to null rather than crashing the trial.
+    """
+    input_tokens = output_tokens = cache_tokens = None
+    cost: float | None = None
+
+    def _accumulate(current: int | float | None, value: Any) -> int | float | None:
+        if not isinstance(value, (int, float)):
+            return current
+        return value if current is None else current + value
+
+    for llm in llms:
+        metrics = getattr(llm, "metrics", None)
+        if metrics is None:
+            continue
+        cost = _accumulate(cost, getattr(metrics, "accumulated_cost", None))
+        token_usage = getattr(metrics, "accumulated_token_usage", None)
+        if token_usage is None:
+            continue
+        input_tokens = _accumulate(input_tokens, getattr(token_usage, "prompt_tokens", None))
+        output_tokens = _accumulate(output_tokens, getattr(token_usage, "completion_tokens", None))
+        cache_tokens = _accumulate(cache_tokens, getattr(token_usage, "cache_read_tokens", None))
+
+    return {
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "cache_tokens": cache_tokens,
+        # accumulated_cost is 0.0 when litellm has no pricing for the model
+        # (e.g. an aliased model behind a proxy); treat that as "unknown" so
+        # downstream cost reporting can fall back to provider dashboards.
+        "cost_usd": cost if cost else None,
+    }
+
+
+# Tool observations larger than this (in chars) are kept intact. The OpenHands
+# SDK otherwise hard-truncates every tool result at DEFAULT_TEXT_CONTENT_LIMIT
+# (50_000) — see openhands/sdk/utils/truncate.py — which silently drops the tail
+# of large reads (a 24-page SOP PDF is ~56 KB, big spreadsheet dumps similar).
+# The worldbench baseline feeds tool outputs to the model untruncated (it only
+# shortens them when building compaction summaries), so an aggressive per-call
+# cap is a parity gap that costs OpenHands rubric points on SOP-heavy tasks.
+# 1 MB clears every realistic task read with headroom while still guarding
+# against a pathological multi-MB blob blowing the context window.
+TOOL_OBSERVATION_CHAR_LIMIT = 1_000_000
+
+
+def _raise_tool_observation_limit() -> None:
+    """Lift the SDK's hardcoded 50K tool-observation truncation cap.
+
+    The limit is a module-level constant (the per-message ``enable_truncation``
+    field is deprecated), and ``message.py`` binds it at import time, so both the
+    source module and that imported reference must be patched.
+    """
+    import openhands.sdk.llm.message as _message
+    import openhands.sdk.utils.truncate as _truncate
+
+    _truncate.DEFAULT_TEXT_CONTENT_LIMIT = TOOL_OBSERVATION_CHAR_LIMIT
+    _message.DEFAULT_TEXT_CONTENT_LIMIT = TOOL_OBSERVATION_CHAR_LIMIT
+
+
+def run(config: dict[str, Any]) -> dict[str, Any]:
+    from pydantic import SecretStr
+
+    from openhands.sdk import (
+        LLM,
+        Agent,
+        Conversation,
+        ConversationExecutionStatus,
+        Event,
+        LLMSummarizingCondenser,
+        MessageEvent,
+    )
+    from openhands.sdk.event import ActionEvent, AgentErrorEvent
+    from openhands.sdk.llm.message import content_to_str
+
+    _raise_tool_observation_limit()
+
+    model_name = config["model"]
+    litellm_model, api_key, base_url = _resolve_llm_auth(model_name)
+    llm_kwargs, reasoning_effort = _resolve_llm_kwargs(config)
+
+    # Opt-in per-call request logging (--ak log_completions=true). Point the
+    # folder at the trajectory's log dir so it's downloaded with the trial. Each
+    # logged payload records the kwargs actually sent to the model (e.g.
+    # reasoning_effort), giving a verifiable record of what was requested.
+    if llm_kwargs.pop("log_completions", False):
+        log_root = config.get("logDir") or WORKSPACE_DIR
+        llm_kwargs["log_completions"] = True
+        llm_kwargs["log_completions_folder"] = os.path.join(log_root, "completions")
+        _log(f"completion logging enabled -> {llm_kwargs['log_completions_folder']}")
+
+    _log(
+        f"Running OpenHands loop (model={litellm_model}, "
+        f"base_url={base_url or 'default'})"
+    )
+
+    # timeout=600 matches worldbench's ~10-min per-call ceiling (anthropic-sdk.ts
+    # uses `timeout: 600000 - 1`). The OpenHands LLM default is 300s, and
+    # litellm.Timeout isn't in its retryable set, so a slow proxy call surfaces as
+    # a fatal RuntimeError and drops the trial — doubling the ceiling matches the
+    # baseline and cuts those spurious exclusions.
+    llm = LLM(
+        usage_id="agent",
+        model=litellm_model,
+        api_key=SecretStr(api_key) if api_key else None,
+        base_url=base_url,
+        timeout=600,
+        **llm_kwargs,
+    )
+
+    # Context compaction, for parity with worldbench (which summarizes old turns
+    # to stay within the window). We use the SDK defaults (event-count trigger at
+    # max_size=240, keep_first=2): for these tasks neither harness's compaction
+    # actually fires, but this gives the *reactive* recovery path — on a real
+    # context-window overflow the SDK emits a CondensationRequest and retries with
+    # a summarized history instead of hard-erroring the trial. The summarizer runs
+    # on the same model/proxy as the agent (worldbench summarizes with its agent
+    # model too); a separate LLM instance just keeps its metrics distinct.
+    condenser_llm = LLM(
+        usage_id="condenser",
+        model=litellm_model,
+        api_key=SecretStr(api_key) if api_key else None,
+        base_url=base_url,
+        timeout=600,
+        **llm_kwargs,
+    )
+
+    if reasoning_effort is not None:
+        if litellm_model.startswith("anthropic/"):
+            llm.reasoning_effort = reasoning_effort
+            condenser_llm.reasoning_effort = reasoning_effort
+        else:
+            rb = {"reasoning": {"effort": reasoning_effort}}
+            llm.litellm_extra_body = {**llm.litellm_extra_body, **rb}
+            condenser_llm.litellm_extra_body = {**condenser_llm.litellm_extra_body, **rb}
+
+    # Pass the office-assistant prompt as `system_prompt` (verbatim full
+    # replacement) rather than an AgentContext suffix — otherwise OpenHands'
+    # default SWE-coding system prompt dominates and diverges from worldbench,
+    # which uses the office-assistant prompt as *the* system message.
+    agent = Agent(
+        llm=llm,
+        tools=[],
+        mcp_config={
+            "mcpServers": {
+                "syntara": {
+                    "url": config["mcpUrl"],
+                    "timeout": MCP_TOOL_TIMEOUT,
+                }
+            }
+        },
+        system_prompt=config["systemPrompt"],
+        condenser=LLMSummarizingCondenser(llm=condenser_llm),
+    )
+
+    n_tool_calls = 0
+    n_agent_errors = 0
+    final_output = ""
+    error_message: str | None = None
+
+    def callback(event: Event) -> None:
+        nonlocal n_tool_calls, n_agent_errors, final_output, error_message
+        if isinstance(event, ActionEvent):
+            n_tool_calls += 1
+        elif isinstance(event, AgentErrorEvent):
+            n_agent_errors += 1
+            error_message = getattr(event, "error", None) or str(event)
+        elif isinstance(event, MessageEvent) and event.source == "agent":
+            text = "".join(content_to_str(event.to_llm_message().content))
+            if text:
+                final_output = text
+
+    conversation = Conversation(
+        agent=agent,
+        callbacks=[callback],
+        workspace=WORKSPACE_DIR,
+        persistence_dir=STATE_DIR,
+        max_iteration_per_run=int(config["maxToolCalls"]),
+    )
+
+    saw_error = False
+    try:
+        conversation.send_message(config["instruction"])
+        conversation.run()
+    except Exception as e:  # provider/transport/loop failure
+        saw_error = True
+        error_message = error_message or str(e)
+        _log(f"OpenHands conversation.run() raised: {e}")
+
+    status = getattr(conversation.state, "execution_status", None)
+    status_val = getattr(status, "value", status)
+    infra_failure = saw_error or status_val == ConversationExecutionStatus.ERROR.value
+    did_work = n_tool_calls > 0 or bool(final_output)
+    if saw_error:
+        # conversation.run() raised (e.g. litellm.Timeout / provider error): the
+        # loop was cut short and never completed, so this is a transport/infra
+        # failure even if the agent had already made tool calls.
+        stopped_reason = "error"
+    elif infra_failure and not did_work:
+        # ERROR status with nothing produced — genuine failure.
+        stopped_reason = "error"
+    elif status_val == ConversationExecutionStatus.STUCK.value:
+        stopped_reason = "stuck"
+    elif status_val == ConversationExecutionStatus.FINISHED.value:
+        stopped_reason = "end_turn"
+    else:
+        stopped_reason = "max_tool_calls"
+
+    usage = _usage(llm, condenser_llm)
+
+    return {
+        "agent_id": "openhands_sdk",
+        "model": litellm_model,
+        "final_output": final_output,
+        "n_tool_calls": n_tool_calls,
+        "n_agent_errors": n_agent_errors,
+        "stopped_reason": stopped_reason,
+        "error_message": error_message,
+        "input_tokens": usage["input_tokens"],
+        "output_tokens": usage["output_tokens"],
+        "cache_tokens": usage["cache_tokens"],
+        "cost_usd": usage["cost_usd"],
+    }
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        _log("usage: openhands_runner.py <config.json> <output.json>")
+        sys.exit(2)
+
+    config_path, output_path = sys.argv[1], sys.argv[2]
+    config = json.loads(open(config_path).read())
+    try:
+        result = run(config)
+    except Exception as e:
+        # Hard failure before/around the loop — still emit a trajectory so the
+        # host can surface it as an errored (excluded) trial rather than a crash.
+        result = {
+            "agent_id": "openhands_sdk",
+            "model": config.get("model"),
+            "final_output": "",
+            "n_tool_calls": 0,
+            "stopped_reason": "error",
+            "error_message": str(e),
+            "input_tokens": None,
+            "output_tokens": None,
+            "cache_tokens": None,
+            "cost_usd": None,
+        }
+
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+    with open(output_path, "w") as f:
+        json.dump(result, f, indent=2)
+    _log(
+        f"wrote trajectory to {output_path} "
+        f"(stopped_reason={result.get('stopped_reason')}, "
+        f"n_tool_calls={result.get('n_tool_calls')})"
+    )
+
+
+if __name__ == "__main__":
+    main()

agent_harness/src/agent_harness/sop_verifier.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run bundled SOP Python verifiers inside a Harbor task container."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from typing import Any
+
+WORKDIR = Path("/workdir")
+DATA_DIR = Path("/data")
+INITIAL_DATA_DIR = Path("/initial_data")
+TESTS_DIR = Path("/tests")
+VERIFIER_DIR = Path("/logs/verifier")
+
+SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
+    "slack": ("slack.json", ("slack.json", "slack_data.json")),
+    "google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
+    "google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
+    "jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
+    "shopify": ("shopify_data.json", ("shopify_data.json",)),
+}
+
+
+def _state_path(service: str, seed_name: str) -> Path | None:
+    candidates = [
+        DATA_DIR / service / "final.json",
+        DATA_DIR / service / seed_name,
+        INITIAL_DATA_DIR / service / seed_name,
+    ]
+    return next((p for p in candidates if p.is_file()), None)
+
+
+def _build_compat_external_services(dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
+        src = _state_path(service, seed_name)
+        if src is not None:
+            for compat_name in compat_names:
+                shutil.copy2(src, dest / compat_name)
+
+
+def _coerce_result(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        passed = bool(raw.get("pass", raw.get("passed", False)))
+        score = raw.get("score", 1.0 if passed else 0.0)
+        try:
+            score = float(score)
+        except (TypeError, ValueError):
+            score = 1.0 if passed else 0.0
+        return {
+            "pass": passed,
+            "score": max(0.0, min(1.0, score)),
+            "feedback": str(raw.get("feedback", "")),
+        }
+    passed = bool(raw)
+    return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
+
+
+def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
+    rubric_id = str(rubric.get("id") or "rubric")
+    code = rubric.get("verifier_code")
+    if not isinstance(code, str) or not code.strip():
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": "rubric has no verifier_code",
+        }
+
+    namespace: dict[str, Any] = {"__builtins__": __builtins__}
+    try:
+        exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
+        verify = namespace.get("verify")
+        if not callable(verify):
+            raise RuntimeError("verifier_code did not define verify()")
+        result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
+        return {"id": rubric_id, **result}
+    except Exception:
+        return {
+            "id": rubric_id,
+            "pass": False,
+            "score": 0.0,
+            "feedback": traceback.format_exc(),
+        }
+
+
+def main() -> None:
+    rubrics_path = TESTS_DIR / "rubrics.json"
+    if not rubrics_path.is_file():
+        print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
+        sys.exit(1)
+
+    rubrics = json.loads(rubrics_path.read_text())
+    if not isinstance(rubrics, list):
+        print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
+        compat_dir = Path(tmp)
+        _build_compat_external_services(compat_dir)
+        results = [_run_one(r, compat_dir) for r in rubrics]
+
+    total = len(results)
+    passed = sum(1 for r in results if r.get("pass"))
+    average_score = round(
+        sum(float(r.get("score", 0.0)) for r in results) / total,
+        4,
+    ) if total else 0.0
+
+    print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
+    for result in results:
+        status = "PASS" if result.get("pass") else "FAIL"
+        feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
+        print(f"  [{status}] {result.get('id')}: {feedback}")
+
+    output = {
+        "passed": passed == total,
+        "rubrics_passed": passed,
+        "rubrics_total": total,
+        "score": average_score,
+        "rubric_results": results,
+    }
+    (TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
+
+    VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
+    (VERIFIER_DIR / "reward.txt").write_text(str(average_score))
+
+
+if __name__ == "__main__":
+    main()