Add Handbook.md benchmark tasks

docker/packages/mcp_proxy/tests/test_datadir_namespacing.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+
+from mcp_proxy.commands.mcp import _build_subprocess_env
+
+
+def test_inputdir_namespaced_per_server(tmp_path, monkeypatch):
+    """When INPUTDIR is set in env, _build_subprocess_env appends server_name."""
+    monkeypatch.setenv("INPUTDIR", str(tmp_path))
+    env = _build_subprocess_env(tmp_path, server_name="google_mail")
+    assert env["INPUTDIR"] == str(tmp_path / "google_mail")
+    assert Path(env["INPUTDIR"]).is_dir()
+
+
+def test_inputdir_defaults_to_base_dir(tmp_path, monkeypatch):
+    """When INPUTDIR is not in env, it defaults to base_dir/server_name."""
+    monkeypatch.delenv("INPUTDIR", raising=False)
+    env = _build_subprocess_env(tmp_path, server_name="google_mail")
+    assert env["INPUTDIR"] == str(tmp_path / "google_mail")
+    assert Path(env["INPUTDIR"]).is_dir()
+
+
+def test_outputdir_from_env(tmp_path, monkeypatch):
+    """When OUTPUTDIR is set in env, it is namespaced per server."""
+    monkeypatch.setenv("OUTPUTDIR", str(tmp_path))
+    env = _build_subprocess_env(tmp_path, server_name="slack")
+    assert env["OUTPUTDIR"] == str(tmp_path / "slack")
+    assert Path(env["OUTPUTDIR"]).is_dir()
+
+
+def test_outputdir_defaults_to_base_dir(tmp_path, monkeypatch):
+    """When OUTPUTDIR is not in env, it defaults to base_dir/server_name."""
+    monkeypatch.delenv("OUTPUTDIR", raising=False)
+    env = _build_subprocess_env(tmp_path, server_name="google_mail")
+    assert env["OUTPUTDIR"] == str(tmp_path / "google_mail")
+    assert Path(env["OUTPUTDIR"]).is_dir()
+
+
+def test_all_three_dirs_always_set(tmp_path, monkeypatch):
+    """INPUTDIR, OUTPUTDIR, and BUNDLE_OUTPUT_DIR are always present in the returned env."""
+    monkeypatch.delenv("INPUTDIR", raising=False)
+    monkeypatch.delenv("OUTPUTDIR", raising=False)
+    env = _build_subprocess_env(tmp_path, server_name="jira")
+    assert "INPUTDIR" in env
+    assert "OUTPUTDIR" in env
+    assert "BUNDLE_OUTPUT_DIR" in env
+
+
+def test_bundle_output_dir_namespaced_per_service(tmp_path, monkeypatch):
+    """BUNDLE_OUTPUT_DIR is per-service so each service writes its own
+    services/<name>/state.json, matching the nested input bundle layout."""
+    monkeypatch.setenv("OUTPUTDIR", str(tmp_path))
+
+    env_a = _build_subprocess_env(tmp_path, server_name="google_mail")
+    env_b = _build_subprocess_env(tmp_path, server_name="shopify")
+
+    assert env_a["BUNDLE_OUTPUT_DIR"] == str(tmp_path / "services" / "google_mail")
+    assert env_b["BUNDLE_OUTPUT_DIR"] == str(tmp_path / "services" / "shopify")
+
+
+def test_bundle_output_dir_defaults_to_base_dir(tmp_path, monkeypatch):
+    """Without OUTPUTDIR in env, BUNDLE_OUTPUT_DIR is rooted at
+    base_dir/services/<name>."""
+    monkeypatch.delenv("OUTPUTDIR", raising=False)
+    env = _build_subprocess_env(tmp_path, server_name="emails_mock")
+    assert env["BUNDLE_OUTPUT_DIR"] == str(tmp_path / "services" / "emails_mock")
+    assert Path(env["BUNDLE_OUTPUT_DIR"]).is_dir()

docker/packages/mcp_proxy/tests/test_env_compartmentalization.py

@@ -0,0 +1,107 @@
+"""Regression tests for proxy-level env compartmentalization.
+
+This is the structural defense against agent-side secret exfiltration via
+mcp tools, like core's bash.
+"""
+
+from mcp_proxy.commands.mcp import _build_subprocess_env
+
+
+def test_declared_credential_is_forwarded(tmp_path, monkeypatch):
+    """A service that declares a credential by exact name in its mcp.json
+    ``env`` list does receive it — opt-in bypasses the denylist."""
+    monkeypatch.setenv("BRAVE_API_KEY", "secret-for-web")
+    env = _build_subprocess_env(tmp_path, server_name="web", declared_secrets=["BRAVE_API_KEY"])
+    assert env["BRAVE_API_KEY"] == "secret-for-web"
+
+
+def test_secret_isolation_between_services(tmp_path, monkeypatch):
+    """Same proxy env, two services with different declarations — only the
+    declaring service sees the secret."""
+    monkeypatch.setenv("BRAVE_API_KEY", "secret-do-not-leak")
+
+    web_env = _build_subprocess_env(tmp_path, server_name="web", declared_secrets=["BRAVE_API_KEY"])
+    core_env = _build_subprocess_env(tmp_path, server_name="core", declared_secrets=())
+
+    assert "BRAVE_API_KEY" in web_env
+    assert "BRAVE_API_KEY" not in core_env
+
+
+def test_credential_shaped_vars_are_dropped(tmp_path, monkeypatch):
+    pollute = {
+        # KEY
+        "BRAVE_API_KEY": "x",
+        "MY_PRIVATE_KEY_PATH": "/etc/key",
+        "WORLDBENCH_FOO_API_KEY": "x",
+        "GITHUB_TOKEN": "ghp_x",
+        "AWS_SECRET_ACCESS_KEY": "aws-x",
+        "stripe_secret": "x",  # case-insensitive
+        "DB_PASSWORD": "hunter2",
+        "MYSQL_PASSWD": "old-school-x",
+        "SSH_PASSPHRASE": "x",
+        "GPG_PASSPHRASE": "x",
+        "GOOGLE_CREDENTIALS": "json-x",
+        "OAUTH_CLIENT_ID": "x",
+        "BEARER_TOKEN": "x",
+        "JWT_SECRET": "x",
+        "JWT_SIGNING_KEY": "x",
+        "SESSION_COOKIE": "x",
+        "AUTH_COOKIE": "x",
+        "TOTP_SECRET": "x",
+        "HOTP_KEY": "x",
+        "DATABASE_URL": "postgres://app_user:hunter2@db.internal:5432/app",
+        "REDIS_URL": "redis://:hunter2@cache.internal:6379/0",
+        "PIP_INDEX_URL": "https://user:token@pypi.example.com/simple",
+        "UV_INDEX_URL": "https://user:token@uv.example.com/simple",
+        "JDBC_DATABASE_URL": "jdbc:postgresql://app:hunter2@db.internal/app",
+        "JDBC_MYSQL_URL": "jdbc:mysql://u:p@db:3306/x",
+        "GIT_REMOTE_URL": "https://ghp_xxxxxxxxxxxx@github.com/org/repo",
+        "REGISTRY_URL": "https://pypi-token-here@pypi.example.com/simple",
+    }
+    for k, v in pollute.items():
+        monkeypatch.setenv(k, v)
+
+    env = _build_subprocess_env(tmp_path, server_name="core", declared_secrets=())
+
+    for k in pollute:
+        assert k not in env, f"{k} leaked despite credential-shaped name"
+
+
+def test_innocuous_vars_pass_through(tmp_path, monkeypatch):
+    """Default-allow: anything without a credential keyword is forwarded.
+    This is what makes the simplified model work — we don't have to enumerate
+    every system var a subprocess might legitimately need."""
+    monkeypatch.setenv("PATH", "/usr/bin:/bin")
+    monkeypatch.setenv("HOME", "/home/model")
+    monkeypatch.setenv("LANG", "en_US.UTF-8")
+    monkeypatch.setenv("WORLDBENCH_TASK_ID", "task-123")
+    monkeypatch.setenv("WORLDBENCH_TOOL_SETS", "core_read")
+    monkeypatch.setenv("RANDOM_CUSTOM_VAR", "value")
+    monkeypatch.setenv("BRAVE_SEARCH_URL", "https://api.search.brave.com/res/v1/web/search")
+    monkeypatch.setenv("PUBLIC_API_BASE", "https://api.example.com/v1")
+    monkeypatch.setenv("DATABASE_URL", "postgres://localhost/dev")  # no userinfo
+
+    env = _build_subprocess_env(tmp_path, server_name="core", declared_secrets=())
+
+    assert env["PATH"] == "/usr/bin:/bin"
+    assert env["HOME"] == "/home/model"
+    assert env["LANG"] == "en_US.UTF-8"
+    assert env["WORLDBENCH_TASK_ID"] == "task-123"
+    assert env["WORLDBENCH_TOOL_SETS"] == "core_read"
+    assert env["RANDOM_CUSTOM_VAR"] == "value"
+    assert env["BRAVE_SEARCH_URL"] == "https://api.search.brave.com/res/v1/web/search"
+    assert env["PUBLIC_API_BASE"] == "https://api.example.com/v1"
+    assert env["DATABASE_URL"] == "postgres://localhost/dev"
+
+
+def test_url_with_userinfo_can_be_explicitly_declared(tmp_path, monkeypatch):
+    """A service that legitimately needs a credentialed URL can opt in by
+    declaring the var name in mcp.json's ``secrets`` — declared names bypass
+    both the name-shape and value-shape checks."""
+    monkeypatch.setenv("DATABASE_URL", "postgres://app_user:hunter2@db.internal:5432/app")
+
+    db_user_env = _build_subprocess_env(tmp_path, server_name="my-db-service", declared_secrets=["DATABASE_URL"])
+    core_env = _build_subprocess_env(tmp_path, server_name="core", declared_secrets=())
+
+    assert "DATABASE_URL" in db_user_env
+    assert "DATABASE_URL" not in core_env

docker/packages/mcp_proxy/tests/test_faketime.py

@@ -0,0 +1,158 @@
+"""Tests for the ``--current-time`` fake-clock wiring.
+
+The proxy runs every MCP service under the ``faketime`` wrapper when a
+``--current-time`` value is supplied, so the service (and anything it spawns,
+notably syntara's executeBash / executePython sandboxes) observes a clock
+anchored to that instant and advancing at real wall-clock rate.
+
+Unit tests here run anywhere — they exercise the pure mapping and the argv
+assembly without launching a process. The integration test is skipped unless
+the ``faketime`` wrapper is actually installed on PATH.
+"""
+
+import shutil
+import subprocess
+import sys
+from datetime import UTC, datetime
+
+import pytest
+
+from mcp_proxy.commands import mcp as mcp_cmd
+from mcp_proxy.commands.mcp import resolve_faketime_launch, to_faketime_spec
+from mcp_proxy.service import HookStep, _service_argv
+
+# ---------------------------------------------------------------------------
+# to_faketime_spec — RFC3339/ISO-8601 -> faketime wrapper's absolute timestamp
+# ---------------------------------------------------------------------------
+
+
+def test_spec_utc_zulu():
+    assert to_faketime_spec("2025-01-15T09:00:00Z") == "2025-01-15 09:00:00"
+
+
+def test_spec_naive_assumed_utc():
+    assert to_faketime_spec("2025-01-15T09:00:00") == "2025-01-15 09:00:00"
+
+
+def test_spec_offset_converted_to_utc():
+    # 09:00 at +05:00 is 04:00 UTC.
+    assert to_faketime_spec("2025-01-15T09:00:00+05:00") == "2025-01-15 04:00:00"
+
+
+def test_spec_negative_offset_crosses_midnight():
+    # 22:00 at -05:00 is 03:00 UTC the next day.
+    assert to_faketime_spec("2025-01-15T22:00:00-05:00") == "2025-01-16 03:00:00"
+
+
+def test_spec_invalid_raises():
+    with pytest.raises(ValueError):
+        to_faketime_spec("not-a-timestamp")
+
+
+def test_spec_empty_raises():
+    # An explicitly-passed empty --current-time is a bad value, not "no clock":
+    # run() routes it here (via `is not None`) so it fails fast rather than
+    # silently falling back to the real clock.
+    with pytest.raises(ValueError):
+        to_faketime_spec("")
+
+
+# ---------------------------------------------------------------------------
+# _service_argv — prepend the faketime wrapper when a prefix is given
+# ---------------------------------------------------------------------------
+
+
+# ``python``/``python3`` resolve to the proxy's own interpreter (see
+# service.resolve_command), so the argv carries ``sys.executable``, not the bare
+# name. The faketime prefix, when present, wraps that resolved command.
+def test_argv_no_prefix():
+    step = HookStep(command="python", args=["-m", "server", "--http"])
+    assert _service_argv(step, None) == [sys.executable, "-m", "server", "--http"]
+
+
+def test_argv_empty_prefix_is_noop():
+    step = HookStep(command="python", args=["-m", "server"])
+    assert _service_argv(step, []) == [sys.executable, "-m", "server"]
+
+
+def test_argv_with_faketime_prefix():
+    step = HookStep(command="python", args=["-m", "server"])
+    prefix = ["faketime", "2025-01-15 09:00:00"]
+    assert _service_argv(step, prefix) == [
+        "faketime",
+        "2025-01-15 09:00:00",
+        sys.executable,
+        "-m",
+        "server",
+    ]
+
+
+# ---------------------------------------------------------------------------
+# resolve_faketime_launch — command prefix + env, with fail-fast
+# ---------------------------------------------------------------------------
+
+
+def test_resolve_returns_prefix_and_utc_env(monkeypatch):
+    monkeypatch.setattr(mcp_cmd.shutil, "which", lambda _name: "/usr/bin/faketime")
+    prefix, env = resolve_faketime_launch("2025-01-15T09:00:00Z")
+    assert prefix == ["faketime", "2025-01-15 09:00:00"]
+    assert env == {"TZ": "UTC"}
+
+
+def test_resolve_fails_fast_when_wrapper_missing(monkeypatch):
+    monkeypatch.setattr(mcp_cmd.shutil, "which", lambda _name: None)
+    with pytest.raises(SystemExit):
+        resolve_faketime_launch("2025-01-15T09:00:00Z")
+
+
+def test_resolve_invalid_timestamp_fails(monkeypatch):
+    monkeypatch.setattr(mcp_cmd.shutil, "which", lambda _name: "/usr/bin/faketime")
+    with pytest.raises(SystemExit):
+        resolve_faketime_launch("not-a-timestamp")
+
+
+# ---------------------------------------------------------------------------
+# Integration — only when the faketime wrapper is installed
+# ---------------------------------------------------------------------------
+
+_HAS_FAKETIME = shutil.which("faketime") is not None
+_SKIP_REASON = "faketime wrapper not installed on PATH"
+
+# 2025-01-15T09:00:00Z
+_FAKE_EPOCH = 1736931600
+_SPEC = "2025-01-15 09:00:00"
+
+
+@pytest.mark.skipif(not _HAS_FAKETIME, reason=_SKIP_REASON)
+def test_faketime_fakes_date():
+    out = subprocess.run(
+        ["faketime", _SPEC, "date", "+%s"],
+        capture_output=True,
+        text=True,
+        env={"TZ": "UTC", "PATH": __import__("os").environ["PATH"]},
+        timeout=30,
+    )
+    assert out.returncode == 0, out.stderr
+    reported = int(out.stdout.strip())
+    # Clock advances from the anchor, so allow a small forward window.
+    assert _FAKE_EPOCH <= reported < _FAKE_EPOCH + 60
+
+
+@pytest.mark.skipif(not _HAS_FAKETIME, reason=_SKIP_REASON)
+def test_faketime_fakes_python():
+    out = subprocess.run(
+        ["faketime", _SPEC, sys.executable, "-c", "import time; print(time.time())"],
+        capture_output=True,
+        text=True,
+        env={"TZ": "UTC", "PATH": __import__("os").environ["PATH"]},
+        timeout=30,
+    )
+    assert out.returncode == 0, out.stderr
+    reported = float(out.stdout.strip())
+    assert _FAKE_EPOCH <= reported < _FAKE_EPOCH + 60
+
+
+def test_fake_epoch_matches_spec():
+    """Guard the integration constants against drift."""
+    dt = datetime(2025, 1, 15, 9, 0, 0, tzinfo=UTC)
+    assert int(dt.timestamp()) == _FAKE_EPOCH

docker/packages/mcp_proxy/tests/test_mcp_utils.py

@@ -0,0 +1,600 @@
+"""Tests for utility functions in commands/mcp.py."""
+
+import json
+import logging
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from types import SimpleNamespace
+from typing import ClassVar
+
+import pytest
+from fastmcp import FastMCP
+
+from mcp_proxy.commands import mcp as mcp_command
+from mcp_proxy.commands.mcp import (
+    HIDDEN_FROM_LISTING_TAG,
+    HideTaggedToolsMiddleware,
+    _find_free_port,
+    _iter_packages,
+    _validate_tool_sets,
+    _wait_for_port,
+    build_proxy_app,
+    discover_mcp_servers,
+)
+from mcp_proxy.service import McpConfig, McpService, resolve_command
+
+# ---------------------------------------------------------------------------
+# _find_free_port
+# ---------------------------------------------------------------------------
+
+
+class TestFindFreePort:
+    def test_returns_int(self):
+        port = _find_free_port()
+        assert isinstance(port, int)
+
+    def test_port_is_in_valid_range(self):
+        port = _find_free_port()
+        assert 1024 <= port <= 65535
+
+    def test_returns_different_ports(self):
+        ports = {_find_free_port() for _ in range(5)}
+        # At least 2 distinct ports (extremely likely)
+        assert len(ports) >= 2
+
+
+# ---------------------------------------------------------------------------
+# _wait_for_port
+# ---------------------------------------------------------------------------
+
+
+class TestWaitForPort:
+    def test_returns_true_when_port_is_listening(self):
+        # Start a simple server
+        server = HTTPServer(("127.0.0.1", 0), BaseHTTPRequestHandler)
+        port = server.server_address[1]
+        thread = threading.Thread(target=server.handle_request, daemon=True)
+        thread.start()
+        try:
+            assert _wait_for_port(port, timeout=5.0) is True
+        finally:
+            server.server_close()
+
+    def test_returns_false_when_port_not_listening(self):
+        # Find a free port and don't listen on it
+        port = _find_free_port()
+        assert _wait_for_port(port, timeout=0.5) is False
+
+
+# ---------------------------------------------------------------------------
+# McpConfig & discover_mcp_servers
+# ---------------------------------------------------------------------------
+
+
+class TestMcpConfig:
+    def test_basic_config(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/my-service"),
+            {"run": {"command": "node", "args": ["index.js"]}},
+        )
+        assert cfg.name == "my-service"
+        assert len(cfg.run) == 1
+        assert cfg.run[0].command == "node"
+        assert cfg.run[0].args == ["index.js"]
+        assert cfg.setup == []
+
+    def test_run_list(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {"run": [{"command": "build"}, {"command": "node", "args": ["index.js"]}]},
+        )
+        assert len(cfg.run) == 2
+        assert cfg.run[0].command == "build"
+        assert cfg.run[1].command == "node"
+
+    def test_missing_run_raises(self):
+        with pytest.raises(ValueError, match="'run' is required"):
+            McpConfig.from_mcp_json(Path("/tmp/svc"), {})
+
+    def test_custom_name(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/my-service"),
+            {"name": "Custom", "run": {"command": "python"}},
+        )
+        assert cfg.name == "Custom"
+
+    def test_setup_single_dict(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {
+                "run": {"command": "node"},
+                "setup": {"command": "npm", "args": ["run", "build"]},
+            },
+        )
+        assert len(cfg.setup) == 1
+        assert cfg.setup[0].command == "npm"
+
+    def test_setup_list(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {
+                "run": {"command": "node"},
+                "setup": [{"command": "a"}, {"command": "b"}],
+            },
+        )
+        assert len(cfg.setup) == 2
+
+    def test_cwd_override(self, tmp_path):
+        cfg = McpConfig.from_mcp_json(
+            tmp_path / "svc",
+            {"run": {"command": "node"}, "cwd": "subdir"},
+        )
+        assert cfg.cwd == (tmp_path / "svc" / "subdir").resolve()
+
+    def test_install_single_dict(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {
+                "run": {"command": "node"},
+                "install": {"command": "uv", "args": ["sync", "--package", "foo"]},
+            },
+        )
+        assert len(cfg.install) == 1
+        assert cfg.install[0].command == "uv"
+        assert cfg.install[0].args == ["sync", "--package", "foo"]
+
+    def test_install_defaults_empty(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {"run": {"command": "node"}},
+        )
+        assert cfg.install == []
+
+    def test_toolsets_parsed(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {
+                "run": {"command": "node"},
+                "toolsets": {"read": ["t1", "t2"], "write": ["t3"]},
+            },
+        )
+        assert cfg.toolsets == {"read": ["t1", "t2"], "write": ["t3"]}
+
+    def test_toolsets_defaults_empty(self):
+        cfg = McpConfig.from_mcp_json(
+            Path("/tmp/svc"),
+            {"run": {"command": "node"}},
+        )
+        assert cfg.toolsets == {}
+
+
+class TestIterPackages:
+    """Iteration is the discovery primitive — it returns every package
+    regardless of any tool_sets filter. ``gen`` uses this directly."""
+
+    def test_iter_packages_returns_each(self, tmp_path):
+        pkg = tmp_path / "svc1"
+        pkg.mkdir()
+        (pkg / "mcp.json").write_text(json.dumps({"run": {"command": "node"}}))
+
+        servers = _iter_packages(tmp_path)
+        assert len(servers) == 1
+        assert servers[0].name == "svc1"
+
+    def test_exits_on_missing_run_command(self, tmp_path):
+        pkg = tmp_path / "bad"
+        pkg.mkdir()
+        (pkg / "mcp.json").write_text(json.dumps({"run": {}}))
+
+        with pytest.raises(SystemExit):
+            _iter_packages(tmp_path)
+
+    def test_exits_on_invalid_json(self, tmp_path):
+        pkg = tmp_path / "broken"
+        pkg.mkdir()
+        (pkg / "mcp.json").write_text("not json")
+
+        with pytest.raises(SystemExit):
+            _iter_packages(tmp_path)
+
+    def test_exits_on_nonexistent_root(self, tmp_path):
+        with pytest.raises(SystemExit):
+            _iter_packages(tmp_path / "nope")
+
+    def test_sorted_by_name(self, tmp_path):
+        for name in ("zebra", "alpha", "middle"):
+            pkg = tmp_path / name
+            pkg.mkdir()
+            (pkg / "mcp.json").write_text(json.dumps({"run": {"command": "python"}}))
+
+        servers = _iter_packages(tmp_path)
+        names = [s.name for s in servers]
+        assert names == ["alpha", "middle", "zebra"]
+
+
+class TestDiscoverMcpServers:
+    def test_filter_by_namespaced_toolset(self, tmp_path):
+        """Only servers declaring the requested namespaced toolset are returned."""
+        (tmp_path / "alpha").mkdir()
+        (tmp_path / "alpha" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t1"]}})
+        )
+        (tmp_path / "beta").mkdir()
+        (tmp_path / "beta" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"write": ["t2"]}})
+        )
+        (tmp_path / "gamma").mkdir()
+        (tmp_path / "gamma" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t3"]}})
+        )
+
+        servers = discover_mcp_servers(tmp_path, tool_sets=["alpha_read", "gamma_read"])
+        assert [s.name for s in servers] == ["alpha", "gamma"]
+
+    def test_no_filter_returns_nothing(self, tmp_path):
+        """tool_sets=None / [] / [""] all mean 'no servers' — callers must opt in
+        explicitly. Use _iter_packages when you need every package."""
+        for name in ("alpha", "beta"):
+            pkg = tmp_path / name
+            pkg.mkdir()
+            (pkg / "mcp.json").write_text(json.dumps({"run": {"command": "node"}}))
+
+        assert discover_mcp_servers(tmp_path, tool_sets=None) == []
+        assert discover_mcp_servers(tmp_path, tool_sets=[]) == []
+        assert discover_mcp_servers(tmp_path, tool_sets=[""]) == []
+
+    def test_filter_multiple_namespaced_toolsets(self, tmp_path):
+        """Servers declaring any of the requested namespaced toolsets are included."""
+        (tmp_path / "alpha").mkdir()
+        (tmp_path / "alpha" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t1"]}})
+        )
+        (tmp_path / "beta").mkdir()
+        (tmp_path / "beta" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"write": ["t2"]}})
+        )
+        (tmp_path / "gamma").mkdir()
+        (tmp_path / "gamma" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"debug": ["t3"]}})
+        )
+
+        servers = discover_mcp_servers(tmp_path, tool_sets=["alpha_read", "beta_write"])
+        assert [s.name for s in servers] == ["alpha", "beta"]
+
+    def test_filter_namespaced_toolset(self, tmp_path):
+        """Namespaced form 'pkg_toolset' selects only that package+toolset."""
+        (tmp_path / "google_mail").mkdir()
+        (tmp_path / "google_mail" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t1"], "write": ["t2"]}})
+        )
+        (tmp_path / "slack").mkdir()
+        (tmp_path / "slack" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t3"], "write": ["t4"]}})
+        )
+        (tmp_path / "jira").mkdir()
+        (tmp_path / "jira" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}})  # no toolsets
+        )
+
+        # Only google_mail should be found; slack has no google_mail_read toolset
+        servers = discover_mcp_servers(tmp_path, tool_sets=["google_mail_read"])
+        assert [s.name for s in servers] == ["google_mail"]
+
+    def test_returns_mcp_service_objects(self, tmp_path):
+        """discover_mcp_servers returns McpService instances wrapping McpConfig."""
+        pkg = tmp_path / "svc"
+        pkg.mkdir()
+        (pkg / "mcp.json").write_text(json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t"]}}))
+
+        servers = discover_mcp_servers(tmp_path, tool_sets=["svc_read"])
+        assert len(servers) == 1
+        assert isinstance(servers[0], McpService)
+        assert isinstance(servers[0].config, McpConfig)
+
+    def test_bare_package_name_is_rejected(self, tmp_path):
+        """Bare package names are not valid tool_sets — they must be namespaced
+        toolset names (``google_mail_read``), so validation rejects them."""
+        (tmp_path / "google_mail").mkdir()
+        (tmp_path / "google_mail" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "node"}, "toolsets": {"read": ["t1"]}})
+        )
+        # "google_mail" is a package name, not a toolset name — unknown tool set.
+        with pytest.raises(SystemExit):
+            discover_mcp_servers(tmp_path, tool_sets=["google_mail"])
+
+    def _core_and_shim(self, tmp_path):
+        """Create fake ``core`` and ``syntara`` (compat shim) packages."""
+        (tmp_path / "core").mkdir()
+        (tmp_path / "core" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "python"}, "toolsets": {"read": ["readFile"]}})
+        )
+        (tmp_path / "syntara").mkdir()
+        (tmp_path / "syntara" / "mcp.json").write_text(
+            json.dumps({"run": {"command": "python"}, "toolsets": {"read": ["readFile"]}})
+        )
+
+    def test_core_and_syntara_shim_together_warns(self, tmp_path, caplog):
+        """core + the syntara compat shim are redundant but allowed — selecting both
+        mounts both surfaces and warns. REMOVE with the syntara package after 2026-06-18."""
+        self._core_and_shim(tmp_path)
+        with caplog.at_level(logging.WARNING, logger="mcp_proxy"):
+            servers = discover_mcp_servers(tmp_path, tool_sets=["core_read", "syntara_read"])
+        assert sorted(s.name for s in servers) == ["core", "syntara"]
+        assert any("both 'core' and the legacy 'syntara'" in r.message for r in caplog.records)
+
+    def test_core_alone_is_allowed(self, tmp_path):
+        self._core_and_shim(tmp_path)
+        assert [s.name for s in discover_mcp_servers(tmp_path, tool_sets=["core_read"])] == ["core"]
+
+    def test_syntara_shim_alone_is_allowed(self, tmp_path):
+        self._core_and_shim(tmp_path)
+        assert [s.name for s in discover_mcp_servers(tmp_path, tool_sets=["syntara_read"])] == ["syntara"]
+
+
+class TestValidateToolSets:
+    @staticmethod
+    def _write_pkg(root, name, toolsets):
+        """Materialize a minimal ``packages/<name>/mcp.json`` declaring *toolsets*."""
+        pkg = root / name
+        pkg.mkdir()
+        (pkg / "mcp.json").write_text(json.dumps({"run": {"command": "node"}, "toolsets": toolsets}))
+
+    def test_valid_tool_sets_pass(self, tmp_path):
+        """Namespaced tool set names that a package declares do not raise."""
+        self._write_pkg(tmp_path, "google_mail", {"read": []})
+        self._write_pkg(tmp_path, "slack", {"state": []})
+        _validate_tool_sets(["google_mail_read", "slack_state"], tmp_path)
+
+    def test_unknown_tool_set_exits(self, tmp_path):
+        """Unknown tool set names cause sys.exit."""
+        self._write_pkg(tmp_path, "slack", {"read": []})
+        with pytest.raises(SystemExit):
+            _validate_tool_sets(["nonexistent"], tmp_path)
+
+    def test_partial_unknown_exits(self, tmp_path):
+        """Even one unknown tool set among valid ones causes exit."""
+        self._write_pkg(tmp_path, "slack", {"read": []})
+        with pytest.raises(SystemExit):
+            _validate_tool_sets(["slack_read", "bad_name"], tmp_path)
+
+    def test_empty_request_passes(self, tmp_path):
+        """No requested tool sets is vacuously valid (nothing unknown)."""
+        self._write_pkg(tmp_path, "slack", {"read": []})
+        _validate_tool_sets([], tmp_path)  # should not raise
+
+
+class TestBuildProxyApp:
+    def test_exits_when_selected_service_fails_startup(self, tmp_path, monkeypatch, caplog):
+        """Requested services that die during startup abort the whole proxy."""
+
+        class FakeProc:
+            returncode = 7
+
+            def poll(self):
+                return self.returncode
+
+            def terminate(self):
+                pass
+
+        class FakeService:
+            name = "jira"
+            secrets: ClassVar[list[str]] = []
+            config = SimpleNamespace(startup_timeout=0.1)
+
+            def run_install(self, env):
+                return True
+
+            def run_setup(self, env):
+                return True
+
+            def run_pre_steps(self, env):
+                return True
+
+            def start_service(self, env, port, proxy_token, command_prefix=None):
+                return FakeProc()
+
+        monkeypatch.setattr(
+            "mcp_proxy.commands.mcp.discover_mcp_servers",
+            lambda packages_root, tool_sets=None: [FakeService()],
+        )
+        monkeypatch.setattr(
+            "mcp_proxy.commands.mcp._build_subprocess_env",
+            lambda base_dir, server_name, declared_secrets=(): {},
+        )
+
+        with pytest.raises(SystemExit):
+            build_proxy_app(tmp_path, tmp_path, tool_sets=["jira_read"])
+
+        assert "Aborting startup because requested MCP services failed to start" in caplog.text
+        assert "jira (exited immediately with code 7)" in caplog.text
+
+
+class TestRunViewerStartup:
+    class FakeApp:
+        def __init__(self):
+            self.run_calls = []
+
+        def custom_route(self, *_args, **_kwargs):
+            def decorator(fn):
+                return fn
+
+            return decorator
+
+        def run(self, **kwargs):
+            self.run_calls.append(kwargs)
+
+    def _stub_proxy_run(self, monkeypatch, tmp_path):
+        app = self.FakeApp()
+        monkeypatch.setenv("WORLDBENCH_ROOT", str(tmp_path))
+        monkeypatch.setenv("WORLDBENCH_PACKAGES_ROOT", str(tmp_path))
+        monkeypatch.delenv("WORLDBENCH_TOOL_SETS", raising=False)
+        monkeypatch.delenv("PORT", raising=False)
+        monkeypatch.setattr(mcp_command, "install_crash_handlers", lambda: None)
+        monkeypatch.setattr(mcp_command, "build_proxy_app", lambda **_kwargs: app)
+        monkeypatch.setattr(mcp_command, "_assert_local_tools_async", lambda _app: None)
+        monkeypatch.setattr(mcp_command, "_kill_child_processes", lambda: None)
+        return app
+
+    def test_does_not_start_viewer_when_viewer_port_is_unset(self, tmp_path, monkeypatch):
+        app = self._stub_proxy_run(monkeypatch, tmp_path)
+        viewer_calls = []
+        monkeypatch.delenv("VIEWER_PORT", raising=False)
+        monkeypatch.setattr(mcp_command, "_start_viewer_server", lambda host, port: viewer_calls.append((host, port)))
+
+        mcp_command.run(method="stdio")
+
+        assert viewer_calls == []
+        assert app.run_calls == [{"show_banner": False}]
+
+    def test_starts_viewer_when_viewer_port_is_set(self, tmp_path, monkeypatch):
+        app = self._stub_proxy_run(monkeypatch, tmp_path)
+        viewer_calls = []
+        monkeypatch.setenv("VIEWER_PORT", "8765")
+        monkeypatch.setattr(mcp_command, "_start_viewer_server", lambda host, port: viewer_calls.append((host, port)))
+
+        mcp_command.run(method="stdio")
+
+        assert viewer_calls == [("0.0.0.0", 8765)]
+        assert app.run_calls == [{"show_banner": False}]
+
+
+class TestHideTaggedToolsMiddleware:
+    """The proxy's un-namespaced ``export_state`` / ``import_state`` are
+    infrastructure for the grading harness: they must remain callable so
+    snapshots round-trip, but they must NOT appear in ``tools/list`` (an
+    agent shouldn't even know they exist).
+    """
+
+    @pytest.mark.anyio
+    async def test_tagged_tools_are_hidden_from_list_but_callable(self):
+        app = FastMCP("test")
+        app.add_middleware(HideTaggedToolsMiddleware())
+
+        @app.tool(name="visible")
+        async def visible() -> dict:
+            return {"v": True}
+
+        @app.tool(name="hidden", tags={HIDDEN_FROM_LISTING_TAG})
+        async def hidden() -> dict:
+            return {"h": True}
+
+        listed = {t.name for t in await app.list_tools()}
+        assert listed == {"visible"}, f"hidden tool leaked into list_tools: {listed}"
+
+        # Hidden tool must still be callable — that's the point.
+        hidden_result = await app.call_tool("hidden", {})
+        assert hidden_result.structured_content == {"h": True}
+
+        visible_result = await app.call_tool("visible", {})
+        assert visible_result.structured_content == {"v": True}
+
+    @pytest.fixture
+    def anyio_backend(self):
+        return "asyncio"
+
+
+class TestMcpServiceLifecycle:
+    @pytest.fixture
+    def env(self):
+        """Return a minimal env dict with PATH so subprocesses can find python."""
+        import os
+
+        return dict(os.environ)
+
+    def test_run_install_executes_steps(self, tmp_path, env):
+        """Install steps are executed and create side effects."""
+        marker = tmp_path / "installed.txt"
+        cfg = McpConfig.from_mcp_json(
+            tmp_path,
+            {
+                "run": {"command": "python", "args": ["-c", "pass"]},
+                "install": {"command": "python", "args": ["-c", f"open('{marker}', 'w').write('ok')"]},
+            },
+        )
+        svc = McpService(cfg)
+        assert svc.run_install(env) is True
+        assert marker.read_text() == "ok"
+
+    def test_run_install_returns_true_when_empty(self, tmp_path, env):
+        """No install steps means success."""
+        cfg = McpConfig.from_mcp_json(tmp_path, {"run": {"command": "python"}})
+        svc = McpService(cfg)
+        assert svc.run_install(env) is True
+
+    def test_run_install_returns_false_on_failure(self, tmp_path, env):
+        """Failed install step returns False."""
+        cfg = McpConfig.from_mcp_json(
+            tmp_path,
+            {
+                "run": {"command": "python"},
+                "install": {"command": "python", "args": ["-c", "import sys; sys.exit(1)"]},
+            },
+        )
+        svc = McpService(cfg)
+        assert svc.run_install(env) is False
+
+    def test_run_setup_executes_steps(self, tmp_path, env):
+        """Setup steps are executed and create side effects."""
+        marker = tmp_path / "setup.txt"
+        cfg = McpConfig.from_mcp_json(
+            tmp_path,
+            {
+                "run": {"command": "python"},
+                "setup": {"command": "python", "args": ["-c", f"open('{marker}', 'w').write('done')"]},
+            },
+        )
+        svc = McpService(cfg)
+        assert svc.run_setup(env) is True
+        assert marker.read_text() == "done"
+
+    def test_run_setup_forwards_env(self, tmp_path, env):
+        """Environment variables are forwarded to setup steps."""
+        marker = tmp_path / "env.txt"
+        env["MY_VAR"] = "hello"
+        cfg = McpConfig.from_mcp_json(
+            tmp_path,
+            {
+                "run": {"command": "python"},
+                "setup": {
+                    "command": "python",
+                    "args": ["-c", f"import os; open('{marker}', 'w').write(os.environ.get('MY_VAR', 'MISSING'))"],
+                },
+            },
+        )
+        svc = McpService(cfg)
+        assert svc.run_setup(env) is True
+        assert marker.read_text() == "hello"
+
+    def test_python_steps_use_proxy_interpreter_not_path(self, tmp_path, env):
+        # Empty PATH: a bare `python` would be unfindable, so success proves the
+        # step ran the absolute sys.executable (the PATH-elimination contract).
+        import sys
+
+        marker = tmp_path / "interp.txt"
+        env["PATH"] = ""
+        cfg = McpConfig.from_mcp_json(
+            tmp_path,
+            {
+                "run": {"command": "python"},
+                "setup": {
+                    "command": "python",
+                    "args": ["-c", f"import sys; open('{marker}', 'w').write(sys.executable)"],
+                },
+            },
+        )
+        svc = McpService(cfg)
+        assert svc.run_setup(env) is True, "bare `python` step failed — interpreter not resolved to an absolute path"
+        assert marker.read_text() == sys.executable
+
+
+class TestResolveCommand:
+    def test_bare_python_resolves_to_sys_executable(self):
+        import sys
+
+        assert resolve_command("python") == sys.executable
+        assert resolve_command("python3") == sys.executable
+
+    def test_non_python_commands_pass_through(self):
+        for command in ("node", "bash", "npm", "uv", "/opt/venv/bin/python", "frappe-bench/env/bin/python"):
+            assert resolve_command(command) == command

docker/packages/mcp_proxy/tests/test_resolve_tool_names.py

@@ -0,0 +1,183 @@
+"""Tests for resolve_tool_transforms in mcp_proxy.service."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from mcp_proxy.service import McpConfig, McpService
+
+
+def _make_cfg(tmp_path, tools: list[str], toolsets: dict, name: str | None = None) -> McpService:
+    """Write mcp.json + mcp-tools.generated.json and return an McpService."""
+    mcp_json: dict = {
+        "run": {"command": "python", "args": ["-m", "fake"]},
+        "toolsets": toolsets,
+    }
+    if name is not None:
+        mcp_json["name"] = name
+    (tmp_path / "mcp.json").write_text(json.dumps(mcp_json))
+    tools_data = {
+        "schema_version": "v1",
+        "tools": [{"name": t, "description": "", "inputSchema": {}} for t in tools],
+        "toolsets": toolsets,
+    }
+    (tmp_path / "mcp-tools.generated.json").write_text(json.dumps(tools_data))
+    return McpService(McpConfig.from_mcp_json(tmp_path, mcp_json))
+
+
+def _enabled_tools(transforms: dict) -> dict[str, str]:
+    """Return {original_name: new_name} for enabled tools only."""
+    return {name: t.name for name, t in transforms.items() if t.enabled and t.name}
+
+
+def _disabled_tools(transforms: dict) -> set[str]:
+    """Return set of original names for disabled tools."""
+    return {name for name, t in transforms.items() if not t.enabled}
+
+
+TOOLS = ["echo", "readFile", "writeFile", "bash"]
+TOOLSETS = {
+    "read": ["echo", "readFile"],
+    "write": ["writeFile", "bash"],
+}
+
+
+def test_empty_tool_sets_exposes_all_tools(tmp_path):
+    """An empty filter (or unset env var) means 'expose every introspected tool'."""
+    cfg = _make_cfg(tmp_path, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms([])
+    assert set(_enabled_tools(result).keys()) == set(TOOLS)
+    assert _disabled_tools(result) == set()
+
+
+def test_namespaced_read_filters_correctly(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_read"])
+    assert set(_enabled_tools(result).keys()) == {"echo", "readFile"}
+    assert _disabled_tools(result) == {"writeFile", "bash"}
+
+
+def test_namespaced_write_filters_correctly(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_write"])
+    assert set(_enabled_tools(result).keys()) == {"writeFile", "bash"}
+    assert _disabled_tools(result) == {"echo", "readFile"}
+
+
+def test_multiple_namespaced_toolsets_are_unioned(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_read", "slack_write"])
+    assert set(_enabled_tools(result).keys()) == set(TOOLS)
+    assert _disabled_tools(result) == set()
+
+
+def test_tools_are_namespaced(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_read"])
+    for original, new_name in _enabled_tools(result).items():
+        assert new_name == f"slack__{original}"
+
+
+def test_unnamespaced_package_exposes_bare_tool_names(tmp_path):
+    """When mcp.json sets namespaced=false, tools are exposed without a prefix."""
+    mcp_json = {
+        "run": {"command": "python", "args": ["-m", "fake"]},
+        "namespaced": False,
+        "toolsets": TOOLSETS,
+    }
+    (tmp_path / "mcp.json").write_text(json.dumps(mcp_json))
+    tools_data = {
+        "schema_version": "v1",
+        "tools": [{"name": t, "description": "", "inputSchema": {}} for t in TOOLS],
+        "toolsets": TOOLSETS,
+    }
+    (tmp_path / "mcp-tools.generated.json").write_text(json.dumps(tools_data))
+    cfg = McpService(McpConfig.from_mcp_json(tmp_path, mcp_json))
+    result = cfg.resolve_tool_transforms([])
+    for original, new_name in _enabled_tools(result).items():
+        assert new_name == original
+
+
+def test_no_toolsets_in_mcp_json_exposes_all(tmp_path):
+    """When mcp.json has no toolsets key, all tools are exposed regardless of filter."""
+    cfg = _make_cfg(tmp_path, TOOLS, {})
+    result = cfg.resolve_tool_transforms(["whatever"])
+    assert set(_enabled_tools(result).keys()) == set(TOOLS)
+    assert _disabled_tools(result) == set()
+
+
+def test_unknown_toolset_disables_all(tmp_path):
+    """A namespaced toolset that doesn't exist on this package disables every tool."""
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_nonexistent"])
+    assert _enabled_tools(result) == {}
+    assert _disabled_tools(result) == set(TOOLS)
+
+
+def test_other_packages_namespace_does_not_match(tmp_path):
+    """A toolset namespaced to a different package is ignored — matches no tools."""
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, TOOLS, TOOLSETS)
+    result = cfg.resolve_tool_transforms(["jira_read"])
+    assert _enabled_tools(result) == {}
+    assert _disabled_tools(result) == set(TOOLS)
+
+
+def test_missing_generated_json_raises(tmp_path):
+    mcp_json = {"run": {"command": "python", "args": []}, "toolsets": TOOLSETS}
+    (tmp_path / "mcp.json").write_text(json.dumps(mcp_json))
+    cfg = McpService(McpConfig.from_mcp_json(tmp_path, mcp_json))
+    with pytest.raises(FileNotFoundError, match=r"mcp-tools\.generated\.json not found"):
+        cfg.resolve_tool_transforms(["slack_read"])
+
+
+# ---------------------------------------------------------------------------
+# State toolsets are reachable when explicitly requested. The "all" / "_all"
+# special case (with its hidden-by-default semantics for state/grading tools)
+# was removed when the auto-aggregated cross-package toolsets were dropped.
+# ---------------------------------------------------------------------------
+
+
+STATEFUL_TOOLS = ["echo", "readFile", "export_state", "import_state"]
+STATEFUL_TOOLSETS = {
+    "read": ["echo", "readFile"],
+    "state": ["export_state", "import_state"],
+}
+
+
+def test_namespaced_state_request_exposes_state(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, STATEFUL_TOOLS, STATEFUL_TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_state"])
+    assert set(_enabled_tools(result).keys()) == {"export_state", "import_state"}
+
+
+def test_namespaced_read_does_not_expose_state(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, STATEFUL_TOOLS, STATEFUL_TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_read"])
+    assert set(_enabled_tools(result).keys()) == {"echo", "readFile"}
+    assert _disabled_tools(result) == {"export_state", "import_state"}
+
+
+def test_combined_namespaced_toolsets_union(tmp_path):
+    pkg = tmp_path / "slack"
+    pkg.mkdir()
+    cfg = _make_cfg(pkg, STATEFUL_TOOLS, STATEFUL_TOOLSETS)
+    result = cfg.resolve_tool_transforms(["slack_read", "slack_state"])
+    assert set(_enabled_tools(result).keys()) == set(STATEFUL_TOOLS)

docker/packages/mcp_proxy/tests/test_service_http.py

@@ -0,0 +1,89 @@
+"""Tests for service_http module."""
+
+import pytest
+from starlette.testclient import TestClient
+
+from mcp_proxy.service_http import ProxyTokenMiddleware, _placeholder_html
+
+# ---------------------------------------------------------------------------
+# Placeholder HTML
+# ---------------------------------------------------------------------------
+
+
+class TestPlaceholderHtml:
+    def test_contains_service_name(self):
+        html = _placeholder_html("My Service")
+        assert "My Service" in html
+
+    def test_is_valid_html(self):
+        html = _placeholder_html("Test")
+        assert html.startswith("<!DOCTYPE html>")
+        assert "</html>" in html
+
+
+# ---------------------------------------------------------------------------
+# ProxyTokenMiddleware
+# ---------------------------------------------------------------------------
+
+
+class TestProxyTokenMiddleware:
+    @pytest.fixture
+    def app(self):
+        """Create a minimal Starlette app with ProxyTokenMiddleware."""
+        from starlette.applications import Starlette
+        from starlette.middleware import Middleware
+        from starlette.requests import Request
+        from starlette.responses import PlainTextResponse
+        from starlette.routing import Route
+
+        async def index(request: Request):
+            return PlainTextResponse("ok")
+
+        async def mcp_endpoint(request: Request):
+            return PlainTextResponse("mcp ok")
+
+        return Starlette(
+            routes=[
+                Route("/", index),
+                Route("/mcp", mcp_endpoint),
+                Route("/mcp/test", mcp_endpoint),
+                Route("/api/data", index),
+            ],
+            middleware=[Middleware(ProxyTokenMiddleware)],
+        )
+
+    def test_mcp_path_bypasses_auth(self, app, monkeypatch):
+        monkeypatch.setenv("MCP_PROXY_TOKEN", "secret")
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/mcp")
+        assert resp.status_code == 200
+
+    def test_mcp_subpath_bypasses_auth(self, app, monkeypatch):
+        monkeypatch.setenv("MCP_PROXY_TOKEN", "secret")
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/mcp/test")
+        assert resp.status_code == 200
+
+    def test_non_mcp_blocked_without_token(self, app, monkeypatch):
+        monkeypatch.setenv("MCP_PROXY_TOKEN", "secret")
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/")
+        assert resp.status_code == 403
+
+    def test_non_mcp_allowed_with_correct_token(self, app, monkeypatch):
+        monkeypatch.setenv("MCP_PROXY_TOKEN", "secret")
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/", headers={"x-proxy-token": "secret"})
+        assert resp.status_code == 200
+
+    def test_non_mcp_blocked_with_wrong_token(self, app, monkeypatch):
+        monkeypatch.setenv("MCP_PROXY_TOKEN", "secret")
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/", headers={"x-proxy-token": "wrong"})
+        assert resp.status_code == 403
+
+    def test_no_token_set_allows_all(self, app, monkeypatch):
+        monkeypatch.delenv("MCP_PROXY_TOKEN", raising=False)
+        client = TestClient(app, raise_server_exceptions=False)
+        resp = client.get("/")
+        assert resp.status_code == 200

docker/packages/mcp_proxy/tests/test_setup.py

@@ -0,0 +1,183 @@
+"""Tests for the setup pass the proxy runs during ``mcp`` startup.
+
+There is no standalone setup command: ``build_proxy_app`` discovers MCP
+servers and runs each one's ``install`` then ``setup`` hook from
+``mcp.json`` (forwarding WORLDBENCH_* env vars) before starting the
+server. Servers are selected by WORLDBENCH_TOOL_SETS: only servers whose
+mcp.json declares at least one of the requested namespaced toolsets are
+included. An empty / unset value runs no setup hooks.
+
+These tests exercise that same discover -> install -> setup pass in
+process, using the exact primitives ``build_proxy_app`` calls.
+"""
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from mcp_proxy.commands.mcp import _build_subprocess_env, discover_mcp_servers
+
+
+class SetupCommandTests(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp()
+        self.packages_root = Path(self.temp_dir) / "packages"
+        self.packages_root.mkdir()
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+
+    def _create_server(self, name, setup_hook=None, toolsets=None):
+        """Create a fake MCP server package with an optional setup hook.
+
+        Always declares a ``"default"`` toolset (so :meth:`_run_setup` can
+        opt every package in by default). *toolsets* is merged on top, in
+        case a test wants to declare additional toolsets to filter against.
+        """
+        pkg_dir = self.packages_root / name
+        pkg_dir.mkdir(parents=True)
+
+        config = {"run": {"command": "python", "args": ["-c", "pass"]}}
+        if setup_hook is not None:
+            config["setup"] = setup_hook
+        config["toolsets"] = {"default": [], **(toolsets or {})}
+
+        (pkg_dir / "mcp.json").write_text(json.dumps(config))
+        return pkg_dir
+
+    def _run_setup(self, env_overrides=None):
+        """Run the startup setup pass in process and return the failed servers.
+
+        Mirrors the discover -> install -> setup loop ``build_proxy_app``
+        runs before booting (the only place setup hooks fire now). Returns
+        the list of server names whose install or setup hook failed — empty
+        on success.
+
+        Defaults WORLDBENCH_TOOL_SETS to every fake package's ``_default``
+        toolset so each test gets full coverage without spelling them out.
+        Override via *env_overrides* when a test wants narrower filtering.
+        """
+        default_sets = " ".join(f"{p.name}_default" for p in sorted(self.packages_root.iterdir()) if p.is_dir())
+        env = {
+            "WORLDBENCH_ROOT": self.temp_dir,
+            "WORLDBENCH_PACKAGES_ROOT": str(self.packages_root),
+            "WORLDBENCH_TOOL_SETS": default_sets,
+        }
+        if env_overrides:
+            env.update(env_overrides)
+
+        failed: list[str] = []
+        with mock.patch.dict(os.environ, env):
+            base_dir = Path(os.environ["WORLDBENCH_ROOT"]).resolve()
+            packages_root = Path(os.environ["WORLDBENCH_PACKAGES_ROOT"])
+            tool_sets = os.environ["WORLDBENCH_TOOL_SETS"].split()
+            for cfg in discover_mcp_servers(packages_root, tool_sets=tool_sets):
+                hook_env = _build_subprocess_env(base_dir, cfg.name, declared_secrets=cfg.secrets)
+                if not cfg.run_install(hook_env) or not cfg.run_setup(hook_env):
+                    failed.append(cfg.name)
+        return failed
+
+    def test_runs_setup_hook(self):
+        """Setup hook from mcp.json is executed."""
+        marker = Path(self.temp_dir) / "setup_ran.txt"
+        self._create_server(
+            "svc",
+            setup_hook={
+                "command": "python",
+                "args": ["-c", f"open('{marker}', 'w').write('ok')"],
+            },
+        )
+
+        failed = self._run_setup()
+        assert failed == [], failed
+        assert marker.exists()
+        assert marker.read_text() == "ok"
+
+    def test_skips_server_without_setup_hook(self):
+        """Servers without a setup hook are skipped gracefully."""
+        self._create_server("no-setup")
+
+        failed = self._run_setup()
+        assert failed == [], failed
+
+    def test_forwards_env_vars(self):
+        """WORLDBENCH_* env vars are forwarded to the setup hook."""
+        marker = Path(self.temp_dir) / "env_check.txt"
+        self._create_server(
+            "svc",
+            setup_hook={
+                "command": "python",
+                "args": [
+                    "-c",
+                    f"import os; open('{marker}', 'w').write(os.environ.get('WORLDBENCH_TASK_ID', 'MISSING'))",
+                ],
+            },
+        )
+
+        failed = self._run_setup({"WORLDBENCH_TASK_ID": "test-123"})
+        assert failed == [], failed
+        assert marker.read_text() == "test-123"
+
+    def test_respects_server_filter(self):
+        """Only servers declaring the requested toolset are set up."""
+        marker_a = Path(self.temp_dir) / "a_ran.txt"
+        marker_b = Path(self.temp_dir) / "b_ran.txt"
+        self._create_server(
+            "alpha",
+            setup_hook={
+                "command": "python",
+                "args": ["-c", f"open('{marker_a}', 'w').write('ok')"],
+            },
+            toolsets={"read": ["tool_a"]},
+        )
+        self._create_server(
+            "beta",
+            setup_hook={
+                "command": "python",
+                "args": ["-c", f"open('{marker_b}', 'w').write('ok')"],
+            },
+            toolsets={"write": ["tool_b"]},
+        )
+
+        failed = self._run_setup({"WORLDBENCH_TOOL_SETS": "alpha_read"})
+        assert failed == [], failed
+        assert marker_a.exists()
+        assert not marker_b.exists()
+
+    def test_fails_if_setup_hook_fails(self):
+        """A non-zero exit from a setup hook is reported as a failed server."""
+        self._create_server(
+            "bad",
+            setup_hook={
+                "command": "python",
+                "args": ["-c", "import sys; sys.exit(1)"],
+            },
+        )
+
+        failed = self._run_setup()
+        assert failed == ["bad"], failed
+
+    def test_multi_step_setup(self):
+        """Setup hooks can be a list of steps."""
+        marker1 = Path(self.temp_dir) / "step1.txt"
+        marker2 = Path(self.temp_dir) / "step2.txt"
+        self._create_server(
+            "svc",
+            setup_hook=[
+                {"command": "python", "args": ["-c", f"open('{marker1}', 'w').write('1')"]},
+                {"command": "python", "args": ["-c", f"open('{marker2}', 'w').write('2')"]},
+            ],
+        )
+
+        failed = self._run_setup()
+        assert failed == [], failed
+        assert marker1.read_text() == "1"
+        assert marker2.read_text() == "2"
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/packages/mcp_proxy/tests/test_toolset_configs.py

@@ -0,0 +1,25 @@
+"""Repository-level checks for package toolset declarations."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+STATE_TOOLS = {"export_state", "import_state"}
+
+
+def test_state_tools_only_appear_in_state_toolsets():
+    repo_root = Path(__file__).resolve().parents[3]
+    offenders: list[str] = []
+
+    for mcp_json in sorted((repo_root / "packages").glob("*/mcp.json")):
+        package = mcp_json.parent.name
+        data = json.loads(mcp_json.read_text())
+        for toolset, tools in data.get("toolsets", {}).items():
+            if toolset == "state":
+                continue
+            leaked = sorted(STATE_TOOLS.intersection(tools))
+            if leaked:
+                offenders.append(f"{package}:{toolset} -> {', '.join(leaked)}")
+
+    assert offenders == []

docker/packages/mcp_proxy/tests/test_viewer.py

@@ -0,0 +1,124 @@
+"""Tests for the viewer reverse-proxy app."""
+
+import httpx
+import pytest
+from starlette.testclient import TestClient
+
+from mcp_proxy.viewer import _DISPLAY_NAMES, _render_shell, create_viewer_app
+
+# ---------------------------------------------------------------------------
+# Shell rendering
+# ---------------------------------------------------------------------------
+
+
+class TestRenderShell:
+    def test_empty_registry(self):
+        html = _render_shell({})
+        assert "No services available" in html
+
+    def test_single_service(self):
+        html = _render_shell({"jira": 9000})
+        assert "Services" in html
+        assert "1 services" in html
+        assert 'data-service="jira"' in html
+        assert 'src="/__viewer__/set?app=jira"' in html
+
+    def test_multiple_services(self):
+        html = _render_shell({"jira": 9000, "slack": 9001, "google_mail": 9002})
+        assert "3 services" in html
+        for name in ("jira", "slack", "google_mail"):
+            assert f'data-service="{name}"' in html
+
+    def test_display_names_used(self):
+        html = _render_shell({"jira": 9000})
+        assert _DISPLAY_NAMES["jira"] in html  # "Issues"
+
+    def test_unknown_service_gets_title_case(self):
+        html = _render_shell({"my_custom_service": 9000})
+        assert "My Custom Service" in html
+
+    def test_services_sorted(self):
+        html = _render_shell({"slack": 9001, "jira": 9000})
+        jira_pos = html.index('data-service="jira"')
+        slack_pos = html.index('data-service="slack"')
+        assert jira_pos < slack_pos
+
+    def test_first_service_is_default_iframe(self):
+        html = _render_shell({"slack": 9001, "jira": 9000})
+        # sorted → jira comes first
+        assert 'src="/__viewer__/set?app=jira"' in html
+
+
+# ---------------------------------------------------------------------------
+# Viewer app routes
+# ---------------------------------------------------------------------------
+
+
+class TestViewerApp:
+    @pytest.fixture
+    def app(self):
+        return create_viewer_app(
+            service_registry={"test_svc": 9999, "other": 8888},
+            proxy_token="secret123",
+        )
+
+    @pytest.fixture
+    def client(self, app):
+        return TestClient(app, raise_server_exceptions=False)
+
+    def test_index_returns_html(self, client):
+        resp = client.get("/")
+        assert resp.status_code == 200
+        assert "Services" in resp.text
+
+    def test_reverse_proxy_unknown_service_returns_404(self, client):
+        resp = client.get("/__viewer__/set?app=nonexistent")
+        assert resp.status_code == 404
+        assert "Unknown service" in resp.text
+
+    def test_iframe_fallback_uses_server_state_when_cookie_blocked(self, app, monkeypatch):
+        """When embedded in an iframe, browsers may block the cookie.  The
+        viewer should fall back to server-side state when any referer is
+        present (indicating a redirect, not a direct navigation)."""
+
+        async def mock_request(self, *args, **kwargs):
+            return httpx.Response(200, text="service content")
+
+        monkeypatch.setattr(httpx.AsyncClient, "request", mock_request)
+        client = TestClient(app, raise_server_exceptions=False)
+
+        # First, select a service via /__viewer__/set to populate server state.
+        resp = client.get("/__viewer__/set?app=test_svc", follow_redirects=False)
+        assert resp.status_code == 307
+
+        # Now simulate an iframe request with NO cookie but a referer present
+        # (browser stripped the path but kept the origin).
+        resp = client.get(
+            "/",
+            headers={"referer": "http://some-outer-host/"},
+            cookies={},  # no cookie — blocked by browser
+        )
+        # Should proxy to the service, not re-render the shell
+        assert resp.status_code == 200
+        assert "service content" in resp.text
+        assert "Services" not in resp.text  # not the shell HTML
+
+    def test_iframe_fallback_not_triggered_without_referer(self, client):
+        """Without a referer, the server state fallback should NOT activate —
+        direct navigation should always show the shell."""
+        resp = client.get("/", cookies={})
+        assert resp.status_code == 200
+        assert "Services" in resp.text
+
+    def test_reverse_proxy_connection_error_returns_502(self, app, monkeypatch):
+        # Mock httpx to simulate a connection error regardless of port state
+        async def mock_request(self, *args, **kwargs):
+            raise httpx.ConnectError("Connection refused")
+
+        monkeypatch.setattr(httpx.AsyncClient, "request", mock_request)
+        client = TestClient(app, raise_server_exceptions=False)
+        # Set the service cookie first, then make a proxied request
+        client.cookies.set("__viewer_service", "test_svc")
+        resp = client.get("/some-page", headers={"referer": "http://testserver/"})
+        assert resp.status_code == 502
+        assert "Service unavailable" in resp.text