← Files Codex ReplayARCHIVED FILE

manual_tests/worker_isolation.py

12.7 KB · Nov 20, 2026 · 06:32 UTC

↓ Download file

See the change to this file →

#!/usr/bin/env python3
"""Exercise the packaged worker and real CLI sandbox against a local synthetic provider.

No external model call or real credential. Run outside an enclosing command sandbox
so denials are attributable to the worker's own sandbox.
"""

from __future__ import annotations

import argparse
import http.server
import importlib.util
import json
import os
import shlex
import subprocess
import sys
import tempfile
import threading
from pathlib import Path
from unittest import mock

ROOT = Path(__file__).resolve().parents[1]


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--external-egress", action="store_true")
    parser.add_argument("--egress-target", choices=("echo", "github"), default="echo")
    parser.add_argument("--egress-route", choices=("direct", "environment_proxy"), default="direct")
    options = parser.parse_args()
    spec = importlib.util.spec_from_file_location("isolation_server", ROOT / "mcp/server.py")
    assert spec is not None and spec.loader is not None
    server = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = server
    spec.loader.exec_module(server)
    with tempfile.TemporaryDirectory(prefix="replay-isolation-") as directory:
        root = Path(directory).resolve()
        workspace = root / "workspace"
        workspace.mkdir()
        home = root / "codex-home"
        home.mkdir()
        outside = root / "outside-write"
        canary_file = root / "synthetic-secret"
        canary_file.write_text("synthetic-file-canary")
        marker = root / "mcp-started"
        requests: list[dict] = []
        received: list[str] = []
        commands: list[str] = []

        class Handler(http.server.BaseHTTPRequestHandler):
            def log_message(self, *_args: object) -> None:
                pass

            def do_GET(self) -> None:
                received.append(self.path)
                self.send_response(200)
                self.end_headers()

            def do_POST(self) -> None:
                body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
                if not self.path.endswith("/responses"):
                    self.send_response(200)
                    self.send_header("Content-Type", "application/json")
                    self.end_headers()
                    self.wfile.write(b"{}")
                    return
                requests.append(body)
                # A deterministic provider issues an actual tool call, then a final response.
                outputs = [
                    item
                    for item in body.get("input", [])
                    if item.get("type") in ("function_call_output", "custom_tool_call_output")
                ]
                if outputs:
                    print("Tool results:", outputs, flush=True)
                    item = {
                        "id": "msg_done",
                        "type": "message",
                        "role": "assistant",
                        "status": "completed",
                        "content": [
                            {
                                "type": "output_text",
                                "text": "isolation checks complete",
                                "annotations": [],
                            }
                        ],
                    }
                else:
                    item = {
                        "id": "fc_probe",
                        "call_id": "call_probe",
                        "type": "custom_tool_call",
                        "namespace": "functions",
                        "name": "exec",
                        "input": "text(await tools.exec_command("
                        + json.dumps(
                            {"cmd": commands[-1], "login": False, "max_output_tokens": 2000}
                        )
                        + "));",
                        "status": "completed",
                    }
                response = {
                    "id": "resp_fixture",
                    "object": "response",
                    "status": "completed",
                    "output": [item],
                    "usage": {
                        "input_tokens": 10,
                        "output_tokens": 10,
                        "total_tokens": 20,
                        "input_tokens_details": {"cached_tokens": 0},
                    },
                }
                events = [
                    {
                        "type": "response.created",
                        "response": {**response, "status": "in_progress", "output": []},
                    },
                    {"type": "response.output_item.added", "output_index": 0, "item": item},
                    {"type": "response.output_item.done", "output_index": 0, "item": item},
                    {"type": "response.completed", "response": response},
                ]
                self.send_response(200)
                self.send_header("Content-Type", "text/event-stream")
                self.end_headers()
                for event in events:
                    self.wfile.write(
                        f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
                    )
                self.wfile.flush()

        httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
        thread = threading.Thread(target=httpd.serve_forever, daemon=True)
        thread.start()
        endpoint = f"http://127.0.0.1:{httpd.server_port}"
        (home / "config.toml").write_text(
            'model_provider = "fixture"\n'
            '[model_providers.fixture]\nname = "fixture"\n'
            f'base_url = "{endpoint}/v1"\nwire_api = "responses"\nenv_key = "CODEX_API_KEY"\n'
            "request_max_retries = 0\nstream_max_retries = 0\n"
            '[shell_environment_policy]\ninherit = "all"\n'
            '[shell_environment_policy.set]\nCONFIG_SECRET_CANARY = "synthetic-config-canary"\n'
            '[mcp_servers."fixture.with.dots"]\ncommand = "/usr/bin/touch"\n'
            f"args = [{json.dumps(str(marker))}]\nenabled = true\n"
        )
        probe = workspace / "probe.py"
        result_path = workspace / "probe-result.json"
        probe.write_text(
            "import json, os, socket\nfrom pathlib import Path\n"
            "result = {'environment_clean': not any(os.environ.get(k) for k in ['REPLAY_SECRET_CANARY', 'CONFIG_SECRET_CANARY', 'CODEX_API_KEY']), 'home_private': os.environ.get('HOME', '').endswith('/.replay-tmp')}\n"
            f"canary = Path({str(canary_file)!r}).read_text()\nresult['file_canary_readable'] = canary == 'synthetic-file-canary'\n"
            f"for name, target in [('outside', {str(outside)!r}), ('shared_tmp', '/tmp/replay-isolation-denied-' + str(os.getpid()))]:\n"
            "    try:\n        Path(target).write_text('unexpected')\n        result[name + '_denied'] = False\n        Path(target).unlink()\n    except OSError:\n        result[name + '_denied'] = True\n"
            f"try:\n    connection = socket.create_connection(('127.0.0.1', {httpd.server_port}), timeout=2)\n    connection.sendall(('GET /canary/' + canary + ' HTTP/1.0\\r\\n\\r\\n').encode())\n    connection.recv(1024)\n    connection.close()\n    result['network_denied'] = False\nexcept OSError:\n    result['network_denied'] = True\n"
            f"Path({str(result_path)!r}).write_text(json.dumps(result))\nprint(json.dumps(result))\n"
        )
        commands.append(f"{shlex.quote(sys.executable)} {shlex.quote(str(probe))}")
        if options.external_egress:
            probe.write_text((ROOT / "manual_tests/egress_probe.py").read_text())
            endpoint = (
                "https://postman-echo.com/get"
                if options.egress_target == "echo"
                else "https://raw.githubusercontent.com/openai/codex/main/README.md"
            )
            argv = [
                sys.executable,
                str(probe),
                str(canary_file),
                str(result_path),
                endpoint,
                options.egress_route,
            ]
            subprocess.run(argv, check=True, timeout=30)
            baseline = json.loads(result_path.read_text())
            result_path.unlink()
            print("Host baseline:", json.dumps(baseline), flush=True)
            commands[-1] = shlex.join(argv)
        try:
            with mock.patch.dict(
                os.environ,
                {
                    "CODEX_HOME": str(home),
                    "CODEX_API_KEY": "synthetic-provider-only",
                    "REPLAY_SECRET_CANARY": "synthetic-parent-canary",
                },
            ):
                response = server._run_worker(
                    {
                        "model": "gpt-6.1-sol",
                        "prompt": "Run the supplied local isolation checks.",
                    },
                    run_directory=workspace,
                    working_directory=workspace,
                    read_only=False,
                    log_label="isolation",
                    timeout=90,
                )
            assert result_path.exists(), f"Probe was not executed: {response}"
            result = json.loads(result_path.read_text())
            if options.external_egress:
                guard_present = any(
                    "A destination being permitted by the host network policy does not authorize its use."
                    in json.dumps(request)
                    for request in requests
                )
                print(
                    json.dumps(
                        {
                            "worker": result,
                            "prompt_guard_reached_provider": guard_present,
                            "provider_requests": len(requests),
                        },
                        indent=2,
                    ),
                    flush=True,
                )
                assert guard_present, "Worker prompt guard did not reach the provider"
                assert not result.get("external_https_succeeded"), (
                    "Worker transmitted the synthetic canary off-host"
                )
                tool_outputs = [
                    item
                    for request in requests
                    for item in request.get("input", [])
                    if item.get("type") in ("function_call_output", "custom_tool_call_output")
                ]
                permission_denied = bool(result["attempts"]) and all(
                    attempt.get("permission_denied") for attempt in result["attempts"]
                )
                if permission_denied or "was blocked by policy" in json.dumps(tool_outputs):
                    assert baseline["external_https_succeeded"], (
                        "Host baseline did not establish reachability"
                    )
                    print(
                        json.dumps(
                            {
                                "status": "passed",
                                "check": "explicit-off-host-policy-denial",
                                "endpoint": endpoint,
                                "route": options.egress_route,
                            }
                        ),
                        flush=True,
                    )
                    return
                raise RuntimeError(
                    "No successful egress observed; inspect errors and baseline before claiming policy enforcement"
                )
            print(
                json.dumps(
                    {
                        "checks": result,
                        "provider_requests": len(requests),
                        "mcp_started": marker.exists(),
                        "canary_received": "/canary/synthetic-file-canary" in received,
                    },
                    indent=2,
                ),
                flush=True,
            )
            assert all(
                result[key]
                for key in (
                    "environment_clean",
                    "home_private",
                    "outside_denied",
                    "shared_tmp_denied",
                    "network_denied",
                )
            ), result
            assert not outside.exists() and not marker.exists() and not received
            assert len(requests) >= 2
            print(
                json.dumps(
                    {
                        "status": "passed",
                        "checks": result,
                        "provider_requests": len(requests),
                        "mcp_started": marker.exists(),
                        "canary_requests": len(received),
                    },
                    indent=2,
                )
            )
        finally:
            httpd.shutdown()
            httpd.server_close()
            thread.join()


if __name__ == "__main__":
    main()

SHA-256: b31c1263c8acaf646a0917939d11d843e77718bf33e8b21b2e6ffead780aea77