feat(e2e): parallelize e2e tests with pytest-xdist (default -n 5) (#102)

Closes #101

Add pytest-xdist for parallel e2e test execution with configurable
concurrency. Default to 5 workers; override via E2E_PARALLEL env var
(accepts a number or 'auto' for CPU-count matching). Make session-scoped
mock inference route fixtures worker-safe by incorporating the xdist
worker_id into route names and routing hints.

Co-authored-by: John Myers <johntmyers@users.noreply.github.com>

Author: johntmyers

e2e/python/conftest.py

@@ -72,12 +72,25 @@ def inference_client(sandbox_client: SandboxClient) -> InferenceRouteClient:
     return InferenceRouteClient.from_sandbox_client(sandbox_client)
 
 
+@pytest.fixture(scope="session")
+def _worker_suffix(worker_id: str) -> str:
+    """Return a suffix for worker-unique resource names.
+
+    Uses the built-in ``worker_id`` fixture from pytest-xdist which returns
+    ``"gw0"``, ``"gw1"``, etc. for workers, or ``"master"`` for non-xdist runs.
+    """
+    if worker_id == "master":
+        return ""
+    return f"-{worker_id}"
+
+
 @pytest.fixture(scope="session")
 def mock_inference_route(
     inference_client: InferenceRouteClient,
+    _worker_suffix: str,
 ) -> Iterator[str]:
-    name = "e2e-mock-local"
-    routing_hint = "e2e_mock_local"
+    name = f"e2e-mock-local{_worker_suffix}"
+    routing_hint = f"e2e_mock_local{_worker_suffix}"
     # Clean up any leftover route from a previous run.
     try:
         inference_client.delete(name)
@@ -93,7 +106,7 @@ def mock_inference_route(
         model_id="mock/test-model",
         enabled=True,
     )
-    yield name
+    yield routing_hint
     try:
         inference_client.delete(name)
     except grpc.RpcError:
@@ -103,9 +116,10 @@ def mock_inference_route(
 @pytest.fixture(scope="session")
 def mock_anthropic_route(
     inference_client: InferenceRouteClient,
+    _worker_suffix: str,
 ) -> Iterator[str]:
-    name = "e2e-mock-anthropic"
-    routing_hint = "e2e_mock_anthropic"
+    name = f"e2e-mock-anthropic{_worker_suffix}"
+    routing_hint = f"e2e_mock_anthropic{_worker_suffix}"
     try:
         inference_client.delete(name)
     except grpc.RpcError:
@@ -120,7 +134,7 @@ def mock_anthropic_route(
         model_id="mock/claude-test",
         enabled=True,
     )
-    yield name
+    yield routing_hint
     try:
         inference_client.delete(name)
     except grpc.RpcError:
@@ -130,10 +144,11 @@ def mock_anthropic_route(
 @pytest.fixture(scope="session")
 def mock_disallowed_route(
     inference_client: InferenceRouteClient,
+    _worker_suffix: str,
 ) -> Iterator[str]:
     """Route that exists but is NOT in any sandbox's allowed_routes."""
-    name = "e2e-mock-disallowed"
-    routing_hint = "e2e_mock_disallowed"
+    name = f"e2e-mock-disallowed{_worker_suffix}"
+    routing_hint = f"e2e_mock_disallowed{_worker_suffix}"
     try:
         inference_client.delete(name)
     except grpc.RpcError:
@@ -148,7 +163,7 @@ def mock_disallowed_route(
         model_id="mock/disallowed-model",
         enabled=True,
     )
-    yield name
+    yield routing_hint
     try:
         inference_client.delete(name)
     except grpc.RpcError:

e2e/python/test_inference_routing.py

@@ -177,7 +177,9 @@ def test_inference_call_routed_to_backend(
     4. Forward locally via sandbox router to the policy-allowed backend
     5. Return the mock response from the configured route
     """
-    spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy())
+    spec = datamodel_pb2.SandboxSpec(
+        policy=_inference_routing_policy(mock_inference_route)
+    )
 
     def call_chat_completions() -> str:
         import json
@@ -226,7 +228,9 @@ def test_non_inference_request_denied(
     undeclared endpoint should be denied with 403 when inference routing
     is configured — only recognized inference API patterns are routed.
     """
-    spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy())
+    spec = datamodel_pb2.SandboxSpec(
+        policy=_inference_routing_policy(mock_inference_route)
+    )
 
     def make_non_inference_request() -> str:
         import ssl
@@ -265,7 +269,7 @@ def test_inference_anthropic_messages_protocol(
     policy = sandbox_pb2.SandboxPolicy(
         version=1,
         inference=sandbox_pb2.InferencePolicy(
-            allowed_routes=["e2e_mock_anthropic"],
+            allowed_routes=[mock_anthropic_route],
         ),
         filesystem=_BASE_FILESYSTEM,
         landlock=_BASE_LANDLOCK,
@@ -323,8 +327,10 @@ def test_inference_route_filtering_by_allowed_routes(
     allowed route should succeed, while inference requests that can't
     match any allowed route get an error from the sandbox router.
     """
-    # Policy only allows e2e_mock_local, NOT e2e_mock_disallowed
-    spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy())
+    # Policy only allows the mock_inference_route, NOT mock_disallowed_route
+    spec = datamodel_pb2.SandboxSpec(
+        policy=_inference_routing_policy(mock_inference_route)
+    )
 
     def call_allowed_route() -> str:
         import json

pyproject.toml

@@ -31,6 +31,7 @@ dev = [
     "pytest>=8.0",
     "pytest-asyncio>=0.23",
     "pytest-cov>=4.0",
+    "pytest-xdist>=3.0",
     "ruff>=0.4",
     "ty>=0.0.1a6",
     "maturin>=1.5,<2.0",

tasks/test.toml

@@ -29,10 +29,10 @@ run = "uv run pytest python/"
 hide = true
 
 ["test:e2e:sandbox"]
-description = "Run sandbox end-to-end tests"
+description = "Run sandbox end-to-end tests (E2E_PARALLEL=N or 'auto'; default 5)"
 depends = ["python:proto", "cluster"]
 env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
-run = "uv run pytest -o python_files='test_*.py' e2e/python"
+run = "uv run pytest -o python_files='test_*.py' -n ${E2E_PARALLEL:-5} e2e/python"
 hide = true
 
 ["test:e2e:port-forward"]