fix(sandbox): eliminate SSH transport race causing flaky E2E tests (#69)

Author: drew

architecture/build-containers.md

@@ -386,7 +386,7 @@ Container builds use Docker BuildKit with local cache directories:
 - `build/scripts/docker-build-component.sh` stores per-component caches in `.cache/buildkit/<component>`.
 - `build/scripts/docker-build-cluster.sh` stores the cluster image cache in `.cache/buildkit/cluster`.
 - `mise run python:build:multiarch` stores per-platform wheel caches in `.cache/buildkit/python-wheels/<platform>` for local builds when using a `docker-container` buildx driver.
-- Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry, cargo target, and sccache local disk directories, keyed by image name and `TARGETARCH`, with `sharing=locked` to prevent concurrent cache corruption in parallel CI builds. The cargo target mount gives cargo a persistent `target/` directory for true incremental rebuilds on source-only changes. sccache uses memcached in CI (`SCCACHE_MEMCACHED_ENDPOINT`) and falls back to the local disk cache mount for local dev builds, providing a second layer of caching at the compilation unit level.
+- Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry, cargo target, and sccache local disk directories. Cargo target cache mounts are keyed by image name, `TARGETARCH`, and a computed scope hash derived from `Cargo.lock` plus a Rust toolchain hint, with `sharing=locked` to prevent concurrent cache corruption in parallel CI builds. This reduces reuse of stale `target/` artifacts across dependency or toolchain changes while preserving incremental rebuilds within a compatible scope. sccache uses memcached in CI (`SCCACHE_MEMCACHED_ENDPOINT`) and falls back to the local disk cache mount for local dev builds, providing a second layer of caching at the compilation unit level.
 - When the active buildx driver is `docker` (not `docker-container`), local cache import/export flags are skipped automatically because the docker driver cannot export local caches. In CI, cache export is also skipped.
 - For local single-arch builds, the scripts auto-select a builder with the native `docker` driver (matching the active Docker context) so images land directly in the Docker image store without slow tarball export.
 

build/python.toml

@@ -17,14 +17,36 @@ run = """
 #!/usr/bin/env bash
 set -euo pipefail
 
+sha256_16() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1" | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+sha256_16_stdin() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
 PLATFORMS=${DOCKER_PLATFORMS:-linux/amd64,linux/arm64}
 CARGO_VERSION=${NAVIGATOR_CARGO_VERSION:-}
 if [ -z "$CARGO_VERSION" ]; then
   CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
 fi
 
+LOCK_HASH=$(sha256_16 Cargo.lock)
+RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-stable}
+CACHE_SCOPE_INPUT="v1|python-wheels|base|${LOCK_HASH}|${RUST_SCOPE}"
+CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin)
+
 VERSION_ARGS=(
   --build-arg "NAVIGATOR_CARGO_VERSION=${CARGO_VERSION}"
+  --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}"
 )
 
 if ! docker buildx inspect multiarch >/dev/null 2>&1; then
@@ -108,19 +130,41 @@ run = """
 #!/usr/bin/env bash
 set -euo pipefail
 
+sha256_16() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1" | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+sha256_16_stdin() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
 CARGO_VERSION=${NAVIGATOR_CARGO_VERSION:-}
 OSXCROSS_IMAGE_REF=${OSXCROSS_IMAGE:-crazymax/osxcross:latest}
 if [ -z "$CARGO_VERSION" ]; then
   CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
 fi
 
+LOCK_HASH=$(sha256_16 Cargo.lock)
+RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-stable}
+CACHE_SCOPE_INPUT="v1|python-wheels-macos|base|${LOCK_HASH}|${RUST_SCOPE}"
+CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin)
+
 mkdir -p target/wheels
 
 docker build \
   -f deploy/docker/Dockerfile.python-wheels-macos \
   --target wheels \
   --build-arg "OSXCROSS_IMAGE=${OSXCROSS_IMAGE_REF}" \
   --build-arg "NAVIGATOR_CARGO_VERSION=${CARGO_VERSION}" \
+  --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \
   --output type=local,dest=target/wheels \
   .
 

build/scripts/docker-build-component.sh

@@ -18,6 +18,39 @@
 #   IMAGE_REGISTRY     - Registry prefix for image name (e.g. ghcr.io/org/repo)
 set -euo pipefail
 
+sha256_16() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1" | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+sha256_16_stdin() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+detect_rust_scope() {
+  local dockerfile="$1"
+  local rust_from
+  rust_from=$(grep -E '^FROM --platform=\$BUILDPLATFORM rust:[^ ]+' "$dockerfile" | head -n1 | sed -E 's/^FROM --platform=\$BUILDPLATFORM rust:([^ ]+).*/\1/' || true)
+  if [[ -n "${rust_from}" ]]; then
+    echo "rust-${rust_from}"
+    return
+  fi
+
+  if grep -q "rustup.rs" "$dockerfile"; then
+    echo "rustup-stable"
+    return
+  fi
+
+  echo "no-rust"
+}
+
 COMPONENT=${1:?"Usage: docker-build-component.sh <component> [variant] [extra-args...]"}
 shift
 
@@ -98,11 +131,17 @@ if [[ -n "${SCCACHE_MEMCACHED_ENDPOINT:-}" ]]; then
   SCCACHE_ARGS=(--build-arg "SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}")
 fi
 
+LOCK_HASH=$(sha256_16 Cargo.lock)
+RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-$(detect_rust_scope "${DOCKERFILE}")}
+CACHE_SCOPE_INPUT="v1|${COMPONENT}|${VARIANT:-base}|${LOCK_HASH}|${RUST_SCOPE}"
+CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "${CACHE_SCOPE_INPUT}" | sha256_16_stdin)
+
 docker buildx build \
   ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} \
   ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \
   ${CACHE_ARGS[@]+"${CACHE_ARGS[@]}"} \
   ${SCCACHE_ARGS[@]+"${SCCACHE_ARGS[@]}"} \
+  --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \
   -f "${DOCKERFILE}" \
   -t "${IMAGE_NAME}:${IMAGE_TAG}" \
   --provenance=false \

build/scripts/docker-publish-multiarch.sh

@@ -22,6 +22,39 @@
 #   AWS_REGION               - AWS region (default: us-west-2)
 set -euo pipefail
 
+sha256_16() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1" | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+sha256_16_stdin() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum | awk '{print substr($1, 1, 16)}'
+  else
+    shasum -a 256 | awk '{print substr($1, 1, 16)}'
+  fi
+}
+
+detect_rust_scope() {
+  local dockerfile="$1"
+  local rust_from
+  rust_from=$(grep -E '^FROM --platform=\$BUILDPLATFORM rust:[^ ]+' "$dockerfile" | head -n1 | sed -E 's/^FROM --platform=\$BUILDPLATFORM rust:([^ ]+).*/\1/' || true)
+  if [[ -n "${rust_from}" ]]; then
+    echo "rust-${rust_from}"
+    return
+  fi
+
+  if grep -q "rustup.rs" "$dockerfile"; then
+    echo "rustup-stable"
+    return
+  fi
+
+  echo "no-rust"
+}
+
 # ---------------------------------------------------------------------------
 # Parse arguments
 # ---------------------------------------------------------------------------
@@ -115,6 +148,7 @@ resolve_dockerfile() {
 # so Rust compiles natively and only the final stage runs on the target arch.
 # ---------------------------------------------------------------------------
 echo "Building multi-arch component images..."
+LOCK_HASH=$(sha256_16 Cargo.lock)
 for component in sandbox server; do
   echo "Building ${IMAGE_PREFIX}${component} for ${PLATFORMS}..."
   BUILD_ARGS=""
@@ -125,6 +159,10 @@ for component in sandbox server; do
     BUILD_ARGS="${BUILD_ARGS} --build-arg SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}"
   fi
   DOCKERFILE=$(resolve_dockerfile "${component}")
+  RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-$(detect_rust_scope "${DOCKERFILE}")}
+  CACHE_SCOPE_INPUT="v1|${component}|base|${LOCK_HASH}|${RUST_SCOPE}"
+  CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "${CACHE_SCOPE_INPUT}" | sha256_16_stdin)
+  BUILD_ARGS="${BUILD_ARGS} --build-arg CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}"
   FULL_IMAGE="${REGISTRY}/${IMAGE_PREFIX}${component}"
   docker buildx build \
     --platform "${PLATFORMS}" \

crates/navigator-sandbox/src/lib.rs

@@ -404,9 +404,13 @@ pub async fn run_sandbox(
         let netns_fd = ssh_netns_fd;
         let ca_paths = ca_file_paths.clone();
         let provider_env_clone = provider_env.clone();
+
+        let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel();
+
         tokio::spawn(async move {
             if let Err(err) = ssh::run_ssh_server(
                 addr,
+                ssh_ready_tx,
                 policy_clone,
                 workdir_clone,
                 secret,
@@ -421,6 +425,28 @@ pub async fn run_sandbox(
                 tracing::error!(error = %err, "SSH server failed");
             }
         });
+
+        // Wait for the SSH server to bind its socket before spawning the
+        // entrypoint process. This prevents exec requests from racing against
+        // SSH server startup when Kubernetes marks the pod Ready.
+        match timeout(Duration::from_secs(10), ssh_ready_rx).await {
+            Ok(Ok(Ok(()))) => {
+                info!("SSH server is ready to accept connections");
+            }
+            Ok(Ok(Err(err))) => {
+                return Err(err.context("SSH server failed during startup"));
+            }
+            Ok(Err(_)) => {
+                return Err(miette::miette!(
+                    "SSH server task panicked before signaling ready"
+                ));
+            }
+            Err(_) => {
+                return Err(miette::miette!(
+                    "SSH server did not start within 10 seconds"
+                ));
+            }
+        }
     }
 
     #[cfg(target_os = "linux")]

crates/navigator-sandbox/src/ssh.rs

@@ -26,18 +26,17 @@ use tracing::{info, warn};
 
 const PREFACE_MAGIC: &str = "NSSH1";
 
-#[allow(clippy::too_many_arguments)]
-pub async fn run_ssh_server(
+/// Perform SSH server initialization: generate a host key, build the config,
+/// and bind the TCP listener. Extracted so that startup errors can be forwarded
+/// through the readiness channel rather than being silently logged.
+async fn ssh_server_init(
     listen_addr: SocketAddr,
-    policy: SandboxPolicy,
-    workdir: Option<String>,
-    handshake_secret: String,
-    handshake_skew_secs: u64,
-    netns_fd: Option<RawFd>,
-    proxy_url: Option<String>,
-    ca_file_paths: Option<(PathBuf, PathBuf)>,
-    provider_env: HashMap<String, String>,
-) -> Result<()> {
+    ca_file_paths: &Option<(PathBuf, PathBuf)>,
+) -> Result<(
+    TcpListener,
+    Arc<russh::server::Config>,
+    Option<Arc<(PathBuf, PathBuf)>>,
+)> {
     let mut rng = OsRng;
     let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?;
 
@@ -48,10 +47,41 @@ pub async fn run_ssh_server(
     config.keys.push(host_key);
 
     let config = Arc::new(config);
-    let ca_paths = ca_file_paths.map(Arc::new);
+    let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone()));
     let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?;
     info!(addr = %listen_addr, "SSH server listening");
 
+    Ok((listener, config, ca_paths))
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn run_ssh_server(
+    listen_addr: SocketAddr,
+    ready_tx: tokio::sync::oneshot::Sender<Result<()>>,
+    policy: SandboxPolicy,
+    workdir: Option<String>,
+    handshake_secret: String,
+    handshake_skew_secs: u64,
+    netns_fd: Option<RawFd>,
+    proxy_url: Option<String>,
+    ca_file_paths: Option<(PathBuf, PathBuf)>,
+    provider_env: HashMap<String, String>,
+) -> Result<()> {
+    let (listener, config, ca_paths) = match ssh_server_init(listen_addr, &ca_file_paths).await {
+        Ok(v) => {
+            // Signal that the SSH server has bound the socket and is ready to
+            // accept connections. The parent task awaits this before spawning
+            // the entrypoint process, ensuring exec requests won't race
+            // against server startup.
+            let _ = ready_tx.send(Ok(()));
+            v
+        }
+        Err(err) => {
+            let _ = ready_tx.send(Err(err));
+            return Ok(());
+        }
+    };
+
     loop {
         let (stream, peer) = listener.accept().await.into_diagnostic()?;
         stream.set_nodelay(true).into_diagnostic()?;

crates/navigator-server/src/grpc.rs

@@ -1407,10 +1407,13 @@ fn is_valid_env_key(key: &str) -> bool {
 /// Maximum number of attempts when establishing the SSH transport to a sandbox.
 /// The sandbox SSH server may not be listening yet when the pod is marked Ready,
 /// so we retry transient connection failures with exponential backoff.
-const SSH_CONNECT_MAX_ATTEMPTS: u32 = 4;
+const SSH_CONNECT_MAX_ATTEMPTS: u32 = 6;
 
 /// Initial backoff duration between SSH connection retries (doubles each attempt).
-const SSH_CONNECT_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200);
+const SSH_CONNECT_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_millis(250);
+
+/// Maximum backoff duration between SSH connection retries (caps exponential growth).
+const SSH_CONNECT_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2);
 
 /// Returns `true` if the gRPC status represents a transient SSH connection error
 /// that is worth retrying (e.g. the sandbox SSH server is not yet listening).
@@ -1453,7 +1456,8 @@ async fn stream_exec_over_ssh(
         let mut result = None;
         for attempt in 0..SSH_CONNECT_MAX_ATTEMPTS {
             if attempt > 0 {
-                let backoff = SSH_CONNECT_INITIAL_BACKOFF * 2u32.pow(attempt - 1);
+                let backoff = (SSH_CONNECT_INITIAL_BACKOFF * 2u32.pow(attempt - 1))
+                    .min(SSH_CONNECT_MAX_BACKOFF);
                 warn!(
                     sandbox_id = %sandbox_id,
                     attempt = attempt + 1,

deploy/docker/Dockerfile.python-wheels

@@ -25,6 +25,7 @@ FROM base AS builder
 ARG TARGETARCH
 ARG BUILDARCH
 ARG NAVIGATOR_CARGO_VERSION
+ARG CARGO_TARGET_CACHE_SCOPE=default
 
 ARG SCCACHE_MEMCACHED_ENDPOINT
 
@@ -59,7 +60,7 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
 # The cargo-target mount gives cargo a persistent target/ dir for incremental rebuilds.
 RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \
     --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \
-    --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH},sharing=locked,target=/build/target \
+    --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
     --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \
     . cross-build.sh && cargo_cross_build --release -p navigator-cli 2>/dev/null || true
 
@@ -83,7 +84,7 @@ RUN touch crates/navigator-cli/src/main.rs \
 
 RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \
     --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \
-    --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH},sharing=locked,target=/build/target \
+    --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
     --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \
     . cross-build.sh && \
     export_cross_env && \

deploy/docker/Dockerfile.python-wheels-macos

@@ -9,6 +9,7 @@ FROM python:${PYTHON_VERSION}-slim AS builder
 
 ARG TARGETARCH
 ARG NAVIGATOR_CARGO_VERSION
+ARG CARGO_TARGET_CACHE_SCOPE=default
 
 ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}"
 ENV LD_LIBRARY_PATH="/osxcross/lib"
@@ -65,7 +66,7 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
 # Build dependencies only (cached unless Cargo.toml/lock changes).
 RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \
     --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \
-    --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH},sharing=locked,target=/build/target \
+    --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
     cargo build --release --target aarch64-apple-darwin -p navigator-cli 2>/dev/null || true
 
 # Copy actual source code and Python packaging files.
@@ -90,7 +91,7 @@ RUN touch crates/navigator-cli/src/main.rs \
 
 RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \
     --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \
-    --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH},sharing=locked,target=/build/target \
+    --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
     if [ -n "${NAVIGATOR_CARGO_VERSION:-}" ]; then \
       sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${NAVIGATOR_CARGO_VERSION}"'"/}' Cargo.toml; \
     fi && \