feat(cli): detect port conflicts before gateway start, add sandbox delete --all, and improve spinner spacing (#225)

Author: drew

.agents/skills/debug-navigator-cluster/SKILL.md

@@ -132,6 +132,21 @@ If `/readyz` fails, k3s is still starting or has crashed. Check container logs (
 
 If pods are in `CrashLoopBackOff`, `ImagePullBackOff`, or `Pending`, investigate those pods specifically.
 
+Also check for node pressure conditions that cause the kubelet to evict pods and reject scheduling:
+
+```bash
+# Check node conditions (DiskPressure, MemoryPressure, PIDPressure)
+docker exec openshell-cluster-<name> sh -lc 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get nodes -o jsonpath="{range .items[*]}{.metadata.name}{range .status.conditions[*]} {.type}={.status}{end}{\"\n\"}{end}"'
+
+# Check disk usage inside the container
+docker exec openshell-cluster-<name> df -h /
+
+# Check memory usage
+docker exec openshell-cluster-<name> free -h
+```
+
+If any pressure condition is `True`, pods will be evicted and new ones rejected. The bootstrap now detects `HEALTHCHECK_NODE_PRESSURE` markers from the health-check script and aborts early with a clear diagnosis. To fix: free disk/memory on the host, then recreate the gateway.
+
 ### Step 4: Check OpenShell Server StatefulSet
 
 The OpenShell server is deployed via a HelmChart CR as a StatefulSet with persistent storage. Check its status:
@@ -305,6 +320,8 @@ If DNS is broken, all image pulls from the distribution registry will fail, as w
 | Port conflict | Another service on 6443 or the configured gateway host port (default 8080) | Stop conflicting service or use `--port` on `openshell gateway start` to pick a different host port |
 | gRPC connect refused to `127.0.0.1:443` in CI | Docker daemon is remote (`DOCKER_HOST=tcp://...`) but metadata still points to loopback | Verify metadata endpoint host matches `DOCKER_HOST` and includes non-loopback host |
 | DNS failures inside container | Entrypoint DNS detection failed | Check `/etc/rancher/k3s/resolv.conf` and container startup logs |
+| Node DiskPressure / MemoryPressure / PIDPressure | Insufficient disk, memory, or PIDs on host | Free disk (`docker system prune -a --volumes`), increase memory, or expand host resources. Bootstrap auto-detects via `HEALTHCHECK_NODE_PRESSURE` marker |
+| Pods evicted with "The node had condition: [DiskPressure]" | Host disk full, kubelet evicting pods | Free disk space on host, then `openshell gateway destroy <name> && openshell gateway start` |
 | `metrics-server` errors in logs | Normal k3s noise, not the root cause | These errors are benign — look for the actual failing health check component |
 | Stale NotReady nodes from previous deploys | Volume reused across container recreations | The deploy flow now auto-cleans stale nodes; if it still fails, manually delete NotReady nodes (see Step 3) or choose "Recreate" when prompted |
 | gRPC `UNIMPLEMENTED` for newer RPCs in push mode | Helm values still point at older pulled images instead of the pushed refs | Verify rendered `navigator-helmchart.yaml` uses the expected push refs (`server`, `sandbox`, `pki-job`) and not `:latest` |
@@ -363,6 +380,12 @@ run docker exec "${CONTAINER}" sh -lc "${KCFG} kubectl get --raw='/readyz'" 2>&1
 echo "=== Nodes ==="
 run docker exec "${CONTAINER}" sh -lc "${KCFG} kubectl get nodes -o wide" 2>&1
 
+echo "=== Node Conditions ==="
+run docker exec "${CONTAINER}" sh -lc "${KCFG} kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{range .status.conditions[*]} {.type}={.status}{end}{\"\n\"}{end}'" 2>&1
+
+echo "=== Disk Usage ==="
+run docker exec "${CONTAINER}" df -h / 2>&1
+
 echo "=== All Pods ==="
 run docker exec "${CONTAINER}" sh -lc "${KCFG} kubectl get pods -A -o wide" 2>&1
 

crates/navigator-bootstrap/src/docker.rs

@@ -15,7 +15,8 @@ use bollard::models::{
 };
 use bollard::query_parameters::{
     CreateContainerOptions, CreateImageOptions, InspectContainerOptions, InspectNetworkOptions,
-    RemoveContainerOptions, RemoveImageOptions, RemoveVolumeOptions, StartContainerOptions,
+    ListContainersOptionsBuilder, RemoveContainerOptions, RemoveImageOptions, RemoveVolumeOptions,
+    StartContainerOptions,
 };
 use futures::StreamExt;
 use miette::{IntoDiagnostic, Result, WrapErr};
@@ -479,6 +480,77 @@ pub async fn ensure_container(
     Ok(())
 }
 
+/// Information about a container that is holding a port we need.
+#[derive(Debug, Clone)]
+pub struct PortConflict {
+    /// Name of the container holding the port (without leading `/`).
+    pub container_name: String,
+    /// The host port that conflicts.
+    pub host_port: u16,
+}
+
+/// Check whether any *other* running container already binds the host ports
+/// that the gateway needs.  Returns a list of conflicts (empty if none).
+///
+/// Docker silently fails to attach networking when a port is already bound,
+/// leaving the new container with only a loopback interface.  Detecting this
+/// up-front lets us give a clear error instead of a cryptic "no default route"
+/// failure 30 seconds later.
+pub async fn check_port_conflicts(
+    docker: &Docker,
+    name: &str,
+    gateway_port: u16,
+    kube_port: Option<u16>,
+) -> Result<Vec<PortConflict>> {
+    let our_container = container_name(name);
+    let needed_ports: Vec<u16> = std::iter::once(gateway_port).chain(kube_port).collect();
+
+    let containers = docker
+        .list_containers(Some(
+            ListContainersOptionsBuilder::new()
+                // Only running containers can hold port bindings.
+                .all(false)
+                .build(),
+        ))
+        .await
+        .into_diagnostic()
+        .wrap_err("failed to list containers for port conflict check")?;
+
+    let mut conflicts = Vec::new();
+    for container in &containers {
+        // Skip our own container (it may already exist from a previous run).
+        let names = container.names.as_deref().unwrap_or_default();
+        let is_ours = names
+            .iter()
+            .any(|n| n.trim_start_matches('/') == our_container);
+        if is_ours {
+            continue;
+        }
+
+        let ports = container.ports.as_deref().unwrap_or_default();
+        for port in ports {
+            if let Some(public) = port.public_port {
+                if needed_ports.contains(&public) {
+                    let cname = names
+                        .first()
+                        .map(|n| n.trim_start_matches('/').to_string())
+                        .unwrap_or_else(|| {
+                            container
+                                .id
+                                .clone()
+                                .unwrap_or_else(|| "<unknown>".to_string())
+                        });
+                    conflicts.push(PortConflict {
+                        container_name: cname,
+                        host_port: public,
+                    });
+                }
+            }
+        }
+    }
+    Ok(conflicts)
+}
+
 pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
     let container_name = container_name(name);
 

crates/navigator-bootstrap/src/errors.rs

@@ -133,6 +133,12 @@ const FAILURE_PATTERNS: &[FailurePattern] = &[
         match_mode: MatchMode::Any,
         diagnose: diagnose_oom_killed,
     },
+    // Node resource pressure (DiskPressure, MemoryPressure, PIDPressure)
+    FailurePattern {
+        matchers: &["HEALTHCHECK_NODE_PRESSURE"],
+        match_mode: MatchMode::Any,
+        diagnose: diagnose_node_pressure,
+    },
     // TLS/certificate issues
     FailurePattern {
         matchers: &[
@@ -178,10 +184,18 @@ fn diagnose_corrupted_state(gateway_name: &str) -> GatewayFailureDiagnosis {
 fn diagnose_no_default_route(_gateway_name: &str) -> GatewayFailureDiagnosis {
     GatewayFailureDiagnosis {
         summary: "Docker networking issue".to_string(),
-        explanation: "The gateway container has no network route. This is usually caused by \
-            stale Docker networks or Docker Desktop networking issues."
+        explanation: "The gateway container has no network route. This can happen when \
+            another container is already bound to the same host port (Docker silently \
+            skips network attachment), or due to stale Docker networks."
             .to_string(),
         recovery_steps: vec![
+            RecoveryStep::with_command(
+                "Check for containers using the same port",
+                "docker ps --format '{{.Names}}\\t{{.Ports}}'",
+            ),
+            RecoveryStep::new(
+                "Stop any container holding the gateway port (default 8080), then retry",
+            ),
             RecoveryStep::with_command("Prune unused Docker networks", "docker network prune -f"),
             RecoveryStep::new("Restart Docker Desktop (if on Mac/Windows)"),
             RecoveryStep::new("Then retry: openshell gateway start"),
@@ -237,11 +251,22 @@ fn diagnose_k3s_dns_proxy_failure(gateway_name: &str) -> GatewayFailureDiagnosis
     GatewayFailureDiagnosis {
         summary: "Cluster DNS resolution failed".to_string(),
         explanation: "The gateway cluster started but its internal DNS proxy cannot resolve \
-            external hostnames. This is typically caused by stale Docker networking state \
-            or Docker Desktop DNS configuration issues."
+            external hostnames. Docker's embedded DNS inside the container cannot reach \
+            an upstream resolver. This is typically caused by Docker not being configured \
+            with the host's DNS servers, stale Docker networking state, or (on Desktop) \
+            DNS configuration issues."
             .to_string(),
         recovery_steps: vec![
-            RecoveryStep::new("Restart Docker Desktop"),
+            RecoveryStep::with_command(
+                "Check your host's DNS servers",
+                "resolvectl status | grep 'DNS Servers' -A2",
+            ),
+            RecoveryStep::with_command(
+                "Configure Docker to use those DNS servers \
+                 (add to /etc/docker/daemon.json, then restart Docker)",
+                "echo '{\"dns\": [\"<your-dns-server-ip>\"]}' | sudo tee /etc/docker/daemon.json \
+                 && sudo systemctl restart docker",
+            ),
             RecoveryStep::with_command("Prune Docker networks", "docker network prune -f"),
             RecoveryStep::with_command(
                 "Destroy and recreate the gateway",
@@ -289,6 +314,34 @@ fn diagnose_oom_killed(_gateway_name: &str) -> GatewayFailureDiagnosis {
     }
 }
 
+fn diagnose_node_pressure(gateway_name: &str) -> GatewayFailureDiagnosis {
+    GatewayFailureDiagnosis {
+        summary: "Node under resource pressure".to_string(),
+        explanation: "The cluster node is reporting a resource pressure condition \
+            (DiskPressure, MemoryPressure, or PIDPressure). When a node is under \
+            pressure the kubelet evicts running pods and rejects new pod scheduling, \
+            so the gateway will never become healthy until the pressure is resolved."
+            .to_string(),
+        recovery_steps: vec![
+            RecoveryStep::with_command("Check available disk space on the host", "df -h /"),
+            RecoveryStep::with_command(
+                "Free disk space by pruning unused Docker resources",
+                "docker system prune -a --volumes",
+            ),
+            RecoveryStep::with_command("Check available memory on the host", "free -h"),
+            RecoveryStep::new(
+                "Increase Docker resource allocation \
+                (Docker Desktop → Settings → Resources), or free resources on the host",
+            ),
+            RecoveryStep::with_command(
+                "Destroy and recreate the gateway after freeing resources",
+                format!("openshell gateway destroy {gateway_name} && openshell gateway start"),
+            ),
+        ],
+        retryable: false,
+    }
+}
+
 fn diagnose_certificate_issue(gateway_name: &str) -> GatewayFailureDiagnosis {
     GatewayFailureDiagnosis {
         summary: "TLS certificate issue".to_string(),
@@ -437,4 +490,59 @@ mod tests {
         let diagnosis = diagnose_failure("test", "Try again later", None);
         assert!(diagnosis.is_none());
     }
+
+    #[test]
+    fn test_diagnose_node_pressure_disk() {
+        let diagnosis = diagnose_failure(
+            "test",
+            "HEALTHCHECK_NODE_PRESSURE: DiskPressure\n\
+             The cluster node is under resource pressure.",
+            None,
+        );
+        assert!(diagnosis.is_some());
+        let d = diagnosis.unwrap();
+        assert!(
+            d.summary.contains("pressure"),
+            "expected pressure diagnosis, got: {}",
+            d.summary
+        );
+        assert!(!d.retryable);
+    }
+
+    #[test]
+    fn test_diagnose_node_pressure_from_container_logs() {
+        let diagnosis = diagnose_failure(
+            "test",
+            "gateway health check reported unhealthy",
+            Some("HEALTHCHECK_NODE_PRESSURE: MemoryPressure"),
+        );
+        assert!(diagnosis.is_some());
+        let d = diagnosis.unwrap();
+        assert!(
+            d.summary.contains("pressure"),
+            "expected pressure diagnosis, got: {}",
+            d.summary
+        );
+    }
+
+    #[test]
+    fn test_diagnose_dns_failure_from_namespace_timeout() {
+        // When wait_for_namespace detects DNS failure, the error message itself
+        // (not container logs) contains the DNS markers. The diagnose_failure
+        // function must match these from the error_message parameter alone,
+        // since container_logs may be None in this path.
+        let diagnosis = diagnose_failure(
+            "test",
+            "K8s namespace not ready\n\nCaused by:\n    dial tcp: lookup registry: Try again\n    DNS resolution is failing inside the gateway container.",
+            None,
+        );
+        assert!(diagnosis.is_some());
+        let d = diagnosis.unwrap();
+        assert!(
+            d.summary.contains("DNS"),
+            "expected DNS diagnosis, got: {}",
+            d.summary
+        );
+        assert!(d.retryable);
+    }
 }