fix(cluster): run helm/kubectl inside container via docker exec (#255)

cluster-deploy-fast.sh ran helm and kubectl directly on the host,
which requires a local kubeconfig pointing at the k3s API server.
Since the cluster container does not expose port 6443, these commands
always fail with 'connection refused'.

Install helm in the cluster container image and run all helm/kubectl
commands via docker exec with KUBECONFIG pre-configured. The local
chart directory is copied into the container before helm upgrade.

Author: drew

deploy/docker/Dockerfile.cluster

@@ -31,6 +31,7 @@
 
 ARG K3S_VERSION=v1.35.2-k3s1
 ARG K9S_VERSION=v0.50.18
+ARG HELM_VERSION=v3.17.3
 ARG NVIDIA_CONTAINER_TOOLKIT_VERSION=1.18.2-1
 
 # ---------------------------------------------------------------------------
@@ -50,6 +51,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi
     chmod +x /tmp/k9s && \
     rm -rf /var/lib/apt/lists/*
 
+# ---------------------------------------------------------------------------
+# Stage 1c: Download helm binary for in-container chart upgrades
+# ---------------------------------------------------------------------------
+FROM ubuntu:24.04 AS helm
+ARG HELM_VERSION
+ARG TARGETARCH
+RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
+    curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \
+      | tar xz --strip-components=1 -C /tmp "linux-${TARGETARCH}/helm" && \
+    chmod +x /tmp/helm && \
+    rm -rf /var/lib/apt/lists/*
+
 # ---------------------------------------------------------------------------
 # Stage 2: Install NVIDIA container toolkit on Ubuntu
 # ---------------------------------------------------------------------------
@@ -99,6 +112,9 @@ COPY --from=k3s /bin/ /bin/
 # Copy k9s binary for interactive cluster debugging via `openshell doctor exec -- k9s`
 COPY --from=k9s /tmp/k9s /usr/local/bin/k9s
 
+# Copy helm binary for in-container chart upgrades (used by cluster-deploy-fast.sh)
+COPY --from=helm /tmp/helm /usr/local/bin/helm
+
 # Copy iptables/nftables tooling (xtables-nft-multi, iptables-detect.sh, etc.)
 # These are in /bin/aux/ in the k3s image and must be on PATH.
 # Note: the Ubuntu iptables package provides /usr/sbin/iptables, but k3s

tasks/scripts/cluster-deploy-fast.sh

@@ -36,6 +36,14 @@ if ! docker ps -q --filter "name=^${CONTAINER_NAME}$" --filter "health=healthy"
   exit 1
 fi
 
+# Run a command inside the cluster container with KUBECONFIG pre-configured.
+cluster_exec() {
+  docker exec "${CONTAINER_NAME}" sh -c "KUBECONFIG=/etc/rancher/k3s/k3s.yaml $*"
+}
+
+# Path inside the container where the chart is copied for helm upgrades.
+CONTAINER_CHART_DIR=/tmp/openshell-chart
+
 build_gateway=0
 build_sandbox=0
 needs_helm_upgrade=0
@@ -376,23 +384,27 @@ fi
 if [[ "${needs_helm_upgrade}" == "1" ]]; then
   helm_start=$(date +%s)
   echo "Upgrading helm release..."
-  helm_wait_args=()
+  helm_wait_args=""
   if [[ "${DEPLOY_FAST_HELM_WAIT}" == "1" ]]; then
-    helm_wait_args+=(--wait)
+    helm_wait_args="--wait"
   fi
 
+  # Copy the local chart source into the container so helm can read it.
+  docker exec "${CONTAINER_NAME}" rm -rf "${CONTAINER_CHART_DIR}"
+  docker cp deploy/helm/openshell "${CONTAINER_NAME}:${CONTAINER_CHART_DIR}"
+
   # grpcEndpoint must be explicitly set to https:// because the chart always
   # terminates mTLS (there is no server.tls.enabled toggle). Without this,
   # a prior Helm override or chart default change could silently regress
   # sandbox callbacks to plaintext.
   # Retrieve the existing handshake secret from the running release, or generate
   # a new one if this is the first deploy with the mandatory secret.
-  EXISTING_SECRET=$(helm get values openshell -n openshell -o json 2>/dev/null \
-    | grep -o '"sshHandshakeSecret":"[^"]*"' \
-    | cut -d'"' -f4) || true
+  EXISTING_SECRET=$(cluster_exec "helm get values openshell -n openshell -o json 2>/dev/null \
+    | grep -o '\"sshHandshakeSecret\":\"[^\"]*\"' \
+    | cut -d'\"' -f4") || true
   SSH_HANDSHAKE_SECRET="${EXISTING_SECRET:-$(openssl rand -hex 32)}"
 
-  helm upgrade openshell deploy/helm/openshell \
+  cluster_exec "helm upgrade openshell ${CONTAINER_CHART_DIR} \
     --namespace openshell \
     --set image.repository=${IMAGE_REPO_BASE}/gateway \
     --set image.tag=${IMAGE_TAG} \
@@ -403,20 +415,20 @@ if [[ "${needs_helm_upgrade}" == "1" ]]; then
     --set server.tls.clientCaSecretName=openshell-server-client-ca \
     --set server.tls.clientTlsSecretName=openshell-client-tls \
     --set server.sshHandshakeSecret=${SSH_HANDSHAKE_SECRET} \
-    "${helm_wait_args[@]}"
+    ${helm_wait_args}"
   helm_end=$(date +%s)
   log_duration "Helm upgrade" "${helm_start}" "${helm_end}"
 fi
 
 if [[ "${#pushed_images[@]}" -gt 0 ]]; then
   rollout_start=$(date +%s)
   echo "Restarting deployment to pick up updated images..."
-  if kubectl get statefulset/openshell -n openshell >/dev/null 2>&1; then
-    kubectl rollout restart statefulset/openshell -n openshell
-    kubectl rollout status statefulset/openshell -n openshell
-  elif kubectl get deployment/openshell -n openshell >/dev/null 2>&1; then
-    kubectl rollout restart deployment/openshell -n openshell
-    kubectl rollout status deployment/openshell -n openshell
+  if cluster_exec "kubectl get statefulset/openshell -n openshell" >/dev/null 2>&1; then
+    cluster_exec "kubectl rollout restart statefulset/openshell -n openshell"
+    cluster_exec "kubectl rollout status statefulset/openshell -n openshell"
+  elif cluster_exec "kubectl get deployment/openshell -n openshell" >/dev/null 2>&1; then
+    cluster_exec "kubectl rollout restart deployment/openshell -n openshell"
+    cluster_exec "kubectl rollout status deployment/openshell -n openshell"
   else
     echo "Warning: no openshell workload found to roll out in namespace 'openshell'."
   fi