fix(gateway): delete finalized ephemeral sandboxes while connected (#3984)

Start driver cleanup after terminal finalization and retain disconnect fallback. Add detached success and failure e2e coverage across supervisor-based drivers.

Closes #3938

Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com>

Author: johntmyers

.agents/skills/launch-openshell-gator/SKILL.md

@@ -158,7 +158,7 @@ sandbox_name="gator-pr-${pr_number}-supervised"
   "Review and monitor PR #${pr_number} through the gator-gate workflow. Scope this invocation only to PR #${pr_number}."
 ```
 
-The launcher queries the gateway's selected compute driver, builds the gator image in the matching Docker or Podman image store, stages the immutable payload, imports provider profiles, configures provider credentials and refresh, and starts the agent supervisor as the sandbox's canonical main process. The detached main process survives loss of the host CLI connection and reconnects to a restarted gateway. Unless `--keep` is set, the sandbox is marked ephemeral so the gateway deletes it after the supervisor exits. `CONTAINER_ENGINE`, when set, must match the gateway driver.
+The launcher queries the gateway's selected compute driver, builds the gator image in the matching Docker or Podman image store, stages the immutable payload, imports provider profiles, configures provider credentials and refresh, and starts the agent supervisor as the sandbox's canonical main process. The detached main process survives loss of the host CLI connection and reconnects to a restarted gateway. Unless `--keep` is set, the sandbox is marked ephemeral so the gateway deletes it after the canonical main process exits and its terminal result is finalized. `CONTAINER_ENGINE`, when set, must match the gateway driver.
 
 The launcher streams image-build and provisioning output until the detached workload is ready, then exits. Use `openshell logs <sandbox-name>` or the TUI for runtime output.
 

crates/openshell-server/src/compute/mod.rs

@@ -2135,6 +2135,7 @@ impl ComputeRuntime {
         }
     }
 
+    #[cfg(test)]
     pub(crate) async fn delete_sandbox(
         &self,
         workspace: &str,
@@ -4725,6 +4726,39 @@ impl ComputeRuntime {
         Ok(())
     }
 
+    /// Start ephemeral cleanup only after the finalize RPC has recorded the
+    /// terminal result and marked its supervisor session finalized.
+    pub async fn cleanup_finalized_ephemeral_sandbox(
+        &self,
+        sandbox_id: &str,
+        instance_id: &str,
+    ) -> Result<(), String> {
+        let _guard = self.sync_lock.lock().await;
+        let Some(sandbox) = self
+            .store
+            .get_message::<Sandbox>(sandbox_id)
+            .await
+            .map_err(|error| error.to_string())?
+        else {
+            return Ok(());
+        };
+        let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown);
+        if phase != SandboxPhase::Completed && !is_failed_main_process_result(&sandbox) {
+            return Ok(());
+        }
+        let Some(status) = sandbox.status.as_ref() else {
+            return Ok(());
+        };
+        if status.exit_code.is_none()
+            || (!status.main_process_instance_id.is_empty()
+                && status.main_process_instance_id != instance_id)
+        {
+            return Ok(());
+        }
+        self.schedule_ephemeral_sandbox_delete(&sandbox);
+        Ok(())
+    }
+
     fn schedule_ephemeral_sandbox_delete(&self, sandbox: &Sandbox) {
         if provisioning_deadline::timed_out(sandbox) {
             return;
@@ -4740,10 +4774,10 @@ impl ComputeRuntime {
         }
 
         let runtime = self.clone();
-        let workspace = sandbox.object_workspace().to_string();
+        let sandbox_id = sandbox.object_id().to_string();
         let name = sandbox.object_name().to_string();
         tokio::spawn(async move {
-            if let Err(error) = runtime.delete_sandbox(&workspace, &name).await {
+            if let Err(error) = runtime.delete_sandbox_by_id(&sandbox_id, &name).await {
                 tracing::warn!(
                     sandbox_name = %name,
                     error = %error,
@@ -9300,7 +9334,83 @@ mod tests {
             .unwrap();
         assert_eq!(driver.delete_calls(), 0);
         runtime
-            .supervisor_session_disconnected("sb-1", true)
+            .cleanup_finalized_ephemeral_sandbox("sb-1", "instance-1")
+            .await
+            .unwrap();
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while driver.delete_calls() == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("terminal finalization should delete before the supervisor disconnects");
+    }
+
+    #[tokio::test]
+    async fn finalized_ephemeral_cleanup_skips_retained_and_restarting_sandboxes() {
+        for (retention, restart_policy, exit_code) in [
+            (None, SandboxRestartPolicy::Never, 0),
+            (Some("ephemeral"), SandboxRestartPolicy::OnFailure, 9),
+        ] {
+            let driver = ControlledDriver::new();
+            let runtime = test_runtime(driver.clone()).await;
+            let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning);
+            if let Some(retention) = retention {
+                sandbox.metadata.as_mut().unwrap().annotations.insert(
+                    "openshell.nvidia.com/retention".to_string(),
+                    retention.to_string(),
+                );
+            }
+            sandbox.spec = Some(SandboxSpec {
+                restart_policy: restart_policy as i32,
+                ..Default::default()
+            });
+            runtime.store.put_message(&sandbox).await.unwrap();
+            runtime
+                .supervisor_session_connected("sb-1", "instance-1")
+                .await
+                .unwrap();
+            runtime
+                .report_main_process_exit("sb-1", "instance-1", exit_code)
+                .await
+                .unwrap();
+            runtime
+                .finalize_main_process_exit("sb-1", "instance-1")
+                .await
+                .unwrap();
+            runtime
+                .cleanup_finalized_ephemeral_sandbox("sb-1", "instance-1")
+                .await
+                .unwrap();
+            tokio::task::yield_now().await;
+            assert_eq!(driver.delete_calls(), 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn finalized_failed_ephemeral_sandbox_deletes_while_connected() {
+        let driver = ControlledDriver::new();
+        let runtime = test_runtime(driver.clone()).await;
+        let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning);
+        sandbox.metadata.as_mut().unwrap().annotations.insert(
+            "openshell.nvidia.com/retention".to_string(),
+            "ephemeral".to_string(),
+        );
+        runtime.store.put_message(&sandbox).await.unwrap();
+        runtime
+            .supervisor_session_connected("sb-1", "instance-1")
+            .await
+            .unwrap();
+        runtime
+            .report_main_process_exit("sb-1", "instance-1", 9)
+            .await
+            .unwrap();
+        runtime
+            .finalize_main_process_exit("sb-1", "instance-1")
+            .await
+            .unwrap();
+        runtime
+            .cleanup_finalized_ephemeral_sandbox("sb-1", "instance-1")
             .await
             .unwrap();
         tokio::time::timeout(Duration::from_secs(1), async {
@@ -9309,7 +9419,7 @@ mod tests {
             }
         })
         .await
-        .expect("terminal finalization should release ephemeral cleanup");
+        .expect("failed canonical main should delete its ephemeral sandbox");
     }
 
     #[tokio::test]

crates/openshell-server/src/supervisor_session.rs

@@ -2072,10 +2072,18 @@ pub async fn handle_finalize_main_process_exit(
         .finalize_main_process_exit(&report.sandbox_id, &report.instance_id)
         .await
         .map_err(Status::failed_precondition)?;
-    if !state
+    let session_finalized = state
         .supervisor_sessions
-        .finalize_main_process_exit(&report.sandbox_id)
-    {
+        .finalize_main_process_exit(&report.sandbox_id);
+    // The session can close between durable result validation and this mark.
+    // Schedule cleanup in either case so a disconnect with an unfinalized
+    // in-memory session cannot strand the ephemeral sandbox.
+    state
+        .compute
+        .cleanup_finalized_ephemeral_sandbox(&report.sandbox_id, &report.instance_id)
+        .await
+        .map_err(Status::internal)?;
+    if !session_finalized {
         return Err(Status::failed_precondition(
             "supervisor session is not connected",
         ));

e2e/rust/Cargo.toml

@@ -98,6 +98,11 @@ name = "local_driver_token_restart"
 path = "tests/local_driver_token_restart.rs"
 required-features = ["e2e"]
 
+[[test]]
+name = "ephemeral_cleanup"
+path = "tests/ephemeral_cleanup.rs"
+required-features = ["e2e"]
+
 [[test]]
 name = "podman_gateway_start"
 path = "tests/podman_gateway_start.rs"

e2e/rust/e2e-podman.sh

@@ -22,6 +22,7 @@ source "${ROOT}/e2e/support/conformance.sh"
 # stabilized and can be added here.
 PODMAN_CI_TESTS=(
   bypass_detection
+  ephemeral_cleanup
   core_dump_hardening
   credential_gating
   default_image

e2e/rust/e2e-vm.sh

@@ -411,6 +411,7 @@ run_e2e_test() {
 if [ -n "${E2E_TEST_OVERRIDE}" ]; then
   run_e2e_test "${E2E_TEST_OVERRIDE}"
 else
+  run_e2e_test ephemeral_cleanup
   run_e2e_test host_gateway_alias
   run_e2e_test vm_overlay
   run_e2e_test vm_gateway_start