diff --git a/docs/user-guide/en/README.md b/docs/user-guide/en/README.md index 7b63a6c4d0..5833807a93 100644 --- a/docs/user-guide/en/README.md +++ b/docs/user-guide/en/README.md @@ -14,7 +14,7 @@ ANOLISA provides a complete server-side runtime for AI Agent workloads. Componen │ anolisa-cli · cosh · os-skills │ ├──────────────────────────────────┬─────────────────────────────────┤ │ Token Saving │ Runtime │ -│ tokenless · agent-memory │ skillfs · ws-ckpt │ +│ tokenless · agent-memory │ blaze · skillfs · ws-ckpt │ ├──────────────────────────────────┼─────────────────────────────────┤ │ Agent Observability │ Agent Security │ │ agentsight │ agent-sec-core │ @@ -68,6 +68,7 @@ ANOLISA provides a complete server-side runtime for AI Agent workloads. Componen | Document | Component | Description | |----------|-----------|-------------| +| [Blaze Runtime Slots](runtime/blaze/QUICKSTART.md) | blaze | Bounded background storage slots with optional backend prefork | | [Workspace Checkpoints](runtime/ws-ckpt.md) | ws-ckpt | Instant snapshot/rollback via btrfs COW | | [Skill Filesystem](runtime/skillfs.md) | skillfs | FUSE virtual views with progressive disclosure | diff --git a/docs/user-guide/en/runtime/blaze/QUICKSTART.md b/docs/user-guide/en/runtime/blaze/QUICKSTART.md new file mode 100644 index 0000000000..87b2708053 --- /dev/null +++ b/docs/user-guide/en/runtime/blaze/QUICKSTART.md @@ -0,0 +1,215 @@ +# Blaze Runtime Slots + +[中文版](../../../zh/runtime/blaze/QUICKSTART.md) + +Blaze can prepare independent runtime slots in the background so a compatible +sandbox create can reuse prepared storage and, optionally, an already started +backend. The feature is bounded, disabled by default, and continues through +the existing create flow whenever no compatible slot is ready. + +## Requirements + +- Linux with root privileges for the selected sandbox backend +- Rust 1.88 or newer for source builds +- a Blaze policy whose `[pool]` section sets `enabled = true` +- stable daemon state, storage-instance, and runtime directories across restart + +## Installation + +### ANOLISA CLI + +Blaze is a Labs component. The source tree contains an ANOLISA component +manifest, but a configured component repository may not publish a `blaze` +candidate. Preview resolution before applying the system installation: + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +### RPM + +On an RPM repository that publishes Blaze: + +```bash +sudo yum install blaze +``` + +### Source build + +```bash +cd src/blaze +cargo build --release --locked +``` + +## Enable Background Capacity + +Set a non-zero target in the daemon configuration: + +```toml +[storage] +provider = "file" +images_dir = "/var/lib/blaze/images" +instances_dir = "/var/lib/blaze/instances" +pool_size = 2 +prefork = false + +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +Enable eligibility in each policy that may use a prepared slot: + +```toml +[pool] +enabled = true +# Optional. When omitted, pool.default_warm_ttl from config.toml applies. +warm_ttl = "15m" +``` + +`storage.pool_size` is the target for background runtime slots. Policy `min`, +`target`, `max`, and `reset_mode` are reserved policy schema metadata. The +runtime-slot worker does not consume them, and `/v1/pools` does not apply them +from policy. None of them resizes background runtime capacity. The public reset +operation currently returns `501`, so a complete lifecycle return-to-pool +workflow is not connected. + +`pool_size = 0` disables construction. Duration values must include a positive +unit: `s`, `m`, `h`, or `d`. + +## Start Blaze + +Use the packaged service: + +```bash +sudo systemctl enable --now blazed +``` + +For a source checkout, run the built daemon with a configuration whose +`policy.dir` points to readable policies: + +```bash +sudo ./target/release/blazed daemon start --config examples/config.toml +``` + +The example configuration points `policy.dir` to +`/etc/anolisa/blaze/policies`. A package installs policies there; for a source +checkout, copy the example policies to that directory or edit the configuration +to use the checkout path. + +## Create and Claim a Slot + +The first eligible create fixes one build shape for this daemon run and wakes +the background worker. It does not wait for the worker to fill the target, so +that request normally continues through the existing create flow. Later +compatible requests claim a ready slot when one is available: + +```bash +curl -X POST --unix-socket /run/blaze/api.sock \ + http://localhost/v1/sandboxes \ + -H 'Content-Type: application/json' \ + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' +``` + +The existing `start_path` field is a generic warm-start classification. A +background runtime-slot claim reports this shape: + +```json +{ + "start_path": "warm", + "instance": { + "start_path": "warm", + "runtime_location": "warm-pool" + } +} +``` + +A `"cold"` result is not an error: this request did not use an applicable warm +source. + +```mermaid +flowchart LR + A["POST /v1/sandboxes"] --> B{"Policy eligible?"} + B -- "No" --> C["Existing create flow"] + B -- "Yes" --> D["Configure or match build shape"] + D --> K{"Prototype accepted?"} + K -- "No" --> C + K -- "Yes" --> E{"Compatible slot ready?"} + E -- "No" --> C + E -- "Yes" --> F["Record ownership handoff"] + F --> G["Publish lifecycle owner"] + G --> H["Return Running sandbox"] + K -- "Yes" --> I["Wake background worker"] + I --> J["Build toward pool_size"] +``` + +## Prefork Modes + +| `storage.prefork` | Prepared slot | Work performed after claim | +| --- | --- | --- | +| `false` | Independent storage | Start the backend and wait for guest readiness when the backend exposes a guest endpoint | +| `true` | Independent storage plus a running backend | Check backend liveness at claim; when a guest endpoint exists, readiness was already checked before the slot became ready | + +Every slot owns its own storage snapshot. A slot is never made ready by sharing +another sandbox's mutable storage. + +## Capacity and Expiry + +The target counts ready slots, active builds, leases being handed off, and +pool-owned slots awaiting cleanup. An ambiguous handoff has no selected +cleanup owner, but remains accounted for until reconciliation. A sandbox stops +consuming this target after lifecycle ownership is durably established. + +The worker removes ready slots after their effective `warm_ttl`. Claim checks +expiry for every slot and backend liveness when the slot holds a prefork +backend. Pool cleanup failures remain owned and are retried; they continue to +consume target capacity until cleanup succeeds. + +## Restart and Shutdown + +Before opening its API listeners, Blaze reconciles every runtime-slot ownership +record with durable sandbox lifecycle state. It cleans unclaimed slots instead +of rebuilding the old ready queue. Any ambiguous or inconsistent record, or +runtime reconciliation step that cannot complete, stops startup. After runtime +reconciliation succeeds, ordinary sandbox lifecycle reconciliation runs; one +sandbox cleanup failure is retained and reported but does not prevent listener +startup. + +Keep `daemon.state_dir`, `storage.instances_dir`, the selected storage provider, +and backend availability consistent across restart. Changing those values can +prevent the daemon from identifying and cleaning resources created by the +previous run. + +During graceful shutdown, Blaze stops new slot construction, joins the worker, +and attempts bounded cleanup for every pool-owned slot. Lifecycle-owned +sandboxes follow the normal sandbox cleanup path. + +## Current Boundaries + +- One compatible build shape is accepted per daemon run. Requests with another + image, backend, policy shape, or runtime configuration continue through the + existing create flow. +- Capacity starts on the first eligible create; daemon startup does not prefill + slots. +- Restart cleans unclaimed slots and does not restore them as ready. +- There is no public status, drain, or refill endpoint for background runtime + slots. `/v1/pools` and the health response's `storage_pool` object refer to + other pool contracts. +- A configured target improves the chance of a warm claim but does not + guarantee one for every request. +- A claimed background slot is single-use: destroy releases its resources and + the worker builds replacement capacity instead of returning that sandbox to + the ready queue. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Every response has `start_path: "cold"` | Confirm `pool_size > 0`, policy `enabled = true`, and identical request/build inputs; then allow time for background construction | +| Slot construction keeps retrying | Inspect daemon logs for storage, backend start, or guest-readiness errors | +| Capacity appears below target | Cleanup or an unresolved handoff may still count toward the target; inspect daemon logs | +| Daemon stops during startup reconciliation | Restore the provider and directory configuration used by the previous run, then inspect the reported ownership record | + +For the ownership and recovery rationale, see +[Runtime Slot Ownership](../../../../../src/blaze/docs/design/runtime-slot-ownership.md). diff --git a/docs/user-guide/zh/README.md b/docs/user-guide/zh/README.md index 371c72b26f..da1d0ce502 100644 --- a/docs/user-guide/zh/README.md +++ b/docs/user-guide/zh/README.md @@ -14,7 +14,7 @@ ANOLISA 为 AI Agent 提供完整的服务端运行时能力。通过 `anolisa` │ anolisa-cli · cosh · os-skills │ ├──────────────────────────────────┬─────────────────────────────────┤ │ Token 节省 │ 运行时 │ -│ tokenless · agent-memory │ skillfs · ws-ckpt │ +│ tokenless · agent-memory │ blaze · skillfs · ws-ckpt │ ├──────────────────────────────────┼─────────────────────────────────┤ │ Agent 可观测 │ Agent 安全 │ │ agentsight │ agent-sec-core │ @@ -68,6 +68,7 @@ ANOLISA 为 AI Agent 提供完整的服务端运行时能力。通过 `anolisa` | 文档 | 组件 | 说明 | |------|------|------| +| [Blaze Runtime 槽位](runtime/blaze/QUICKSTART.md) | blaze | 有界后台存储槽位，可选后端 prefork | | [工作区快照](runtime/ws-ckpt.md) | ws-ckpt | 秒级快照创建/回滚，基于 btrfs COW | | [技能文件系统](runtime/skillfs.md) | skillfs | FUSE 虚拟视图、渐进披露 | diff --git a/docs/user-guide/zh/runtime/blaze/QUICKSTART.md b/docs/user-guide/zh/runtime/blaze/QUICKSTART.md new file mode 100644 index 0000000000..083f80ab8d --- /dev/null +++ b/docs/user-guide/zh/runtime/blaze/QUICKSTART.md @@ -0,0 +1,200 @@ +# Blaze Runtime 槽位 + +[English](../../../en/runtime/blaze/QUICKSTART.md) + +Blaze 可以在后台准备彼此独立的 runtime 槽位，让兼容的 sandbox 创建请求 +复用已准备的存储，并可选择复用已经启动的后端。该功能有容量上限、默认 +关闭；没有兼容槽位就绪时，请求会继续走已有创建流程。 + +## 环境要求 + +- Linux；所选 sandbox 后端需要 root 权限 +- 从源码构建时需要 Rust 1.88 或更高版本 +- Blaze 策略的 `[pool]` 设置 `enabled = true` +- daemon 状态目录、存储 instance 目录和 runtime 目录在重启前后保持稳定 + +## 安装 + +### ANOLISA CLI + +Blaze 当前是 Labs 组件。源码树中包含 ANOLISA 组件清单，但配置的组件仓库 +不一定发布 `blaze` 候选包。执行系统级安装前，先预览解析结果： + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +### RPM + +如果 RPM 仓库发布了 Blaze： + +```bash +sudo yum install blaze +``` + +### 从源码构建 + +```bash +cd src/blaze +cargo build --release --locked +``` + +## 启用后台容量 + +在 daemon 配置中设置非零目标： + +```toml +[storage] +provider = "file" +images_dir = "/var/lib/blaze/images" +instances_dir = "/var/lib/blaze/instances" +pool_size = 2 +prefork = false + +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +在允许使用准备槽位的策略中启用该能力： + +```toml +[pool] +enabled = true +# 可选。省略时使用 config.toml 中的 pool.default_warm_ttl。 +warm_ttl = "15m" +``` + +`storage.pool_size` 是后台 runtime 槽位的目标。策略中的 `min`、`target`、 +`max`、`reset_mode` 是预留的 policy schema metadata。runtime-slot worker +不会读取它们，`/v1/pools` 也不会从 policy 自动应用这些值；它们都不会调整 +后台 runtime 容量。公开 reset 操作目前返回 `501`，因此完整的 lifecycle +return-to-pool 流程尚未接通。 + +`pool_size = 0` 会关闭构建。duration 必须是带正数单位的值：`s`、`m`、 +`h` 或 `d`。 + +## 启动 Blaze + +使用软件包提供的服务： + +```bash +sudo systemctl enable --now blazed +``` + +使用源码 checkout 时，确保配置中的 `policy.dir` 指向可读的策略目录，然后 +运行构建出的 daemon： + +```bash +sudo ./target/release/blazed daemon start --config examples/config.toml +``` + +示例配置中的 `policy.dir` 指向 `/etc/anolisa/blaze/policies`。软件包会在该 +目录安装策略；使用源码 checkout 时，应把示例策略复制到该目录，或者修改 +配置以使用 checkout 中的路径。 + +## 创建并取用槽位 + +首个符合条件的创建请求会固定本次 daemon 运行使用的一组构建参数，并唤醒 +后台 worker。该请求不会等待 worker 填满目标，因此通常会继续走已有创建 +流程。之后的兼容请求会在有槽位就绪时取用它： + +```bash +curl -X POST --unix-socket /run/blaze/api.sock \ + http://localhost/v1/sandboxes \ + -H 'Content-Type: application/json' \ + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' +``` + +现有 `start_path` 字段是通用的 warm-start 分类。后台 runtime 槽位会返回 +以下结构： + +```json +{ + "start_path": "warm", + "instance": { + "start_path": "warm", + "runtime_location": "warm-pool" + } +} +``` + +`"cold"` 结果不是错误，只表示本次请求没有使用适用的 warm 来源。 + +```mermaid +flowchart LR + A["POST /v1/sandboxes"] --> B{"策略符合条件？"} + B -- "否" --> C["已有创建流程"] + B -- "是" --> D["配置或匹配构建参数"] + D --> K{"构建参数已接受？"} + K -- "否" --> C + K -- "是" --> E{"兼容槽位已就绪？"} + E -- "否" --> C + E -- "是" --> F["记录 ownership 交接"] + F --> G["发布 lifecycle owner"] + G --> H["返回 Running sandbox"] + K -- "是" --> I["唤醒后台 worker"] + I --> J["向 pool_size 补充"] +``` + +## Prefork 模式 + +| `storage.prefork` | 已准备的槽位 | 取用后仍需完成的工作 | +| --- | --- | --- | +| `false` | 独立存储 | 启动后端；后端提供 guest endpoint 时等待 guest readiness | +| `true` | 独立存储和运行中的后端 | 取用时检查后端存活；存在 guest endpoint 时，槽位进入 ready 前已经检查 readiness | + +每个槽位都持有自己的存储快照，不会通过共享另一个 sandbox 的可变存储进入 +ready。 + +## 容量与过期 + +目标会统计 ready 槽位、正在构建的槽位、正在交接的 lease，以及等待清理的 +pool-owned 槽位。未决交接没有选定 cleanup owner，但在核对完成前仍计入 +目标容量。生命周期 ownership 持久化成功后，该 sandbox 不再占用此目标。 + +ready 槽位超过有效 `warm_ttl` 后，worker 会将其清理。请求取用槽位时也会 +检查过期时间；只有槽位持有 prefork 后端时才检查后端存活。pool cleanup +失败的资源会继续被持有并重试；清理成功前，它们仍占用目标容量。 + +## 重启与关闭 + +Blaze 打开 API listener 前，会把每条 runtime 槽位 ownership 记录与 +sandbox 的持久化生命周期状态进行核对。它会清理未交接槽位，而不是重建 +旧的 ready 队列。存在无法明确归属或互相矛盾的记录，或者 runtime 核对 +步骤无法完成时，daemon 会停止启动。runtime 核对成功后才执行普通 sandbox +生命周期核对；其中单个 sandbox 清理失败会被保留和报告，但不会阻止 +listener 启动。 + +重启前后应保持 `daemon.state_dir`、`storage.instances_dir`、所选存储 +provider 和后端可用性一致。修改这些值可能导致 daemon 无法识别和清理 +上一次运行创建的资源。 + +正常关闭时，Blaze 会停止创建新槽位、等待 worker 结束，并在有界时间内 +尝试清理所有 pool-owned 槽位。lifecycle-owned sandbox 仍走普通 sandbox +清理路径。 + +## 当前边界 + +- 每次 daemon 运行只接受一组兼容构建参数。使用其他 image、backend、 + policy 参数或 runtime 配置的请求会继续走已有创建流程。 +- 容量由首个符合条件的创建请求启动；daemon 启动时不会预填充槽位。 +- 重启会清理未交接槽位，不会把它们恢复为 ready。 +- 后台 runtime 槽位目前没有公开的 status、drain 或 refill 接口。 + `/v1/pools` 和健康响应中的 `storage_pool` 对象属于其他 pool contract。 +- 配置目标只能提高 warm claim 的概率，不能保证每个请求都命中。 +- 取用的后台槽位只使用一次：destroy 会释放其资源，worker 重新构建补充 + 容量，不会把该 sandbox 放回 ready 队列。 + +## 故障排查 + +| 现象 | 检查项 | +| --- | --- | +| 所有响应都是 `start_path: "cold"` | 确认 `pool_size > 0`、策略 `enabled = true` 且请求和构建参数一致；然后为后台构建预留时间 | +| 槽位构建持续重试 | 查看 daemon 日志中的存储、后端启动或 guest readiness 错误 | +| 容量看起来低于目标 | 清理或未决交接可能仍占用目标；查看 daemon 日志 | +| daemon 在启动核对时停止 | 恢复上一次运行使用的 provider 和目录配置，再检查日志指出的 ownership 记录 | + +ownership 与恢复设计参见 +[Runtime 槽位 Ownership](../../../../../src/blaze/docs/design/runtime-slot-ownership.md)。 diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index c34acdec25..0d74dbac37 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -29,7 +29,14 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw - **Daemon-only API model**: No CLI client for sandbox operations. All instance/pool/template management is done via HTTP endpoints on UDS (`/run/blaze/api.sock`) or TCP (`:14159`). The CLI subcommands (`daemon start`, `daemon reload`, `daemon doctor`) only manage daemon lifecycle. - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. -- **Lifecycle state machine**: 8 states (Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed). State transitions are enforced by `blaze_core::lifecycle`. Do not bypass via direct field mutation. +- **Lifecycle state machine**: The persisted model contains 13 states. Current + managed routes implement create, destroy, checkpoint capture/list/prune, + rollback, hibernate, resume, and + RecoveryRequired cleanup. Checkpoint capture returns `501` without advancing + state when either the backend or storage provider lacks capture support. + Reset remains reserved and returns `501` without advancing state. + Transitions are enforced by `blaze_core::lifecycle`; do not bypass them + through direct field mutation. - **MockSpawner fallback**: When the configured backend binary is missing or fails `probe()`, the daemon auto-downgrades to `MockSpawner` with a warning. This keeps API/integration tests functional without a real backend. ## Adding a New Backend diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 36b5e15874..152ccdc5c6 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -133,14 +133,17 @@ version = "0.3.0" dependencies = [ "anyhow", "async-trait", + "base64", "blaze-core", "chrono", "clap", "http-body-util", "hyper", "hyper-util", + "libc", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", @@ -151,6 +154,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -253,6 +265,35 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -308,6 +349,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -327,11 +379,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -822,6 +885,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -986,6 +1060,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -1123,6 +1198,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1159,6 +1240,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index c031e1ef38..1a23d8ad02 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1.0" # Async runtime tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["net"] } +tokio-util = { version = "0.7", features = ["net", "rt"] } hyper = { version = "1", features = ["full"] } hyper-util = { version = "0.1", features = ["full"] } http-body-util = "0.1" @@ -42,6 +42,11 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } semver = "1.0" +base64 = "0.22" +sha2 = "0.10" + +# Platform +libc = "0.2" # Internal crates blaze-core = { path = "crates/blaze-core" } diff --git a/src/blaze/README.md b/src/blaze/README.md index 8f4b0c00a1..8af9ab62e9 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -13,21 +13,52 @@ Designed as the per-host agent for E2B-style orchestrator platforms. - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **Policy-driven backend selection** — workload class → backend priority list -- **Lifecycle state machine** — 8 states (Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed) -- **Warm pool management** — pre-warmed instances with TTL-based GC +- **Lifecycle state machine** — 13 states: Pending, Creating, Running, Paused, + Checkpointed, Restoring, Hibernating, Hibernated, Resuming, + RecoveryRequired, Reset, Warm, and Destroyed +- **Guest operations** — bounded command execution and file transfer for + running backends that expose a guest endpoint +- **Runtime slot capacity** — independent storage slots with optional backend + prefork and TTL-based cleanup - **Template registry** — in-memory template tracking with idle eviction - **Kernel hook registry** — state tracking for pre/post hooks - **Prometheus metrics** — request counts, instance gauges, pool sizes - **Spawners** — FirecrackerSpawner, BubblewrapSpawner, MockSpawner +- **Optional VM networking** — isolated namespace, tap, veth, and NAT per Firecracker VM -## Quick Start +## Installation + +Blaze is a Labs component. This source tree contains its ANOLISA component +manifest and RPM packaging, but not every configured component repository +publishes a `blaze` candidate. Preview repository resolution before applying +the system installation: + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +On an RPM repository that publishes Blaze: + +```bash +sudo yum install blaze +``` + +For a developer source build: ```bash -# Build cd src/blaze -cargo build --release +cargo build --release --locked +``` + +## Quick Start + +```bash +# Choose one startup method; do not run both at the same time. +# Packaged installation +sudo systemctl enable --now blazed -# Run daemon (dev: override policy.dir to use local examples) +# Source build alternative (override policy.dir to use local examples) sudo ./target/release/blazed daemon start --config examples/config.toml # Note: the default config sets policy.dir = /etc/anolisa/blaze/policies. # For source-checkout testing, create a symlink or override: @@ -38,11 +69,15 @@ sudo ./target/release/blazed daemon start --config examples/config.toml curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # Create a sandbox -curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/instances \ +curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ - -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' ``` +The quick-start request uses an example policy with Firecracker guest transport +disabled, so an image without the compatible guest agent does not wait for guest +readiness. Enable the transport only for images that run that agent. + ## Configuration The daemon reads a TOML config file (default: `/etc/anolisa/blaze/config.toml`) @@ -58,6 +93,30 @@ and a policies directory containing per-workload-class policy files. See `src/blaze/examples/` for annotated sample configurations. +### API Request Limits + +The daemon accepts request bodies up to 1 MiB by default. It checks both +declared `Content-Length` values and streamed body frames, and returns HTTP +413 when the configured limit is exceeded. Override the limit with a positive +byte count: + +```toml +[api] +max_body_bytes = 1048576 +``` + +Guest files are limited to 16 MiB after base64 decoding. A full-size write is +larger on the wire because JSON and base64 add overhead, so the default 1 MiB +request limit intentionally rejects it. Set at least 22 MiB when callers need +the full decoded limit: + +```toml +[api] +max_body_bytes = 23068672 +``` + +The daemon checks both the HTTP request size and the decoded file size. + ### VM Resource Configuration Blaze resolves vCPU and memory settings using a three-layer fallback chain: @@ -76,8 +135,20 @@ memory = "512Mi" [backend.firecracker] vcpus = 4 # overrides [vm].vcpus for Firecracker only memory = "1Gi" # overrides [vm].memory for Firecracker only +enable_network = false ``` +Set `enable_network = true` to create an isolated network slot for each +Firecracker VM. Explicit sandbox destroy and compensated startup failure remove +the namespace, tap, and veth after process termination. A destroy retried after +a daemon restart can reconstruct the recorded slot; there is no background +cleanup scan. Slot creation and deletion use a host-wide lock so independent +daemon processes cannot allocate the same host device names concurrently. +When a loaded Firecracker policy enables this option, backend probing also +checks the required commands and host privileges. The checks are skipped when +networking is disabled. Upstream routing and DNS remain host operator +responsibilities. + ### Storage Configuration The `[storage]` section controls the sandbox storage backend: @@ -88,39 +159,248 @@ provider = "file" # Storage provider selection. Currently supported: "file # "auto" probes available providers in priority order (currently equivalent to "file"). # Other values will log a warning and fall back to file. images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) -# flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) +pool_size = 0 # Background runtime slots; zero disables construction +prefork = false # Start the backend before a slot becomes ready +flush_interval = "disabled" # Set a positive duration to synchronize running slots +flush_timeout = "30s" # Maximum duration of one provider synchronization attempt + +[pool] +default_warm_ttl = "30m" # Used when an eligible policy omits warm_ttl +gc_interval = "5m" # Expiry and capacity maintenance interval ``` The `file` provider uses standard filesystem operations for sandbox storage. The `auto` provider probes available backends in priority order (currently equivalent to `file`). Unrecognized values will log a warning and fall back to `file`. +When periodic synchronization is enabled, one provider failure or timeout does +not block later sandboxes. The slot remains owned so a later sweep or destroy +can retry it. The daemon stops and joins the synchronization worker before +draining connections and releasing runtime resources. + +See [Storage Synchronization](docs/design/storage-synchronization.md) for +selection, retry, and shutdown behavior. + +When `pool_size` is non-zero, the first eligible create request fixes one +compatible build shape and starts background construction. Every slot owns +storage; a slot owns a running backend only when `prefork` is enabled. +`pool_size` limits pool-owned and in-flight slots, not sandboxes that have +already completed lifecycle handoff. Incompatible requests continue through +the existing create flow. A policy is eligible only when its `[pool]` section +sets `enabled = true`; its optional `warm_ttl` overrides `default_warm_ttl`. + +On restart, the daemon cleans unclaimed slot journals before serving requests; +it does not restore old slots to the ready queue. That cleanup uses the +currently configured provider and storage/runtime roots, so those settings +must continue to identify the same owned directories across a restart. The +`/v1/pools` endpoints below expose a separate lifecycle recycling-pool +contract and do not expose this background runtime capacity. Public reset +currently returns `501`, so no production path returns a used sandbox to that +pool. ## API Endpoints | Method | Path | Description | |--------|------|-------------| | GET | `/v1/health` | Health check | -| GET | `/v1/instances` | List all instances | -| POST | `/v1/instances` | Create a new sandbox instance | -| GET | `/v1/instances/{id}` | Get instance details | -| POST | `/v1/instances/{id}/checkpoint` | Checkpoint an instance | -| POST | `/v1/instances/{id}/reset` | Reset instance to checkpoint | -| POST | `/v1/instances/{id}/destroy` | Destroy an instance | -| GET | `/v1/pools` | List warm pools | -| GET | `/v1/pools/{backend}/{class}` | Get pool status | -| POST | `/v1/pools/{backend}/{class}/drain` | Drain a pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a pool | -| GET | `/v1/templates` | List templates | -| GET | `/v1/templates/{id}` | Inspect a template | +| GET | `/v1/sandboxes` | List all sandboxes | +| POST | `/v1/sandboxes` | Create a sandbox | +| GET | `/v1/sandboxes/{id}` | Get sandbox details | +| DELETE | `/v1/sandboxes/{id}` | Destroy a sandbox | +| POST | `/v1/sandboxes/{id}/exec` | Execute a guest command | +| POST | `/v1/sandboxes/{id}/read` | Read a guest file | +| POST | `/v1/sandboxes/{id}/write` | Replace a guest file | +| POST | `/v1/sandboxes/{id}/checkpoint` | Capture a full checkpoint when the backend and storage provider support it | +| GET | `/v1/sandboxes/{id}/checkpoints` | List committed checkpoints and HEAD reachability | +| POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | Replace a running sandbox from a verified checkpoint | +| POST | `/v1/sandboxes/{id}/hibernate` | Persist VM state and release the live backend | +| POST | `/v1/sandboxes/{id}/resume` | Resume a hibernated sandbox and wait for enabled guest transport | +| POST | `/v1/sandboxes/{id}/checkpoints/prune` | Remove checkpoint branches outside retained lineages | +| GET | `/v1/instances` | Alias for listing sandboxes | +| POST | `/v1/instances` | Alias for creating a sandbox | +| GET | `/v1/instances/{id}` | Alias for sandbox details | +| DELETE | `/v1/instances/{id}` | Alias for destroying a sandbox | +| POST | `/v1/instances/{id}/destroy` | Compatible destroy action | +| POST | `/v1/instances/{id}/exec` | Compatible guest command action | +| POST | `/v1/instances/{id}/read` | Compatible guest file read action | +| POST | `/v1/instances/{id}/write` | Compatible guest file write action | +| POST | `/v1/instances/{id}/checkpoint` | Compatible full-checkpoint action | +| GET | `/v1/instances/{id}/checkpoints` | Compatible checkpoint-list action | +| POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | Compatible checkpoint-restore action | +| POST | `/v1/instances/{id}/hibernate` | Compatible sandbox-hibernation action | +| POST | `/v1/instances/{id}/resume` | Compatible sandbox-resume action | +| POST | `/v1/instances/{id}/checkpoints/prune` | Compatible checkpoint-prune action | +| POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | +| GET | `/v1/pools` | List lifecycle recycling pools | +| GET | `/v1/pools/{backend}/{class}` | Get lifecycle recycling-pool status | +| POST | `/v1/pools/{backend}/{class}/drain` | Drain a lifecycle recycling pool | +| PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a lifecycle recycling pool | +| GET | `/v1/templates` | List in-memory template registry entries | +| GET | `/v1/templates/{id}` | Inspect an in-memory template registry entry | | POST | `/v1/templates/gc` | Trigger template GC | +| GET | `/v1/runtime-templates` | List published runtime artifact sets | +| GET | `/v1/runtime-templates/{name}` | Inspect a published runtime artifact set | +| POST | `/v1/runtime-templates/import` | Publish artifacts from the configured import root | | GET | `/v1/policies` | List loaded policies | | GET | `/v1/hooks` | List kernel hooks | | GET | `/v1/metrics` | Prometheus metrics | | POST | `/v1/admin/reload` | Hot-reload policies | +The `/v1/runtime-templates` routes manage a durable artifact catalog that is +separate from the existing in-memory `/v1/templates` registry. Importing an +entry does not make sandbox creation select it. See +[Runtime template catalog](docs/design/runtime-template-catalog.md) for the +accepted artifacts, configuration limits, and publication rules. + +### Managed lifecycle and recovery + +Create and destroy record their operation before changing storage or backend +resources. A successful create finishes in `Running`; a successful destroy +finishes in `Destroyed`. If compensation cannot release every owned resource, +the sandbox remains visible as `RecoveryRequired` so destroy can be retried. + +Runtime-slot reconciliation completes before this lifecycle pass. An +inventory, journal, or runtime cleanup error stops startup. After it succeeds, +the daemon reconciles each sandbox independently. A completed hibernation is +retained for resume. An interrupted hibernate or resume is retained as +`RecoveryRequired` for explicit destroy instead of being mistaken for a live +runtime. Failure to clean up one of the remaining sandboxes does not prevent +the other records from being processed or the API from starting. + +During graceful shutdown, the daemon first stops accepting work and drains +accepted connections. It then attempts bounded cleanup for every persisted +record and retained backend owner. One cleanup failure does not skip the +remaining sandboxes, and all unresolved records are reported. + +Create and destroy journals record the operation and start time. Checkpoint +journals also record the generated checkpoint ID and the latest durable +boundary the daemon confirmed. Checkpoint listing separately reports which +catalog entries and HEAD update are actually visible after an interruption. +An interrupted create is cleaned up rather than resumed, and an existing +backend process is not adopted after restart. Failed recovery does not run in +a background retry loop. + +Checkpoint capture is available only when both the selected backend and the +configured storage provider report full-capture support. Otherwise the daemon +returns `501` before creating a journal, pausing the backend, or changing the +checkpoint catalog. A supported capture: + +1. pauses the backend and captures full VM and memory artifacts; +2. flushes the live storage slot and copies the full root filesystem; +3. publishes a verified checkpoint and advances HEAD; and +4. resumes the backend and confirms guest readiness before returning + `Running`. + +The file storage provider copies the complete root filesystem into each +checkpoint. This uses more capacity than a shared-base format, but each +checkpoint remains independent of later changes to the live slot. + +Failures detected before calling the catalog publication step resume the +backend and discard the incomplete stage. If publication or HEAD has an +uncertain outcome, or the backend cannot resume, the sandbox becomes +`RecoveryRequired` while runtime ownership and committed checkpoint data +remain available for explicit cleanup. Listing uses the same per-sandbox +operation lock as capture, guest operations, and destroy. Destroy removes +transaction scratch but preserves committed checkpoint history. + +Checkpoint restore is available only when the current storage provider and the +checkpoint's backend implement restore, and the current backend version exactly +matches the version recorded at capture. The daemon verifies the selected +checkpoint, its parent chain, and all artifact hashes before changing runtime +state. + +The file provider stages a separate rootfs copy while the current backend is +still running. After the old backend stops, the daemon selects that copy, +starts and owns the replacement backend, moves HEAD to the selected checkpoint, +and only then releases the previous rootfs. A failure before backend shutdown +keeps the original runtime running. A failure after shutdown retains the +resources that actually exist and marks the sandbox `RecoveryRequired`, so a +later destroy can finish cleanup without losing process ownership. + +`last_checkpoint` continues to mean the most recent completed capture. Restore +moves catalog HEAD but does not rewrite capture history. + +Hibernation is available only when the running backend supports pause and full +snapshot capture and its configured adapter can restore the same backend +version. These checks happen before the lifecycle journal changes. A successful +hibernate: + +1. records intent, pauses the backend, and writes VM state and memory into a + hidden staging directory; +2. flushes the retained storage slot and records artifact sizes and SHA-256 + digests in a manifest; +3. synchronizes the complete image before stopping the backend; +4. publishes the hibernation directory and commits `Hibernated`. + +Resume verifies the manifest identity, exact file set, and artifact digests +before starting a replacement backend. The manager owns that backend before +waiting for optional guest readiness and commits `Running` only after a final +liveness check. A failure before the original backend stops resumes it and +rechecks enabled guest transport. A clean resume failure returns to +`Hibernated`; if cleanup cannot be confirmed, the replacement owner and +operation journal remain available through `RecoveryRequired`. + +The storage slot remains allocated while hibernated. A successful resume also +retains the latest hibernation image until the next hibernate replaces it or an +explicit destroy removes it. The daemon does not automatically complete an +interrupted hibernate or resume after restart. + +Checkpoint pruning removes committed entries outside the HEAD lineage and any +lineage still referenced by durable sandbox state. Each candidate is first +renamed to a hidden tombstone, so a retry, sandbox destroy, or startup +reconciliation can finish an interrupted deletion. Pruning uses the same +per-sandbox operation lock as capture and rejects an unfinished lifecycle +operation. + +Runtime reset remains reserved and returns `501` without changing runtime or +persisted state. + +### Guest operations + +Guest operations are available only while a sandbox is `Running` and its +backend reports a guest endpoint. A cold create that reports such an endpoint +waits for the guest agent to answer before publishing `Running`. Backends with +guest support disabled skip that wait, and later guest-operation requests +return HTTP 409. A prefork runtime waits for guest readiness before its slot +becomes ready when the backend exposes a guest endpoint. Claim checks backend +liveness without repeating guest readiness. A storage-only slot waits after it +starts its backend. + +Guest operations and lifecycle changes use the same per-sandbox operation +lock. A request may wait for an earlier lifecycle action. After it obtains the +lock, the manager checks `Running` again; if destroy or another state change +won the race, the guest request fails without contacting the old runtime. + +The endpoints accept JSON: + +```json +{"cmd":"uname -a","cwd":"/","env":{"LANG":"C"},"timeout":10} +``` + +```json +{"path":"/tmp/input","data_b64":"aGVsbG8="} +``` + +`read` takes only `path`; successful file reads and command output use standard +base64 in the response. Exec timeouts must be from 1 through 20 seconds. Guest +files are limited to 16 MiB after decoding, and response frames are bounded. + +An exec or write failure before request delivery is an ordinary transport +failure. A bounded wait that expires before delivery uses +`"code": "guest_timeout"`. If delivery began but the daemon cannot determine +the result, the API returns HTTP 504 with +`"code": "guest_outcome_unknown"`; callers must reconcile state instead of +automatically replaying the operation. Reads do not change guest state and +remain safe for caller-directed retry. An oversized read response returns +HTTP 502 with `"code": "guest_response_too_large"`. For exec or write after +delivery starts, an oversized or otherwise untrusted response instead leaves +the outcome unknown. An oversized caller request returns HTTP 413. + +Each request is fully buffered. The limits bound one request, not the sum of +concurrent requests, so callers should also bound guest-operation concurrency. +Streaming files, interactive terminals, and session reuse are not supported. + #### Health Check -`GET /v1/health` returns daemon status including storage pool readiness: +`GET /v1/health` returns daemon status including provider storage-pool +readiness: ```json { @@ -130,6 +410,13 @@ The `file` provider uses standard filesystem operations for sandbox storage. The } ``` +The `storage_pool` object does not report background runtime-slot capacity. + +## Documentation + +- [Runtime slot user guide](../../docs/user-guide/en/runtime/blaze/QUICKSTART.md) +- [Runtime slot ownership design](docs/design/runtime-slot-ownership.md) + ## Project Layout ``` @@ -137,6 +424,7 @@ src/blaze/ ├── crates/ │ ├── blaze-core/ # Library: policy, lifecycle, pool, template, kernel, config │ └── blazed/ # Binary: daemon, API server, spawners, metrics +├── docs/design/ # Component design documents ├── examples/ # config.toml, policies/ ├── dist/ # blazed.service, blaze.spec, tmpfiles └── manifests/ # Component metadata @@ -146,6 +434,7 @@ src/blaze/ - Rust 1.88+ (see `src/blaze/rust-toolchain.toml`) - Linux host with root privileges for sandbox backends +- `ip`, `iptables`, `sysctl`, and network namespace privileges when VM + networking is enabled ## License - diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index fddef048c4..d629c10984 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -12,21 +12,49 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **策略驱动后端选择** — workload class → 后端优先级列表 -- **生命周期状态机** — 8 种状态（Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed） -- **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC +- **生命周期状态机** — 13 种状态：Pending、Creating、Running、Paused、 + Checkpointed、Restoring、Hibernating、Hibernated、Resuming、 + RecoveryRequired、Reset、Warm 和 Destroyed +- **Guest 操作** — 对提供 guest endpoint 的运行中后端执行有界命令和文件传输 +- **Runtime 槽位容量** — 独立存储槽位、可选后端 prefork 和基于 TTL 的清理 - **模板注册表** — 内存中模板追踪，支持空闲驱逐 - **内核 hook 注册** — 前/后置 hook 状态追踪 - **Prometheus 指标** — 请求计数、实例 gauge、池大小 - **Spawner 后端** — FirecrackerSpawner、BubblewrapSpawner、MockSpawner +- **可选 VM 网络** — 每台 Firecracker VM 独立使用 netns、tap、veth 和 NAT -## 快速开始 +## 安装 + +Blaze 当前是 Labs 组件。源码树中包含 ANOLISA 组件清单和 RPM 打包文件，但 +配置的组件仓库不一定发布 `blaze` 候选包。执行系统级安装前，先预览仓库 +解析结果： + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +如果 RPM 仓库发布了 Blaze： + +```bash +sudo yum install blaze +``` + +开发者从源码构建： ```bash -# 构建 cd src/blaze -cargo build --release +cargo build --release --locked +``` + +## 快速开始 + +```bash +# 选择一种启动方式，不要同时运行两种方式。 +# 软件包安装 +sudo systemctl enable --now blazed -# 运行 daemon（开发环境：覆盖 policy.dir 使用本地示例） +# 源码构建方式（覆盖 policy.dir 使用本地示例） sudo ./target/release/blazed daemon start --config examples/config.toml # 注意：默认配置设置 policy.dir = /etc/anolisa/blaze/policies。 # 源码开发测试时，创建符号链接或覆盖： @@ -37,11 +65,15 @@ sudo ./target/release/blazed daemon start --config examples/config.toml curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # 创建 sandbox -curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/instances \ +curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ - -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' ``` +快速开始使用关闭 Firecracker guest transport 的示例策略，因此没有兼容 +guest agent 的镜像不会等待 guest 就绪。只有镜像运行了对应 agent 时才应 +启用该 transport。 + ## 配置 daemon 读取 TOML 配置文件（默认：`/etc/anolisa/blaze/config.toml`） @@ -57,6 +89,28 @@ daemon 读取 TOML 配置文件（默认：`/etc/anolisa/blaze/config.toml`） 参见 `src/blaze/examples/` 获取带注释的示例配置。 +### API 请求上限 + +daemon 默认接收不超过 1 MiB 的请求体。它会同时检查声明的 +`Content-Length` 和逐帧到达的数据；超过配置上限时返回 HTTP 413。 +可以通过正整数字节数覆盖默认值： + +```toml +[api] +max_body_bytes = 1048576 +``` + +Guest 文件在 base64 解码后最多为 16 MiB。完整的 16 MiB 写入经过 JSON 和 +base64 编码后会变大，所以默认 1 MiB 请求上限会拒绝它。调用者确实需要完整 +文件上限时，应至少配置 22 MiB： + +```toml +[api] +max_body_bytes = 23068672 +``` + +daemon 会同时检查 HTTP 请求大小和解码后的文件大小。 + ### VM 资源配置 Blaze 使用三层回退链解析 vCPU 和内存设置： @@ -75,8 +129,17 @@ memory = "512Mi" [backend.firecracker] vcpus = 4 # 仅对 Firecracker 覆盖 [vm].vcpus memory = "1Gi" # 仅对 Firecracker 覆盖 [vm].memory +enable_network = false ``` +设置 `enable_network = true` 后，每台 Firecracker VM 会获得独立的网络 +slot。显式销毁 sandbox 和启动失败补偿会在进程确认终止后删除对应的 netns、 +tap 和 veth。daemon 重启后再次销毁时可以根据记录恢复清理，但不会在后台 +自动扫描。slot 创建和删除使用主机级锁，避免多个 daemon 同时分配相同的主机 +设备名。加载的 Firecracker 策略启用该选项时，backend probe 还会检查所需 +命令和主机权限；网络关闭时跳过这些检查。上游路由和 DNS 仍由主机运维方 +配置。 + ### 存储配置 `[storage]` 部分控制 sandbox 存储后端： @@ -87,39 +150,213 @@ provider = "file" # 存储 provider 选择。当前支持："file"、"auto # "auto" 按优先级探测可用 provider（当前等同于 "file"）。 # 其他值将记录告警并回退到 file。 images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] 预热存储槽位数（尚未启用） -# prefork = false # [Reserved] 是否在槽位中预启动 VM（尚未启用） -# flush_interval = "30s" # [Reserved] 脏数据刷盘周期（尚未启用） +pool_size = 0 # 后台运行槽位数；0 表示不构建 +prefork = false # 槽位就绪前是否启动后端 +flush_interval = "disabled" # 设置正数 duration 后同步 running slot +flush_timeout = "30s" # 单次 provider 同步的最长时间 + +[pool] +default_warm_ttl = "30m" # 符合条件的策略未设置 warm_ttl 时使用 +gc_interval = "5m" # 过期检查和容量维护间隔 ``` `file` provider 使用标准文件系统操作管理 sandbox 存储。`auto` 按优先级探测可用 provider（当前等同于 `file`）。无法识别的值将记录告警并回退到 `file`。 +启用周期同步后，单个 provider 失败或超时不会阻塞后续 sandbox。slot 会继续 +由 daemon 持有，后续 sweep 或 destroy 可以重试。daemon 会先停止并等待同步 +任务退出，再排空连接和释放 runtime 资源。 + +[存储同步](docs/design/storage-synchronization_zh.md)进一步说明选择、重试和 +关闭行为。 + +`pool_size` 非零时，首个符合条件的创建请求会固定一组兼容构建参数，并启动 +后台补充。每个槽位都持有存储；只有启用 `prefork` 时，槽位才同时持有已经 +运行的后端。`pool_size` 限制池内及正在构建或交接的槽位，不限制已经完成 +生命周期交接的 sandbox。参数不兼容的请求会继续走已有创建流程。 +只有策略的 `[pool]` 设置 `enabled = true` 时，该策略才符合条件；策略中 +可选的 `warm_ttl` 会覆盖 `default_warm_ttl`。 + +daemon 重启时会在接收请求前清理尚未交接的槽位记录，不会把旧槽位重新放回 +ready 队列。该清理使用重启后当前配置的 provider 以及存储和运行目录，因此 +这些设置必须继续指向同一组已有目录。下文的 `/v1/pools` 描述的是另一套 +生命周期回收 pool 的管理接口，不展示这里的后台运行容量。公开 reset 当前 +返回 `501`，因此没有生产路径把已经使用的 sandbox 放回该 pool。 ## API 端点 | 方法 | 路径 | 说明 | |--------|------|-------------| | GET | `/v1/health` | 健康检查 | -| GET | `/v1/instances` | 列出所有实例 | -| POST | `/v1/instances` | 创建新 sandbox 实例 | -| GET | `/v1/instances/{id}` | 获取实例详情 | -| POST | `/v1/instances/{id}/checkpoint` | 对实例做 checkpoint | -| POST | `/v1/instances/{id}/reset` | 将实例重置到 checkpoint | -| POST | `/v1/instances/{id}/destroy` | 销毁实例 | -| GET | `/v1/pools` | 列出 warm pool | -| GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | -| POST | `/v1/pools/{backend}/{class}/drain` | 排空 pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | 调整 pool 大小 | -| GET | `/v1/templates` | 列出模板 | -| GET | `/v1/templates/{id}` | 查看模板详情 | +| GET | `/v1/sandboxes` | 列出所有 sandbox | +| POST | `/v1/sandboxes` | 创建 sandbox | +| GET | `/v1/sandboxes/{id}` | 获取 sandbox 详情 | +| DELETE | `/v1/sandboxes/{id}` | 销毁 sandbox | +| POST | `/v1/sandboxes/{id}/exec` | 执行 guest 命令 | +| POST | `/v1/sandboxes/{id}/read` | 读取 guest 文件 | +| POST | `/v1/sandboxes/{id}/write` | 替换 guest 文件 | +| POST | `/v1/sandboxes/{id}/checkpoint` | 后端和存储 provider 支持时捕获完整 checkpoint | +| GET | `/v1/sandboxes/{id}/checkpoints` | 列出已提交的 checkpoint 及其 HEAD 可达性 | +| POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | 用经过校验的 checkpoint 替换正在运行的 sandbox | +| POST | `/v1/sandboxes/{id}/hibernate` | 持久化 VM 状态并释放正在运行的后端 | +| POST | `/v1/sandboxes/{id}/resume` | 恢复休眠 sandbox，并等待已启用的 guest 通信就绪 | +| POST | `/v1/sandboxes/{id}/checkpoints/prune` | 删除保留 lineage 之外的 checkpoint 分支 | +| GET | `/v1/instances` | 列出 sandbox 的兼容入口 | +| POST | `/v1/instances` | 创建 sandbox 的兼容入口 | +| GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | +| DELETE | `/v1/instances/{id}` | 销毁 sandbox 的兼容入口 | +| POST | `/v1/instances/{id}/destroy` | 保留的销毁 action | +| POST | `/v1/instances/{id}/exec` | Guest 命令兼容入口 | +| POST | `/v1/instances/{id}/read` | Guest 文件读取兼容入口 | +| POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | +| POST | `/v1/instances/{id}/checkpoint` | 捕获完整 checkpoint 的兼容入口 | +| GET | `/v1/instances/{id}/checkpoints` | 列出 checkpoint 的兼容入口 | +| POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | 恢复 checkpoint 的兼容入口 | +| POST | `/v1/instances/{id}/hibernate` | 休眠 sandbox 的兼容入口 | +| POST | `/v1/instances/{id}/resume` | 恢复休眠 sandbox 的兼容入口 | +| POST | `/v1/instances/{id}/checkpoints/prune` | 删除 checkpoint 分支的兼容入口 | +| POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | +| GET | `/v1/pools` | 列出生命周期回收 pool | +| GET | `/v1/pools/{backend}/{class}` | 获取生命周期回收 pool 状态 | +| POST | `/v1/pools/{backend}/{class}/drain` | 排空生命周期回收 pool | +| PUT | `/v1/pools/{backend}/{class}/sizing` | 调整生命周期回收 pool 大小 | +| GET | `/v1/templates` | 列出内存模板 registry 条目 | +| GET | `/v1/templates/{id}` | 查看内存模板 registry 条目 | | POST | `/v1/templates/gc` | 触发模板 GC | +| GET | `/v1/runtime-templates` | 列出已发布的 runtime artifact | +| GET | `/v1/runtime-templates/{name}` | 查看已发布的 runtime artifact | +| POST | `/v1/runtime-templates/import` | 从配置的导入根目录发布 artifact | | GET | `/v1/policies` | 列出已加载策略 | | GET | `/v1/hooks` | 列出内核 hook | | GET | `/v1/metrics` | Prometheus 指标 | | POST | `/v1/admin/reload` | 热加载策略 | +`/v1/runtime-templates` 路由管理持久 artifact 目录，与已有的内存 +`/v1/templates` registry 相互独立。导入条目不会让 sandbox create 自动 +选择它。接受的 artifact、配置上限和发布规则参见 +[Runtime 模板目录](docs/design/runtime-template-catalog_zh.md)。 + +### 生命周期管理与恢复 + +创建和销毁会在修改存储或后端资源之前记录当前操作。创建成功后状态为 +`Running`，销毁成功后状态为 `Destroyed`。如果失败补偿不能释放全部已有 +资源，sandbox 会保留为可查询的 `RecoveryRequired`，后续可以再次执行销毁。 + +runtime 槽位核对会在该生命周期处理之前完成；inventory、journal 或 runtime +清理失败会停止启动。runtime 核对成功后，daemon 会逐个处理未结束的 +sandbox。已经完成休眠的 sandbox 会保留下来等待恢复；中断的休眠或恢复操作 +会以 `RecoveryRequired` 保留，等待显式销毁，而不会被误认为仍在运行。其余 +sandbox 中的单条清理失败不会阻止其他记录继续处理，也不会阻止 API 启动。 + +正常关闭时，daemon 会先停止接收新请求并等待已有连接结束，再为每条持久化 +记录和仍持有的后端资源执行有界清理。单条清理失败不会跳过其余 sandbox， +所有未完成记录都会汇总报告。 + +创建和销毁的操作记录会保存操作类型及开始时间。checkpoint 还会记录生成的 +checkpoint ID，以及 daemon 已确认的最近一次持久化边界。checkpoint 列表会 +另外报告中断后实际可见的 catalog 记录和 HEAD 更新。中断的创建会被清理而 +不是从原位置继续，重启后也不会接管先前的后端进程。恢复失败后目前没有后台 +循环自动重试。 + +只有所选后端和当前存储 provider 都声明支持完整捕获时，checkpoint 才可用。 +否则 daemon 会在创建操作记录、暂停后端或修改 checkpoint catalog 之前返回 +`501`。一次成功的捕获会： + +1. 暂停后端，并捕获完整的 VM 状态和内存； +2. 刷新当前存储 slot，并复制完整根文件系统； +3. 发布经过校验的 checkpoint，并推进 HEAD； +4. 恢复后端，确认 guest 已经就绪后再返回 `Running`。 + +file storage provider 会把完整根文件系统复制到每个 checkpoint。与共享 base +的格式相比，这会占用更多空间，但每个 checkpoint 都不依赖 live slot 后续的 +变化。 + +如果在调用 catalog 发布步骤之前发现失败，daemon 会恢复后端并删除未完成的 +stage。如果发布或 HEAD 的结果无法确定，或者后端无法恢复，sandbox 会进入 +`RecoveryRequired`；runtime ownership 和已经提交的 checkpoint 数据仍会 +保留，供后续显式清理。列出 checkpoint 与捕获、guest 操作及销毁共用同一个 +sandbox 操作锁。销毁会删除事务临时文件，但保留已经提交的 checkpoint 历史。 + +只有当前存储 provider 和 checkpoint 对应的后端都实现恢复，并且当前后端版本 +与捕获时记录的版本完全一致，daemon 才会开始恢复。修改 runtime 之前，daemon +会先校验所选 checkpoint、完整父链和全部 artifact hash。 + +file provider 会在旧后端仍然运行时准备一份独立的 rootfs。旧后端停止后， +daemon 才选择这份 rootfs，启动并持有新的后端，随后把 HEAD 指向所选 +checkpoint，最后释放旧 rootfs。旧后端停止前发生失败时，原 runtime 会继续 +运行；停止后发生任何无法确认的失败时，daemon 会保留实际存在的资源，并把 +sandbox 标记为 `RecoveryRequired`，后续 destroy 仍能找到并清理这些资源。 + +`last_checkpoint` 始终表示最近一次成功捕获。恢复只移动 catalog HEAD，不会 +改写捕获历史。 + +只有运行中后端支持暂停和完整快照，并且配置的 adapter 能恢复相同的后端版本 +时，休眠才可用。这些检查会在生命周期操作记录发生变化前完成。一次成功的 +休眠会： + +1. 记录操作意图、暂停后端，并把 VM 状态和内存写入隐藏的暂存目录； +2. 刷新保留的存储 slot，并在 manifest 中记录文件大小和 SHA-256 摘要； +3. 在停止后端之前同步完整的休眠镜像； +4. 发布休眠目录，并提交 `Hibernated` 状态。 + +恢复会在启动替换后端前校验 manifest 身份、完整文件集合和文件摘要。manager +会先取得新后端的归属，再等待可选的 guest 通信就绪；只有最后一次存活检查 +通过后才提交 `Running`。旧后端停止前发生失败时，daemon 会恢复旧后端，并 +重新检查已经启用的 guest 通信。可以完整清理的恢复失败会回到 +`Hibernated`；如果无法确认清理结果，替换后端归属和操作记录会通过 +`RecoveryRequired` 保留下来。 + +休眠期间，存储 slot 会继续保留。恢复成功后，最近一次休眠镜像也会保留到 +下一次休眠替换它，或显式销毁将其删除。daemon 重启后不会自动完成中断的 +休眠或恢复操作。 + +checkpoint pruning 会删除 HEAD lineage 和持久 sandbox 状态所引用 lineage 之外 +的已提交记录。每个待删除目录会先被重命名为隐藏 tombstone，因此重试、 +sandbox 销毁或启动恢复都可以完成被中断的删除。pruning 与 capture 共用 +sandbox 操作锁，并拒绝存在未完成生命周期操作的 sandbox。 + +runtime reset 仍是预留接口，会返回 `501`，且不会修改 runtime 或持久化状态。 + +### Guest 操作 + +只有 sandbox 处于 `Running` 且后端报告了 guest endpoint 时，才能执行 +guest 操作。冷启动后端如果报告了该 endpoint，创建流程会等待 guest agent +响应后才发布 `Running`。关闭 guest 支持的后端会跳过等待，后续 guest +操作返回 HTTP 409。后端提供 guest endpoint 时，prefork 槽位会在进入 +ready 前等待 guest readiness；取用时再次检查后端存活，但不重复 guest +readiness。仅含存储的槽位会在启动后端后等待 guest readiness。 + +Guest 操作和生命周期变更使用同一个 sandbox 操作锁。请求可能等待先开始的 +生命周期操作；取得锁后，manager 会再次检查 `Running`。如果 destroy 或 +其他状态变更先完成，guest 请求不会访问旧 runtime，而是直接失败。 + +接口接收以下 JSON： + +```json +{"cmd":"uname -a","cwd":"/","env":{"LANG":"C"},"timeout":10} +``` + +```json +{"path":"/tmp/input","data_b64":"aGVsbG8="} +``` + +`read` 只需要 `path`；文件读取结果和命令输出使用标准 base64。Exec timeout +范围为 1 至 20 秒。Guest 文件解码后最多为 16 MiB，响应帧也有固定上限。 + +如果 exec 或 write 在请求送达前失败，它是普通通信失败；送达前等待超时使用 +`"code": "guest_timeout"`。如果已经开始送达，但 daemon 无法确定结果，API +返回 HTTP 504 和 `"code": "guest_outcome_unknown"`；调用者应先核对状态， +不能自动重放。read 不改变 guest 状态，可以由调用者决定是否重试。Guest +read 返回内容过大时，API 返回 HTTP 502 和 +`"code": "guest_response_too_large"`。exec 或 write 已经开始送达后，如果 +返回内容过大或不可信，结果仍归为 unknown。调用者的请求过大时返回 HTTP +413。 + +每个请求都会完整缓冲。上限约束的是单个请求，而不是所有并发请求之和，因此 +调用方还需要限制 guest 操作并发数。当前不支持文件流式传输、交互式终端和 +会话复用。 + #### 健康检查 -`GET /v1/health` 返回 daemon 状态，包含存储池就绪信息： +`GET /v1/health` 返回 daemon 状态，包含 provider 存储池就绪信息： ```json { @@ -129,6 +366,13 @@ images_dir = "/var/lib/blaze/images" } ``` +`storage_pool` 对象不报告后台 runtime 槽位容量。 + +## 文档 + +- [Runtime 槽位用户指南](../../docs/user-guide/zh/runtime/blaze/QUICKSTART.md) +- [Runtime 槽位 ownership 设计](docs/design/runtime-slot-ownership.md) + ## 项目结构 ``` @@ -136,6 +380,7 @@ src/blaze/ ├── crates/ │ ├── blaze-core/ # 库：策略、生命周期、池、模板、内核、配置 │ └── blazed/ # 二进制：daemon、API server、spawner、指标 +├── docs/design/ # 组件设计文档 ├── examples/ # config.toml、policies/ ├── dist/ # blazed.service、blaze.spec、tmpfiles └── manifests/ # 组件元数据 @@ -145,6 +390,7 @@ src/blaze/ - Rust 1.88+（参见 `src/blaze/rust-toolchain.toml`） - 具有 root 权限的 Linux 主机（sandbox 后端需要） +- 启用 VM 网络时需要 `ip`、`iptables`、`sysctl` 和 netns 管理权限 ## 许可证 diff --git a/src/blaze/crates/blaze-core/src/backend.rs b/src/blaze/crates/blaze-core/src/backend.rs index e213e24397..475ed61bac 100644 --- a/src/blaze/crates/blaze-core/src/backend.rs +++ b/src/blaze/crates/blaze-core/src/backend.rs @@ -101,6 +101,75 @@ pub struct SpawnRequest { pub vm: Option, } +/// Backend identity and snapshot semantics accepted by a restore adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestoreCapability { + /// Concrete backend implementation that can consume the checkpoint. + pub backend: BackendKind, + /// Exact backend version required by versioned snapshot formats. + pub version: Option, + /// Snapshot flavor accepted by the adapter. + pub snapshot_kind: SnapshotKind, +} + +/// Complete input for restoring an owned backend instance. +#[derive(Debug, Clone)] +pub struct RestoreRequest { + /// Stable sandbox identifier. + pub instance_id: Uuid, + /// Provider-owned runtime directory that preserves backend resource paths. + pub run_dir: PathBuf, + /// Backend executable selected from the current daemon configuration. + pub binary_path: PathBuf, + /// Storage resources reconstructed for this sandbox. + pub storage: StorageSlot, + /// VM-state artifact from a committed checkpoint. + pub snapshot_path: PathBuf, + /// Guest-memory artifact from the same checkpoint. + pub mem_path: PathBuf, + /// Backend identity frozen into the checkpoint metadata. + pub checkpoint_backend: BackendKind, + /// Backend version frozen into the checkpoint metadata. + pub expected_version: Option, + /// Snapshot flavor frozen into the checkpoint metadata. + pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable run-directory guest transport. + pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, +} + +/// Snapshot flavor requested from a backend. +/// +/// The file provider currently requires self-contained artifacts, so only +/// full snapshots are exposed until a restore-independent delta format exists. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SnapshotKind { + /// Self-contained VM and memory snapshot. + Full, +} + +/// Paths and semantics for one snapshot operation. +#[derive(Debug, Clone)] +pub struct SnapshotRequest { + /// Destination for VM state. + pub snapshot_path: PathBuf, + /// Destination for guest memory. + pub mem_path: PathBuf, + /// Snapshot flavor. + pub kind: SnapshotKind, +} + +/// Backend-reported snapshot artifacts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotResult { + /// Written VM-state path. + pub snapshot_path: PathBuf, + /// Written memory path. + pub mem_path: PathBuf, +} + /// Probed availability of a single backend on this host. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BackendStatus { @@ -205,4 +274,12 @@ mod tests { let err = select_backend(&priority, &available).expect_err("must fail"); assert!(matches!(err, BlazeError::BackendUnavailable { .. })); } + + #[test] + fn snapshot_kind_serializes_as_a_stable_lowercase_value() { + assert_eq!( + serde_json::to_value(SnapshotKind::Full).expect("snapshot kind"), + serde_json::json!("full") + ); + } } diff --git a/src/blaze/crates/blaze-core/src/checkpoint.rs b/src/blaze/crates/blaze-core/src/checkpoint.rs new file mode 100644 index 0000000000..4c11a28e30 --- /dev/null +++ b/src/blaze/crates/blaze-core/src/checkpoint.rs @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Pure checkpoint records and manifest validation. +//! +//! This module deliberately contains no filesystem or path handling. The +//! daemon owns checkpoint persistence, hashing, publication, and cleanup. + +use std::collections::HashSet; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +use crate::backend::{BackendKind, SnapshotKind}; + +/// Current on-disk checkpoint metadata format. +pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; + +/// Self-contained artifacts required for every committed checkpoint. +pub const REQUIRED_ARTIFACTS: [&str; 3] = ["vmstate.snap", "memory.snap", "rootfs.snap"]; + +/// One content digest recorded in a checkpoint manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointArtifact { + /// File name relative to the checkpoint directory. + pub name: String, + /// Logical file size in bytes. + pub size_bytes: u64, + /// Lowercase SHA-256 digest. + pub sha256: String, +} + +/// Durable checkpoint identity and integrity manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointMetadata { + /// Metadata schema version. + pub format_version: u32, + /// Stable `ckpt-` identifier. + pub id: String, + /// Previous checkpoint on this branch. + #[serde(default)] + pub parent: Option, + /// Sandbox that owns the checkpoint. + pub sandbox_id: Uuid, + /// Policy that selected the captured runtime. + pub policy_name: String, + /// Image identity selected by the policy. + pub image_digest: String, + /// Backend that produced the runtime artifacts. + pub backend: BackendKind, + /// Backend version captured by the daemon, when available. + #[serde(default)] + pub backend_version: Option, + /// UTC publication time. + pub created_at: DateTime, + /// Backend snapshot semantics. + pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable guest transport. + pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, + /// Integrity records for all required artifacts. + pub artifacts: Vec, +} + +/// Read-only API view of a checkpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointInfo { + /// Checkpoint identifier. + pub id: String, + /// Parent checkpoint. + pub parent: Option, + /// Publication time. + pub created_at: DateTime, + /// Sum of logical artifact sizes. + pub size_bytes: u64, + /// Whether this checkpoint is the current HEAD. + pub is_head: bool, + /// Whether this checkpoint is reachable from HEAD. + pub on_head_chain: bool, +} + +/// Values supplied by the daemon when publishing a populated stage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitCheckpoint { + /// Parent checkpoint, if the sandbox already has a HEAD. + pub parent: Option, + /// Policy that selected the captured runtime. + pub policy_name: String, + /// Image identity selected by the policy. + pub image_digest: String, + /// Backend that produced the artifacts. + pub backend: BackendKind, + /// Backend version captured by the caller, when available. + pub backend_version: Option, + /// Backend snapshot semantics. + pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable guest transport. + pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, +} + +/// Pure validation failure for a checkpoint identifier or manifest. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CheckpointValidationError { + /// A checkpoint identifier is not canonical `ckpt-`. + #[error("invalid checkpoint identifier {checkpoint_id:?}: {reason}")] + InvalidIdentifier { + checkpoint_id: String, + reason: String, + }, + + /// A manifest uses a schema version this daemon cannot interpret. + #[error("checkpoint {checkpoint_id} uses unsupported format {actual}; expected {expected}")] + UnsupportedFormat { + checkpoint_id: String, + actual: u32, + expected: u32, + }, + + /// The durable identity does not agree with the catalog location. + #[error("checkpoint manifest identity mismatch: {reason}")] + IdentityMismatch { reason: String }, + + /// A manifest field needed to identify or reproduce a capture is empty. + #[error("checkpoint {checkpoint_id} has invalid {field}: {reason}")] + InvalidField { + checkpoint_id: String, + field: &'static str, + reason: String, + }, + + /// The artifact manifest is incomplete, duplicated, or malformed. + #[error("checkpoint {checkpoint_id} has an invalid artifact manifest: {reason}")] + InvalidArtifacts { + checkpoint_id: String, + reason: String, + }, + + /// A requested artifact name is outside the frozen format. + #[error("artifact name {name:?} is not part of the checkpoint format")] + InvalidArtifactName { name: String }, +} + +/// Validate a canonical `ckpt-` identifier. +pub fn validate_checkpoint_id(checkpoint_id: &str) -> Result { + let raw = checkpoint_id + .strip_prefix("ckpt-") + .ok_or_else(|| invalid_identifier(checkpoint_id, "missing ckpt- prefix"))?; + let uuid = Uuid::parse_str(raw) + .map_err(|error| invalid_identifier(checkpoint_id, error.to_string()))?; + if checkpoint_id != format!("ckpt-{uuid}") { + return Err(invalid_identifier( + checkpoint_id, + "identifier is not in canonical hyphenated lowercase form", + )); + } + Ok(uuid) +} + +/// Validate a name before the daemon resolves it inside a checkpoint stage. +pub fn validate_artifact_name(name: &str) -> Result<(), CheckpointValidationError> { + if REQUIRED_ARTIFACTS.contains(&name) { + Ok(()) + } else { + Err(CheckpointValidationError::InvalidArtifactName { + name: name.to_string(), + }) + } +} + +/// Validate daemon-supplied values before constructing a durable manifest. +pub fn validate_commit_checkpoint( + checkpoint_id: &str, + input: &CommitCheckpoint, +) -> Result<(), CheckpointValidationError> { + validate_checkpoint_id(checkpoint_id)?; + if let Some(parent) = &input.parent { + validate_checkpoint_id(parent)?; + if parent == checkpoint_id { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "parent", + reason: "a checkpoint cannot be its own parent".to_string(), + }); + } + } + validate_runtime_identity( + checkpoint_id, + &input.policy_name, + &input.image_digest, + input.backend, + input.backend_version.as_deref(), + ) +} + +/// Validate a parsed manifest against the catalog location that contained it. +/// +/// Artifact content hashes are intentionally not checked here: reading files +/// belongs to the daemon. This function only validates the pure record. +pub fn validate_checkpoint_manifest( + metadata: &CheckpointMetadata, + expected_sandbox_id: Uuid, + expected_checkpoint_id: &str, +) -> Result<(), CheckpointValidationError> { + validate_checkpoint_id(expected_checkpoint_id)?; + validate_checkpoint_id(&metadata.id)?; + if metadata.format_version != CHECKPOINT_FORMAT_VERSION { + return Err(CheckpointValidationError::UnsupportedFormat { + checkpoint_id: metadata.id.clone(), + actual: metadata.format_version, + expected: CHECKPOINT_FORMAT_VERSION, + }); + } + if metadata.id != expected_checkpoint_id { + return Err(CheckpointValidationError::IdentityMismatch { + reason: format!( + "manifest id {:?} does not match catalog id {expected_checkpoint_id:?}", + metadata.id + ), + }); + } + if metadata.sandbox_id != expected_sandbox_id { + return Err(CheckpointValidationError::IdentityMismatch { + reason: format!( + "manifest sandbox {} does not match catalog sandbox {expected_sandbox_id}", + metadata.sandbox_id + ), + }); + } + if let Some(parent) = &metadata.parent { + validate_checkpoint_id(parent)?; + if parent == &metadata.id { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: metadata.id.clone(), + field: "parent", + reason: "a checkpoint cannot be its own parent".to_string(), + }); + } + } + validate_runtime_identity( + &metadata.id, + &metadata.policy_name, + &metadata.image_digest, + metadata.backend, + metadata.backend_version.as_deref(), + )?; + validate_artifact_manifest(&metadata.id, &metadata.artifacts) +} + +fn validate_runtime_identity( + checkpoint_id: &str, + policy_name: &str, + image_digest: &str, + backend: BackendKind, + backend_version: Option<&str>, +) -> Result<(), CheckpointValidationError> { + if policy_name.trim().is_empty() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "policy_name", + reason: "value is empty".to_string(), + }); + } + if image_digest.trim().is_empty() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "image_digest", + reason: "value is empty".to_string(), + }); + } + if backend_version.is_some_and(|version| version.trim().is_empty()) { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "backend_version", + reason: "present version is empty".to_string(), + }); + } + if backend == BackendKind::Firecracker && backend_version.is_none() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "backend_version", + reason: "Firecracker captures require a backend version".to_string(), + }); + } + Ok(()) +} + +fn validate_artifact_manifest( + checkpoint_id: &str, + artifacts: &[CheckpointArtifact], +) -> Result<(), CheckpointValidationError> { + if artifacts.len() != REQUIRED_ARTIFACTS.len() { + return Err(invalid_artifacts( + checkpoint_id, + format!( + "expected {} artifacts, found {}", + REQUIRED_ARTIFACTS.len(), + artifacts.len() + ), + )); + } + + let mut names = HashSet::with_capacity(artifacts.len()); + for artifact in artifacts { + validate_artifact_name(&artifact.name).map_err(|_| { + invalid_artifacts( + checkpoint_id, + format!("unexpected artifact {:?}", artifact.name), + ) + })?; + if !names.insert(artifact.name.as_str()) { + return Err(invalid_artifacts( + checkpoint_id, + format!("duplicate artifact {:?}", artifact.name), + )); + } + if artifact.sha256.len() != 64 + || !artifact + .sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(invalid_artifacts( + checkpoint_id, + format!("artifact {:?} has an invalid SHA-256 digest", artifact.name), + )); + } + } + if REQUIRED_ARTIFACTS + .iter() + .any(|required| !names.contains(required)) + { + return Err(invalid_artifacts( + checkpoint_id, + "one or more required artifacts are missing", + )); + } + Ok(()) +} + +fn invalid_identifier(checkpoint_id: &str, reason: impl Into) -> CheckpointValidationError { + CheckpointValidationError::InvalidIdentifier { + checkpoint_id: checkpoint_id.to_string(), + reason: reason.into(), + } +} + +fn invalid_artifacts(checkpoint_id: &str, reason: impl Into) -> CheckpointValidationError { + CheckpointValidationError::InvalidArtifacts { + checkpoint_id: checkpoint_id.to_string(), + reason: reason.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn artifact(name: &str, fill: char) -> CheckpointArtifact { + CheckpointArtifact { + name: name.to_string(), + size_bytes: 10, + sha256: std::iter::repeat_n(fill, 64).collect(), + } + } + + fn metadata() -> CheckpointMetadata { + let sandbox_id = Uuid::new_v4(); + CheckpointMetadata { + format_version: CHECKPOINT_FORMAT_VERSION, + id: format!("ckpt-{}", Uuid::new_v4()), + parent: None, + sandbox_id, + policy_name: "default".to_string(), + image_digest: "sha256:image".to_string(), + backend: BackendKind::Mock, + backend_version: Some("mock-v1".to_string()), + created_at: Utc::now(), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, + artifacts: vec![ + artifact("vmstate.snap", 'a'), + artifact("memory.snap", 'b'), + artifact("rootfs.snap", 'c'), + ], + } + } + + #[test] + fn canonical_identifier_round_trips() { + let uuid = Uuid::new_v4(); + assert_eq!( + validate_checkpoint_id(&format!("ckpt-{uuid}")).expect("valid identifier"), + uuid + ); + } + + #[test] + fn noncanonical_identifier_is_rejected() { + let uuid = Uuid::new_v4(); + assert!( + validate_checkpoint_id(&format!("ckpt-{}", uuid.to_string().to_uppercase())).is_err() + ); + assert!(validate_checkpoint_id(&format!("ckpt-{}", uuid.simple())).is_err()); + assert!(validate_checkpoint_id("../checkpoint").is_err()); + } + + #[test] + fn valid_manifest_passes_pure_validation() { + let metadata = metadata(); + validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect("valid manifest"); + } + + #[test] + fn manifest_identity_must_match_catalog_location() { + let metadata = metadata(); + let error = validate_checkpoint_manifest(&metadata, Uuid::new_v4(), &metadata.id) + .expect_err("sandbox mismatch must fail"); + assert!(matches!( + error, + CheckpointValidationError::IdentityMismatch { .. } + )); + } + + #[test] + fn manifest_requires_the_exact_artifact_set() { + let mut metadata = metadata(); + metadata.artifacts[2].name = "memory.snap".to_string(); + let error = validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect_err("duplicate artifact must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidArtifacts { .. } + )); + } + + #[test] + fn manifest_rejects_noncanonical_digest() { + let mut metadata = metadata(); + metadata.artifacts[0].sha256 = "A".repeat(64); + assert!( + validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id).is_err() + ); + } + + #[test] + fn commit_input_rejects_self_parent_and_empty_identity() { + let id = format!("ckpt-{}", Uuid::new_v4()); + let input = CommitCheckpoint { + parent: Some(id.clone()), + policy_name: String::new(), + image_digest: String::new(), + backend: BackendKind::Mock, + backend_version: None, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + network_slot: None, + }; + assert!(validate_commit_checkpoint(&id, &input).is_err()); + } + + #[test] + fn firecracker_checkpoint_records_require_a_backend_version() { + let mut metadata = metadata(); + metadata.backend = BackendKind::Firecracker; + metadata.backend_version = None; + let error = validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect_err("missing version must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidField { + field: "backend_version", + .. + } + )); + + let input = CommitCheckpoint { + parent: metadata.parent, + policy_name: metadata.policy_name, + image_digest: metadata.image_digest, + backend: metadata.backend, + backend_version: metadata.backend_version, + snapshot_kind: metadata.snapshot_kind, + expose_guest_socket: metadata.expose_guest_socket, + network_slot: metadata.network_slot, + }; + let error = validate_commit_checkpoint(&metadata.id, &input) + .expect_err("missing version must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidField { + field: "backend_version", + .. + } + )); + } +} diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index fb10c93e0c..674e60bc21 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -4,10 +4,12 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use std::time::Duration; use serde::{Deserialize, Serialize}; use crate::error::{BlazeError, ConfigErrorSource, Result}; +use crate::policy::parse_duration; /// Top-level daemon configuration. #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -16,6 +18,8 @@ pub struct DaemonConfig { pub daemon: DaemonSection, #[serde(default)] pub listen: ListenSection, + #[serde(default)] + pub api: ApiSection, /// Backend name → binary path mapping (e.g. `firecracker = "/usr/bin/firecracker"`). #[serde(default)] pub backends: HashMap, @@ -28,6 +32,8 @@ pub struct DaemonConfig { #[serde(default)] pub template: TemplateSection, #[serde(default)] + pub runtime_templates: RuntimeTemplateSection, + #[serde(default)] pub metrics: MetricsSection, } @@ -60,6 +66,22 @@ pub struct ListenSection { pub http_addr: String, } +/// HTTP API request limits. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiSection { + /// Maximum number of bytes collected from one HTTP request body. + #[serde(default = "default_max_body_bytes")] + pub max_body_bytes: usize, +} + +impl Default for ApiSection { + fn default() -> Self { + Self { + max_body_bytes: default_max_body_bytes(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PolicySection { #[serde(default = "default_policy_dir")] @@ -121,6 +143,45 @@ impl Default for TemplateSection { } } +/// Published runtime artifact catalog and its local import boundary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuntimeTemplateSection { + /// Directory containing atomically published runtime artifact sets. + #[serde(default = "default_runtime_template_dir")] + pub dir: PathBuf, + /// Optional root containing operator-prepared import sources. + /// + /// Imports are disabled when this value is absent. API callers provide a + /// relative path below this root rather than an arbitrary daemon path. + #[serde(default)] + pub import_root: Option, + /// Maximum number of regular files accepted from one source directory. + #[serde(default = "default_runtime_template_max_files")] + pub max_files: usize, + /// Maximum final artifact and generated metadata bytes for one import. + #[serde(default = "default_runtime_template_max_bytes")] + pub max_bytes: u64, + /// Maximum serialized size of one published `template.json`. + #[serde(default = "default_runtime_template_max_metadata_bytes")] + pub max_metadata_bytes: u64, + /// Maximum aggregate bytes retained by the published catalog. + #[serde(default = "default_runtime_template_max_total_bytes")] + pub max_total_bytes: u64, +} + +impl Default for RuntimeTemplateSection { + fn default() -> Self { + Self { + dir: default_runtime_template_dir(), + import_root: None, + max_files: default_runtime_template_max_files(), + max_bytes: default_runtime_template_max_bytes(), + max_metadata_bytes: default_runtime_template_max_metadata_bytes(), + max_total_bytes: default_runtime_template_max_total_bytes(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MetricsSection { #[serde(default = "default_prometheus_socket")] @@ -150,21 +211,24 @@ pub struct StorageSection { #[serde(default = "default_storage_provider")] pub provider: String, - /// Warm pool target size (0 = no pool). - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Warm runtime target size (0 disables background construction). #[serde(default)] pub pool_size: usize, - /// Whether to pre-start VMs in pool slots. - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Whether to pre-start backends in warm runtime slots. #[serde(default)] pub prefork: bool, - /// Interval for flushing dirty data. - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Interval for synchronizing provider-owned runtime data. + /// + /// The literal `disabled` turns off periodic synchronization. #[serde(default = "default_flush_interval")] pub flush_interval: String, + /// Maximum duration of one provider synchronization attempt. + #[serde(default = "default_flush_timeout")] + pub flush_timeout: String, + /// Logical size of file-provider root filesystem slots. #[serde(default = "default_rootfs_size")] pub rootfs_size: u64, @@ -183,12 +247,40 @@ impl Default for StorageSection { pool_size: 0, prefork: false, flush_interval: default_flush_interval(), + flush_timeout: default_flush_timeout(), rootfs_size: default_rootfs_size(), mem_size: default_mem_size(), } } } +/// Parsed periodic storage synchronization policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageFlushSchedule { + /// Do not run periodic synchronization. + Disabled, + /// Run one sweep after every configured interval. + Every(Duration), +} + +impl StorageSection { + /// Parse the periodic synchronization setting. + pub fn flush_schedule(&self) -> Result { + if self.flush_interval == "disabled" { + return Ok(StorageFlushSchedule::Disabled); + } + parse_duration(&self.flush_interval) + .map(StorageFlushSchedule::Every) + .ok_or_else(|| invalid_storage_duration("flush_interval", &self.flush_interval, true)) + } + + /// Parse the maximum duration of one provider synchronization attempt. + pub fn flush_timeout_duration(&self) -> Result { + parse_duration(&self.flush_timeout) + .ok_or_else(|| invalid_storage_duration("flush_timeout", &self.flush_timeout, false)) + } +} + impl DaemonConfig { /// Load and parse a daemon configuration file at `path`. pub fn load(path: &Path) -> Result { @@ -201,16 +293,94 @@ impl DaemonConfig { /// Validate cross-field invariants that serde cannot express. pub fn validate(&self) -> Result<()> { - validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir) + if self.api.max_body_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "api.max_body_bytes must be greater than zero".to_string(), + ), + }); + } + for (name, value) in [ + ("pool.default_warm_ttl", self.pool.default_warm_ttl.as_str()), + ("pool.gc_interval", self.pool.gc_interval.as_str()), + ] { + if parse_duration(value).is_none() { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "{name} must be a positive duration with an s, m, h, or d suffix, got \ + {value:?}" + )), + }); + } + } + let runtime_root = self.daemon.state_dir.join("runtime-pool"); + validate_runtime_storage_paths( + &runtime_root, + &self.storage.images_dir, + &self.storage.instances_dir, + )?; + self.storage.flush_schedule()?; + self.storage.flush_timeout_duration()?; + validate_runtime_template_paths( + &self.runtime_templates.dir, + self.runtime_templates.import_root.as_deref(), + &runtime_root, + &self.storage.images_dir, + &self.storage.instances_dir, + &self.template.dir, + )?; + if self.runtime_templates.max_files == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_files must be greater than zero".to_string(), + ), + }); + } + if self.runtime_templates.max_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_bytes must be greater than zero".to_string(), + ), + }); + } + if self.runtime_templates.max_metadata_bytes == 0 + || self.runtime_templates.max_metadata_bytes > self.runtime_templates.max_bytes + { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_metadata_bytes must be greater than zero and no \ + larger than max_bytes" + .to_string(), + ), + }); + } + if self.runtime_templates.max_total_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_total_bytes must be greater than zero".to_string(), + ), + }); + } + Ok(()) + } +} + +fn invalid_storage_duration(name: &str, value: &str, allow_disabled: bool) -> BlazeError { + let expected = if allow_disabled { + "a positive duration or \"disabled\"" + } else { + "a positive duration" + }; + BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "storage.{name} ({value:?}) must be {expected}" + )), } } /// Reject storage roots whose ownership domains overlap. pub fn validate_storage_paths(images_dir: &Path, instances_dir: &Path) -> Result<()> { - if images_dir == instances_dir - || images_dir.starts_with(instances_dir) - || instances_dir.starts_with(images_dir) - { + if paths_overlap(images_dir, instances_dir) { return Err(BlazeError::ConfigError { source: ConfigErrorSource::InvalidValue(format!( "storage.images_dir ({}) and storage.instances_dir ({}) must be disjoint", @@ -222,6 +392,109 @@ pub fn validate_storage_paths(images_dir: &Path, instances_dir: &Path) -> Result Ok(()) } +/// Reject runtime and storage roots whose ownership domains overlap. +/// +/// Callers should invoke this once with configured paths and again with +/// canonical paths after creating the roots. The second check catches aliases +/// introduced by symbolic links. +pub fn validate_runtime_storage_paths( + runtime_root: &Path, + images_dir: &Path, + instances_dir: &Path, +) -> Result<()> { + validate_storage_paths(images_dir, instances_dir)?; + for (name, storage_root) in [ + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ] { + if paths_overlap(runtime_root, storage_root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime slot root ({}) and {name} ({}) must be disjoint", + runtime_root.display(), + storage_root.display() + )), + }); + } + } + Ok(()) +} + +fn validate_runtime_template_paths( + dir: &Path, + import_root: Option<&Path>, + runtime_root: &Path, + images_dir: &Path, + instances_dir: &Path, + template_dir: &Path, +) -> Result<()> { + validate_absolute_root(dir, "runtime_templates.dir")?; + if let Some(import_root) = import_root { + validate_absolute_root(import_root, "runtime_templates.import_root")?; + } + + let mut roots = vec![ + ("runtime slot root", runtime_root), + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ("template.dir", template_dir), + ]; + if let Some(import_root) = import_root { + roots.push(("runtime_templates.import_root", import_root)); + } + for (label, root) in roots { + if paths_overlap(dir, root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime_templates.dir ({}) and {label} ({}) must be disjoint", + dir.display(), + root.display() + )), + }); + } + } + + if let Some(import_root) = import_root { + for (label, root) in [ + ("runtime slot root", runtime_root), + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ("template.dir", template_dir), + ] { + if paths_overlap(import_root, root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime_templates.import_root ({}) and {label} ({}) must be disjoint", + import_root.display(), + root.display() + )), + }); + } + } + } + Ok(()) +} + +fn validate_absolute_root(path: &Path, label: &str) -> Result<()> { + if !path.is_absolute() + || path + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "{label} ({}) must be an absolute path without parent components", + path.display() + )), + }); + } + Ok(()) +} + +fn paths_overlap(left: &Path, right: &Path) -> bool { + left == right || left.starts_with(right) || right.starts_with(left) +} + // ----- defaults ----- fn default_log_level() -> String { @@ -239,6 +512,9 @@ fn default_policy_dir() -> PathBuf { fn default_on_load_error() -> PolicyLoadErrorMode { PolicyLoadErrorMode::Fail } +fn default_max_body_bytes() -> usize { + 1024 * 1024 +} fn default_pool_warm_ttl() -> String { "30m".to_string() } @@ -254,6 +530,21 @@ fn default_template_gc_interval() -> String { fn default_template_idle_ttl() -> String { "1h".to_string() } +fn default_runtime_template_dir() -> PathBuf { + PathBuf::from("/var/lib/blaze/runtime-templates") +} +fn default_runtime_template_max_files() -> usize { + 32 +} +fn default_runtime_template_max_bytes() -> u64 { + 256 * 1024 * 1024 * 1024 +} +fn default_runtime_template_max_metadata_bytes() -> u64 { + 1024 * 1024 +} +fn default_runtime_template_max_total_bytes() -> u64 { + 1024 * 1024 * 1024 * 1024 +} fn default_prometheus_socket() -> PathBuf { PathBuf::from("/run/blaze/metrics.sock") } @@ -267,6 +558,9 @@ fn default_storage_provider() -> String { "file".to_string() } fn default_flush_interval() -> String { + "disabled".to_string() +} +fn default_flush_timeout() -> String { "30s".to_string() } fn default_rootfs_size() -> u64 { @@ -285,8 +579,14 @@ mod tests { let cfg: DaemonConfig = toml::from_str("").expect("empty parses to defaults"); assert_eq!(cfg.daemon.log_level, "info"); assert_eq!(cfg.policy.on_load_error, PolicyLoadErrorMode::Fail); + assert_eq!(cfg.api.max_body_bytes, 1024 * 1024); assert!(cfg.backends.is_empty()); assert_ne!(cfg.storage.images_dir, cfg.storage.instances_dir); + assert_eq!( + cfg.storage.flush_schedule().expect("flush schedule"), + StorageFlushSchedule::Disabled + ); + assert!(cfg.runtime_templates.import_root.is_none()); } #[test] @@ -311,6 +611,58 @@ mod tests { assert_eq!(cfg.backends.len(), 2); } + #[test] + fn parses_api_body_limit() { + let cfg: DaemonConfig = toml::from_str( + r#" + [api] + max_body_bytes = 4096 + "#, + ) + .expect("api config"); + assert_eq!(cfg.api.max_body_bytes, 4096); + cfg.validate().expect("positive body limit"); + } + + #[test] + fn rejects_zero_api_body_limit() { + let cfg: DaemonConfig = toml::from_str( + r#" + [api] + max_body_bytes = 0 + "#, + ) + .expect("api config"); + let error = cfg.validate().expect_err("zero body limit"); + assert!( + error + .to_string() + .contains("api.max_body_bytes must be greater than zero") + ); + } + + #[test] + fn rejects_invalid_runtime_pool_durations() { + for (field, value) in [ + ("pool.default_warm_ttl", "30"), + ("pool.default_warm_ttl", "0s"), + ("pool.gc_interval", "soon"), + ("pool.gc_interval", "0m"), + ] { + let mut cfg = DaemonConfig::default(); + if field == "pool.default_warm_ttl" { + cfg.pool.default_warm_ttl = value.to_string(); + } else { + cfg.pool.gc_interval = value.to_string(); + } + + let error = cfg.validate().expect_err("invalid pool duration"); + + assert!(error.to_string().contains(field)); + assert!(error.to_string().contains(value)); + } + } + #[test] fn rejects_equal_or_nested_storage_roots() { for (images, instances) in [ @@ -325,4 +677,134 @@ mod tests { assert!(error.to_string().contains("must be disjoint")); } } + + #[test] + fn accepts_sibling_runtime_and_storage_roots() { + validate_runtime_storage_paths( + Path::new("/var/lib/blaze/runtime-pool"), + Path::new("/var/lib/blaze/images"), + Path::new("/var/lib/blaze/instances"), + ) + .expect("sibling ownership roots are disjoint"); + } + + #[test] + fn rejects_equal_or_nested_runtime_and_storage_roots() { + for (runtime, images, instances) in [ + ( + "/var/lib/blaze/images", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze/images/runtime", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze/instances/runtime", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ] { + let error = validate_runtime_storage_paths( + Path::new(runtime), + Path::new(images), + Path::new(instances), + ) + .expect_err("overlapping runtime and storage roots"); + assert!(matches!(error, BlazeError::ConfigError { .. })); + assert!(error.to_string().contains("must be disjoint")); + } + } + + #[test] + fn storage_flush_schedule_accepts_disabled_or_positive_duration() { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_interval = "disabled".into(); + cfg.validate().expect("disabled schedule"); + assert_eq!( + cfg.storage.flush_schedule().expect("schedule"), + StorageFlushSchedule::Disabled + ); + + cfg.storage.flush_interval = "15s".into(); + cfg.validate().expect("positive schedule"); + assert_eq!( + cfg.storage.flush_schedule().expect("schedule"), + StorageFlushSchedule::Every(Duration::from_secs(15)) + ); + } + + #[test] + fn storage_flush_schedule_rejects_invalid_values() { + for interval in ["0s", "not-a-duration"] { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_interval = interval.into(); + let error = cfg.validate().expect_err("invalid flush interval"); + assert!( + error.to_string().contains("storage.flush_interval"), + "{error}" + ); + } + + for timeout in ["0s", "disabled", "not-a-duration"] { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_timeout = timeout.into(); + let error = cfg.validate().expect_err("invalid flush timeout"); + assert!( + error.to_string().contains("storage.flush_timeout"), + "{error}" + ); + } + } + + #[test] + fn rejects_unsafe_runtime_template_boundaries() { + let mut relative = DaemonConfig::default(); + relative.runtime_templates.dir = PathBuf::from("runtime-templates"); + assert!(relative.validate().is_err()); + + let mut parent = DaemonConfig::default(); + parent.runtime_templates.dir = PathBuf::from("/var/lib/blaze/../runtime-templates"); + assert!(parent.validate().is_err()); + + let mut overlapping = DaemonConfig::default(); + overlapping.runtime_templates.import_root = + Some(PathBuf::from("/var/lib/blaze/runtime-templates/imports")); + assert!(overlapping.validate().is_err()); + + for owned_root in [ + "/var/lib/blaze/runtime-pool/catalog", + "/var/lib/blaze/images/catalog", + "/var/lib/blaze/instances/catalog", + "/var/lib/blaze/templates/catalog", + ] { + let mut config = DaemonConfig::default(); + config.runtime_templates.dir = PathBuf::from(owned_root); + assert!(config.validate().is_err()); + } + + let mut source_overlap = DaemonConfig::default(); + source_overlap.runtime_templates.import_root = + Some(PathBuf::from("/var/lib/blaze/images/imports")); + assert!(source_overlap.validate().is_err()); + + let mut unbounded = DaemonConfig::default(); + unbounded.runtime_templates.max_files = 0; + assert!(unbounded.validate().is_err()); + + let mut metadata = DaemonConfig::default(); + metadata.runtime_templates.max_metadata_bytes = metadata.runtime_templates.max_bytes + 1; + assert!(metadata.validate().is_err()); + + let mut total = DaemonConfig::default(); + total.runtime_templates.max_total_bytes = 0; + assert!(total.validate().is_err()); + } } diff --git a/src/blaze/crates/blaze-core/src/error.rs b/src/blaze/crates/blaze-core/src/error.rs index 8c2f34c372..9f0db3c7f4 100644 --- a/src/blaze/crates/blaze-core/src/error.rs +++ b/src/blaze/crates/blaze-core/src/error.rs @@ -29,6 +29,10 @@ pub enum BlazeError { #[error("invalid sandbox state transition: {from} -> {to}")] InvalidStateTransition { from: String, to: String }, + /// A lifecycle caller tried to replace an unfinished durable operation. + #[error("sandbox operation already in progress: active={active}, requested={requested}")] + OperationInProgress { active: String, requested: String }, + #[error("template registry error: {msg}")] TemplateError { msg: String }, diff --git a/src/blaze/crates/blaze-core/src/guest_protocol.rs b/src/blaze/crates/blaze-core/src/guest_protocol.rs new file mode 100644 index 0000000000..282bfc2066 --- /dev/null +++ b/src/blaze/crates/blaze-core/src/guest_protocol.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Wire DTOs shared with a compatible sandbox guest agent. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Firecracker vsock port used by the compatible sandbox guest agent. +pub const DEFAULT_GUEST_PORT: u32 = 5000; + +/// Maximum accepted JSON response line, excluding the newline delimiter. +pub const DEFAULT_MAX_RESPONSE_BYTES: usize = 32 * 1024 * 1024; + +/// Guest operation names implemented by `sandbox-agent`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum GuestOp { + /// Check whether the guest agent can serve requests. + Ping, + /// Execute one shell command. + Exec, + /// Read one guest file. + Read, + /// Replace one guest file. + Write, +} + +/// One newline-delimited request sent after the Firecracker CONNECT handshake. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GuestRequest { + /// Correlation identifier echoed by the guest. + pub id: String, + /// Requested guest operation. + pub op: GuestOp, + /// Shell command for [`GuestOp::Exec`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub cmd: Option, + /// Working directory for command execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Environment additions for command execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub env: Option>, + /// Guest-side timeout in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + /// Guest path for file operations. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + /// Standard-base64 file bytes for [`GuestOp::Write`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub data_b64: Option, +} + +/// One newline-delimited response returned by the compatible guest agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GuestResponse { + /// Correlation identifier copied from the request. + pub id: String, + /// Whether the operation completed successfully. + pub ok: bool, + /// Guest error message when `ok` is false. + #[serde(default)] + pub err: Option, + /// Command exit status. + #[serde(default)] + pub rc: Option, + /// Standard-base64 command stdout. + #[serde(default)] + pub stdout_b64: Option, + /// Standard-base64 command stderr. + #[serde(default)] + pub stderr_b64: Option, + /// Standard-base64 file bytes. + #[serde(default)] + pub data_b64: Option, +} + +impl GuestRequest { + /// Build a request with operation-specific fields initially absent. + pub fn new(id: String, op: GuestOp) -> Self { + Self { + id, + op, + cmd: None, + cwd: None, + env: None, + timeout: None, + path: None, + data_b64: None, + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn request_uses_wire_operation_names_and_omits_absent_fields() { + let request = GuestRequest::new("request-1".to_string(), GuestOp::Exec); + + assert_eq!( + serde_json::to_value(request).expect("serialize request"), + json!({ + "id": "request-1", + "op": "exec", + }) + ); + } + + #[test] + fn response_requires_outcome_and_defaults_optional_fields() { + assert!(serde_json::from_value::(json!({"id": "request-1"})).is_err()); + assert!(serde_json::from_value::(json!({"ok": true})).is_err()); + let response: GuestResponse = serde_json::from_value(json!({ + "id": "request-1", + "ok": true + })) + .expect("deserialize response"); + + assert_eq!(response.id, "request-1"); + assert!(response.ok); + assert!(response.err.is_none()); + assert!(response.rc.is_none()); + assert!(response.stdout_b64.is_none()); + assert!(response.stderr_b64.is_none()); + assert!(response.data_b64.is_none()); + } +} diff --git a/src/blaze/crates/blaze-core/src/lib.rs b/src/blaze/crates/blaze-core/src/lib.rs index d95a9db18d..cd1e1908b6 100644 --- a/src/blaze/crates/blaze-core/src/lib.rs +++ b/src/blaze/crates/blaze-core/src/lib.rs @@ -9,6 +9,8 @@ //! - [`config`]: daemon TOML configuration //! - [`policy`]: workload class + policy file schema //! - [`backend`]: backend kinds + selection / fallback +//! - [`checkpoint`]: pure checkpoint records and manifest validation +//! - [`guest_protocol`]: guest-agent wire DTOs //! - [`lifecycle`]: sandbox state machine + JSON persistence //! - [`pool`]: warm-pool key/stat/manager //! - [`template`]: template registry + refcnt + GC @@ -16,8 +18,10 @@ //! - [`error`]: unified [`BlazeError`] error enum pub mod backend; +pub mod checkpoint; pub mod config; pub mod error; +pub mod guest_protocol; pub mod kernel; pub mod lifecycle; pub mod policy; diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index cecf6bbb2a..dcf29be1d1 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 //! Sandbox lifecycle state machine + JSON persistence. -use std::fs; +use std::fs::{self, File}; +use std::io::Write; use std::path::{Path, PathBuf}; use chrono::{DateTime, Utc}; @@ -21,6 +22,15 @@ pub enum SandboxState { Running, Paused, Checkpointed, + /// The previous backend is stopped while replacement resources are owned. + Restoring, + /// A live backend is being converted into durable hibernation artifacts. + Hibernating, + /// Durable hibernation artifacts and storage are retained without a backend. + Hibernated, + /// A backend is being started from retained hibernation artifacts. + Resuming, + RecoveryRequired, Reset, Warm, Destroyed, @@ -34,6 +44,11 @@ impl SandboxState { SandboxState::Running => "running", SandboxState::Paused => "paused", SandboxState::Checkpointed => "checkpointed", + SandboxState::Restoring => "restoring", + SandboxState::Hibernating => "hibernating", + SandboxState::Hibernated => "hibernated", + SandboxState::Resuming => "resuming", + SandboxState::RecoveryRequired => "recovery-required", SandboxState::Reset => "reset", SandboxState::Warm => "warm", SandboxState::Destroyed => "destroyed", @@ -41,6 +56,185 @@ impl SandboxState { } } +/// Persisted multi-step operation used for crash diagnosis and recovery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperationKind { + /// Sandbox creation is acquiring resources or starting a backend. + Create, + /// A point-in-time checkpoint is being captured and published. + Checkpoint, + /// A running sandbox is being replaced from a selected checkpoint. + Restore, + /// A live backend is being stopped after durable artifacts are prepared. + Hibernate, + /// A backend is being started from retained hibernation artifacts. + Resume, + /// Runtime resources are being destroyed. + Destroy, +} + +impl OperationKind { + pub const fn as_str(&self) -> &'static str { + match self { + OperationKind::Create => "create", + OperationKind::Checkpoint => "checkpoint", + OperationKind::Restore => "restore", + OperationKind::Hibernate => "hibernate", + OperationKind::Resume => "resume", + OperationKind::Destroy => "destroy", + } + } +} + +impl std::fmt::Display for OperationKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Durable boundary reached by a multi-step lifecycle operation. +/// +/// The journal keeps this separate from [`SandboxState`]: state describes +/// externally visible runtime availability, while the phase identifies the +/// last resource-ownership or catalog boundary committed before interruption. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperationPhase { + /// A staging directory exists, but the backend has not been paused. + CheckpointPreparing, + /// The backend is paused while snapshot artifacts are being written. + CheckpointPaused, + /// A complete checkpoint directory is visible, but HEAD is unchanged. + CheckpointPublished, + /// HEAD references the checkpoint; runtime resume is not yet committed. + CheckpointHeadUpdated, + /// Restore intent is durable, but the current runtime is still owned. + RestorePreparing, + /// Replacement storage is staged without changing the live rootfs. + RestoreStorageStaged, + /// The current backend has been confirmed stopped. + RestoreBackendStopped, + /// Staged storage is active while the predecessor remains recoverable. + RestoreStorageActivated, + /// A replacement backend has started and is owned by the runtime. + RestoreBackendStarted, + /// HEAD references the restored checkpoint; storage and lifecycle commits remain. + RestoreHeadUpdated, + /// The storage replacement is committed and can no longer be aborted. + RestoreStorageCommitted, + /// Hibernate intent is durable, but the backend is still running. + HibernatePreparing, + /// The backend is paused while hibernation artifacts are written. + HibernatePaused, + /// Hibernation artifacts are complete and durable. + HibernateArtifactsSynced, + /// The live backend has stopped and no longer owns runtime resources. + HibernateBackendStopped, + /// The replacement hibernation directory is durably visible. + HibernatePublished, + /// Resume intent is durable and no backend has started. + ResumePreparing, + /// Backend ownership intent is durable before restore starts. + ResumeBackendStarting, + /// A restored backend is owned, but readiness is not yet confirmed. + ResumeBackendStarted, + /// The restored backend and optional guest transport are ready. + ResumeBackendReady, +} + +impl OperationPhase { + const fn as_str(self) -> &'static str { + match self { + OperationPhase::CheckpointPreparing => "checkpoint-preparing", + OperationPhase::CheckpointPaused => "checkpoint-paused", + OperationPhase::CheckpointPublished => "checkpoint-published", + OperationPhase::CheckpointHeadUpdated => "checkpoint-head-updated", + OperationPhase::RestorePreparing => "restore-preparing", + OperationPhase::RestoreStorageStaged => "restore-storage-staged", + OperationPhase::RestoreBackendStopped => "restore-backend-stopped", + OperationPhase::RestoreStorageActivated => "restore-storage-activated", + OperationPhase::RestoreBackendStarted => "restore-backend-started", + OperationPhase::RestoreHeadUpdated => "restore-head-updated", + OperationPhase::RestoreStorageCommitted => "restore-storage-committed", + OperationPhase::HibernatePreparing => "hibernate-preparing", + OperationPhase::HibernatePaused => "hibernate-paused", + OperationPhase::HibernateArtifactsSynced => "hibernate-artifacts-synced", + OperationPhase::HibernateBackendStopped => "hibernate-backend-stopped", + OperationPhase::HibernatePublished => "hibernate-published", + OperationPhase::ResumePreparing => "resume-preparing", + OperationPhase::ResumeBackendStarting => "resume-backend-starting", + OperationPhase::ResumeBackendStarted => "resume-backend-started", + OperationPhase::ResumeBackendReady => "resume-backend-ready", + } + } + + const fn operation_kind(self) -> OperationKind { + match self { + OperationPhase::CheckpointPreparing + | OperationPhase::CheckpointPaused + | OperationPhase::CheckpointPublished + | OperationPhase::CheckpointHeadUpdated => OperationKind::Checkpoint, + OperationPhase::RestorePreparing + | OperationPhase::RestoreStorageStaged + | OperationPhase::RestoreBackendStopped + | OperationPhase::RestoreStorageActivated + | OperationPhase::RestoreBackendStarted + | OperationPhase::RestoreHeadUpdated + | OperationPhase::RestoreStorageCommitted => OperationKind::Restore, + OperationPhase::HibernatePreparing + | OperationPhase::HibernatePaused + | OperationPhase::HibernateArtifactsSynced + | OperationPhase::HibernateBackendStopped + | OperationPhase::HibernatePublished => OperationKind::Hibernate, + OperationPhase::ResumePreparing + | OperationPhase::ResumeBackendStarting + | OperationPhase::ResumeBackendStarted + | OperationPhase::ResumeBackendReady => OperationKind::Resume, + } + } + + const fn rank(self) -> u8 { + match self { + OperationPhase::CheckpointPreparing => 0, + OperationPhase::CheckpointPaused => 1, + OperationPhase::CheckpointPublished => 2, + OperationPhase::CheckpointHeadUpdated => 3, + OperationPhase::RestorePreparing => 0, + OperationPhase::RestoreStorageStaged => 1, + OperationPhase::RestoreBackendStopped => 2, + OperationPhase::RestoreStorageActivated => 3, + OperationPhase::RestoreBackendStarted => 4, + OperationPhase::RestoreHeadUpdated => 5, + OperationPhase::RestoreStorageCommitted => 6, + OperationPhase::HibernatePreparing => 0, + OperationPhase::HibernatePaused => 1, + OperationPhase::HibernateArtifactsSynced => 2, + OperationPhase::HibernateBackendStopped => 3, + OperationPhase::HibernatePublished => 4, + OperationPhase::ResumePreparing => 0, + OperationPhase::ResumeBackendStarting => 1, + OperationPhase::ResumeBackendStarted => 2, + OperationPhase::ResumeBackendReady => 3, + } + } +} + +/// Durable journal entry for one active lifecycle operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OperationJournal { + /// Operation being performed. + pub kind: OperationKind, + /// UTC time at which the operation became externally visible. + pub started_at: DateTime, + /// Checkpoint selected by this operation, when applicable. + #[serde(default)] + pub checkpoint_id: Option, + /// Last durably committed operation boundary. + #[serde(default)] + pub phase: Option, +} + impl std::fmt::Display for SandboxState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) @@ -56,6 +250,20 @@ pub enum StartPath { Warm, } +/// Canonical directory family that owns backend runtime artifacts. +/// +/// This is independent from [`StartPath`]: a sandbox can be activated through +/// a warm lifecycle transition while retaining its original sandbox directory. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RuntimeLocation { + /// Runtime artifacts live under the sandbox's lifecycle state directory. + #[default] + Sandbox, + /// Runtime artifacts live under the daemon's warm-slot directory. + WarmPool, +} + /// Durable knowledge about whether a backend may still own a live process. /// /// `Unknown` is the safe default for state written by older daemon versions. @@ -82,9 +290,21 @@ pub struct SandboxInstance { pub created_at: DateTime, pub updated_at: DateTime, pub policy_name: String, + /// Canonical directory family containing backend runtime artifacts. + #[serde(default)] + pub runtime_location: RuntimeLocation, + /// Stable nonce that links a claimed warm runtime to its slot journal. + #[serde(default)] + pub runtime_owner_token: Option, /// Last durably known backend ownership state. #[serde(default)] pub backend_ownership: BackendOwnership, + /// Active multi-step operation, if any. + #[serde(default)] + pub operation: Option, + /// Last checkpoint whose capture completed and returned the sandbox to running. + #[serde(default)] + pub last_checkpoint: Option, } impl SandboxInstance { @@ -109,15 +329,274 @@ impl SandboxInstance { created_at: now, updated_at: now, policy_name, + runtime_location: RuntimeLocation::Sandbox, + runtime_owner_token: None, backend_ownership: BackendOwnership::NotStarted, + operation: None, + last_checkpoint: None, } } - /// Apply a state transition. Returns + /// Adopt a ready runtime slot into a recoverable create operation. + pub fn new_warm_claim( + id: Uuid, + backend: BackendKind, + workload_class: WorkloadClass, + image_digest: String, + policy_name: String, + backend_ownership: BackendOwnership, + runtime_owner_token: Uuid, + ) -> Result { + if !matches!( + backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Running + ) { + return Err(BlazeError::BackendError { + msg: format!( + "ready runtime slot {id} has invalid backend ownership {backend_ownership:?}" + ), + }); + } + let mut instance = Self::new( + backend, + workload_class, + image_digest, + StartPath::Warm, + policy_name, + ); + instance.id = id; + instance.state = SandboxState::Warm; + instance.runtime_location = RuntimeLocation::WarmPool; + instance.runtime_owner_token = Some(runtime_owner_token); + instance.backend_ownership = backend_ownership; + instance.begin_operation(OperationKind::Create)?; + instance.transition(SandboxState::Creating)?; + Ok(instance) + } + + /// Record a new operation before starting its first owned-resource + /// mutation. An unfinished journal must be recovered rather than silently + /// replaced by a later request. + pub fn begin_operation(&mut self, kind: OperationKind) -> Result<()> { + if let Some(active) = &self.operation { + return Err(BlazeError::OperationInProgress { + active: active.kind.to_string(), + requested: kind.to_string(), + }); + } + self.operation = Some(OperationJournal { + kind, + started_at: Utc::now(), + checkpoint_id: None, + phase: None, + }); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Record checkpoint intent before pausing the backend. + pub fn begin_checkpoint_operation(&mut self, checkpoint_id: String) -> Result<()> { + self.begin_operation(OperationKind::Checkpoint)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.checkpoint_id = Some(checkpoint_id); + operation.phase = Some(OperationPhase::CheckpointPreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active checkpoint journal without replacing its identity. + pub fn advance_checkpoint_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Checkpoint, phase) + } + + /// Record restore intent without changing the last completed checkpoint. + pub fn begin_restore_operation(&mut self, checkpoint_id: String) -> Result<()> { + self.begin_operation(OperationKind::Restore)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.checkpoint_id = Some(checkpoint_id); + operation.phase = Some(OperationPhase::RestorePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active restore journal without replacing its identity. + pub fn advance_restore_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Restore, phase) + } + + /// Record hibernation intent before pausing the backend. + pub fn begin_hibernate_operation(&mut self) -> Result<()> { + self.begin_operation(OperationKind::Hibernate)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.phase = Some(OperationPhase::HibernatePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active hibernation journal without replacing its identity. + pub fn advance_hibernate_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Hibernate, phase) + } + + /// Record resume intent before preparing a replacement backend. + pub fn begin_resume_operation(&mut self) -> Result<()> { + self.begin_operation(OperationKind::Resume)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.phase = Some(OperationPhase::ResumePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active resume journal without replacing its identity. + pub fn advance_resume_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Resume, phase) + } + + fn advance_operation_phase( + &mut self, + requested_kind: OperationKind, + phase: OperationPhase, + ) -> Result<()> { + if phase.operation_kind() != requested_kind { + return Err(BlazeError::InvalidStateTransition { + from: requested_kind.to_string(), + to: phase.as_str().to_string(), + }); + } + let operation = self + .operation + .as_mut() + .ok_or_else(|| BlazeError::OperationInProgress { + active: "none".to_string(), + requested: requested_kind.to_string(), + })?; + if operation.kind != requested_kind { + return Err(BlazeError::OperationInProgress { + active: operation.kind.to_string(), + requested: requested_kind.to_string(), + }); + } + if let Some(current) = operation.phase + && (current.operation_kind() != requested_kind || phase.rank() < current.rank()) + { + return Err(BlazeError::InvalidStateTransition { + from: current.as_str().to_string(), + to: phase.as_str().to_string(), + }); + } + operation.phase = Some(phase); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Transfer an interrupted lifecycle operation to destroy recovery. + /// + /// Cleanup is the only operation allowed to supersede an unfinished + /// journal because it releases, rather than acquires, owned resources. + pub fn begin_destroy_recovery(&mut self) { + if self.operation.as_ref().map(|operation| operation.kind) == Some(OperationKind::Destroy) { + return; + } + self.operation = Some(OperationJournal { + kind: OperationKind::Destroy, + started_at: Utc::now(), + checkpoint_id: None, + phase: None, + }); + self.updated_at = Utc::now(); + } + + /// Clear the marker before atomically persisting the final state. + pub fn finish_operation(&mut self) { + self.operation = None; + self.updated_at = Utc::now(); + } + + /// Return whether lifecycle metadata proves that no runtime owner remains. + pub fn is_clean_terminal(&self) -> bool { + self.state == SandboxState::Destroyed + && self.operation.is_none() + && matches!( + self.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) + } + + /// Apply a state transition. + /// + /// Restore transitions additionally require the durable backend-stop and + /// storage-commit boundaries before changing externally visible state. + /// Returns /// [`BlazeError::InvalidStateTransition`] when the move is not part /// of the lifecycle state graph. pub fn transition(&mut self, target: SandboxState) -> Result<()> { - if !is_valid_transition(self.state, target) { + let restore_boundary_reached = match (self.state, target) { + (_, SandboxState::Restoring) => { + self.restore_phase_reached(OperationPhase::RestoreBackendStopped) + } + (SandboxState::Restoring, SandboxState::Running) => { + self.restore_phase_reached(OperationPhase::RestoreStorageCommitted) + } + _ => true, + }; + let hibernate_boundary_reached = match (self.state, target) { + (SandboxState::Running, SandboxState::Hibernating) => self.operation_phase_reached( + OperationKind::Hibernate, + OperationPhase::HibernatePreparing, + ), + (SandboxState::Hibernating, SandboxState::Hibernated) => { + self.backend_ownership == BackendOwnership::Stopped + && self.operation_phase_reached( + OperationKind::Hibernate, + OperationPhase::HibernatePublished, + ) + } + (SandboxState::Hibernating, SandboxState::Running) => { + self.backend_ownership == BackendOwnership::Running + && self + .operation + .as_ref() + .is_some_and(|operation| operation.kind == OperationKind::Hibernate) + } + (SandboxState::Hibernated, SandboxState::Resuming) => { + self.backend_ownership == BackendOwnership::Stopped + && self.operation_phase_reached( + OperationKind::Resume, + OperationPhase::ResumePreparing, + ) + } + (SandboxState::Resuming, SandboxState::Running) => { + self.backend_ownership == BackendOwnership::Running + && self.operation_phase_reached( + OperationKind::Resume, + OperationPhase::ResumeBackendReady, + ) + } + (SandboxState::Resuming, SandboxState::Hibernated) => { + self.backend_ownership == BackendOwnership::Stopped + && self + .operation + .as_ref() + .is_some_and(|operation| operation.kind == OperationKind::Resume) + } + _ => true, + }; + if !restore_boundary_reached + || !hibernate_boundary_reached + || !is_valid_transition(self.state, target) + { return Err(BlazeError::InvalidStateTransition { from: self.state.to_string(), to: target.to_string(), @@ -146,17 +625,103 @@ impl SandboxInstance { Ok(()) } + fn restore_phase_reached(&self, minimum: OperationPhase) -> bool { + self.operation_phase_reached(OperationKind::Restore, minimum) + } + + fn operation_phase_reached( + &self, + operation_kind: OperationKind, + minimum: OperationPhase, + ) -> bool { + minimum.operation_kind() == operation_kind + && self.operation.as_ref().is_some_and(|operation| { + operation.kind == operation_kind + && operation.phase.is_some_and(|phase| { + phase.operation_kind() == operation_kind && phase.rank() >= minimum.rank() + }) + }) + } + /// Persist this instance to `{state_dir}/{id}/state.json`. Atomic /// rename via `state.json.tmp` to avoid torn reads on daemon restart. pub fn persist(&self, state_dir: &Path) -> Result<()> { - let dir = state_dir.join(self.id.to_string()); - fs::create_dir_all(&dir)?; - let final_path = dir.join("state.json"); - let tmp_path = dir.join("state.json.tmp"); + self.persist_with(state_dir, |tmp_path, final_path, json| { + let mut file = File::create(tmp_path)?; + file.write_all(json)?; + file.write_all(b"\n")?; + file.sync_all()?; + drop(file); + fs::rename(tmp_path, final_path)?; + Ok(()) + }) + } + + fn persist_with(&self, state_dir: &Path, publish: F) -> Result<()> + where + F: FnOnce(&Path, &Path, &[u8]) -> Result<()>, + { + self.persist_with_directory_sync(state_dir, publish, |directory| { + File::open(directory)?.sync_all()?; + Ok(()) + }) + } + + fn persist_with_directory_sync( + &self, + state_dir: &Path, + publish: F, + mut sync_directory: S, + ) -> Result<()> + where + F: FnOnce(&Path, &Path, &[u8]) -> Result<()>, + S: FnMut(&Path) -> Result<()>, + { + let owner_dir = state_dir.join(self.id.to_string()); let json = serde_json::to_vec_pretty(self)?; - fs::write(&tmp_path, &json)?; - fs::rename(&tmp_path, &final_path)?; - Ok(()) + fs::create_dir_all(state_dir)?; + + match fs::symlink_metadata(&owner_dir) { + Ok(metadata) if metadata.file_type().is_dir() => { + let final_path = owner_dir.join("state.json"); + let tmp_path = owner_dir.join("state.json.tmp"); + let result = publish(&tmp_path, &final_path, &json); + if result.is_err() { + let _ = fs::remove_file(&tmp_path); + } + result?; + sync_directory(&owner_dir) + } + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "lifecycle owner {} is not a real directory", + owner_dir.display() + ), + ) + .into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let staging_dir = + state_dir.join(format!(".state-{}-{}.tmp", self.id, Uuid::new_v4())); + fs::create_dir(&staging_dir)?; + let final_path = staging_dir.join("state.json"); + let tmp_path = staging_dir.join("state.json.tmp"); + if let Err(error) = publish(&tmp_path, &final_path, &json) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error); + } + if let Err(error) = sync_directory(&staging_dir) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error); + } + if let Err(error) = fs::rename(&staging_dir, &owner_dir) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error.into()); + } + sync_directory(state_dir) + } + Err(error) => Err(error.into()), + } } /// Reload an instance previously persisted via [`Self::persist`]. @@ -169,11 +734,17 @@ impl SandboxInstance { } fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { - use SandboxState::{Checkpointed, Creating, Destroyed, Paused, Pending, Reset, Running, Warm}; + use SandboxState::{ + Checkpointed, Creating, Destroyed, Hibernated, Hibernating, Paused, Pending, + RecoveryRequired, Reset, Restoring, Resuming, Running, Warm, + }; if to == Destroyed { // `* → destroyed` is always valid (terminal sink). return from != Destroyed; } + if to == RecoveryRequired { + return !matches!(from, Destroyed | RecoveryRequired); + } match (from, to) { (Pending, Creating) => true, (Creating, Running) => true, @@ -181,6 +752,15 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { (Running, Reset) => true, (Paused, Checkpointed) => true, (Paused, Running) => true, // resume + (Checkpointed, Running) => true, + (Running, Restoring) => true, + (Restoring, Running) => true, + (Running, Hibernating) => true, + (Hibernating, Hibernated) => true, + (Hibernating, Running) => true, + (Hibernated, Resuming) => true, + (Resuming, Running) => true, + (Resuming, Hibernated) => true, (Reset, Warm) => true, (Warm, Creating) => true, // pool reuse / warm path _ => false, @@ -189,6 +769,8 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { #[cfg(test)] mod tests { + use std::cell::Cell; + use super::*; fn fresh() -> SandboxInstance { @@ -209,6 +791,7 @@ mod tests { SandboxState::Running, SandboxState::Paused, SandboxState::Checkpointed, + SandboxState::Running, SandboxState::Destroyed, ] { inst.transition(target).expect("legal transition"); @@ -228,6 +811,100 @@ mod tests { assert_eq!(inst.start_path, StartPath::Warm); } + #[test] + fn restore_state_requires_owned_replacement_boundaries() { + let mut inst = fresh(); + let error = inst + .transition(SandboxState::Restoring) + .expect_err("pending sandbox cannot restore"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Pending); + + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect("begin restore"); + inst.advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect("stage storage"); + let error = inst + .transition(SandboxState::Restoring) + .expect_err("running remains visible until the backend is stopped"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Running); + + inst.advance_restore_phase(OperationPhase::RestoreBackendStopped) + .expect("stop backend"); + inst.transition(SandboxState::Restoring) + .expect("restore starts"); + inst.advance_restore_phase(OperationPhase::RestoreStorageActivated) + .expect("activate storage"); + inst.advance_restore_phase(OperationPhase::RestoreBackendStarted) + .expect("start backend"); + inst.advance_restore_phase(OperationPhase::RestoreHeadUpdated) + .expect("update head"); + let error = inst + .transition(SandboxState::Running) + .expect_err("storage commit precedes the final running state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Restoring); + + inst.advance_restore_phase(OperationPhase::RestoreStorageCommitted) + .expect("commit storage"); + inst.transition(SandboxState::Running) + .expect("restore commits"); + } + + #[test] + fn hibernation_state_requires_durable_ownership_boundaries() { + let mut inst = fresh(); + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.backend_ownership = BackendOwnership::Running; + + let error = inst + .transition(SandboxState::Hibernating) + .expect_err("hibernate requires a journal"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.begin_hibernate_operation().expect("begin hibernate"); + inst.transition(SandboxState::Hibernating) + .expect("hibernate starts"); + inst.advance_hibernate_phase(OperationPhase::HibernateArtifactsSynced) + .expect("artifacts durable"); + let error = inst + .transition(SandboxState::Hibernated) + .expect_err("a live backend prevents hibernated state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.backend_ownership = BackendOwnership::Stopped; + inst.advance_hibernate_phase(OperationPhase::HibernatePublished) + .expect("publish hibernation"); + inst.transition(SandboxState::Hibernated) + .expect("hibernate commits"); + inst.finish_operation(); + + let error = inst + .transition(SandboxState::Resuming) + .expect_err("resume requires a journal"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.begin_resume_operation().expect("begin resume"); + inst.transition(SandboxState::Resuming) + .expect("resume starts"); + inst.advance_resume_phase(OperationPhase::ResumeBackendStarted) + .expect("backend started"); + inst.backend_ownership = BackendOwnership::Running; + let error = inst + .transition(SandboxState::Running) + .expect_err("readiness must precede running state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.advance_resume_phase(OperationPhase::ResumeBackendReady) + .expect("backend ready"); + inst.transition(SandboxState::Running) + .expect("resume commits"); + } + #[test] fn destroy_is_always_legal_except_from_destroyed() { let mut inst = fresh(); @@ -239,6 +916,62 @@ mod tests { )); } + #[test] + fn clean_terminal_requires_safe_ownership_and_no_operation() { + let mut destroyed = fresh(); + destroyed + .transition(SandboxState::Destroyed) + .expect("destroyed"); + for ownership in [BackendOwnership::NotStarted, BackendOwnership::Stopped] { + destroyed.backend_ownership = ownership; + assert!(destroyed.is_clean_terminal()); + } + + destroyed.backend_ownership = BackendOwnership::Running; + assert!(!destroyed.is_clean_terminal()); + + let mut unfinished = fresh(); + unfinished + .begin_operation(OperationKind::Create) + .expect("begin create"); + unfinished + .transition(SandboxState::Destroyed) + .expect("destroyed"); + unfinished.backend_ownership = BackendOwnership::Stopped; + assert!(!unfinished.is_clean_terminal()); + + let mut running = fresh(); + running + .transition(SandboxState::Creating) + .expect("creating"); + running.transition(SandboxState::Running).expect("running"); + running.backend_ownership = BackendOwnership::Stopped; + assert!(!running.is_clean_terminal()); + } + + #[test] + fn recovery_required_can_finish_but_cannot_be_reentered() { + let mut inst = fresh(); + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.transition(SandboxState::RecoveryRequired) + .expect("recovery required"); + + let repeated = inst.transition(SandboxState::RecoveryRequired); + assert!(matches!( + repeated, + Err(BlazeError::InvalidStateTransition { .. }) + )); + + inst.transition(SandboxState::Destroyed) + .expect("destroyed from recovery"); + let terminal = inst.transition(SandboxState::RecoveryRequired); + assert!(matches!( + terminal, + Err(BlazeError::InvalidStateTransition { .. }) + )); + } + #[test] fn illegal_pending_to_running() { let mut inst = fresh(); @@ -278,4 +1011,443 @@ mod tests { assert_eq!(loaded.state, SandboxState::Creating); assert_eq!(loaded.policy_name, inst.policy_name); } + + #[test] + fn failed_first_persist_removes_an_empty_owner_directory() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + + let error = instance + .persist_with(tmp.path(), |_tmp_path, _final_path, _json| { + Err(std::io::Error::other("injected publication failure").into()) + }) + .expect_err("publication must fail"); + + assert!(error.to_string().contains("injected publication failure")); + assert!(!tmp.path().join(instance.id.to_string()).exists()); + assert_eq!( + std::fs::read_dir(tmp.path()) + .expect("state directory") + .count(), + 0 + ); + } + + #[test] + fn first_persist_syncs_the_staged_owner_and_state_root() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let sync_count = Cell::new(0); + + instance + .persist_with_directory_sync( + tmp.path(), + |tmp_path, final_path, json| { + std::fs::write(tmp_path, json)?; + std::fs::rename(tmp_path, final_path)?; + Ok(()) + }, + |_| { + sync_count.set(sync_count.get() + 1); + Ok(()) + }, + ) + .expect("publish lifecycle"); + + assert_eq!(sync_count.get(), 2); + assert_eq!( + SandboxInstance::load(tmp.path(), instance.id) + .expect("published lifecycle") + .id, + instance.id + ); + } + + #[test] + fn parent_sync_failure_preserves_the_published_owner() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let sync_count = Cell::new(0); + + let error = instance + .persist_with_directory_sync( + tmp.path(), + |tmp_path, final_path, json| { + std::fs::write(tmp_path, json)?; + std::fs::rename(tmp_path, final_path)?; + Ok(()) + }, + |_| { + let next = sync_count.get() + 1; + sync_count.set(next); + if next == 2 { + Err(std::io::Error::other("injected parent sync failure").into()) + } else { + Ok(()) + } + }, + ) + .expect_err("parent sync result is uncertain"); + + assert!(error.to_string().contains("injected parent sync failure")); + assert_eq!( + SandboxInstance::load(tmp.path(), instance.id) + .expect("published lifecycle remains") + .id, + instance.id + ); + } + + #[test] + fn failed_persist_preserves_a_directory_with_owned_artifacts() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let owner_dir = tmp.path().join(instance.id.to_string()); + std::fs::create_dir_all(&owner_dir).expect("owner dir"); + std::fs::write(owner_dir.join("backend.pid"), b"owner").expect("owner marker"); + + instance + .persist_with(tmp.path(), |_tmp_path, _final_path, _json| { + Err(std::io::Error::other("injected publication failure").into()) + }) + .expect_err("publication must fail"); + + assert!(owner_dir.join("backend.pid").exists()); + } + + #[test] + fn legacy_state_without_optional_fields_deserializes() { + let inst = fresh(); + let value = serde_json::json!({ + "id": inst.id, + "state": "running", + "backend": "mock", + "workload_class": "agent-rl", + "image_digest": "sha256:old", + "start_path": "cold", + "created_at": inst.created_at, + "updated_at": inst.updated_at, + "policy_name": "legacy" + }); + let loaded: SandboxInstance = serde_json::from_value(value).expect("legacy state"); + assert!(loaded.operation.is_none()); + assert!(loaded.last_checkpoint.is_none()); + assert_eq!(loaded.backend_ownership, BackendOwnership::Unknown); + assert_eq!(loaded.runtime_location, RuntimeLocation::Sandbox); + assert!(loaded.runtime_owner_token.is_none()); + } + + #[test] + fn warm_start_classification_does_not_move_runtime_artifacts() { + let mut instance = fresh(); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.transition(SandboxState::Reset).expect("reset"); + instance.transition(SandboxState::Warm).expect("warm"); + instance + .transition(SandboxState::Creating) + .expect("warm creating"); + + assert_eq!(instance.start_path, StartPath::Warm); + assert_eq!(instance.runtime_location, RuntimeLocation::Sandbox); + } + + #[test] + fn runtime_location_round_trips() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + let owner_token = Uuid::new_v4(); + instance.runtime_location = RuntimeLocation::WarmPool; + instance.runtime_owner_token = Some(owner_token); + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + + assert_eq!(loaded.runtime_location, RuntimeLocation::WarmPool); + assert_eq!(loaded.runtime_owner_token, Some(owner_token)); + } + + #[test] + fn warm_claim_starts_one_durable_create_operation() { + let id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + + let instance = SandboxInstance::new_warm_claim( + id, + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:warm".into(), + "warm-policy".into(), + BackendOwnership::Running, + owner_token, + ) + .expect("warm claim"); + + assert_eq!(instance.id, id); + assert_eq!(instance.state, SandboxState::Creating); + assert_eq!(instance.start_path, StartPath::Warm); + assert_eq!(instance.runtime_location, RuntimeLocation::WarmPool); + assert_eq!(instance.runtime_owner_token, Some(owner_token)); + assert_eq!(instance.backend_ownership, BackendOwnership::Running); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); + } + + #[test] + fn warm_claim_rejects_an_unstable_backend_owner() { + let error = SandboxInstance::new_warm_claim( + Uuid::new_v4(), + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:warm".into(), + "warm-policy".into(), + BackendOwnership::Starting, + Uuid::new_v4(), + ) + .expect_err("starting backend cannot be claimed"); + + assert!(matches!(error, BlazeError::BackendError { .. })); + } + + #[test] + fn create_journal_round_trips() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + instance.persist(tmp.path()).expect("persist"); + + let mut loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + assert_eq!( + loaded.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); + loaded.finish_operation(); + assert!(loaded.operation.is_none()); + } + + #[test] + fn unfinished_journal_cannot_be_overwritten() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone().expect("journal"); + + let error = instance + .begin_operation(OperationKind::Destroy) + .expect_err("unfinished operation must be preserved"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "destroy" + )); + assert_eq!(instance.operation, Some(journal)); + } + + #[test] + fn checkpoint_journal_preserves_identity_and_phase() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect("begin checkpoint"); + instance + .advance_checkpoint_phase(OperationPhase::CheckpointPublished) + .expect("advance checkpoint"); + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + let journal = loaded.operation.expect("checkpoint journal"); + assert_eq!(journal.kind, OperationKind::Checkpoint); + assert_eq!( + journal.checkpoint_id.as_deref(), + Some("ckpt-00000000-0000-0000-0000-000000000001") + ); + assert_eq!(journal.phase, Some(OperationPhase::CheckpointPublished)); + } + + #[test] + fn checkpoint_journal_rejects_phase_regression() { + let mut instance = fresh(); + instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect("begin checkpoint"); + instance + .advance_checkpoint_phase(OperationPhase::CheckpointPublished) + .expect("advance checkpoint"); + + let error = instance + .advance_checkpoint_phase(OperationPhase::CheckpointPaused) + .expect_err("checkpoint phase must remain a durable lower bound"); + + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!( + instance + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + } + + #[test] + fn checkpoint_journal_cannot_replace_an_active_operation() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone(); + + let error = instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect_err("checkpoint must not replace create"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "checkpoint" + )); + assert_eq!(instance.operation, journal); + } + + #[test] + fn restore_journal_round_trips_without_overwriting_last_checkpoint() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + let completed = "ckpt-00000000-0000-0000-0000-000000000001".to_string(); + let selected = "ckpt-00000000-0000-0000-0000-000000000002".to_string(); + instance.last_checkpoint = Some(completed.clone()); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance + .begin_restore_operation(selected.clone()) + .expect("begin restore"); + instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect("stage storage"); + instance + .advance_restore_phase(OperationPhase::RestoreBackendStopped) + .expect("stop backend"); + instance + .transition(SandboxState::Restoring) + .expect("restoring"); + + for phase in [ + OperationPhase::RestoreStorageActivated, + OperationPhase::RestoreBackendStarted, + OperationPhase::RestoreHeadUpdated, + OperationPhase::RestoreStorageCommitted, + ] { + instance + .advance_restore_phase(phase) + .expect("advance restore"); + assert_eq!( + instance.last_checkpoint.as_deref(), + Some(completed.as_str()) + ); + } + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + let journal = loaded.operation.expect("restore journal"); + assert_eq!(journal.kind, OperationKind::Restore); + assert_eq!(journal.checkpoint_id.as_deref(), Some(selected.as_str())); + assert_eq!(journal.phase, Some(OperationPhase::RestoreStorageCommitted)); + assert_eq!(loaded.last_checkpoint.as_deref(), Some(completed.as_str())); + assert_eq!( + serde_json::to_value(journal.kind).expect("serialize kind"), + serde_json::json!("restore") + ); + assert_eq!( + serde_json::to_value(journal.phase).expect("serialize phase"), + serde_json::json!("restore-storage-committed") + ); + } + + #[test] + fn restore_journal_rejects_phase_regression() { + let mut instance = fresh(); + let completed = "ckpt-00000000-0000-0000-0000-000000000001".to_string(); + instance.last_checkpoint = Some(completed.clone()); + instance + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000002".to_string()) + .expect("begin restore"); + instance + .advance_restore_phase(OperationPhase::RestoreStorageActivated) + .expect("advance restore"); + + let error = instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect_err("restore phase must remain a durable lower bound"); + + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!( + instance + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::RestoreStorageActivated) + ); + assert_eq!(instance.last_checkpoint, Some(completed)); + } + + #[test] + fn operation_journals_reject_phases_from_the_other_operation() { + let mut checkpoint = fresh(); + checkpoint + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect("begin checkpoint"); + let checkpoint_journal = checkpoint.operation.clone(); + let checkpoint_error = checkpoint + .advance_checkpoint_phase(OperationPhase::RestoreBackendStopped) + .expect_err("checkpoint cannot record restore progress"); + assert!(matches!( + checkpoint_error, + BlazeError::InvalidStateTransition { .. } + )); + assert_eq!(checkpoint.operation, checkpoint_journal); + + let mut restore = fresh(); + restore + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000002".to_string()) + .expect("begin restore"); + let restore_journal = restore.operation.clone(); + let restore_error = restore + .advance_restore_phase(OperationPhase::CheckpointPublished) + .expect_err("restore cannot record checkpoint progress"); + assert!(matches!( + restore_error, + BlazeError::InvalidStateTransition { .. } + )); + assert_eq!(restore.operation, restore_journal); + } + + #[test] + fn restore_journal_cannot_replace_an_active_operation() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone(); + + let error = instance + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect_err("restore must not replace create"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "restore" + )); + assert_eq!(instance.operation, journal); + } } diff --git a/src/blaze/crates/blaze-core/src/policy.rs b/src/blaze/crates/blaze-core/src/policy.rs index bfcbac3aa8..b3e86888af 100644 --- a/src/blaze/crates/blaze-core/src/policy.rs +++ b/src/blaze/crates/blaze-core/src/policy.rs @@ -184,15 +184,15 @@ impl PolicyFile { }); } - // Validate [pool].warm_ttl format (e.g. "30s", "30m", "1h", "1d"; pure numbers are illegal). + // Validate an explicit pool TTL while allowing the daemon default to apply when omitted. if let Some(pool) = self.pool.as_ref() - && parse_duration(&pool.warm_ttl).is_none() + && let Some(warm_ttl) = pool.warm_ttl.as_deref() + && parse_duration(warm_ttl).is_none() { return Err(BlazeError::PolicyEvalError { reason: format!( "policy \"{policy_name}\": [pool].warm_ttl must be a duration like \"30s\", \"30m\", \"1h\", \"1d\", got \"{warm_ttl}\"", policy_name = self.policy_name, - warm_ttl = pool.warm_ttl ), }); } @@ -297,16 +297,13 @@ pub struct PolicyPool { pub target: u32, #[serde(default)] pub max: u32, - #[serde(default = "default_warm_ttl")] - pub warm_ttl: String, + /// Optional TTL override; the daemon-wide pool default applies when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub warm_ttl: Option, #[serde(default)] pub reset_mode: ResetMode, } -fn default_warm_ttl() -> String { - "30m".to_string() -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PolicyCheckpoint { #[serde(default)] @@ -358,9 +355,14 @@ pub struct FirecrackerConfig { /// Guest kernel command line. #[serde(default = "default_fc_boot_args")] pub boot_args: String, - /// Enable virtio-vsock (Phase 2+; parsed today but not yet wired). + /// Enable guest operations over virtio-vsock. + /// + /// This is disabled by default and requires a compatible guest service. #[serde(default)] pub enable_vsock: bool, + /// Create an isolated network namespace, veth uplink, tap, and NAT. + #[serde(default)] + pub enable_network: bool, /// Capture guest ttyS0 output (Firecracker stdout) to `serial.log`. #[serde(default)] pub serial_log: bool, @@ -377,6 +379,7 @@ impl Default for FirecrackerConfig { Self { boot_args: default_fc_boot_args(), enable_vsock: false, + enable_network: false, serial_log: false, vcpus: None, memory: None, @@ -817,6 +820,7 @@ memory = "4G" [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" +enable_network = true serial_log = true [hooks.on_create] @@ -831,6 +835,38 @@ sequence = ["template-reg:bind-mm-template"] assert_eq!(pf.match_.workload_class, WorkloadClass::AgentRl); assert_eq!(pf.select.backend_priority[0], BackendKind::KataFc); assert!(pf.pool.as_ref().expect("pool").enabled); + assert!( + pf.backend + .firecracker + .as_ref() + .expect("firecracker") + .enable_network + ); + assert_eq!( + pf.pool.as_ref().and_then(|pool| pool.warm_ttl.as_deref()), + Some("30m") + ); + } + + #[test] + fn omitted_pool_ttl_remains_unset_for_daemon_resolution() { + let raw = r#" +manifest_version = 1 +policy_name = "test" + +[match] +workload_class = "agent-tool" + +[select] +backend_priority = ["mock"] + +[pool] +enabled = true +"#; + let policy: PolicyFile = toml::from_str(raw).expect("parse"); + + policy.validate().expect("omitted TTL is valid"); + assert!(policy.pool.as_ref().expect("pool").warm_ttl.is_none()); } #[test] @@ -929,6 +965,7 @@ memory = "2G" .expect("firecracker config"); assert_eq!(fc.vcpus, Some(2)); assert_eq!(fc.memory, Some("2G".to_string())); + assert!(!fc.enable_network); } #[test] diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index aa4961a6ca..396754698e 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -5,7 +5,8 @@ //! (warm pools, copy-on-write, content-addressable dedup) but present //! a uniform interface to the daemon layer. -use std::path::PathBuf; +use std::fs::File; +use std::path::{Path, PathBuf}; use async_trait::async_trait; use thiserror::Error; @@ -32,6 +33,19 @@ pub struct StorageSlot { pub instance_dir: PathBuf, } +/// Stable handle for one provider-owned rootfs restore transaction. +/// +/// Callers must keep this handle from staging through activation and +/// finalization. Providers must validate both fields against durable state +/// before changing storage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageRestoreTransaction { + /// Stable sandbox identifier whose rootfs is being replaced. + pub instance_id: String, + /// Unique transaction identifier used to reject stale handles. + pub transaction_id: uuid::Uuid, +} + /// Pool readiness status. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct PoolStatus { @@ -53,6 +67,40 @@ pub struct AcquireOpts { pub mem_size: u64, } +/// One already-open runtime-template artifact. +/// +/// The open file object binds later materialization to the object that was +/// validated by the catalog, even if its catalog path is replaced afterward. +#[derive(Debug)] +pub struct RuntimeTemplateArtifact { + /// Stable source object positioned at the beginning of the artifact. + pub file: File, + /// Exact byte length recorded by the template manifest. + pub size_bytes: u64, + /// Lowercase SHA-256 digest recorded by the template manifest. + pub sha256: String, +} + +/// Self-contained artifacts needed to restore one runtime template. +#[derive(Debug)] +pub struct RuntimeTemplateStorage { + /// Backend VM-state snapshot. + pub vmstate: RuntimeTemplateArtifact, + /// Guest-memory snapshot. + pub memory: RuntimeTemplateArtifact, + /// Independent root filesystem snapshot. + pub rootfs: RuntimeTemplateArtifact, +} + +/// Provider-owned storage produced from one runtime template. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeTemplateStorageSlot { + /// Writable storage owned by the new sandbox. + pub storage: StorageSlot, + /// Provider-owned backend VM-state snapshot. + pub snapshot_path: PathBuf, +} + /// Storage allocation failure with an optional residual slot owner. /// /// A provider returns `residual` only when rollback could not remove resources @@ -107,6 +155,26 @@ pub trait StorageProvider: Send + Sync { opts: &AcquireOpts, ) -> std::result::Result; + /// Materialize a self-contained runtime template into a new owned slot. + /// + /// Providers must not retain paths into the catalog. Every artifact used + /// by the restored sandbox must be copied into provider-owned storage. + async fn acquire_runtime_template( + &self, + opts: &AcquireOpts, + source: RuntimeTemplateStorage, + ) -> std::result::Result { + let _ = (opts, source); + Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: "storage provider does not support runtime templates".to_string(), + })) + } + + /// Report whether runtime-template materialization is implemented. + fn supports_runtime_templates(&self) -> bool { + false + } + /// Release a storage slot (cleanup all associated resources). async fn release(&self, slot: StorageSlot) -> Result<()>; @@ -119,6 +187,28 @@ pub trait StorageProvider: Send + Sync { self.release(slot).await } + /// Report whether pool-owned slots can be inventoried and released by ID. + /// + /// Returning true promises both a complete [`Self::list_owned_ids`] + /// inventory for the provider's currently configured root and an + /// idempotent [`Self::release_by_id`] that can retry complete, missing, + /// and partially created slots. It does not identify a provider or root + /// recorded by another subsystem. + fn supports_runtime_pool_recovery(&self) -> bool { + false + } + + /// List every stable slot identifier currently owned by the provider. + /// + /// Implementations must reject entries that cannot be classified as a + /// provider-owned slot. Callers use this inventory only when + /// [`Self::supports_runtime_pool_recovery`] returns true. + async fn list_owned_ids(&self) -> Result> { + Err(BlazeError::StorageError { + msg: "storage provider does not expose stable slot inventory".to_string(), + }) + } + /// Reconstruct a previously allocated slot from a stable instance id. /// /// Implementations must derive every returned path from their configured @@ -126,11 +216,98 @@ pub trait StorageProvider: Send + Sync { async fn reconstruct(&self, instance_id: &str) -> Result; /// Flush dirty data to persistent storage (implementation may be no-op). + /// + /// The daemon may cancel this future when its configured attempt deadline + /// or shutdown signal wins. Cancellation must retain slot ownership and + /// leave a later synchronization or cleanup attempt safe. async fn flush_dirty(&self, slot: &StorageSlot) -> Result<()>; + /// Report whether this provider can capture a self-contained checkpoint. + /// + /// The default is conservative so existing providers do not advertise a + /// data path they have not implemented. + fn supports_checkpoint_capture(&self) -> bool { + false + } + + /// Capture the slot's writable root filesystem at `target`. + async fn capture_checkpoint(&self, slot: &StorageSlot, target: &Path) -> Result<()> { + let _ = (slot, target); + Err(BlazeError::StorageError { + msg: "storage provider does not support checkpoint capture".to_string(), + }) + } + + /// Report whether this provider can restore a self-contained checkpoint. + /// + /// The default is conservative so existing providers cannot enter a + /// partially implemented replacement flow. + fn supports_checkpoint_restore(&self) -> bool { + false + } + + /// Copy a checkpoint rootfs into provider-owned staging storage. + /// + /// Staging must leave the live rootfs unchanged so callers may prepare the + /// replacement before stopping the current runtime. + async fn stage_checkpoint_restore( + &self, + slot: &StorageSlot, + source: &Path, + ) -> Result { + let _ = (slot, source); + Err(checkpoint_restore_unsupported()) + } + + /// Select the staged rootfs while retaining the previous rootfs. + /// + /// A successful activation must remain abortable until + /// [`Self::commit_checkpoint_restore`] starts. + async fn activate_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Finalize an activated rootfs and release its retained predecessor. + async fn commit_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Restore the predecessor retained by a staged or activated transaction. + async fn abort_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Resolve an interrupted restore transaction after process restart. + /// + /// Implementations choose the outcome from durable transaction state: + /// work not yet committed should roll back, while a durable commit intent + /// should finish committing. + async fn reconcile_checkpoint_restore(&self, instance_id: &str) -> Result<()> { + let _ = instance_id; + Err(checkpoint_restore_unsupported()) + } + /// Query warm pool status. fn pool_status(&self) -> PoolStatus; /// Drain all ready slots from the warm pool. async fn drain_pool(&self) -> Result; } + +fn checkpoint_restore_unsupported() -> BlazeError { + BlazeError::StorageError { + msg: "storage provider does not support checkpoint restore".to_string(), + } +} diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index f4c426ef74..790a3c65e7 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -34,6 +34,10 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } +libc = { workspace = true } +base64 = { workspace = true } +sha2 = { workspace = true } [dev-dependencies] tempfile = "3" +tokio = { workspace = true, features = ["test-util"] } diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index c78985cc94..5b5b3d7386 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -5,44 +5,74 @@ //! than a router framework — the surface is small (~17 endpoints) and //! the cost of a fresh dependency outweighs the readability win. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::convert::Infallible; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use std::time::Duration; -use blaze_core::BlazeError; -use blaze_core::backend::{BackendKind, BackendStatus, SpawnRequest, select_backend}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use blaze_core::backend::{BackendKind, BackendStatus, select_backend}; use blaze_core::kernel::HookKind; -use blaze_core::lifecycle::{BackendOwnership, SandboxInstance, SandboxState, StartPath}; +use blaze_core::lifecycle::{SandboxInstance, SandboxState, StartPath}; use blaze_core::policy::{ImageMetadata, RuntimeDecision, WorkloadClass, parse_duration}; use blaze_core::pool::{PoolConfig, PoolKey}; -use blaze_core::storage::AcquireOpts; -use http_body_util::{BodyExt, Full}; -use hyper::body::{Bytes, Incoming}; +use http_body_util::Full; +use hyper::body::{Body, Bytes, Incoming}; use hyper::header::CONTENT_TYPE; use hyper::{Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::json; +use tokio::task::JoinSet; +use tokio::time::Instant; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; +use crate::guest::MAX_GUEST_FILE_BYTES; +use crate::request_body; +use crate::sandbox::{ + CreateSandbox, HibernateSandbox, RestoreSandbox, RestoreSandboxResult, ResumeSandbox, +}; use crate::state::ServerState; +const MAX_EXEC_TIMEOUT_SECS: u32 = 20; + /// Top-level request handler. Always returns `Ok(Response)`; internal /// errors are turned into JSON error bodies so hyper never sees a panic. pub async fn handle( req: Request, state: Arc, ) -> std::result::Result>, Infallible> { + handle_request(req, state).await +} + +async fn handle_request( + req: Request, + state: Arc, +) -> std::result::Result>, Infallible> +where + B: Body + Unpin, + B::Error: std::fmt::Display, +{ state.metrics.inc(&state.metrics.requests_total); let method = req.method().clone(); let path = req.uri().path().to_string(); let query = req.uri().query().unwrap_or("").to_string(); - let response = match collect_body(req).await { - Ok(body) => dispatch(&method, &path, &query, body, &state).await, - Err(e) => Err(e), + let limit = state + .config + .lock() + .map(|config| config.api.max_body_bytes) + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into())); + let response = match limit { + Ok(limit) => match request_body::collect(req, limit).await { + Ok(body) => dispatch(&method, &path, &query, body, &state).await, + Err(error) => Err(error), + }, + Err(error) => Err(error), }; let resp = match response { @@ -52,9 +82,11 @@ pub async fn handle( Ok(resp) } -async fn collect_body(req: Request) -> Result> { - let collected = req.into_body().collect().await?; - Ok(collected.to_bytes().to_vec()) +const fn max_base64_len(decoded_bytes: usize) -> usize { + decoded_bytes + .saturating_add(2) + .saturating_div(3) + .saturating_mul(4) } async fn dispatch( @@ -73,12 +105,42 @@ async fn dispatch( match (m, parts.as_slice()) { ("GET", ["v1", "health"]) => health(state), - ("GET", ["v1", "instances"]) => list_instances(state), - ("POST", ["v1", "instances"]) => create_instance(state, &body).await, - ("GET", ["v1", "instances", id]) => get_instance(state, id), - ("POST", ["v1", "instances", id, "checkpoint"]) => checkpoint(state, id).await, + ("GET", ["v1", "instances"]) | ("GET", ["v1", "sandboxes"]) => list_instances(state), + ("POST", ["v1", "instances"]) | ("POST", ["v1", "sandboxes"]) => { + create_instance(state, &body).await + } + ("GET", ["v1", "instances", id]) | ("GET", ["v1", "sandboxes", id]) => { + get_instance(state, id) + } + ("POST", ["v1", "sandboxes", id, "exec"]) | ("POST", ["v1", "instances", id, "exec"]) => { + exec_instance(state, id, &body).await + } + ("POST", ["v1", "sandboxes", id, "read"]) | ("POST", ["v1", "instances", id, "read"]) => { + read_instance_file(state, id, &body).await + } + ("POST", ["v1", "sandboxes", id, "write"]) | ("POST", ["v1", "instances", id, "write"]) => { + write_instance_file(state, id, &body).await + } + ("POST", ["v1", "instances", id, "checkpoint"]) + | ("POST", ["v1", "sandboxes", id, "checkpoint"]) => checkpoint(state, id).await, + ("GET", ["v1", "instances", id, "checkpoints"]) + | ("GET", ["v1", "sandboxes", id, "checkpoints"]) => list_checkpoints(state, id).await, + ("POST", ["v1", "instances", id, "rollback", checkpoint_id]) + | ("POST", ["v1", "sandboxes", id, "rollback", checkpoint_id]) => { + rollback(state, id, checkpoint_id).await + } + ("POST", ["v1", "instances", id, "hibernate"]) + | ("POST", ["v1", "sandboxes", id, "hibernate"]) => hibernate(state, id).await, + ("POST", ["v1", "instances", id, "resume"]) + | ("POST", ["v1", "sandboxes", id, "resume"]) => resume(state, id).await, + ("POST", ["v1", "instances", id, "checkpoints", "prune"]) + | ("POST", ["v1", "sandboxes", id, "checkpoints", "prune"]) => { + prune_checkpoints(state, id).await + } ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, - ("POST", ["v1", "instances", id, "destroy"]) => destroy_instance(state, id).await, + ("DELETE", ["v1", "instances", id]) + | ("DELETE", ["v1", "sandboxes", id]) + | ("POST", ["v1", "instances", id, "destroy"]) => destroy_instance(state, id).await, ("GET", ["v1", "pools"]) => list_pools(state), ("GET", ["v1", "pools", backend, class]) => pool_status(state, backend, class), ("POST", ["v1", "pools", backend, class, "drain"]) => drain_pool(state, backend, class), @@ -88,6 +150,11 @@ async fn dispatch( ("POST", ["v1", "templates", "gc"]) => gc_templates(state), ("GET", ["v1", "templates"]) => list_templates(state), ("GET", ["v1", "templates", id]) => inspect_template(state, id), + ("GET", ["v1", "runtime-templates"]) => list_runtime_templates(state).await, + ("GET", ["v1", "runtime-templates", name]) => get_runtime_template(state, name).await, + ("POST", ["v1", "runtime-templates", "import"]) => { + import_runtime_template(state, &body).await + } ("GET", ["v1", "policies"]) => list_policies(state), ("GET", ["v1", "hooks"]) => list_hooks(state), ("GET", ["v1", "metrics"]) => metrics(state), @@ -161,55 +228,51 @@ struct CreateInstanceResp { } fn list_instances(state: &Arc) -> Result>> { - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - let list: Vec<&SandboxInstance> = map.values().collect(); - json_ok(&list) + json_ok(&state.manager.list()?) } fn get_instance(state: &Arc, id: &str) -> Result>> { - let uuid = parse_uuid(id)?; - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - let inst = map - .get(&uuid) - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; - json_ok(inst) + json_ok(&state.manager.get(parse_uuid(id)?)?) } async fn create_instance(state: &Arc, body: &[u8]) -> Result>> { let req: CreateInstanceReq = serde_json::from_slice(body) .map_err(|e| BlazeDaemonError::BadRequest(format!("invalid create body: {e}")))?; - let img = ImageMetadata { + let image = ImageMetadata { digest: req.image_digest.clone(), workload_class: Some(req.workload_class), kernel_version: req.kernel_version.clone(), }; - - // 1. Policy evaluation. - let decision = { + let mut decision = { let engine = state .policy .lock() .map_err(|_| BlazeDaemonError::Internal("policy lock poisoned".into()))?; - match engine.evaluate(&req.labels, &img) { - Ok(d) => d, - Err(e) => { + match engine.evaluate(&req.labels, &image) { + Ok(decision) => decision, + Err(error) => { state.metrics.inc(&state.metrics.policy_eval_failures); - return Err(e.into()); + return Err(error.into()); } } }; + if let Some(pool) = decision.pool.as_mut() + && pool.warm_ttl.is_none() + { + let default_warm_ttl = state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .pool + .default_warm_ttl + .clone(); + pool.warm_ttl = Some(default_warm_ttl); + } - // 2. Backend selection. Constrain availability to the daemon's active - // spawner — only the backend that was actually probed at boot can execute. + // Constrain availability to the implementation selected at daemon boot. let availability: Vec = { - let cfg = state + let config = state .config .lock() .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))?; @@ -219,10 +282,10 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result, body: &[u8]) -> Result b, - Err(e) => { - if state.active_backend == BackendKind::Mock { - *decision.backend_priority.first().ok_or_else(|| { - BlazeDaemonError::Internal("policy has empty backend_priority".into()) - })? - } else { - return Err(e.into()); - } + Ok(backend) => backend, + Err(_) if state.active_backend == BackendKind::Mock => { + *decision.backend_priority.first().ok_or_else(|| { + BlazeDaemonError::Internal("policy has empty backend_priority".into()) + })? } + Err(error) => return Err(error.into()), }; - // Policy chooses an allowed backend preference. Runtime ownership must - // record the spawner that actually serves the request. In portable Mock - // mode those can differ because Mock is an explicit local fallback. let runtime_backend = if state.active_backend == BackendKind::Mock { BackendKind::Mock } else { policy_backend }; - - let pool_key = PoolKey::new( - runtime_backend, - decision.workload_class, - req.image_digest.clone(), - ); - if decision.pool_eligible { - if let Some((instance, selected_backend)) = activate_warm_instance(state, &pool_key).await? - { - state.metrics.inc(&state.metrics.pool_hits); - return json_created(&CreateInstanceResp { - start_path: instance.start_path, - instance, - decision, - selected_backend, - }); - } - state.metrics.inc(&state.metrics.pool_misses); - } - - let start_path = StartPath::Cold; - let mut instance = SandboxInstance::new( - runtime_backend, - decision.workload_class, - req.image_digest.clone(), - start_path, - decision.policy_name.clone(), - ); - instance.transition(SandboxState::Creating)?; - let operation_lock = state.operation_lock(instance.id); - let _operation = operation_lock.lock().await; - - let (binary_path, rootfs_size, mem_size) = { - let cfg = state - .config - .lock() - .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))?; - ( - cfg.backends - .get(state.active_backend.as_str()) - .cloned() - .unwrap_or_default(), - cfg.storage.rootfs_size, - cfg.storage.mem_size, - ) - }; - // Publish ownership before allocation. A restart can now discover this - // stable ID and release either an absent slot or a completed allocation. - instance.persist(&state.state_dir)?; - if let Some(error) = retain_instance_state(state, instance.clone()) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "create {}: {error}", - instance.id - ))); - } - let storage = match state - .storage - .acquire(&AcquireOpts { - instance_id: instance.id.to_string(), - rootfs_size, - mem_size, + let binary_path = state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(state.active_backend.as_str()) + .cloned() + .unwrap_or_default(); + + let created = state + .manager + .create(CreateSandbox { + decision: decision.clone(), + image_digest: req.image_digest, + runtime_backend, + binary_path, }) - .await - { - Ok(storage) => storage, - Err(error) => { - let (source, residual) = error.into_parts(); - return Err(retain_failed_acquire( - state, - &mut instance, - residual, - source.into(), - )); - } - }; - crate::failpoint::pause("create-after-storage-acquire").await; - - instance.backend_ownership = BackendOwnership::Starting; - if let Err(error) = instance.persist(&state.state_dir) { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - error.into(), - ) - .await); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(error), - ) - .await); - } - - let work_dir = state.state_dir.join(instance.id.to_string()); - let spawner = match state.spawner_for(state.active_backend) { - Some(spawner) => spawner, - None => { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(format!( - "active backend {} has no registered spawner", - state.active_backend - )), - ) - .await); - } - }; - let spawn = match crate::failpoint::backend("create-spawn") { - Ok(()) => { - spawner - .spawn(SpawnRequest { - instance_id: instance.id, - run_dir: work_dir, - binary_path, - storage: storage.clone(), - backend: decision.backend.clone(), - vm: decision.vm.clone(), - }) - .await - } - Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), - }; - let actual_backend = match spawn { - Ok(backend_instance) => { - instance.backend_ownership = BackendOwnership::Running; - let real_backend = backend_instance.backend(); - let mut backend_instance = Some(backend_instance); - let registered = match state.backend_instances.lock() { - Ok(mut instances) => { - instances.insert( - instance.id, - backend_instance - .take() - .expect("backend instance is present"), - ); - true - } - Err(_) => false, - }; - if !registered { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - backend_instance, - false, - BlazeDaemonError::Internal("backend_instances lock poisoned".to_string()), - ) - .await); - } - real_backend - } - Err(error) => { - let (source, backend) = error.into_parts(); - instance.backend_ownership = if backend.is_some() { - BackendOwnership::Running - } else { - BackendOwnership::Stopped - }; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - backend, - false, - source.into(), - ) - .await); - } - }; - if let Err(error) = instance.transition(SandboxState::Running) { - return Err( - cleanup_failed_create(state, &mut instance, storage, None, true, error.into()).await, - ); - } - if let Err(error) = crate::failpoint::state("create-state-commit") - .and_then(|_| instance.persist(&state.state_dir).map_err(Into::into)) - { - return Err(cleanup_failed_create(state, &mut instance, storage, None, true, error).await); - } - - let inserted = match state.instances.lock() { - Ok(mut instances) => { - instances.insert(instance.id, instance.clone()); - true - } - Err(_) => false, - }; - if !inserted { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - true, - BlazeDaemonError::Internal("instances lock poisoned".to_string()), - ) - .await); - } - state.metrics.inc(&state.metrics.instances_created); - + .await?; json_created(&CreateInstanceResp { - instance, + start_path: created.instance.start_path, + instance: created.instance, decision, - start_path, - selected_backend: actual_backend, + selected_backend: created.selected_backend, }) } -async fn activate_warm_instance( - state: &Arc, - key: &PoolKey, -) -> Result> { - let candidate = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))? - .lookup(key); - let Some(id) = candidate else { - return Ok(None); - }; - let operation_lock = state.operation_lock(id); - let _operation = operation_lock.lock().await; - - let instance = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .get(&id) - .cloned(); - let backend = state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .get(&id) - .cloned(); - - let invalid_reason = match (&instance, &backend) { - (None, _) => Some("lifecycle metadata is missing".to_string()), - (_, None) => Some("backend owner is missing".to_string()), - (Some(instance), Some(_)) if instance.state != SandboxState::Warm => { - Some(format!("lifecycle state is {}", instance.state)) - } - (Some(instance), Some(backend)) if backend.backend() != instance.backend => Some(format!( - "backend owner is {}, metadata is {}", - backend.backend(), - instance.backend - )), - (_, Some(backend)) => match backend.try_wait().await { - Ok(None) => None, - Ok(Some(status)) => Some(format!("backend exited: {status:?}")), - Err(error) => Some(format!("backend liveness check failed: {error}")), - }, - }; - - if let Some(reason) = invalid_reason { - quarantine_warm_instance(state, key, id, instance, backend, &reason).await; - return Ok(None); - } - - let original = instance.expect("validated warm metadata"); - match state.storage.reconstruct(&id.to_string()).await { - Ok(_) => {} - Err(error @ BlazeError::StorageIncomplete { .. }) => { - quarantine_warm_instance( - state, - key, - id, - Some(original), - backend, - &format!("storage validation failed: {error}"), - ) - .await; - return Ok(None); - } - Err(error) => { - return Err(restore_warm_claim(state, key, original, error.into())); - } - } - - crate::failpoint::pause("warm-before-state-commit").await; - let mut instance = original.clone(); - let selected_backend = backend.expect("validated warm backend").backend(); - if let Err(error) = instance - .transition(SandboxState::Creating) - .and_then(|_| instance.transition(SandboxState::Running)) - .and_then(|_| instance.persist(&state.state_dir)) - { - return Err(restore_warm_claim(state, key, original, error.into())); - } - state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .insert(id, instance.clone()); - Ok(Some((instance, selected_backend))) +async fn checkpoint(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + json_ok(&state.manager.checkpoint(uuid).await?) } -fn restore_warm_claim( - state: &Arc, - key: &PoolKey, - instance: SandboxInstance, - cause: BlazeDaemonError, -) -> BlazeDaemonError { - let id = instance.id; - let mut errors = Vec::new(); - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("restore warm state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance) { - errors.push(error); - } - match state.pool.lock() { - Ok(mut pool) => pool.restore_lookup(key.clone(), id), - Err(poisoned) => { - poisoned.into_inner().restore_lookup(key.clone(), id); - errors.push("pool lock poisoned while restoring warm claim".to_string()); - } - } - let details = if errors.is_empty() { - "warm claim restored for retry".to_string() - } else { - format!("warm claim restored with errors: {}", errors.join("; ")) - }; - BlazeDaemonError::RecoveryRequired(format!("{cause}; instance {id}: {details}")) +async fn list_checkpoints(state: &Arc, id: &str) -> Result>> { + json_ok(&state.manager.list_checkpoints(parse_uuid(id)?).await?) } -async fn quarantine_warm_instance( +async fn rollback( state: &Arc, - key: &PoolKey, - id: Uuid, - mut instance: Option, - backend: Option, - reason: &str, -) { - match state.pool.lock() { - Ok(mut pool) => pool.quarantine(key, id), - Err(poisoned) => poisoned.into_inner().quarantine(key, id), - } - tracing::warn!(instance = %id, reason, "warm instance validation failed"); - - let backend_stopped = match backend.as_ref() { - Some(backend) => match backend.kill().await { - Ok(()) => true, - Err(error) => { - tracing::error!(instance = %id, %error, "quarantined backend cleanup failed"); - false - } - }, - None => match instance.as_ref() { - Some(instance) - if matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) => - { - true - } - Some(instance) => match state.spawner_for(instance.backend) { - Some(spawner) => match spawner - .cleanup_orphan(id, &state.state_dir.join(id.to_string())) - .await - { - Ok(()) => true, - Err(error) => { - tracing::error!( - instance = %id, - %error, - "quarantined orphan cleanup failed" - ); - false - } - }, - None => { - tracing::error!( - instance = %id, - backend = %instance.backend, - "quarantined backend has no recovery spawner" - ); - false - } + id: &str, + checkpoint_id: &str, +) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(instance.backend.as_str()) + .cloned() + .unwrap_or_default(); + let restored: RestoreSandboxResult = state + .manager + .restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint_id.to_string(), + binary_path, }, - None => false, - }, - }; - if !backend_stopped { - return; - } - let Some(instance) = instance.as_mut() else { - tracing::error!( - instance = %id, - "quarantined lifecycle metadata missing; retaining storage" - ); - return; - }; - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.persist(&state.state_dir) { - tracing::error!(instance = %id, %error, "quarantined stop state commit failed"); - return; - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - tracing::error!(instance = %id, %error, "quarantined stop state retention failed"); - return; - } - if let Err(error) = state.storage.release_by_id(&id.to_string()).await { - tracing::error!(instance = %id, %error, "quarantined storage cleanup failed"); - return; - } - - if instance.state != SandboxState::Destroyed - && let Err(error) = instance.transition(SandboxState::Destroyed) - { - tracing::error!(instance = %id, %error, "quarantined lifecycle cleanup failed"); - return; - } - if let Err(error) = instance.persist(&state.state_dir) { - tracing::error!(instance = %id, %error, "quarantined state commit failed"); - return; - } - match state.instances.lock() { - Ok(mut instances) => { - instances.insert(id, instance.clone()); - } - Err(poisoned) => { - poisoned.into_inner().insert(id, instance.clone()); - } - } - match state.backend_instances.lock() { - Ok(mut instances) => { - instances.remove(&id); - } - Err(poisoned) => { - poisoned.into_inner().remove(&id); - } - } -} - -async fn cleanup_failed_create( - state: &Arc, - instance: &mut SandboxInstance, - storage: blaze_core::storage::StorageSlot, - backend: Option, - registered: bool, - original: BlazeDaemonError, -) -> BlazeDaemonError { - let mut cleanup_errors = Vec::new(); - let backend = if registered { - match state.backend_instances.lock() { - Ok(mut instances) => instances.remove(&instance.id), - Err(poisoned) => poisoned.into_inner().remove(&instance.id), - } - } else { - backend - }; - let mut backend_stopped = matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ); - if registered && backend.is_none() { - backend_stopped = false; - cleanup_errors.push("registered backend owner is missing".to_string()); - } - if let Some(backend) = backend.as_ref() { - match backend.kill().await { - Ok(()) => { - backend_stopped = true; - instance.backend_ownership = BackendOwnership::Stopped; - } - Err(error) => { - backend_stopped = false; - cleanup_errors.push(format!("backend termination failed: {error}")); - } - } - } - - let mut storage_released = false; - if backend_stopped { - match state.storage.release(storage).await { - Ok(()) => storage_released = true, - Err(error) => cleanup_errors.push(format!("storage release failed: {error}")), - } - } else { - cleanup_errors.push("storage retained until backend termination succeeds".to_string()); - } - - if backend_stopped && storage_released { - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.transition(SandboxState::Destroyed) { - let mut recovery_errors = vec![format!("lifecycle update failed: {error}")]; - if let Err(persist_error) = instance.persist(&state.state_dir) { - recovery_errors.push(format!("state persistence failed: {persist_error}")); - } - if let Some(retain_error) = retain_instance_state(state, instance.clone()) { - recovery_errors.push(retain_error); - } - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {}", - recovery_errors.join("; ") - )); - } - if let Err(error) = instance.persist(&state.state_dir) { - let mut recovery_errors = vec![format!("state persistence failed: {error}")]; - if let Some(retain_error) = retain_instance_state(state, instance.clone()) { - recovery_errors.push(retain_error); - } - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {}", - recovery_errors.join("; ") - )); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {error}" - )); - } - state.metrics.inc(&state.metrics.instances_destroyed); - return original; - } - - if let Some(backend) = backend - && let Some(error) = retain_backend_owner(state, instance.id, backend) - { - cleanup_errors.push(error); - } - if let Err(error) = instance.persist(&state.state_dir) { - cleanup_errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - cleanup_errors.push(error); - } - BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup incomplete: {}", - cleanup_errors.join("; ") - )) -} - -fn retain_failed_acquire( - state: &Arc, - instance: &mut SandboxInstance, - residual: Option, - original: BlazeDaemonError, -) -> BlazeDaemonError { - if residual.is_some() { - let mut errors = Vec::new(); - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - errors.push(error); - } - let suffix = if errors.is_empty() { - "residual storage retained for destroy retry".to_string() - } else { - format!( - "residual storage retained with recovery errors: {}", - errors.join("; ") - ) - }; - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; instance {}: {suffix}", - instance.id - )); - } - - let mut errors = Vec::new(); - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.transition(SandboxState::Destroyed) { - errors.push(format!("lifecycle update failed: {error}")); - } - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - errors.push(error); - } - if errors.is_empty() { - original - } else { - BlazeDaemonError::RecoveryRequired(format!( - "{original}; acquire rollback completed but {}", - errors.join("; ") - )) - } + ) + .await?; + json_ok(&json!({ + "instance_id": restored.instance.id, + "checkpoint_id": restored.checkpoint_id, + "restored": true, + "state": restored.instance.state, + })) } -fn retain_backend_owner( - state: &Arc, - id: Uuid, - backend: crate::spawner::DynBackendInstance, -) -> Option { - match state.backend_instances.lock() { - Ok(mut instances) => { - instances.insert(id, backend); - None - } - Err(poisoned) => { - poisoned.into_inner().insert(id, backend); - Some("backend owner retained in poisoned runtime map".to_string()) - } - } +async fn hibernate(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = configured_backend_path(state, instance.backend)?; + json_ok( + &state + .manager + .hibernate(uuid, HibernateSandbox { binary_path }) + .await?, + ) } -fn retain_instance_state(state: &Arc, instance: SandboxInstance) -> Option { - match state.instances.lock() { - Ok(mut instances) => { - instances.insert(instance.id, instance); - None - } - Err(poisoned) => { - poisoned.into_inner().insert(instance.id, instance); - Some("instance state retained in poisoned lifecycle map".to_string()) - } - } +async fn resume(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = configured_backend_path(state, instance.backend)?; + json_ok( + &state + .manager + .resume(uuid, ResumeSandbox { binary_path }) + .await?, + ) } -async fn checkpoint(state: &Arc, id: &str) -> Result>> { - let uuid = parse_uuid(id)?; - let operation_lock = state.operation_lock(uuid); - let _operation = operation_lock.lock().await; - let mut map = state - .instances +fn configured_backend_path( + state: &ServerState, + backend: BackendKind, +) -> Result { + Ok(state + .config .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - let inst = map - .get_mut(&uuid) - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; - - if inst.state == SandboxState::Running { - inst.transition(SandboxState::Paused)?; - } - inst.transition(SandboxState::Checkpointed)?; - inst.persist(&state.state_dir)?; + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(backend.as_str()) + .cloned() + .unwrap_or_default()) +} - let checkpoint_id = format!("ckpt-{}-{}", inst.id, chrono::Utc::now().timestamp()); +async fn prune_checkpoints(state: &Arc, id: &str) -> Result>> { + let removed = state.manager.prune_checkpoints(parse_uuid(id)?).await?; json_ok(&json!({ - "checkpoint_id": checkpoint_id, - "instance_id": inst.id, + "removed": removed, + "count": removed.len(), })) } async fn reset_instance(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let _operation = state + .manager + .lock_quiescent_state(uuid, SandboxState::Running) + .await?; + + Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {uuid} cannot be reset until its backend can reset runtime and storage state" + ))) +} + +#[cfg(test)] +async fn return_to_pool_for_test(state: &Arc, id: &str) -> Result<()> { let uuid = parse_uuid(id)?; let operation_lock = state.operation_lock(uuid); let _operation = operation_lock.lock().await; @@ -922,9 +447,6 @@ async fn reset_instance(state: &Arc, id: &str) -> Result, id: &str) -> Result, id: &str) -> Result, id: &str) -> Result>> { let uuid = parse_uuid(id)?; - let operation_lock = state.operation_lock(uuid); - let _operation = operation_lock.lock().await; - let mut original = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .get(&uuid) - .cloned() - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; - let backend = state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .get(&uuid) - .cloned(); - - let stop_result = match crate::failpoint::backend("destroy-kill") { - Ok(()) => { - if let Some(backend) = backend.as_ref() { - backend.kill().await - } else if matches!( - original.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) { - Ok(()) - } else { - match state.spawner_for(original.backend) { - Some(spawner) => { - spawner - .cleanup_orphan(uuid, &state.state_dir.join(uuid.to_string())) - .await - } - None => Err(BlazeError::BackendError { - msg: format!( - "no recovery spawner registered for persisted backend {}", - original.backend - ), - }), - } - } - } - Err(error) => Err(error), - }; - if let Err(error) = stop_result { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend termination failed: {error}; owner and storage retained" + state.manager.destroy(uuid).await?; + json_ok(&json!({ + "destroyed": true, + "instance_id": uuid, + })) +} + +#[derive(Debug, Deserialize)] +struct ExecRequest { + cmd: String, + #[serde(default)] + cwd: Option, + #[serde(default)] + env: Option>, + #[serde(default)] + timeout: Option, +} + +async fn exec_instance( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: ExecRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid exec body: {error}")))?; + if request.cmd.is_empty() { + return Err(BlazeDaemonError::BadRequest( + "exec command is required".to_string(), + )); + } + let timeout = request.timeout.unwrap_or(MAX_EXEC_TIMEOUT_SECS); + if timeout == 0 || timeout > MAX_EXEC_TIMEOUT_SECS { + return Err(BlazeDaemonError::BadRequest(format!( + "exec timeout must be between 1 and {MAX_EXEC_TIMEOUT_SECS} seconds" ))); } + let result = state + .manager + .exec( + parse_uuid(id)?, + request.cmd, + request.cwd, + request.env, + timeout, + ) + .await?; + json_ok(&json!({ + "exit_code": result.exit_code, + "stdout_b64": BASE64.encode(result.stdout), + "stderr_b64": BASE64.encode(result.stderr), + })) +} - original.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = original.persist(&state.state_dir) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but stop state persistence failed: {error}; storage retained" - ))); +#[derive(Debug, Deserialize)] +struct FileRequest { + path: String, + #[serde(default)] + data_b64: Option, +} + +async fn read_instance_file( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: FileRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid read body: {error}")))?; + let data = state + .manager + .read_file(parse_uuid(id)?, request.path) + .await?; + json_ok(&json!({"data_b64": BASE64.encode(data)})) +} + +async fn write_instance_file( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: FileRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid write body: {error}")))?; + let encoded = request + .data_b64 + .ok_or_else(|| BlazeDaemonError::BadRequest("data_b64 is required".to_string()))?; + let data = decode_guest_file(&encoded, MAX_GUEST_FILE_BYTES)?; + state + .manager + .write_file(parse_uuid(id)?, request.path, &data) + .await?; + json_ok(&json!({"written": true, "bytes": data.len()})) +} + +fn decode_guest_file(encoded: &str, limit: usize) -> Result> { + let encoded_limit = max_base64_len(limit); + if encoded.len() > encoded_limit { + return Err(crate::guest::GuestError::PayloadTooLarge { + actual: encoded.len(), + limit: encoded_limit, + } + .into()); } - if let Some(error) = retain_instance_state(state, original.clone()) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but lifecycle retention failed: {error}; storage retained" - ))); + let data = BASE64 + .decode(encoded) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid base64: {error}")))?; + if data.len() > limit { + return Err(crate::guest::GuestError::PayloadTooLarge { + actual: data.len(), + limit, + } + .into()); } + Ok(data) +} - if let Err(error) = state.storage.release_by_id(&uuid.to_string()).await { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but storage release failed: {error}; lifecycle retained for retry" - ))); +/// Stop every tracked sandbox after the daemon has stopped accepting work. +/// +/// Cleanup starts concurrently for all known owners and shares one deadline. +/// This lets independent owners finish after another owner fails or stalls +/// without multiplying the daemon's shutdown time by the sandbox count. +/// Timed-out sandbox tasks are cancelled and joined. Runtime-pool shutdown +/// retains control of its nested worker and joins that worker itself. +pub(crate) async fn shutdown_instances(state: &Arc, budget: Duration) -> Result<()> { + let ids = state.manager.owned_instance_ids()?; + + let deadline = Instant::now() + budget; + let mut tasks = JoinSet::new(); + let mut task_owners = HashMap::new(); + for id in ids { + let state = state.clone(); + let task = tasks.spawn(async move { state.manager.destroy(id).await.map(|_| ()) }); + task_owners.insert(task.id(), id.to_string()); + } + let pool_state = state.clone(); + let mut pool_shutdown = Box::pin(async move { + pool_state + .manager + .shutdown_runtime_pool_until(deadline) + .await + }); + let mut pool_pending = true; + let mut deadline_sleep = Box::pin(tokio::time::sleep_until(deadline)); + + let mut failures = BTreeMap::new(); + let mut deadline_expired = false; + while !tasks.is_empty() || pool_pending { + tokio::select! { + result = &mut pool_shutdown, if pool_pending => { + pool_pending = false; + if let Err(error) = result { + failures.insert("runtime-pool".to_string(), error.to_string()); + } + } + task = tasks.join_next_with_id(), if !tasks.is_empty() => { + match task { + Some(Ok((task_id, result))) => { + let Some(id) = task_owners.remove(&task_id) else { + failures.insert( + format!("task-{task_id}"), + "cleanup result had no tracked sandbox owner".to_string(), + ); + continue; + }; + if let Err(error) = result { + failures.insert(id, error.to_string()); + } + } + Some(Err(error)) => { + let key = task_owners + .remove(&error.id()) + .unwrap_or_else(|| format!("task-{}", error.id())); + failures.insert(key, format!("cleanup task failed: {error}")); + } + None => {} + } + } + _ = &mut deadline_sleep => { + deadline_expired = true; + break; + } + } } - let mut destroyed = original; - if destroyed.state != SandboxState::Destroyed { - destroyed.transition(SandboxState::Destroyed)?; + if deadline_expired { + let mut timed_out_owners = task_owners.values().cloned().collect::>(); + tasks.abort_all(); + while let Some(result) = tasks.join_next_with_id().await { + match result { + Ok((task_id, result)) => { + let Some(id) = task_owners.remove(&task_id) else { + failures.insert( + format!("task-{task_id}"), + "cleanup result had no tracked sandbox owner".to_string(), + ); + continue; + }; + timed_out_owners.remove(&id); + if let Err(error) = result { + failures.insert(id, error.to_string()); + } + } + Err(error) => { + let owner = task_owners.remove(&error.id()); + let key = owner + .clone() + .unwrap_or_else(|| format!("task-{}", error.id())); + if !error.is_cancelled() { + if let Some(id) = owner { + timed_out_owners.remove(&id); + } + failures.insert(key, format!("cleanup task failed: {error}")); + } + } + } + } + for id in timed_out_owners { + failures.entry(id).or_insert_with(|| { + format!("cleanup did not finish within the shared {budget:?} budget") + }); + } + if pool_pending { + pool_pending = false; + if let Err(error) = pool_shutdown.await { + failures.insert("runtime-pool".to_string(), error.to_string()); + } + } } - destroyed.persist(&state.state_dir)?; - state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .insert(uuid, destroyed.clone()); - state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .remove(&uuid); + debug_assert!(!pool_pending); - state.metrics.inc(&state.metrics.instances_destroyed); - json_ok(&json!({ - "destroyed": true, - "instance_id": destroyed.id, - })) + if failures.is_empty() { + Ok(()) + } else { + let failures = failures + .into_iter() + .map(|(owner, error)| format!("{owner}: {error}")) + .collect::>(); + Err(BlazeDaemonError::RecoveryRequired(format!( + "daemon shutdown left {} runtime cleanup operation(s) incomplete: {}", + failures.len(), + failures.join("; ") + ))) + } } // --------------------------------------------------------------------------- @@ -1196,6 +869,39 @@ fn inspect_template(state: &Arc, id: &str) -> Result) -> Result>> { + json_ok(&state.manager.list_runtime_templates().await?) +} + +async fn get_runtime_template( + state: &Arc, + name: &str, +) -> Result>> { + json_ok(&state.manager.get_runtime_template(name.to_string()).await?) +} + +#[derive(Debug, Deserialize)] +struct ImportRuntimeTemplateRequest { + name: String, + source: PathBuf, + #[serde(default)] + description: String, +} + +async fn import_runtime_template( + state: &Arc, + body: &[u8], +) -> Result>> { + let request: ImportRuntimeTemplateRequest = serde_json::from_slice(body).map_err(|error| { + BlazeDaemonError::BadRequest(format!("invalid runtime template import body: {error}")) + })?; + let imported = state + .manager + .import_runtime_template(request.name, request.source, request.description) + .await?; + json_response(StatusCode::CREATED, &imported) +} + fn gc_templates(state: &Arc) -> Result>> { let idle_ttl = { let cfg = state @@ -1275,10 +981,13 @@ fn json_response(status: StatusCode, value: &T) -> Result Response> { let status = StatusCode::from_u16(err.status_code()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - let body = json!({ + let mut body = json!({ "error": err.to_string(), "status": status.as_u16(), }); + if let Some(code) = err.api_code() { + body["code"] = json!(code); + } let bytes = serde_json::to_vec_pretty(&body) .unwrap_or_else(|_| br#"{"error":"serialize_failed"}"#.to_vec()); Response::builder() @@ -1300,14 +1009,20 @@ fn _hookkind_marker(_k: HookKind) {} #[cfg(test)] mod tests { use std::collections::HashMap; + use std::future; use std::path::{Path, PathBuf}; - use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; use async_trait::async_trait; - use blaze_core::backend::BackendKind; + use blaze_core::BlazeError; + use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; + #[cfg(feature = "test-failpoints")] + use blaze_core::lifecycle::OperationPhase; + use blaze_core::lifecycle::{BackendOwnership, OperationKind, RuntimeLocation}; use blaze_core::policy::{ BackendConfigs, FallbackOnMissingHook, PolicyEngine, PolicyFile, PolicyHooks, PolicyMatch, PolicyPool, PolicySelect, ResetMode, WorkloadClass, @@ -1317,8 +1032,12 @@ mod tests { AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, }; use blaze_core::template::TemplateRegistry; + use http_body_util::BodyExt; use crate::file_provider::FileStorageProvider; + use crate::runtime_pool::PoolPrototype; + #[cfg(target_os = "linux")] + use crate::spawner::BubblewrapSpawner; use crate::spawner::{ BackendInstance, BackendSpawner, DynBackendInstance, DynSpawner, MockSpawner, SpawnFailure, SpawnResult, SpawnerRegistry, @@ -1338,6 +1057,7 @@ mod tests { config.daemon.state_dir = temp.path().join("state"); config.storage.images_dir = temp.path().join("images"); config.storage.instances_dir = temp.path().join("instances"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); std::fs::create_dir_all(&config.storage.images_dir).expect("images"); std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); @@ -1364,7 +1084,7 @@ mod tests { min: 0, target: 0, max: 1, - warm_ttl: "30m".into(), + warm_ttl: Some("30m".into()), reset_mode: ResetMode::FullRecreate, }), checkpoint: None, @@ -1390,21 +1110,28 @@ mod tests { active_backend: BackendKind, storage: Arc, ) -> Arc { - Arc::new(ServerState::build( - config, - PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), - registry, - active_backend, - storage, - )) + Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(vec![policy]), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + registry, + active_backend, + storage, + ) + .expect("build server state"), + ) } #[cfg(feature = "test-failpoints")] fn mock_state(temp: &tempfile::TempDir, pooled: bool) -> Arc { - let config = test_config(temp); + mock_state_from_config(test_config(temp), pooled) + } + + #[cfg(feature = "test-failpoints")] + fn mock_state_from_config(config: DaemonConfig, pooled: bool) -> Arc { let storage: Arc = Arc::new(FileStorageProvider::with_images( config.storage.images_dir.clone(), config.storage.instances_dir.clone(), @@ -1431,6 +1158,224 @@ mod tests { .expect("created json") } + async fn wait_for_ready_runtime(state: &Arc) -> Uuid { + let runtime_root = state.state_dir.join("runtime-pool"); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if state.manager.runtime_pool_status().ready > 0 + && let Ok(mut entries) = tokio::fs::read_dir(&runtime_root).await + { + while let Ok(Some(entry)) = entries.next_entry().await { + let Ok(instance_id) = Uuid::parse_str(&entry.file_name().to_string_lossy()) + else { + continue; + }; + let Ok(raw) = tokio::fs::read(entry.path().join("ownership.json")).await + else { + continue; + }; + let Ok(ownership) = serde_json::from_slice::(&raw) + else { + continue; + }; + if ownership["phase"]["kind"] == "ready" { + return instance_id; + } + } + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + }) + .await + .expect("runtime pool produces a ready slot") + } + + fn warm_runtime_state( + temp: &tempfile::TempDir, + prefork: bool, + spawner: DynSpawner, + ) -> (Arc, PathBuf) { + let mut config = test_config(temp); + config.storage.pool_size = 1; + config.storage.prefork = prefork; + std::fs::create_dir_all(config.daemon.state_dir.join("runtime-pool")) + .expect("runtime pool root"); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + ( + build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, spawner), + BackendKind::Mock, + storage, + ), + instances_dir, + ) + } + + async fn assert_warm_runtime_round_trip(prefork: bool) { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state(&temp, prefork, Arc::new(MockSpawner)); + + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let claimed = created_json(&state, &test_request()).await; + let claimed_id = + Uuid::parse_str(claimed["instance"]["id"].as_str().expect("claimed ID")).expect("UUID"); + + assert_eq!(claimed_id, ready_id); + assert_eq!(claimed["start_path"], "warm"); + assert_eq!(claimed["instance"]["runtime_location"], "warm-pool"); + assert!(claimed["instance"]["runtime_owner_token"].is_string()); + + state.manager.begin_shutdown(); + assert!( + state + .manager + .destroy(claimed_id) + .await + .expect("destroy claim") + ); + let terminal = state.manager.get(claimed_id).expect("terminal lifecycle"); + assert!(terminal.is_clean_terminal()); + assert!(!instances_dir.join(claimed_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(claimed_id.to_string()) + .exists() + ); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(".cleanup") + .join(claimed_id.to_string()) + .exists() + ); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown remaining owners"); + } + + async fn write_checkpoint_fixture(state: &Arc, id: &str) -> StorageSlot { + let slot = state.storage.reconstruct(id).await.expect("storage slot"); + tokio::fs::write(&slot.rootfs_path, b"checkpoint-rootfs") + .await + .expect("rootfs"); + slot + } + + #[cfg(feature = "test-failpoints")] + async fn cancel_checkpoint_at(state: &Arc, id: Uuid, failpoint: &'static str) { + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = + tokio::spawn( + async move { capture_hook.run(capture_state.manager.checkpoint(id)).await }, + ); + hook.wait_until_paused().await; + capture.abort(); + let cancelled = capture + .await + .expect_err("checkpoint task must be cancelled"); + assert!(cancelled.is_cancelled()); + } + + struct NoCheckpointStorage { + inner: FileStorageProvider, + } + + #[async_trait] + impl StorageProvider for NoCheckpointStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + async fn dispatched_json( + state: &Arc, + method: Method, + path: &str, + body: Vec, + ) -> (StatusCode, serde_json::Value) { + let response = dispatch(&method, path, "", body, state) + .await + .expect("dispatch"); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value = serde_json::from_slice(&body).expect("response json"); + (status, value) + } + + async fn handled_json( + state: &Arc, + method: Method, + path: &str, + body: Vec, + ) -> (StatusCode, serde_json::Value) { + let request = Request::builder() + .method(method) + .uri(path) + .header(hyper::header::CONTENT_LENGTH, body.len()) + .body(Full::new(Bytes::from(body))) + .expect("request"); + let response = handle_request(request, state.clone()) + .await + .expect("infallible response"); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value = serde_json::from_slice(&body).expect("response json"); + (status, value) + } + struct TransientReconstructStorage { inner: FileStorageProvider, fail_reconstruct: AtomicBool, @@ -1508,6 +1453,10 @@ mod tests { let instance = SandboxInstance::load(&self.state_dir, id).expect("ownership published"); assert_eq!(instance.state, SandboxState::Creating); assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); self.observed.store(true, Ordering::Release); self.inner.acquire(opts).await } @@ -1562,6 +1511,108 @@ mod tests { } } + #[derive(Clone, Copy)] + enum ShutdownBehavior { + Complete, + Fail, + Stall, + } + + struct ShutdownOwner { + instance_id: Uuid, + behavior: ShutdownBehavior, + attempts: Arc, + active: Arc, + } + + struct ActiveCleanup(Arc); + + impl ActiveCleanup { + fn enter(active: Arc) -> Self { + active.fetch_add(1, Ordering::AcqRel); + Self(active) + } + } + + impl Drop for ActiveCleanup { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + #[async_trait] + impl BackendInstance for ShutdownOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(None) + } + + async fn kill(&self) -> blaze_core::Result<()> { + self.attempts.fetch_add(1, Ordering::AcqRel); + let _active = ActiveCleanup::enter(self.active.clone()); + match self.behavior { + ShutdownBehavior::Complete => Ok(()), + ShutdownBehavior::Fail => Err(BlazeError::BackendError { + msg: format!("instance {} termination failed", self.instance_id), + }), + ShutdownBehavior::Stall => future::pending().await, + } + } + } + + async fn track_shutdown_owner( + state: &Arc, + behavior: ShutdownBehavior, + attempts: Arc, + active: Arc, + ) -> Uuid { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:shutdown-budget".into(), + StartPath::Cold, + "shutdown-budget-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance.persist(&state.state_dir).expect("persist"); + state + .storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 4096, + mem_size: 4096, + }) + .await + .expect("storage"); + + let id = instance.id; + state + .instances + .lock() + .expect("instances") + .insert(id, instance); + state + .manager + .insert_backend_owner( + id, + Arc::new(ShutdownOwner { + instance_id: id, + behavior, + attempts, + active, + }), + ) + .expect("retain backend owner"); + id + } + struct PartialSpawnSpawner; #[async_trait] @@ -1626,12 +1677,696 @@ mod tests { } } - /// When multiple backend binaries exist on disk but the daemon probed - /// Firecracker at boot, only Firecracker should be reported available - /// and selected — even if policy prioritizes bubblewrap higher. - #[tokio::test] - async fn availability_constrained_to_active_backend() { - // Create temp files to simulate both binaries existing. + struct SelectiveCleanupSpawner { + failed_id: Uuid, + cleanup_count: Arc, + } + + #[async_trait] + impl BackendSpawner for SelectiveCleanupSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + MockSpawner.spawn(request).await + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + self.cleanup_count.fetch_add(1, Ordering::AcqRel); + if instance_id == self.failed_id { + return Err(BlazeError::BackendError { + msg: "cleanup deferred".into(), + }); + } + Ok(()) + } + } + + struct CountingOwner { + instance_id: Uuid, + kill_count: Arc, + killed: AtomicBool, + } + + #[async_trait] + impl BackendInstance for CountingOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(self.killed.load(Ordering::Acquire).then_some(SpawnResult { + instance_id: self.instance_id, + exit_code: Some(0), + signal: None, + })) + } + + async fn kill(&self) -> blaze_core::Result<()> { + if !self.killed.swap(true, Ordering::AcqRel) { + self.kill_count.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } + } + + struct CountingSpawner { + kill_count: Arc, + orphan_cleanup_count: Arc, + } + + #[async_trait] + impl BackendSpawner for CountingSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + Ok(Arc::new(CountingOwner { + instance_id: request.instance_id, + kill_count: self.kill_count.clone(), + killed: AtomicBool::new(false), + })) + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + _instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + self.orphan_cleanup_count.fetch_add(1, Ordering::AcqRel); + Ok(()) + } + } + + struct CaptureOnlyMockSpawner; + + #[async_trait] + impl BackendSpawner for CaptureOnlyMockSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + MockSpawner.spawn(request).await + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + instance_id: Uuid, + run_dir: &Path, + ) -> blaze_core::Result<()> { + MockSpawner.cleanup_orphan(instance_id, run_dir).await + } + } + + struct StalledGuestOwner { + instance_id: Uuid, + socket: PathBuf, + kill_count: Arc, + killed: AtomicBool, + } + + #[async_trait] + impl BackendInstance for StalledGuestOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + fn guest_socket_path(&self) -> &Path { + &self.socket + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(self.killed.load(Ordering::Acquire).then_some(SpawnResult { + instance_id: self.instance_id, + exit_code: Some(0), + signal: None, + })) + } + + async fn kill(&self) -> blaze_core::Result<()> { + if !self.killed.swap(true, Ordering::AcqRel) { + self.kill_count.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } + } + + struct StalledGuestSpawner { + spawned: Arc, + kill_count: Arc, + } + + #[async_trait] + impl BackendSpawner for StalledGuestSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + self.spawned.notify_one(); + Ok(Arc::new(StalledGuestOwner { + instance_id: request.instance_id, + socket: request.run_dir.join("missing-guest.uds"), + kill_count: self.kill_count.clone(), + killed: AtomicBool::new(false), + })) + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + _instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + Ok(()) + } + } + + struct CountingStorage { + inner: FileStorageProvider, + release_count: Arc, + } + + struct PoolWorkerReleaseStorage { + inner: FileStorageProvider, + acquire_count: AtomicUsize, + residual_attempt: usize, + delayed_id: Mutex>, + release_started: Arc, + release_active: Arc, + release_completed: Arc, + release_delay: Duration, + } + + struct ActiveRelease { + active: Arc, + } + + impl Drop for ActiveRelease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } + } + + #[async_trait] + impl StorageProvider for PoolWorkerReleaseStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + let attempt = self.acquire_count.fetch_add(1, Ordering::AcqRel) + 1; + let slot = self.inner.acquire(opts).await?; + if attempt == self.residual_attempt { + *self.delayed_id.lock().expect("delayed ID") = Some(slot.id.clone()); + return Err(StorageAcquireError::with_residual( + BlazeError::StorageError { + msg: "injected pool build residual".to_string(), + }, + slot, + )); + } + Ok(slot) + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + let delayed = + self.delayed_id.lock().expect("delayed ID").as_deref() == Some(instance_id); + if delayed { + self.release_active.fetch_add(1, Ordering::AcqRel); + let _active = ActiveRelease { + active: self.release_active.clone(), + }; + self.release_started.fetch_add(1, Ordering::AcqRel); + tokio::time::sleep(self.release_delay).await; + self.inner.release_by_id(instance_id).await?; + self.release_completed.fetch_add(1, Ordering::AcqRel); + return Ok(()); + } + self.inner.release_by_id(instance_id).await + } + + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> blaze_core::Result> { + self.inner.list_owned_ids().await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + #[async_trait] + impl StorageProvider for CountingStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.release_count.fetch_add(1, Ordering::AcqRel); + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + self.release_count.fetch_add(1, Ordering::AcqRel); + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + struct SelectiveHangingStorage { + inner: FileStorageProvider, + stalled_id: Uuid, + } + + #[async_trait] + impl StorageProvider for SelectiveHangingStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + if instance_id == self.stalled_id.to_string() { + return std::future::pending().await; + } + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + #[cfg(feature = "test-failpoints")] + fn counting_state( + temp: &tempfile::TempDir, + ) -> ( + Arc, + Arc, + Arc, + Arc, + ) { + let config = test_config(temp); + let kill_count = Arc::new(AtomicUsize::new(0)); + let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage: Arc = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: orphan_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + (state, kill_count, orphan_cleanup_count, release_count) + } + + #[tokio::test] + async fn sandbox_collection_and_item_routes_match_instance_routes() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let (status, created) = + dispatched_json(&state, Method::POST, "/v1/sandboxes", test_request()).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(created["instance"]["state"], "running"); + let id = created["instance"]["id"].as_str().expect("instance id"); + + let (_, sandboxes) = + dispatched_json(&state, Method::GET, "/v1/sandboxes", Vec::new()).await; + let (_, instances) = + dispatched_json(&state, Method::GET, "/v1/instances", Vec::new()).await; + assert_eq!(sandboxes, instances); + + let (_, sandbox) = dispatched_json( + &state, + Method::GET, + &format!("/v1/sandboxes/{id}"), + Vec::new(), + ) + .await; + let (_, instance) = dispatched_json( + &state, + Method::GET, + &format!("/v1/instances/{id}"), + Vec::new(), + ) + .await; + assert_eq!(sandbox, instance); + } + + #[tokio::test] + async fn destroy_route_forms_share_managed_cleanup() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let mut ids = Vec::new(); + for _ in 0..3 { + let created = created_json(&state, &test_request()).await; + ids.push( + Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"), + ); + } + let routes = [ + (Method::DELETE, format!("/v1/sandboxes/{}", ids[0]), ids[0]), + (Method::DELETE, format!("/v1/instances/{}", ids[1]), ids[1]), + ( + Method::POST, + format!("/v1/instances/{}/destroy", ids[2]), + ids[2], + ), + ]; + + for (method, path, id) in routes { + let (status, response) = dispatched_json(&state, method, &path, Vec::new()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response["destroyed"], true); + assert_eq!(response["instance_id"], id.to_string()); + assert_eq!( + state.manager.get(id).expect("destroyed state").state, + SandboxState::Destroyed + ); + } + } + + #[tokio::test] + async fn non_prefork_runtime_claim_completes_create_and_destroy() { + assert_warm_runtime_round_trip(false).await; + } + + #[tokio::test] + async fn prefork_runtime_claim_completes_create_and_destroy() { + assert_warm_runtime_round_trip(true).await; + } + + #[tokio::test] + async fn omitted_policy_ttl_is_resolved_in_create_response() { + let temp = tempfile::tempdir().expect("temp"); + let mut config = test_config(&temp); + config.pool.default_warm_ttl = "1h".into(); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir, + )); + let mut policy = test_policy(BackendKind::Mock, true); + policy.pool.as_mut().expect("pool").warm_ttl = None; + let state = build_test_state( + config, + policy, + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let created = created_json(&state, &test_request()).await; + assert_eq!(created["decision"]["pool"]["warm_ttl"], "1h"); + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance ID")) + .expect("UUID"); + assert!(state.manager.destroy(id).await.expect("destroy instance")); + shutdown_instances(&state, Duration::from_millis(100)) + .await + .expect("no owners remain"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn visible_lifecycle_publish_error_keeps_one_cleanup_owner() { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state(&temp, false, Arc::new(MockSpawner)); + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let hook = crate::failpoint::TestFailpoint::new(&["warm-runtime-lifecycle-publish-result"]); + + let error = hook + .run(create_instance(&state, &test_request())) + .await + .expect_err("visible lifecycle publication must be compensated"); + + assert!(error.to_string().contains("publication reported an error")); + let terminal = state.manager.get(ready_id).expect("terminal lifecycle"); + assert!(terminal.is_clean_terminal()); + assert!(state.manager.backend_owner(ready_id).is_none()); + assert!(!instances_dir.join(ready_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .exists() + ); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(".cleanup") + .join(ready_id.to_string()) + .exists() + ); + + state.manager.begin_shutdown(); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown remaining owners"); + } + + #[cfg(unix)] + #[tokio::test] + async fn ambiguous_lifecycle_publish_remains_counted_until_restart() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state( + &temp, + false, + Arc::new(CountingSpawner { + kill_count: Arc::new(AtomicUsize::new(0)), + orphan_cleanup_count: Arc::new(AtomicUsize::new(0)), + }), + ); + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let external = tempfile::tempdir().expect("external lifecycle owner"); + symlink(external.path(), state.state_dir.join(ready_id.to_string())) + .expect("linked lifecycle owner"); + + let error = create_instance(&state, &test_request()) + .await + .expect_err("ambiguous lifecycle publication must stop the claim"); + + assert!(error.to_string().contains("publication was ambiguous")); + let status = state.manager.runtime_pool_status(); + assert_eq!(status.ready, 0); + assert_eq!(status.leased, 0); + assert_eq!(status.unresolved, 1); + assert_eq!(status.deficit, 0); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(state.manager.runtime_pool_status().unresolved, 1); + assert!(instances_dir.join(ready_id.to_string()).exists()); + + state.manager.begin_shutdown(); + let shutdown = shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect_err("shutdown must report the unresolved owner"); + assert!(shutdown.to_string().contains(&ready_id.to_string())); + assert!( + shutdown + .to_string() + .contains("unresolved lifecycle publication") + ); + assert_eq!(state.manager.runtime_pool_status().unresolved, 1); + } + + #[tokio::test] + async fn failed_runtime_claim_retains_one_recoverable_owner() { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = + warm_runtime_state(&temp, false, Arc::new(PartialSpawnSpawner)); + + let _bootstrap_error = create_instance(&state, &test_request()) + .await + .expect_err("bootstrap spawn fails after pool configuration"); + let ready_id = wait_for_ready_runtime(&state).await; + let error = create_instance(&state, &test_request()) + .await + .expect_err("runtime claim spawn fails"); + + assert!(error.to_string().contains("cleanup incomplete")); + let retained = state.manager.get(ready_id).expect("retained lifecycle"); + assert_eq!(retained.runtime_location, RuntimeLocation::WarmPool); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!(!retained.is_clean_terminal()); + let ownership: serde_json::Value = serde_json::from_slice( + &std::fs::read( + state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .join("ownership.json"), + ) + .expect("retained ownership"), + ) + .expect("ownership JSON"); + assert_eq!(ownership["phase"]["kind"], "lifecycle-cleanup"); + + assert!( + state + .manager + .destroy(ready_id) + .await + .expect("retry retained cleanup") + ); + assert!( + state + .manager + .get(ready_id) + .expect("terminal lifecycle") + .is_clean_terminal() + ); + assert!(!instances_dir.join(ready_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .exists() + ); + + state.manager.begin_shutdown(); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown bootstrap owner"); + } + + /// When multiple backend binaries exist on disk but the daemon probed + /// Firecracker at boot, only Firecracker should be reported available + /// and selected — even if policy prioritizes bubblewrap higher. + #[tokio::test] + async fn availability_constrained_to_active_backend() { + // Create temp files to simulate both binaries existing. let tmp = std::env::temp_dir().join("blaze-test-active-backend"); let _ = std::fs::create_dir_all(&tmp); let fc_bin = tmp.join("firecracker"); @@ -1639,215 +2374,2766 @@ mod tests { std::fs::write(&fc_bin, b"fake-fc").unwrap(); std::fs::write(&bwrap_bin, b"fake-bwrap").unwrap(); - // Minimal config with both backends present. + // Minimal config with both backends present. + let mut config = DaemonConfig::default(); + config.daemon.state_dir = tmp.join("state"); + config.runtime_templates.dir = tmp.join("runtime-templates"); + let _ = std::fs::create_dir_all(&config.daemon.state_dir); + config.backends.insert("firecracker".into(), fc_bin.clone()); + config + .backends + .insert("bubblewrap".into(), bwrap_bin.clone()); + + // Policy that prioritizes bubblewrap over firecracker. + let policy_file = PolicyFile { + manifest_version: 1, + policy_name: "test-multi-backend".into(), + priority: 100, + match_: PolicyMatch { + workload_class: WorkloadClass::AgentRl, + image_labels: HashMap::new(), + }, + select: PolicySelect { + backend_priority: vec![BackendKind::Bubblewrap, BackendKind::Firecracker], + kernel_hooks: vec![], + templates: vec![], + fallback_on_missing_hook: FallbackOnMissingHook::default(), + }, + pool: None, + checkpoint: None, + quota: None, + hooks: PolicyHooks::default(), + backend: BackendConfigs::default(), + vm: None, + }; + let engine = PolicyEngine::with_policies(vec![policy_file]); + + // Build state with active_backend = Firecracker (simulating probe + // selected FC at boot) but using MockSpawner for test portability. + let spawner: DynSpawner = Arc::new(MockSpawner); + let storage_dir = tmp.join("storage"); + let _ = std::fs::create_dir_all(&storage_dir); + let storage: Arc = + Arc::new(FileStorageProvider::new(storage_dir)); + let state = Arc::new( + ServerState::build( + config, + engine, + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Firecracker, spawner), + BackendKind::Firecracker, + storage, + ) + .expect("build server state"), + ); + + // Create instance request for AgentRl workload. + let req_body = serde_json::to_vec(&serde_json::json!({ + "workload_class": "agent-rl", + "image_digest": "sha256:abc123", + })) + .unwrap(); + + let resp = create_instance(&state, &req_body).await.unwrap(); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + let resp_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + + // The instance should be created with backend = firecracker, + // NOT bubblewrap (even though bwrap was higher priority in policy) + // because only the active backend is reported as available. + assert_eq!( + resp_json["instance"]["backend"].as_str().unwrap(), + "firecracker", + "instance backend should be the active backend (firecracker), \ + not the higher-priority bubblewrap" + ); + + // Cleanup. + let _ = std::fs::remove_dir_all(&tmp); + } + + #[tokio::test] + async fn checkpoint_rejects_unsupported_storage_without_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(NoCheckpointStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + }); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let request = test_request(); + let created = created_json(&state, &request).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let state_path = state.state_dir.join(id).join("state.json"); + let persisted_before = std::fs::read(&state_path).expect("persisted state"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint without backend and storage capture must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Running + ); + assert!( + state.instances.lock().expect("instances")[&uuid] + .operation + .is_none() + ); + assert_eq!( + std::fs::read(state_path).expect("persisted state"), + persisted_before + ); + assert!(!state.state_dir.join("checkpoints").join(id).exists()); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_rejects_unsupported_backend_without_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let kill_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: Arc::new(AtomicUsize::new(0)), + }), + ), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let state_path = state.state_dir.join(id).join("state.json"); + let persisted_before = std::fs::read(&state_path).expect("persisted state"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint without backend capture must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!( + std::fs::read(state_path).expect("persisted state"), + persisted_before + ); + assert!(!state.state_dir.join("checkpoints").join(id).exists()); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_routes_capture_and_list_live_state() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + + let (status, checkpoint) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoint"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let checkpoint_id = checkpoint["id"].as_str().expect("checkpoint id"); + assert_eq!(checkpoint["snapshot_kind"], "full"); + assert_eq!(checkpoint["sandbox_id"], id); + let captured_rootfs = state + .state_dir + .join("checkpoints") + .join(id) + .join(checkpoint_id) + .join("rootfs.snap"); + assert_eq!( + tokio::fs::read(&captured_rootfs) + .await + .expect("captured rootfs"), + b"checkpoint-rootfs" + ); + + tokio::fs::write(&slot.rootfs_path, b"changed-after-checkpoint") + .await + .expect("mutate live rootfs"); + assert_eq!( + tokio::fs::read(&captured_rootfs) + .await + .expect("independent captured rootfs"), + b"checkpoint-rootfs" + ); + let (status, checkpoints) = dispatched_json( + &state, + Method::GET, + &format!("/v1/instances/{id}/checkpoints"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(checkpoints.as_array().expect("checkpoint list").len(), 1); + assert_eq!(checkpoints[0]["id"], checkpoint_id); + assert_eq!(checkpoints[0]["is_head"], true); + assert_eq!(checkpoints[0]["on_head_chain"], true); + + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!(lifecycle.last_checkpoint.as_deref(), Some(checkpoint_id)); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn hibernate_releases_the_backend_and_resume_survives_restart() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"hibernate-memory") + .await + .expect("write guest state"); + + let (status, hibernated) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/hibernate"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(hibernated["state"], "hibernated"); + assert_eq!(hibernated["backend_ownership"], "stopped"); + assert!(state.manager.backend_owner(uuid).is_none()); + let hibernate_dir = config.daemon.state_dir.join(id).join("hibernate"); + for name in ["manifest.json", "memory.snap", "vmstate.snap"] { + assert!(hibernate_dir.join(name).is_file(), "{name} is missing"); + } + let report = state.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + drop(state); + + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + assert_eq!( + restarted.manager.get(uuid).expect("loaded state").state, + SandboxState::Hibernated + ); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + + let (status, resumed) = dispatched_json( + &restarted, + Method::POST, + &format!("/v1/instances/{id}/resume"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(resumed["state"], "running"); + assert_eq!( + restarted + .manager + .read_file(uuid, "/tmp/value".to_string()) + .await + .expect("read resumed guest state"), + b"hibernate-memory" + ); + assert!( + hibernate_dir.is_dir(), + "the last hibernation image remains available until replacement or destroy" + ); + assert!(restarted.manager.destroy(uuid).await.expect("destroy")); + assert!(!hibernate_dir.exists()); + } + + #[tokio::test] + async fn hibernate_rejects_a_capture_only_backend_before_state_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(CaptureOnlyMockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + + let error = state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("resume capability is required"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[tokio::test] + async fn resume_rejects_corrupted_hibernation_artifacts_without_starting_a_backend() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + tokio::fs::write( + config + .daemon + .state_dir + .join(id) + .join("hibernate/memory.snap"), + b"corrupted", + ) + .await + .expect("corrupt artifact"); + + let error = state + .manager + .resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("corrupted artifact must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert!(lifecycle.operation.is_none()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[tokio::test] + async fn startup_retains_an_interrupted_hibernation_for_explicit_cleanup() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + StartPath::Cold, + "ownership-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance + .begin_hibernate_operation() + .expect("begin hibernation"); + instance + .transition(SandboxState::Hibernating) + .expect("hibernating"); + instance.persist(&config.daemon.state_dir).expect("persist"); + storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 4096, + mem_size: 4096, + }) + .await + .expect("storage"); + let id = instance.id; + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let report = state.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + let retained = state.manager.get(id).expect("retained lifecycle"); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate) + ); + assert!(state.manager.destroy(id).await.expect("explicit destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_snapshot_failure_resumes_the_existing_backend() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-snapshot"]); + + hook.run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("snapshot failure"); + + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert!(lifecycle.operation.is_none()); + let names = std::fs::read_dir(state.state_dir.join(id)) + .expect("instance directory") + .map(|entry| entry.expect("entry").file_name()) + .collect::>(); + assert!( + names + .iter() + .all(|name| !name.to_string_lossy().starts_with(".hibernate.")) + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_compensation_requires_guest_readiness() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = + crate::failpoint::TestFailpoint::new(&["hibernate-snapshot", "resume-guest-ready"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("guest readiness must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_hibernate_stop_retains_the_existing_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-backend-stop"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("uncertain stop must retain ownership"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::HibernateArtifactsSynced) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_publish_failure_retains_stopped_ownership_for_destroy() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-publish"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("publish failure follows backend stop"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::HibernateBackendStopped) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_start_failure_preserves_retryable_hibernation() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = crate::failpoint::TestFailpoint::new(&["resume-backend-start"]); + + hook.run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("resume start failure"); + + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Hibernated); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert!(lifecycle.operation.is_none()); + state + .manager + .resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("retry resume"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_readiness_failure_cleans_the_replacement_backend() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = crate::failpoint::TestFailpoint::new(&["resume-guest-ready"]); + + hook.run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("readiness failure"); + + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Hibernated); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_cleanup_failure_retains_the_replacement_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = + crate::failpoint::TestFailpoint::new(&["resume-guest-ready", "resume-backend-stop"]); + + let error = hook + .run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("failed cleanup must retain ownership"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Resume) + ); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::ResumeBackendStarted) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[tokio::test] + async fn rollback_replaces_runtime_state_without_rewriting_capture_history() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = state.storage.reconstruct(id).await.expect("storage slot"); + + tokio::fs::write(&slot.rootfs_path, b"first-rootfs") + .await + .expect("first rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"first-memory") + .await + .expect("first guest state"); + let (_, first) = dispatched_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/checkpoint"), + Vec::new(), + ) + .await; + let first_id = first["id"].as_str().expect("first checkpoint"); + + tokio::fs::write(&slot.rootfs_path, b"second-rootfs") + .await + .expect("second rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"second-memory") + .await + .expect("second guest state"); + let (_, second) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoint"), + Vec::new(), + ) + .await; + let second_id = second["id"].as_str().expect("second checkpoint"); + + tokio::fs::write(&slot.rootfs_path, b"third-rootfs") + .await + .expect("third rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"third-memory") + .await + .expect("third guest state"); + + let (status, restored) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/rollback/{first_id}"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(restored["instance_id"], id); + assert_eq!(restored["checkpoint_id"], first_id); + assert_eq!(restored["restored"], true); + assert_eq!(restored["state"], "running"); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("restored rootfs"), + b"first-rootfs" + ); + assert_eq!( + state + .manager + .read_file(uuid, "/tmp/value".to_string()) + .await + .expect("restored guest state"), + b"first-memory" + ); + + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!(lifecycle.last_checkpoint.as_deref(), Some(second_id)); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint list") + .iter() + .find(|checkpoint| checkpoint.is_head) + .map(|checkpoint| checkpoint.id.as_str()), + Some(first_id) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + for name in [ + ".rootfs.restore-copying", + ".rootfs.restore-staged", + ".rootfs.restore-backup", + ".rootfs.restore-discard", + ".rootfs.restore.json", + ".rootfs.restore-journal.tmp", + ] { + assert!(!slot.instance_dir.join(name).exists(), "{name} remains"); + } + } + + #[tokio::test] + async fn rollback_rejects_an_unavailable_adapter_before_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(CaptureOnlyMockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + + let error = state + .manager + .restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("restore must require an adapter"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn restore_stage_failure_keeps_the_current_runtime_running() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-storage-stage"]); + + hook.run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("stage failure"); + + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_backend_stop_retains_the_current_owner_and_rootfs() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-backend-stop"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("backend stop outcome must require recovery"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::RestoreStorageStaged) + ); + for name in [ + ".rootfs.restore-staged", + ".rootfs.restore-backup", + ".rootfs.restore.json", + ] { + assert!(!slot.instance_dir.join(name).exists(), "{name} remains"); + } + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_head_update_retains_the_replacement_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"later-checkpoint-rootfs") + .await + .expect("later checkpoint rootfs"); + let latest = state + .manager + .checkpoint(uuid) + .await + .expect("later checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-head-after-rename"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id.clone(), + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("HEAD update must be reported"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("selected rootfs"), + b"checkpoint-rootfs" + ); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::RestoreBackendStarted) + ); + assert_eq!( + lifecycle.last_checkpoint.as_deref(), + Some(latest.id.as_str()) + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("observable checkpoint catalog") + .iter() + .find(|item| item.is_head) + .map(|item| item.id.as_str()), + Some(checkpoint.id.as_str()) + ); + + assert!(state.manager.destroy(uuid).await.expect("destroy")); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn final_state_failure_keeps_the_committed_restore_journal() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-final-state"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id.clone(), + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("final state failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("committed rootfs"), + b"checkpoint-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert_eq!( + lifecycle + .operation + .as_ref() + .map(|operation| (operation.checkpoint_id.as_deref(), operation.phase)), + Some(( + Some(checkpoint.id.as_str()), + Some(OperationPhase::RestoreStorageCommitted) + )) + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint list") + .iter() + .find(|item| item.is_head) + .map(|item| item.id.as_str()), + Some(checkpoint.id.as_str()) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_restore_after_head_is_destroyable() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-after-head"]); + let restore_state = state.clone(); + let restore_hook = hook.clone(); + let restore = tokio::spawn(async move { + restore_hook + .run(restore_state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + }); + hook.wait_until_paused().await; + + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted restore journal"); + assert_eq!(persisted.state, SandboxState::Restoring); + assert_eq!( + persisted.operation.and_then(|operation| operation.phase), + Some(OperationPhase::RestoreHeadUpdated) + ); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(state.manager.backend_owner(uuid).is_some()); + + restore.abort(); + assert!(restore.await.expect_err("cancelled restore").is_cancelled()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + assert!( + !state + .config + .lock() + .expect("config") + .storage + .instances_dir + .join(id) + .exists() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_snapshot_failure_resumes_and_clears_the_journal() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-snapshot"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("snapshot failure"); + + assert!(matches!( + error, + BlazeDaemonError::Core(BlazeError::BackendError { .. }) + )); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!( + SandboxInstance::load(&state.state_dir, uuid) + .expect("persisted lifecycle") + .operation, + None + ); + let checkpoint_dir = state.state_dir.join("checkpoints").join(id); + let staging = std::fs::read_dir(checkpoint_dir) + .expect("checkpoint directory") + .filter_map(std::result::Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().starts_with(".ckpt-")) + .count(); + assert_eq!(staging, 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_prepublication_failure_discards_the_stage() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-publish"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("publication must fail before the store call"); + + assert!(matches!( + error, + BlazeDaemonError::Core(BlazeError::StorageError { .. }) + )); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint catalog") + .is_empty() + ); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_state_failures_retain_the_reached_durable_phase() { + for (failpoint, expected_phase, expected_head) in [ + ( + "checkpoint-published-state", + OperationPhase::CheckpointPublished, + false, + ), + ( + "checkpoint-head-state", + OperationPhase::CheckpointHeadUpdated, + true, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("state commit must fail"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(expected_phase) + ); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("published checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0].is_head, expected_head); + assert!(state.manager.backend_owner(uuid).is_some()); + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_store_boundary_failures_preserve_observable_catalog_truth() { + for (failpoint, expected_phase, expected_head) in [ + ( + "checkpoint-store-publish-after-rename", + OperationPhase::CheckpointPaused, + false, + ), + ( + "checkpoint-store-head-after-rename", + OperationPhase::CheckpointPublished, + true, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("durability boundary must report an uncertain result"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(expected_phase) + ); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("observable checkpoint catalog"); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0].is_head, expected_head); + assert!(state.manager.backend_owner(uuid).is_some()); + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn startup_cleans_a_terminal_checkpoint_prune_tombstone() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + state.manager.destroy(uuid).await.expect("destroy sandbox"); + + let prune_hook = + crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + prune_hook + .run(state.manager.prune_checkpoints(uuid)) + .await + .expect_err("prune must stop after the durable tombstone"); + let checkpoint_dir = state.state_dir.join("checkpoints").join(&id); + assert!( + std::fs::read_dir(&checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + drop(state); + + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); + assert!( + !std::fs::read_dir(&checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + assert!( + restarted + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_prune_removes_an_unreachable_publication() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + let checkpoint_id = state + .manager + .list_checkpoints(uuid) + .await + .expect("unreachable checkpoint") + .pop() + .expect("checkpoint") + .id; + + let (status, pruned) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoints/prune"), + Vec::new(), + ) + .await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(pruned["count"], 1); + assert_eq!(pruned["removed"], json!([checkpoint_id])); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn prune_retry_cleans_a_prior_checkpoint_tombstone() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + let prune_hook = + crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + prune_hook + .run(state.manager.prune_checkpoints(uuid)) + .await + .expect_err("first prune must stop after the durable tombstone"); + + assert!( + state + .manager + .prune_checkpoints(uuid) + .await + .expect("retry prune") + .is_empty() + ); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + assert!( + !std::fs::read_dir(state.state_dir.join("checkpoints").join(&id)) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_resume_failure_keeps_head_and_runtime_ownership() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-resume"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("resume failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointHeadUpdated) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("committed checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert!(checkpoints[0].is_head); + + state.manager.destroy(uuid).await.expect("destroy retry"); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("durable checkpoint history") + .len(), + 1 + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn published_checkpoint_holds_the_operation_lock_until_head_commit() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted checkpoint journal"); + assert_eq!(persisted.state, SandboxState::Paused); + assert_eq!( + persisted.operation.and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + let list_state = state.clone(); + let mut list = tokio::spawn(async move { list_state.manager.list_checkpoints(uuid).await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut list) + .await + .is_err(), + "checkpoint listing must wait for a consistent catalog boundary" + ); + let destroy_state = state.clone(); + let mut destroy = tokio::spawn(async move { destroy_state.manager.destroy(uuid).await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut destroy) + .await + .is_err(), + "destroy must wait for checkpoint ownership" + ); + + hook.release(); + capture + .await + .expect("capture task") + .expect("checkpoint capture"); + let checkpoints = list.await.expect("list task").expect("checkpoint list"); + assert_eq!(checkpoints.len(), 1); + assert!(checkpoints[0].is_head); + assert!(destroy.await.expect("destroy task").expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_published_checkpoint_is_destroyable() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + capture.abort(); + let _ = capture.await; + + let interrupted = state.manager.get(uuid).expect("interrupted lifecycle"); + assert_eq!(interrupted.state, SandboxState::Paused); + assert_eq!( + interrupted.operation.and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + assert!( + !state + .state_dir + .join("checkpoints") + .join(&id) + .join("HEAD") + .exists() + ); + + state + .manager + .destroy(uuid) + .await + .expect("destroy interrupted capture"); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("unreachable checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert!(!checkpoints[0].is_head); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_checkpoint_phases_are_destroyable_before_and_after_restart() { + for (failpoint, expected_state, expected_phase, checkpoint_count) in [ + ( + "checkpoint-after-begin", + SandboxState::Running, + OperationPhase::CheckpointPreparing, + 0, + ), + ( + "checkpoint-after-pause", + SandboxState::Paused, + OperationPhase::CheckpointPaused, + 0, + ), + ( + "checkpoint-after-head", + SandboxState::Paused, + OperationPhase::CheckpointHeadUpdated, + 1, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + cancel_checkpoint_at(&state, uuid, failpoint).await; + let interrupted = state.manager.get(uuid).expect("interrupted lifecycle"); + assert_eq!(interrupted.state, expected_state); + assert_eq!( + interrupted.operation.and_then(|journal| journal.phase), + Some(expected_phase) + ); + + state + .manager + .destroy(uuid) + .await + .expect("same-process destroy"); + let destroyed = state.manager.get(uuid).expect("destroyed lifecycle"); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint history") + .len(), + checkpoint_count + ); + + let restart_temp = tempfile::tempdir().expect("restart temp"); + let config = test_config(&restart_temp); + let restart_state = mock_state_from_config(config.clone(), false); + let created = created_json(&restart_state, &test_request()).await; + let restart_id = created["instance"]["id"] + .as_str() + .expect("restart id") + .to_string(); + let restart_uuid = Uuid::parse_str(&restart_id).expect("restart uuid"); + write_checkpoint_fixture(&restart_state, &restart_id).await; + cancel_checkpoint_at(&restart_state, restart_uuid, failpoint).await; + restart_state + .manager + .backend_owner(restart_uuid) + .expect("backend owner") + .kill() + .await + .expect("simulate daemon exit"); + drop(restart_state); + + let restarted = mock_state_from_config(config, false); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); + let destroyed = restarted + .manager + .get(restart_uuid) + .expect("reconciled lifecycle"); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + assert_eq!( + restarted + .manager + .list_checkpoints(restart_uuid) + .await + .expect("reconciled checkpoint history") + .len(), + checkpoint_count + ); + let checkpoint_dir = restarted.state_dir.join("checkpoints").join(&restart_id); + if checkpoint_dir.exists() { + assert!( + !std::fs::read_dir(checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with('.')) + ); + } + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn guest_operations_wait_for_checkpoint_publication() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + state + .manager + .write_file(uuid, "/tmp/existing".into(), b"before") + .await + .expect("seed guest file"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + + let exec_state = state.clone(); + let mut exec = tokio::spawn(async move { + exec_state + .manager + .exec(uuid, "printf locked".into(), None, None, 5) + .await + }); + let read_state = state.clone(); + let mut read = tokio::spawn(async move { + read_state + .manager + .read_file(uuid, "/tmp/existing".into()) + .await + }); + let write_state = state.clone(); + let mut write = tokio::spawn(async move { + write_state + .manager + .write_file(uuid, "/tmp/after".into(), b"after") + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut exec) + .await + .is_err(), + "guest exec must wait for checkpoint ownership" + ); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut read) + .await + .is_err(), + "guest read must wait for checkpoint ownership" + ); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut write) + .await + .is_err(), + "guest write must wait for checkpoint ownership" + ); + + hook.release(); + capture + .await + .expect("capture task") + .expect("checkpoint capture"); + assert_eq!( + exec.await.expect("exec task").expect("guest exec").stdout, + b"printf locked" + ); + assert_eq!( + read.await.expect("read task").expect("guest read"), + b"before" + ); + write.await.expect("write task").expect("guest write"); + } + + #[tokio::test] + async fn reset_rejects_state_only_pool_return() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let request = test_request(); + let created = created_json(&state, &request).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + + let error = reset_instance(&state, id) + .await + .expect_err("reset without a runtime implementation must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Running + ); + let key = PoolKey::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + ); + assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_rejects_an_unfinished_lifecycle_journal() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let journal = { + let mut instances = state.instances.lock().expect("instances"); + let instance = instances.get_mut(&uuid).expect("instance"); + instance + .begin_operation(OperationKind::Create) + .expect("begin unfinished operation"); + instance.persist(&state.state_dir).expect("persist journal"); + instance.operation.clone().expect("journal") + }; + + let error = checkpoint(&state, id) + .await + .expect_err("unfinished lifecycle work must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].operation, + Some(journal) + ); + assert_eq!( + SandboxInstance::load(&state.state_dir, uuid) + .expect("persisted instance") + .operation, + state.instances.lock().expect("instances")[&uuid].operation + ); + } + + #[tokio::test] + async fn checkpoint_rejects_a_non_running_lifecycle_state() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state.manager.destroy(uuid).await.expect("destroy"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint must require a running instance"); + + assert!(matches!(error, BlazeDaemonError::Conflict(_))); + assert_eq!(error.status_code(), 409); + } + + #[tokio::test] + async fn quiescent_state_guard_serializes_later_destroy() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("id")).expect("uuid"); + let guard = state + .manager + .lock_quiescent_state(id, SandboxState::Running) + .await + .expect("quiescent running state"); + let destroy_state = state.clone(); + let mut destroy = tokio::spawn(async move { destroy_state.manager.destroy(id).await }); + + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut destroy) + .await + .is_err(), + "destroy must wait while a lifecycle operation holds the guard" + ); + drop(guard); + + assert!(destroy.await.expect("destroy task").expect("destroy")); + assert_eq!( + state.manager.get(id).expect("instance").state, + SandboxState::Destroyed + ); + } + + #[tokio::test] + async fn guest_manager_waits_for_the_lifecycle_lock() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + + let operation = state.manager.operation_lock(uuid).lock_owned().await; + let exec_state = state.clone(); + let mut pending_exec = tokio::spawn(async move { + exec_state + .manager + .exec(uuid, "printf guest-lock".into(), None, None, 5) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut pending_exec) + .await + .is_err(), + "guest operation bypassed the lifecycle lock" + ); + drop(operation); + + let exec = pending_exec + .await + .expect("exec task") + .expect("managed exec"); + assert_eq!(exec.stdout, b"printf guest-lock"); + } + + #[tokio::test] + async fn manager_cleanup_releases_tracked_runtime_resources() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"); + { + let mut instances = state.instances.lock().expect("instances"); + let instance = instances.get_mut(&id).expect("instance"); + instance + .transition(SandboxState::Destroyed) + .expect("persisted terminal state"); + instance.backend_ownership = BackendOwnership::Stopped; + instance.finish_operation(); + instance.persist(&state.state_dir).expect("persist"); + } + + let report = state + .manager + .cleanup_owned_instances_with_timeout(Duration::from_secs(1)) + .await; + + assert!(report.failures.is_empty()); + assert_eq!( + state.manager.get(id).expect("instance").state, + SandboxState::Destroyed + ); + assert!(state.manager.backend_owner(id).is_none()); + assert!(!instances_dir.join(id.to_string()).exists()); + } + + #[tokio::test] + async fn warm_claim_validates_runtime_and_quarantines_dead_owner() { + let temp = tempfile::tempdir().expect("temp"); let mut config = DaemonConfig::default(); - config.daemon.state_dir = tmp.join("state"); - let _ = std::fs::create_dir_all(&config.daemon.state_dir); - config.backends.insert("firecracker".into(), fc_bin.clone()); - config - .backends - .insert("bubblewrap".into(), bwrap_bin.clone()); + config.daemon.state_dir = temp.path().join("state"); + config.storage.images_dir = temp.path().join("images"); + config.storage.instances_dir = temp.path().join("instances"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); + std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); + std::fs::create_dir_all(&config.storage.images_dir).expect("images"); + std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); - // Policy that prioritizes bubblewrap over firecracker. - let policy_file = PolicyFile { + let policy = PolicyFile { manifest_version: 1, - policy_name: "test-multi-backend".into(), + policy_name: "warm-validation".into(), priority: 100, match_: PolicyMatch { - workload_class: WorkloadClass::AgentRl, + workload_class: WorkloadClass::AgentTool, image_labels: HashMap::new(), }, select: PolicySelect { - backend_priority: vec![BackendKind::Bubblewrap, BackendKind::Firecracker], + backend_priority: vec![BackendKind::Mock], kernel_hooks: vec![], templates: vec![], fallback_on_missing_hook: FallbackOnMissingHook::default(), }, - pool: None, + pool: Some(PolicyPool { + enabled: true, + min: 0, + target: 0, + max: 1, + warm_ttl: Some("30m".into()), + reset_mode: ResetMode::FullRecreate, + }), checkpoint: None, quota: None, hooks: PolicyHooks::default(), backend: BackendConfigs::default(), vm: None, }; - let engine = PolicyEngine::with_policies(vec![policy_file]); - - // Build state with active_backend = Firecracker (simulating probe - // selected FC at boot) but using MockSpawner for test portability. - let spawner: DynSpawner = Arc::new(MockSpawner); - let storage_dir = tmp.join("storage"); - let _ = std::fs::create_dir_all(&storage_dir); let storage: Arc = - Arc::new(FileStorageProvider::new(storage_dir)); - let state = Arc::new(ServerState::build( + Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(vec![policy]), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ) + .expect("build server state"), + ); + let request = serde_json::to_vec(&json!({ + "workload_class": "agent-tool", + "image_digest": "sha256:warm-validation" + })) + .expect("request"); + + let cold = create_instance(&state, &request) + .await + .expect("cold create"); + let cold: serde_json::Value = + serde_json::from_slice(&cold.into_body().collect().await.expect("body").to_bytes()) + .expect("cold json"); + let id = cold["instance"]["id"].as_str().expect("id").to_string(); + return_to_pool_for_test(&state, &id) + .await + .expect("return to pool"); + + let warm = create_instance(&state, &request) + .await + .expect("warm create"); + let warm: serde_json::Value = + serde_json::from_slice(&warm.into_body().collect().await.expect("body").to_bytes()) + .expect("warm json"); + assert_eq!(warm["instance"]["id"], id); + assert_eq!(warm["start_path"], "warm"); + + return_to_pool_for_test(&state, &id) + .await + .expect("return live owner"); + let owner = state + .manager + .backend_owner(Uuid::parse_str(&id).expect("uuid")) + .expect("owner"); + owner.kill().await.expect("simulate backend exit"); + + let replacement = create_instance(&state, &request) + .await + .expect("cold fallback"); + let replacement: serde_json::Value = serde_json::from_slice( + &replacement + .into_body() + .collect() + .await + .expect("body") + .to_bytes(), + ) + .expect("replacement json"); + assert_ne!(replacement["instance"]["id"], id); + assert_eq!(replacement["start_path"], "cold"); + let key = PoolKey::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:warm-validation".into(), + ); + assert_eq!( + state + .pool + .lock() + .expect("pool") + .stats(&key) + .quarantine_count, + 1 + ); + } + + #[tokio::test] + async fn sandbox_guest_routes_use_owned_runtime() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( config, - engine, - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), - spawners(BackendKind::Firecracker, spawner), - BackendKind::Firecracker, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + + let (status, exec) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/exec"), + serde_json::to_vec(&json!({ + "cmd": "printf routed", + "timeout": 5, + })) + .expect("exec request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(exec["exit_code"], 0); + assert_eq!(exec["stdout_b64"], BASE64.encode(b"printf routed")); + + let encoded = "AAEC/2d1ZXN0"; + let (status, written) = dispatched_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/write"), + serde_json::to_vec(&json!({ + "path": "/tmp/value", + "data_b64": encoded, + })) + .expect("write request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], 9); + + let (status, read) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/read"), + serde_json::to_vec(&json!({"path": "/tmp/value"})).expect("read request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(read["data_b64"], encoded); + + let invalid_timeout = dispatch( + &Method::POST, + &format!("/v1/sandboxes/{id}/exec"), + "", + serde_json::to_vec(&json!({ + "cmd": "true", + "timeout": MAX_EXEC_TIMEOUT_SECS + 1, + })) + .expect("invalid request"), + &state, + ) + .await + .expect_err("timeout above the API limit must fail"); + assert!(matches!(invalid_timeout, BlazeDaemonError::BadRequest(_))); + + assert_eq!( + decode_guest_file(&BASE64.encode(b"1234"), 4).expect("boundary"), + b"1234" + ); + assert!(matches!( + decode_guest_file(&BASE64.encode(b"12345"), 4), + Err(BlazeDaemonError::Guest( + crate::guest::GuestError::PayloadTooLarge { .. } + )) + )); + assert!(matches!( + decode_guest_file("not/base64!", 16), + Err(BlazeDaemonError::BadRequest(_)) )); - // Create instance request for AgentRl workload. - let req_body = serde_json::to_vec(&serde_json::json!({ - "workload_class": "agent-rl", - "image_digest": "sha256:abc123", - })) - .unwrap(); + let (status, destroyed) = dispatched_json( + &state, + Method::DELETE, + &format!("/v1/sandboxes/{id}"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(destroyed["destroyed"], true); + } - let resp = create_instance(&state, &req_body).await.unwrap(); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let resp_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + #[tokio::test] + async fn guest_write_respects_http_and_decoded_limits() { + const EXTENDED_BODY_LIMIT: usize = 22 * 1024 * 1024; - // The instance should be created with backend = firecracker, - // NOT bubblewrap (even though bwrap was higher priority in policy) - // because only the active backend is reported as available. + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let default_limit = config.api.max_body_bytes; + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let path = format!("/v1/sandboxes/{id}/write"); + + let envelope_payload = vec![b'y'; 800 * 1024]; + let envelope_body = serde_json::to_vec(&json!({ + "path": "/tmp/http-envelope", + "data_b64": BASE64.encode(&envelope_payload), + })) + .expect("write request above the default HTTP limit"); + assert!(envelope_body.len() > default_limit); + let (status, error) = + handled_json(&state, Method::POST, &path, envelope_body.clone()).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error["status"], 413); + + state.config.lock().expect("config").api.max_body_bytes = EXTENDED_BODY_LIMIT; + let (status, written) = handled_json(&state, Method::POST, &path, envelope_body).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], envelope_payload.len()); assert_eq!( - resp_json["instance"]["backend"].as_str().unwrap(), - "firecracker", - "instance backend should be the active backend (firecracker), \ - not the higher-priority bubblewrap" + state + .manager + .read_file(uuid, "/tmp/http-envelope".into()) + .await + .expect("read envelope payload"), + envelope_payload ); - // Cleanup. - let _ = std::fs::remove_dir_all(&tmp); + let mut payload = vec![b'z'; MAX_GUEST_FILE_BYTES]; + let body = serde_json::to_vec(&json!({ + "path": "/tmp/max-size", + "data_b64": BASE64.encode(&payload), + })) + .expect("write request"); + assert!(body.len() <= EXTENDED_BODY_LIMIT); + + let (status, written) = handled_json(&state, Method::POST, &path, body).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], MAX_GUEST_FILE_BYTES); + let readback = state + .manager + .read_file(uuid, "/tmp/max-size".into()) + .await + .expect("read maximum file"); + assert_eq!(readback, payload); + drop(readback); + + payload.push(b'z'); + let oversized = serde_json::to_vec(&json!({ + "path": "/tmp/too-large", + "data_b64": BASE64.encode(&payload), + })) + .expect("oversized write request"); + assert!(oversized.len() <= EXTENDED_BODY_LIMIT); + let (status, error) = handled_json(&state, Method::POST, &path, oversized).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error["status"], 413); + } + + #[tokio::test] + async fn write_route_reports_unknown_after_delivery_failure() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .backend_owner(uuid) + .expect("mock owner") + .kill() + .await + .expect("stop mock guest"); + + let socket = temp.path().join("uncertain.uds"); + let listener = tokio::net::UnixListener::bind(&socket).expect("bind guest endpoint"); + state + .manager + .insert_backend_owner( + uuid, + Arc::new(StalledGuestOwner { + instance_id: uuid, + socket, + kill_count: Arc::new(AtomicUsize::new(0)), + killed: AtomicBool::new(false), + }), + ) + .expect("replace backend owner"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept guest request"); + let mut reader = tokio::io::BufReader::new(stream); + let mut connect = String::new(); + reader.read_line(&mut connect).await.expect("read connect"); + assert_eq!(connect, "CONNECT 5000\n"); + reader + .get_mut() + .write_all(b"OK 5000\n") + .await + .expect("write handshake"); + let mut request = String::new(); + reader + .read_line(&mut request) + .await + .expect("read guest request"); + let request: serde_json::Value = + serde_json::from_str(&request).expect("parse guest request"); + assert_eq!(request["op"], "write"); + }); + + let body = serde_json::to_vec(&json!({ + "path": "/tmp/value", + "data_b64": BASE64.encode(b"value"), + })) + .expect("write request"); + let (status, error) = handled_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/write"), + body, + ) + .await; + assert_eq!(status, StatusCode::GATEWAY_TIMEOUT); + assert_eq!(error["code"], "guest_outcome_unknown"); + server.await.expect("guest server"); + } + + #[tokio::test] + async fn unknown_guest_outcome_has_stable_api_code() { + let response = error_response(&BlazeDaemonError::Guest( + crate::guest::GuestError::OutcomeUnknown("response lost".into()), + )); + assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_outcome_unknown"); + assert_eq!(value["status"], 504); + + let response = error_response(&BlazeDaemonError::Guest( + crate::guest::GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + )); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_response_too_large"); + + let response = error_response(&BlazeDaemonError::Guest(crate::guest::GuestError::Timeout( + "connect stalled".into(), + ))); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_timeout"); } #[tokio::test] - async fn warm_claim_validates_runtime_and_quarantines_dead_owner() { + async fn create_publishes_ownership_before_provider_acquire() { let temp = tempfile::tempdir().expect("temp"); - let mut config = DaemonConfig::default(); - config.daemon.state_dir = temp.path().join("state"); - config.storage.images_dir = temp.path().join("images"); - config.storage.instances_dir = temp.path().join("instances"); - std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); - std::fs::create_dir_all(&config.storage.images_dir).expect("images"); - std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); - - let policy = PolicyFile { - manifest_version: 1, - policy_name: "warm-validation".into(), - priority: 100, - match_: PolicyMatch { - workload_class: WorkloadClass::AgentTool, - image_labels: HashMap::new(), - }, - select: PolicySelect { - backend_priority: vec![BackendKind::Mock], - kernel_hooks: vec![], - templates: vec![], - fallback_on_missing_hook: FallbackOnMissingHook::default(), - }, - pool: Some(PolicyPool { - enabled: true, - min: 0, - target: 0, - max: 1, - warm_ttl: "30m".into(), - reset_mode: ResetMode::FullRecreate, - }), - checkpoint: None, - quota: None, - hooks: PolicyHooks::default(), - backend: BackendConfigs::default(), - vm: None, - }; - let storage: Arc = - Arc::new(FileStorageProvider::with_images( + let config = test_config(&temp); + let observed = Arc::new(AtomicBool::new(false)); + let storage: Arc = Arc::new(OwnershipObservingStorage { + inner: FileStorageProvider::with_images( config.storage.images_dir.clone(), config.storage.instances_dir.clone(), - )); - let state = Arc::new(ServerState::build( + ), + state_dir: config.daemon.state_dir.clone(), + observed: observed.clone(), + }); + let state = build_test_state( config, - PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), + test_policy(BackendKind::Mock, false), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, + ); + + created_json(&state, &test_request()).await; + assert!(observed.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn daemon_shutdown_releases_tracked_runtime_resources() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), )); - let request = serde_json::to_vec(&json!({ - "workload_class": "agent-tool", - "image_digest": "sha256:warm-validation" - })) - .expect("request"); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"); - let cold = create_instance(&state, &request) + shutdown_instances(&state, Duration::from_secs(1)) .await - .expect("cold create"); - let cold: serde_json::Value = - serde_json::from_slice(&cold.into_body().collect().await.expect("body").to_bytes()) - .expect("cold json"); - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("return to pool"); + .expect("shutdown cleanup"); - let warm = create_instance(&state, &request) + assert_eq!( + state.manager.get(id).expect("instance").state, + SandboxState::Destroyed + ); + assert!(state.manager.backend_owner(id).is_none()); + assert!(!instances_dir.join(id.to_string()).exists()); + } + + #[tokio::test] + async fn shutdown_cancels_readiness_and_completes_create_compensation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let spawned = Arc::new(tokio::sync::Notify::new()); + let kill_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(StalledGuestSpawner { + spawned: spawned.clone(), + kill_count: kill_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + let create_state = state.clone(); + let create = + tokio::spawn(async move { create_instance(&create_state, &test_request()).await }); + tokio::time::timeout(Duration::from_secs(1), spawned.notified()) .await - .expect("warm create"); - let warm: serde_json::Value = - serde_json::from_slice(&warm.into_body().collect().await.expect("body").to_bytes()) - .expect("warm json"); - assert_eq!(warm["instance"]["id"], id); - assert_eq!(warm["start_path"], "warm"); + .expect("backend started"); - reset_instance(&state, &id) + state.manager.begin_shutdown(); + let error = tokio::time::timeout(Duration::from_secs(1), create) .await - .expect("return live owner"); - let owner = state - .backend_instances - .lock() - .expect("owners") - .get(&Uuid::parse_str(&id).expect("uuid")) - .cloned() - .expect("owner"); - owner.kill().await.expect("simulate backend exit"); + .expect("create cancellation") + .expect("create task") + .expect_err("readiness must be cancelled"); + assert!(matches!( + error, + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) + )); - let replacement = create_instance(&state, &request) + let instance = state + .manager + .list() + .expect("instances") + .into_iter() + .next() + .expect("cancelled create record"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert!(instance.operation.is_none()); + assert_eq!(instance.backend_ownership, BackendOwnership::Stopped); + assert!(state.manager.backend_owner(instance.id).is_none()); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert!(!instances_dir.join(instance.id.to_string()).exists()); + shutdown_instances(&state, Duration::from_millis(100)) .await - .expect("cold fallback"); - let replacement: serde_json::Value = serde_json::from_slice( - &replacement - .into_body() - .collect() - .await - .expect("body") - .to_bytes(), - ) - .expect("replacement json"); - assert_ne!(replacement["instance"]["id"], id); - assert_eq!(replacement["start_path"], "cold"); - let key = PoolKey::new( - BackendKind::Mock, - WorkloadClass::AgentTool, - "sha256:warm-validation".into(), - ); - assert_eq!( - state - .pool - .lock() - .expect("pool") - .stats(&key) - .quarantine_count, - 1 - ); + .expect("no ownership remains"); } #[tokio::test] - async fn create_publishes_ownership_before_provider_acquire() { + async fn daemon_shutdown_joins_an_active_pool_worker_before_returning() { let temp = tempfile::tempdir().expect("temp"); - let config = test_config(&temp); - let observed = Arc::new(AtomicBool::new(false)); - let storage: Arc = Arc::new(OwnershipObservingStorage { + let mut config = test_config(&temp); + config.storage.pool_size = 1; + config.storage.prefork = false; + std::fs::create_dir_all(config.daemon.state_dir.join("runtime-pool")) + .expect("runtime pool root"); + let release_started = Arc::new(AtomicUsize::new(0)); + let release_active = Arc::new(AtomicUsize::new(0)); + let release_completed = Arc::new(AtomicUsize::new(0)); + let storage = Arc::new(PoolWorkerReleaseStorage { inner: FileStorageProvider::with_images( config.storage.images_dir.clone(), config.storage.instances_dir.clone(), ), - state_dir: config.daemon.state_dir.clone(), - observed: observed.clone(), + acquire_count: AtomicUsize::new(0), + residual_attempt: 1, + delayed_id: Mutex::new(None), + release_started: release_started.clone(), + release_active: release_active.clone(), + release_completed: release_completed.clone(), + release_delay: Duration::from_secs(1), }); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + state + .manager + .configure_runtime_pool_for_test(PoolPrototype { + image_digest: "sha256:pool-shutdown".to_string(), + policy_name: "pool-shutdown".to_string(), + workload_class: WorkloadClass::AgentTool, + templates: Vec::new(), + kernel_hooks: Vec::new(), + binary_path: PathBuf::from("/unused"), + runtime_backend: BackendKind::Mock, + backend: BackendConfigs::default(), + vm: None, + warm_ttl: Duration::from_secs(60), + }) + .expect("configure runtime pool"); + tokio::time::timeout(Duration::from_secs(2), async { + while release_active.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("pool worker starts delayed cleanup"); + + let error = shutdown_instances(&state, Duration::from_millis(40)) + .await + .expect_err("pool cleanup exceeds the shared budget"); + + assert!(error.to_string().contains("runtime-pool")); + assert_eq!(release_started.load(Ordering::Acquire), 1); + assert_eq!(release_completed.load(Ordering::Acquire), 0); + assert_eq!( + release_active.load(Ordering::Acquire), + 0, + "shutdown returned while the pool worker was still running" + ); + assert!(!state.manager.runtime_pool_has_tracked_worker()); + let status = state.manager.runtime_pool_status(); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.quarantined, 1); + + state + .manager + .shutdown_runtime_pool_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry releases the retained pool owner"); + assert_eq!(release_completed.load(Ordering::Acquire), 1); + assert_eq!(state.manager.runtime_pool_status().quarantined, 0); + } + + #[tokio::test(start_paused = true)] + async fn daemon_shutdown_joins_all_owners_within_one_budget() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); let state = build_test_state( config, test_policy(BackendKind::Mock, false), @@ -1855,9 +5141,74 @@ mod tests { BackendKind::Mock, storage, ); + let attempts = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let complete = track_shutdown_owner( + &state, + ShutdownBehavior::Complete, + attempts.clone(), + active.clone(), + ) + .await; + let failed = track_shutdown_owner( + &state, + ShutdownBehavior::Fail, + attempts.clone(), + active.clone(), + ) + .await; + let stalled_a = track_shutdown_owner( + &state, + ShutdownBehavior::Stall, + attempts.clone(), + active.clone(), + ) + .await; + let stalled_b = track_shutdown_owner( + &state, + ShutdownBehavior::Stall, + attempts.clone(), + active.clone(), + ) + .await; - created_json(&state, &test_request()).await; - assert!(observed.load(Ordering::Acquire)); + let started = Instant::now(); + let error = shutdown_instances(&state, Duration::from_millis(40)) + .await + .expect_err("stalled cleanup must exhaust the shared budget"); + + assert!( + started.elapsed() < Duration::from_millis(400), + "cleanup did not quiesce promptly after the test deadline" + ); + assert_eq!( + attempts.load(Ordering::Acquire), + 4, + "every owner must receive a cleanup attempt" + ); + assert_eq!( + active.load(Ordering::Acquire), + 0, + "shutdown returned before every cleanup task became quiescent" + ); + let message = error.to_string(); + for id in [failed, stalled_a, stalled_b] { + assert!(message.contains(&id.to_string())); + assert!(state.manager.backend_owner(id).is_some()); + assert!(instances_dir.join(id.to_string()).is_dir()); + } + assert_eq!( + state + .instances + .lock() + .expect("instances") + .get(&complete) + .expect("complete instance") + .state, + SandboxState::Destroyed + ); + assert!(state.manager.backend_owner(complete).is_none()); + assert!(!instances_dir.join(complete.to_string()).exists()); } #[tokio::test] @@ -1881,13 +5232,17 @@ mod tests { let id = cold["instance"]["id"].as_str().expect("id").to_string(); assert_eq!(cold["instance"]["backend"], "mock"); assert_eq!(cold["selected_backend"], "mock"); + assert!(cold["instance"]["operation"].is_null()); - reset_instance(&state, &id).await.expect("return to pool"); + return_to_pool_for_test(&state, &id) + .await + .expect("return to pool"); let warm = created_json(&state, &request).await; assert_eq!(warm["instance"]["id"], id); assert_eq!(warm["instance"]["backend"], "mock"); assert_eq!(warm["selected_backend"], "mock"); assert_eq!(warm["start_path"], "warm"); + assert!(warm["instance"]["operation"].is_null()); } #[tokio::test] @@ -1919,15 +5274,14 @@ mod tests { .next() .cloned() .expect("retained lifecycle"); + assert_eq!(instance.state, SandboxState::RecoveryRequired); assert_eq!(instance.backend_ownership, BackendOwnership::Running); - assert!(instances_dir.join(instance.id.to_string()).is_dir()); - assert!( - state - .backend_instances - .lock() - .expect("owners") - .contains_key(&instance.id) + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) ); + assert!(instances_dir.join(instance.id.to_string()).is_dir()); + assert!(state.manager.backend_owner(instance.id).is_some()); destroy_instance(&state, &instance.id.to_string()) .await @@ -2124,7 +5478,7 @@ mod tests { let request = test_request(); let cold = created_json(&state, &request).await; let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + return_to_pool_for_test(&state, &id).await.expect("warm"); storage.fail_reconstruct.store(true, Ordering::Release); let error = create_instance(&state, &request) @@ -2168,7 +5522,7 @@ mod tests { let request = test_request(); let cold = created_json(&state, &request).await; let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + return_to_pool_for_test(&state, &id).await.expect("warm"); std::fs::remove_file(instances_dir.join(&id).join("mem.bin")).expect("remove artifact"); let replacement = created_json(&state, &request).await; @@ -2181,6 +5535,95 @@ mod tests { ); } + #[cfg(feature = "test-failpoints")] + async fn assert_warm_state_commit_failure_restores_claim(failpoint: &'static str) { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, true); + let request = test_request(); + let cold = created_json(&state, &request).await; + let id = cold["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + return_to_pool_for_test(&state, &id).await.expect("warm"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let key = PoolKey::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + ); + + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let error = hook + .run(create_instance(&state, &request)) + .await + .expect_err("state commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let restored = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(restored.state, SandboxState::Warm); + assert!(restored.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted warm state"); + assert_eq!(persisted.state, SandboxState::Warm); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + let retained_owner = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained_owner)); + assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 1); + assert!(retained_owner.try_wait().await.expect("liveness").is_none()); + + let retried = created_json(&state, &request).await; + assert_eq!(retried["instance"]["id"], id); + assert_eq!(retried["start_path"], "warm"); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted running state"); + assert_eq!(persisted.state, SandboxState::Running); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn warm_intent_commit_failure_restores_the_claim() { + assert_warm_state_commit_failure_restores_claim("warm-intent-state-commit").await; + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn warm_final_commit_failure_restores_the_claim() { + assert_warm_state_commit_failure_restores_claim("warm-final-state-commit").await; + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn guest_readiness_failure_compensates_owned_resources() { + let request = test_request(); + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let hook = crate::failpoint::TestFailpoint::new(&["create-guest-ready"]); + + hook.run(create_instance(&state, &request)) + .await + .expect_err("guest readiness failure"); + + let instance = state + .instances + .lock() + .expect("instances") + .values() + .next() + .cloned() + .expect("destroyed create"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert!(state.manager.backend_owner(instance.id).is_none()); + assert!( + !temp + .path() + .join("instances") + .join(instance.id.to_string()) + .exists() + ); + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn failure_hooks_drive_create_and_destroy_compensation() { @@ -2221,10 +5664,9 @@ mod tests { assert_eq!(commit_instance.state, SandboxState::Destroyed); assert!( commit_state - .backend_instances - .lock() - .expect("owners") - .is_empty() + .manager + .backend_owner(commit_instance.id) + .is_none() ); let destroy_temp = tempfile::tempdir().expect("temp"); @@ -2237,13 +5679,16 @@ mod tests { .await .expect_err("kill boundary"); let uuid = Uuid::parse_str(&id).expect("uuid"); - assert!( - destroy_state - .backend_instances - .lock() - .expect("owners") - .contains_key(&uuid) + let failed_destroy = destroy_state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(failed_destroy.state, SandboxState::RecoveryRequired); + assert_eq!( + failed_destroy + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Destroy) ); + assert!(destroy_state.manager.backend_owner(uuid).is_some()); destroy_instance(&destroy_state, &id) .await .expect("destroy retry"); @@ -2262,11 +5707,153 @@ mod tests { release_state.instances.lock().expect("instances")[&uuid].backend_ownership, BackendOwnership::Stopped ); + assert_eq!( + release_state.instances.lock().expect("instances")[&uuid] + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); destroy_instance(&release_state, &id) .await .expect("release retry"); } + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_intent_failure_does_not_touch_owned_resources() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-intent-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("intent failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 0); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert!(retained.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + assert!(temp.path().join("instances").join(&id).is_dir()); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 1); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Destroyed + ); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_stop_commit_failure_retains_storage_for_retry() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-stop-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("stop commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 0); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!(retained.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + persisted.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!(temp.path().join("instances").join(&id).is_dir()); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 1); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_final_commit_failure_retains_retryable_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-final-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("final commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 1); + assert!(!temp.path().join("instances").join(&id).exists()); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!(retained.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + persisted.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 2); + let destroyed = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn acquire_rollback_failure_retains_a_destroyable_record() { @@ -2290,8 +5877,12 @@ mod tests { .next() .cloned() .expect("recovery record"); - assert_eq!(instance.state, SandboxState::Creating); + assert_eq!(instance.state, SandboxState::RecoveryRequired); assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); assert!( temp.path() .join("instances") @@ -2308,133 +5899,760 @@ mod tests { async fn acquired_slot_is_destroyable_after_restart_before_start_commit() { let temp = tempfile::tempdir().expect("temp"); let config = test_config(&temp); - let instances_dir = config.storage.instances_dir.clone(); - let initial_storage: Arc = Arc::new(FileStorageProvider::with_images( + let instances_dir = config.storage.instances_dir.clone(); + let initial_storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let initial_state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + initial_storage, + ); + let pause_hook = crate::failpoint::TestFailpoint::new(&["create-after-storage-acquire"]); + let create_state = initial_state.clone(); + let create_hook = pause_hook.clone(); + let create = tokio::spawn(async move { + create_hook + .run(create_instance(&create_state, &test_request())) + .await + }); + pause_hook.wait_until_paused().await; + + let instance = initial_state + .instances + .lock() + .expect("instances") + .values() + .next() + .cloned() + .expect("write-ahead instance"); + let id = instance.id; + assert_eq!(instance.state, SandboxState::Creating); + assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); + assert!( + config + .daemon + .state_dir + .join(id.to_string()) + .join("state.json") + .is_file() + ); + assert!(instances_dir.join(id.to_string()).is_dir()); + + create.abort(); + assert!( + create + .await + .expect_err("create task aborted") + .is_cancelled() + ); + drop(initial_state); + + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let restarted_storage: Arc = + Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(RecordingSpawner { + cleanup_count: cleanup_count.clone(), + }), + ), + BackendKind::Mock, + restarted_storage, + ); + assert!( + restarted + .instances + .lock() + .expect("instances") + .contains_key(&id) + ); + + destroy_instance(&restarted, &id.to_string()) + .await + .expect("destroy acquired slot after restart"); + assert_eq!(cleanup_count.load(Ordering::Acquire), 0); + assert_eq!( + restarted.instances.lock().expect("instances")[&id].state, + SandboxState::Destroyed + ); + assert!(!instances_dir.join(id.to_string()).exists()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn warm_activation_and_destroy_are_serialized_per_instance() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, true); + let request = test_request(); + let cold = created_json(&state, &request).await; + let id = cold["instance"]["id"].as_str().expect("id").to_string(); + return_to_pool_for_test(&state, &id).await.expect("warm"); + + let pause_hook = crate::failpoint::TestFailpoint::new(&["warm-before-state-commit"]); + let create_state = state.clone(); + let create_request = request.clone(); + let activation_hook = pause_hook.clone(); + let activation = tokio::spawn(async move { + activation_hook + .run(create_instance(&create_state, &create_request)) + .await + }); + pause_hook.wait_until_paused().await; + let uuid = Uuid::parse_str(&id).expect("uuid"); + assert_eq!( + state.instances.lock().expect("instances")[&uuid] + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Create) + ); + + let destroy_state = state.clone(); + let destroy_id = id.clone(); + let destroy = + tokio::spawn(async move { destroy_instance(&destroy_state, &destroy_id).await }); + tokio::task::yield_now().await; + assert!(!destroy.is_finished(), "destroy must wait for activation"); + + pause_hook.release(); + activation + .await + .expect("activation task") + .expect("activation"); + destroy.await.expect("destroy task").expect("destroy"); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Destroyed + ); + } + + #[tokio::test] + async fn startup_reconciliation_repairs_anomalous_destroyed_records() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + + let clean_stopped_id = Uuid::new_v4(); + let clean_not_started_id = Uuid::new_v4(); + let legacy_id = Uuid::new_v4(); + let active_id = Uuid::new_v4(); + for (id, ownership, active_operation) in [ + (clean_stopped_id, BackendOwnership::Stopped, false), + (clean_not_started_id, BackendOwnership::NotStarted, false), + (legacy_id, BackendOwnership::Unknown, false), + (active_id, BackendOwnership::Running, true), + ] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:destroyed-reconcile".into(), + StartPath::Cold, + "destroyed-reconcile-test".into(), + ); + instance.id = id; + if active_operation { + instance + .begin_operation(OperationKind::Create) + .expect("begin interrupted create"); + } + instance + .transition(SandboxState::Destroyed) + .expect("destroyed"); + instance.backend_ownership = ownership; + instance.persist(&config.daemon.state_dir).expect("persist"); + + if id == legacy_id || id == active_id { + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + } + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(SelectiveCleanupSpawner { + failed_id: legacy_id, + cleanup_count: cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage.clone(), + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, legacy_id); + assert_eq!(cleanup_count.load(Ordering::Acquire), 2); + assert_eq!(release_count.load(Ordering::Acquire), 1); + let retryable = state.manager.get(legacy_id).expect("retryable instance"); + assert_eq!(retryable.state, SandboxState::Destroyed); + assert_eq!(retryable.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + retryable.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!( + config + .storage + .instances_dir + .join(legacy_id.to_string()) + .is_dir() + ); + + drop(state); + let retry_cleanup_count = Arc::new(AtomicUsize::new(0)); + let restarted = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(RecordingSpawner { + cleanup_count: retry_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + let retry_report = restarted.manager.reconcile_startup().await; + + assert_eq!(retry_report.attempted, 1); + assert_eq!(retry_report.completed, 1); + assert!(retry_report.failures.is_empty()); + assert_eq!(retry_cleanup_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 2); + for id in [clean_stopped_id, legacy_id, active_id] { + let instance = restarted.manager.get(id).expect("instance"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert_eq!(instance.backend_ownership, BackendOwnership::Stopped); + assert!(instance.operation.is_none()); + } + let clean_not_started = restarted + .manager + .get(clean_not_started_id) + .expect("clean not-started instance"); + assert_eq!(clean_not_started.state, SandboxState::Destroyed); + assert_eq!( + clean_not_started.backend_ownership, + BackendOwnership::NotStarted + ); + assert!(clean_not_started.operation.is_none()); + assert!( + !config + .storage + .instances_dir + .join(legacy_id.to_string()) + .exists() + ); + assert!( + !config + .storage + .instances_dir + .join(active_id.to_string()) + .exists() + ); + } + + #[tokio::test(start_paused = true)] + async fn startup_reconciliation_times_out_one_record_and_continues() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let stalled_id = Uuid::new_v4(); + let completed_id = Uuid::new_v4(); + let inner = FileStorageProvider::with_images( config.storage.images_dir.clone(), - instances_dir.clone(), - )); - let initial_state = build_test_state( + config.storage.instances_dir.clone(), + ); + for id in [stalled_id, completed_id] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:bounded-reconcile".into(), + StartPath::Cold, + "bounded-reconcile-test".into(), + ); + instance.id = id; + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.persist(&config.daemon.state_dir).expect("persist"); + inner + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + let storage: Arc = + Arc::new(SelectiveHangingStorage { inner, stalled_id }); + let state = build_test_state( config.clone(), test_policy(BackendKind::Mock, false), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, - initial_storage, + storage, ); - let pause_hook = crate::failpoint::TestFailpoint::new(&["create-after-storage-acquire"]); - let create_state = initial_state.clone(); - let create_hook = pause_hook.clone(); - let create = tokio::spawn(async move { - create_hook - .run(create_instance(&create_state, &test_request())) - .await - }); - pause_hook.wait_until_paused().await; - let instance = initial_state - .instances - .lock() - .expect("instances") - .values() - .next() - .cloned() - .expect("write-ahead instance"); - let id = instance.id; - assert_eq!(instance.state, SandboxState::Creating); - assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + let report = state + .manager + .cleanup_owned_instances_with_timeout(Duration::from_millis(20)) + .await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, stalled_id); + assert!(report.failures[0].error.contains("20 ms")); + let stalled = state.manager.get(stalled_id).expect("stalled record"); + assert_eq!(stalled.state, SandboxState::RecoveryRequired); + assert_eq!(stalled.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + stalled.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert_eq!( + state.manager.get(completed_id).expect("completed").state, + SandboxState::Destroyed + ); assert!( config - .daemon - .state_dir - .join(id.to_string()) - .join("state.json") - .is_file() + .storage + .instances_dir + .join(stalled_id.to_string()) + .is_dir() ); - assert!(instances_dir.join(id.to_string()).is_dir()); - - create.abort(); assert!( - create - .await - .expect_err("create task aborted") - .is_cancelled() + !config + .storage + .instances_dir + .join(completed_id.to_string()) + .exists() ); - drop(initial_state); - let cleanup_count = Arc::new(AtomicUsize::new(0)); - let restarted_storage: Arc = - Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - instances_dir.clone(), - )); + drop(state); + let retry_storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); let restarted = build_test_state( config, test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + retry_storage, + ); + let retry_report = restarted.manager.reconcile_startup().await; + + assert_eq!(retry_report.attempted, 1); + assert_eq!(retry_report.completed, 1); + assert!(retry_report.failures.is_empty()); + let recovered = restarted.manager.get(stalled_id).expect("recovered record"); + assert_eq!(recovered.state, SandboxState::Destroyed); + assert_eq!(recovered.backend_ownership, BackendOwnership::Stopped); + assert!(recovered.operation.is_none()); + } + + #[tokio::test] + async fn startup_reconciliation_continues_after_one_cleanup_failure() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let failed_id = Uuid::new_v4(); + let completed_id = Uuid::new_v4(); + for id in [failed_id, completed_id] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:reconcile".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + instance.id = id; + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance.persist(&config.daemon.state_dir).expect("persist"); + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), spawners( BackendKind::Mock, - Arc::new(RecordingSpawner { + Arc::new(SelectiveCleanupSpawner { + failed_id, cleanup_count: cleanup_count.clone(), }), ), BackendKind::Mock, - restarted_storage, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, failed_id); + assert_eq!(cleanup_count.load(Ordering::Acquire), 2); + assert_eq!( + state.instances.lock().expect("instances")[&failed_id].state, + SandboxState::RecoveryRequired + ); + assert_eq!( + state.instances.lock().expect("instances")[&completed_id].state, + SandboxState::Destroyed ); assert!( - restarted - .instances - .lock() - .expect("instances") - .contains_key(&id) + config + .storage + .instances_dir + .join(failed_id.to_string()) + .is_dir() + ); + assert!( + !config + .storage + .instances_dir + .join(completed_id.to_string()) + .exists() ); + let created = created_json(&state, &test_request()).await; + assert_eq!(created["instance"]["state"], "running"); + } - destroy_instance(&restarted, &id.to_string()) + #[cfg(target_os = "linux")] + #[tokio::test] + async fn startup_reconciliation_cleans_pre_spawn_backend_ownership() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let mut instance = SandboxInstance::new( + BackendKind::Bubblewrap, + WorkloadClass::AgentTool, + "sha256:pre-spawn".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + let id = instance.id; + let run_dir = config.daemon.state_dir.join(id.to_string()); + let spawner = Arc::new(BubblewrapSpawner); + spawner + .prepare_spawn(&run_dir) .await - .expect("destroy acquired slot after restart"); - assert_eq!(cleanup_count.load(Ordering::Acquire), 0); + .expect("persist pre-spawn handoff"); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + instance.backend_ownership = BackendOwnership::Starting; + instance + .persist(&config.daemon.state_dir) + .expect("persist starting ownership"); + storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + assert!( + std::fs::read_to_string(run_dir.join("backend.pid")) + .expect("pre-spawn handoff") + .is_empty(), + "crash occurred after the empty handoff was persisted but before spawn" + ); + + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Bubblewrap, false), + spawners(BackendKind::Bubblewrap, spawner), + BackendKind::Bubblewrap, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); assert_eq!( - restarted.instances.lock().expect("instances")[&id].state, + state.instances.lock().expect("instances")[&id].state, SandboxState::Destroyed ); - assert!(!instances_dir.join(id.to_string()).exists()); + assert!(!config.storage.instances_dir.join(id.to_string()).exists()); + assert!(run_dir.join("backend.stopped").is_file()); } - #[cfg(feature = "test-failpoints")] #[tokio::test] - async fn warm_activation_and_destroy_are_serialized_per_instance() { + async fn startup_reconciliation_skips_cleanup_for_known_stopped_states() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, true); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + let config = test_config(&temp); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage: Arc = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + let not_started_id = Uuid::new_v4(); + let stopped_id = Uuid::new_v4(); - let pause_hook = crate::failpoint::TestFailpoint::new(&["warm-before-state-commit"]); - let create_state = state.clone(); - let create_request = request.clone(); - let activation_hook = pause_hook.clone(); - let activation = tokio::spawn(async move { - activation_hook - .run(create_instance(&create_state, &create_request)) + let mut not_started = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:not-started".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + not_started.id = not_started_id; + not_started + .transition(SandboxState::Creating) + .expect("creating"); + not_started + .persist(&config.daemon.state_dir) + .expect("persist"); + + let mut stopped = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:stopped".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + stopped.id = stopped_id; + stopped + .transition(SandboxState::Creating) + .expect("creating"); + stopped.transition(SandboxState::Running).expect("running"); + stopped.backend_ownership = BackendOwnership::Stopped; + stopped.persist(&config.daemon.state_dir).expect("persist"); + + for id in [not_started_id, stopped_id] { + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) .await - }); - pause_hook.wait_until_paused().await; + .expect("storage"); + } + let kill_count = Arc::new(AtomicUsize::new(0)); + let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: orphan_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); - let destroy_state = state.clone(); - let destroy_id = id.clone(); - let destroy = - tokio::spawn(async move { destroy_instance(&destroy_state, &destroy_id).await }); - tokio::task::yield_now().await; - assert!(!destroy.is_finished(), "destroy must wait for activation"); + let report = state.manager.reconcile_startup().await; - pause_hook.release(); - activation - .await - .expect("activation task") - .expect("activation"); - destroy.await.expect("destroy task").expect("destroy"); - let uuid = Uuid::parse_str(&id).expect("uuid"); + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 2); + assert!(report.failures.is_empty()); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 2); assert_eq!( - state.instances.lock().expect("instances")[&uuid].state, + state.instances.lock().expect("instances")[¬_started_id].state, + SandboxState::Destroyed + ); + assert_eq!( + state.instances.lock().expect("instances")[&stopped_id].state, SandboxState::Destroyed ); } + + #[tokio::test] + async fn runtime_template_routes_import_list_and_get_published_artifacts() { + let temp = tempfile::tempdir().expect("temp"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + std::fs::create_dir(&import_root).expect("import root"); + std::fs::create_dir(&source).expect("source"); + std::fs::write(source.join("vmstate.snap"), b"snapshot").expect("snapshot"); + std::fs::write(source.join("mem.bin"), b"memory").expect("memory"); + std::fs::write(source.join("rootfs.ext4"), b"rootfs").expect("rootfs"); + + let mut config = DaemonConfig::default(); + config.daemon.state_dir = temp.path().join("state"); + config.storage.images_dir = temp.path().join("images"); + config.storage.instances_dir = temp.path().join("instances"); + config.template.dir = temp.path().join("templates"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); + config.runtime_templates.import_root = Some(import_root); + for directory in [ + &config.daemon.state_dir, + &config.storage.images_dir, + &config.storage.instances_dir, + &config.template.dir, + &config.runtime_templates.dir, + ] { + std::fs::create_dir_all(directory).expect("directory"); + } + let storage: Arc = + Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(Vec::new()), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ) + .expect("state"), + ); + + let request = serde_json::to_vec(&json!({ + "name": "runtime-base", + "source": "source", + "description": "reusable runtime", + })) + .expect("request"); + let imported = dispatch( + &Method::POST, + "/v1/runtime-templates/import", + "", + request.clone(), + &state, + ) + .await + .expect("import"); + assert_eq!(imported.status(), StatusCode::CREATED); + let imported = serde_json::from_slice::( + &imported + .into_body() + .collect() + .await + .expect("body") + .to_bytes(), + ) + .expect("json"); + assert_eq!(imported["name"], "runtime-base"); + assert_eq!(imported["description"], "reusable runtime"); + + let listed = dispatch( + &Method::GET, + "/v1/runtime-templates", + "", + Vec::new(), + &state, + ) + .await + .expect("list"); + let listed = serde_json::from_slice::( + &listed.into_body().collect().await.expect("body").to_bytes(), + ) + .expect("json"); + assert_eq!(listed.as_array().expect("templates").len(), 1); + assert_eq!(listed[0]["name"], "runtime-base"); + + let fetched = dispatch( + &Method::GET, + "/v1/runtime-templates/runtime-base", + "", + Vec::new(), + &state, + ) + .await + .expect("get"); + let fetched = serde_json::from_slice::( + &fetched + .into_body() + .collect() + .await + .expect("body") + .to_bytes(), + ) + .expect("json"); + assert_eq!(fetched, imported); + + let duplicate = dispatch( + &Method::POST, + "/v1/runtime-templates/import", + "", + request, + &state, + ) + .await + .expect_err("duplicate"); + assert!(matches!(duplicate, BlazeDaemonError::Conflict(_))); + + let legacy = dispatch(&Method::GET, "/v1/templates", "", Vec::new(), &state) + .await + .expect("legacy template registry"); + let legacy = serde_json::from_slice::( + &legacy.into_body().collect().await.expect("body").to_bytes(), + ) + .expect("json"); + assert_eq!(legacy, json!([])); + } } diff --git a/src/blaze/crates/blazed/src/checkpoint_store.rs b/src/blaze/crates/blazed/src/checkpoint_store.rs new file mode 100644 index 0000000000..32b4005c66 --- /dev/null +++ b/src/blaze/crates/blazed/src/checkpoint_store.rs @@ -0,0 +1,1387 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Filesystem-backed checkpoint catalog owned by the daemon. +//! +//! Publication and HEAD updates are separate durability boundaries. A +//! checkpoint can therefore be published but unreachable after an interrupted +//! operation; listing exposes that state and pruning can remove it later. + +use std::collections::{HashMap, HashSet}; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use blaze_core::checkpoint::{ + CHECKPOINT_FORMAT_VERSION, CheckpointArtifact, CheckpointInfo, CheckpointMetadata, + CheckpointValidationError, CommitCheckpoint, REQUIRED_ARTIFACTS, validate_artifact_name, + validate_checkpoint_id, validate_checkpoint_manifest, validate_commit_checkpoint, +}; +use chrono::Utc; +use sha2::{Digest, Sha256}; +use thiserror::Error; +use uuid::Uuid; + +const METADATA_FILE: &str = "metadata.json"; +const HEAD_FILE: &str = "HEAD"; +const STAGING_SUFFIX: &str = ".tmp"; +const TOMBSTONE_SUFFIX: &str = ".tombstone"; +const PRUNE_TOMBSTONE_PREFIX: &str = ".prune."; +const ABORT_TOMBSTONE_PREFIX: &str = ".abort."; + +/// Failure while reading or mutating the daemon checkpoint catalog. +#[derive(Debug, Error)] +pub enum CheckpointStoreError { + /// A checkpoint record failed pure model validation. + #[error(transparent)] + Validation(#[from] CheckpointValidationError), + + /// A catalog filesystem operation failed. + #[error("checkpoint catalog {operation} failed for {}: {source}", path.display())] + Io { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// A metadata file could not be encoded or decoded. + #[error("checkpoint metadata at {} is invalid: {source}", path.display())] + Json { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + + /// The catalog layout violates an invariant required for safe mutation. + #[error("checkpoint catalog invariant failed: {0}")] + Invariant(String), +} + +/// Convenient result type for checkpoint catalog operations. +pub type Result = std::result::Result; + +/// Verified checkpoint metadata and provider-safe artifact paths. +#[derive(Debug)] +pub struct VerifiedCheckpoint { + /// Validated checkpoint manifest. + pub metadata: CheckpointMetadata, + /// Full backend-state snapshot. + pub snapshot_path: PathBuf, + /// Full guest-memory snapshot. + pub memory_path: PathBuf, + /// Self-contained root filesystem snapshot. + pub rootfs_path: PathBuf, +} + +/// Temporary checkpoint directory populated before atomic publication. +#[derive(Debug)] +pub struct CheckpointStage { + id: String, + sandbox_id: Uuid, + path: PathBuf, + final_path: PathBuf, +} + +impl CheckpointStage { + /// Generated checkpoint identifier. + pub fn id(&self) -> &str { + &self.id + } + + /// Resolve one frozen artifact inside the stage. + pub fn artifact_path(&self, name: &str) -> Result { + validate_artifact_name(name)?; + Ok(self.path.join(name)) + } +} + +/// Filesystem-backed checkpoint catalog. +#[derive(Debug, Clone)] +pub struct CheckpointStore { + root: PathBuf, +} + +impl CheckpointStore { + /// Create a catalog rooted at `root` without performing I/O. + pub fn new(root: PathBuf) -> Self { + Self { root } + } + + /// Create and durably expose a unique staging directory. + pub fn begin(&self, sandbox_id: Uuid) -> Result { + let sandbox_dir = self.ensure_sandbox_dir(sandbox_id)?; + let id = format!("ckpt-{}", Uuid::new_v4()); + let path = sandbox_dir.join(format!(".{id}{STAGING_SUFFIX}")); + create_directory(&path, "create staging directory")?; + sync_directory(&sandbox_dir)?; + Ok(CheckpointStage { + final_path: sandbox_dir.join(&id), + id, + sandbox_id, + path, + }) + } + + /// Hash, sync, and atomically publish a populated stage without moving HEAD. + pub fn publish( + &self, + stage: &CheckpointStage, + input: CommitCheckpoint, + ) -> Result { + self.validate_stage(stage)?; + validate_commit_checkpoint(&stage.id, &input)?; + if let Some(parent) = &input.parent { + self.validated_chain_from(stage.sandbox_id, parent)?; + } + ensure_missing(&stage.final_path, "inspect publication target")?; + validate_exact_entries(&stage.path, &REQUIRED_ARTIFACTS)?; + + let mut artifacts = Vec::with_capacity(REQUIRED_ARTIFACTS.len()); + for name in REQUIRED_ARTIFACTS { + let path = require_contained_file(&stage.path, name)?; + sync_file(&path)?; + artifacts.push(hash_artifact(&path, name)?); + } + + let metadata = CheckpointMetadata { + format_version: CHECKPOINT_FORMAT_VERSION, + id: stage.id.clone(), + parent: input.parent, + sandbox_id: stage.sandbox_id, + policy_name: input.policy_name, + image_digest: input.image_digest, + backend: input.backend, + backend_version: input.backend_version, + created_at: Utc::now(), + snapshot_kind: input.snapshot_kind, + expose_guest_socket: input.expose_guest_socket, + network_slot: input.network_slot, + artifacts, + }; + validate_checkpoint_manifest(&metadata, stage.sandbox_id, &stage.id)?; + + let metadata_path = stage.path.join(METADATA_FILE); + write_json_new(&metadata_path, &metadata)?; + sync_directory(&stage.path)?; + rename_path( + &stage.path, + &stage.final_path, + "publish checkpoint directory", + )?; + let sandbox_dir = stage + .final_path + .parent() + .ok_or_else(|| invariant("published checkpoint has no sandbox parent"))?; + checkpoint_store_failpoint("checkpoint-store-publish-after-rename", &stage.final_path)?; + sync_directory(sandbox_dir)?; + Ok(metadata) + } + + /// Remove an unpublished stage owned by this process. + pub fn abort(&self, stage: CheckpointStage) -> Result<()> { + self.abort_staging(stage.sandbox_id, &stage.id) + } + + /// Remove one unpublished stage if it still exists. + /// + /// The stage is first renamed to a tombstone so a process interruption + /// cannot make it appear publishable again. Startup cleanup removes any + /// residual tombstone. + pub fn abort_staging(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result<()> { + validate_checkpoint_id(checkpoint_id)?; + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(()); + }; + let stage = sandbox_dir.join(format!(".{checkpoint_id}{STAGING_SUFFIX}")); + let Some(metadata) = optional_symlink_metadata(&stage, "inspect staging directory")? else { + return Ok(()); + }; + require_plain_directory_metadata(&stage, &metadata, "checkpoint staging directory")?; + require_direct_child(&sandbox_dir, &stage, "checkpoint staging directory")?; + + let tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, checkpoint_id); + rename_path(&stage, &tombstone, "tombstone aborted checkpoint stage")?; + sync_directory(&sandbox_dir)?; + remove_directory(&tombstone, "remove aborted checkpoint tombstone")?; + sync_directory(&sandbox_dir) + } + + /// Read and validate one committed checkpoint and all artifact hashes. + pub fn verify(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result { + let dir = self.committed_dir(sandbox_id, checkpoint_id)?; + validate_exact_entries( + &dir, + &[ + REQUIRED_ARTIFACTS[0], + REQUIRED_ARTIFACTS[1], + REQUIRED_ARTIFACTS[2], + METADATA_FILE, + ], + )?; + let metadata_path = require_contained_file(&dir, METADATA_FILE)?; + let bytes = read_file(&metadata_path, "read checkpoint metadata")?; + let metadata: CheckpointMetadata = + serde_json::from_slice(&bytes).map_err(|source| CheckpointStoreError::Json { + path: metadata_path, + source, + })?; + validate_checkpoint_manifest(&metadata, sandbox_id, checkpoint_id)?; + + for name in REQUIRED_ARTIFACTS { + let expected = metadata + .artifacts + .iter() + .find(|artifact| artifact.name == name) + .ok_or_else(|| { + invariant(format!( + "validated checkpoint {checkpoint_id} has no record for {name}" + )) + })?; + let path = require_contained_file(&dir, name)?; + let actual = hash_artifact(&path, name)?; + if &actual != expected { + return Err(invariant(format!( + "checkpoint {checkpoint_id} artifact {name} failed integrity validation" + ))); + } + } + Ok(metadata) + } + + /// Verify a restore target, its complete ancestry, and its artifact paths. + pub fn verify_restore_target( + &self, + sandbox_id: Uuid, + checkpoint_id: &str, + ) -> Result { + let metadata = self.verify(sandbox_id, checkpoint_id)?; + if let Some(parent) = metadata.parent.as_deref() { + self.validated_chain_from(sandbox_id, parent)?; + } + let directory = self.committed_dir(sandbox_id, checkpoint_id)?; + Ok(VerifiedCheckpoint { + snapshot_path: require_contained_file(&directory, "vmstate.snap")?, + memory_path: require_contained_file(&directory, "memory.snap")?, + rootfs_path: require_contained_file(&directory, "rootfs.snap")?, + metadata, + }) + } + + /// List committed checkpoints and mark the lineage reachable from HEAD. + pub fn list(&self, sandbox_id: Uuid) -> Result> { + let Some(_) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + let catalog = self.load_catalog(sandbox_id)?; + let head = self.read_head(sandbox_id)?; + let on_head_chain = match head.as_deref() { + Some(head) => lineage_from(&catalog, head)?, + None => HashSet::new(), + }; + + let mut checkpoints = Vec::with_capacity(catalog.len()); + for metadata in catalog.into_values() { + let size_bytes = metadata + .artifacts + .iter() + .try_fold(0_u64, |total, artifact| { + total.checked_add(artifact.size_bytes) + }) + .ok_or_else(|| { + invariant(format!( + "checkpoint {} artifact sizes overflow u64", + metadata.id + )) + })?; + checkpoints.push(CheckpointInfo { + id: metadata.id.clone(), + parent: metadata.parent, + created_at: metadata.created_at, + size_bytes, + is_head: head.as_deref() == Some(metadata.id.as_str()), + on_head_chain: on_head_chain.contains(&metadata.id), + }); + } + checkpoints.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + Ok(checkpoints) + } + + /// Prune unreferenced branches while retaining HEAD and explicit lineages. + /// + /// Every candidate is first atomically renamed to a hidden tombstone and + /// the sandbox directory is synced. An interrupted delete is therefore + /// absent from the live catalog and can be completed by + /// [`Self::cleanup_transaction_artifacts`]. + pub fn prune_preserving(&self, sandbox_id: Uuid, protected: &[String]) -> Result> { + let catalog = self.load_catalog(sandbox_id)?; + let mut keep = HashSet::new(); + if let Some(head) = self.read_head(sandbox_id)? { + keep.extend(lineage_from(&catalog, &head)?); + } + for checkpoint_id in protected { + validate_checkpoint_id(checkpoint_id)?; + keep.extend(lineage_from(&catalog, checkpoint_id)?); + } + let mut candidates = catalog + .keys() + .filter(|checkpoint_id| !keep.contains(*checkpoint_id)) + .cloned() + .collect::>(); + candidates.sort(); + if candidates.is_empty() { + return Ok(Vec::new()); + } + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} disappeared")))?; + + let mut removed = Vec::with_capacity(candidates.len()); + for checkpoint_id in candidates { + let checkpoint_dir = self.committed_dir(sandbox_id, &checkpoint_id)?; + let tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, &checkpoint_id); + rename_path(&checkpoint_dir, &tombstone, "tombstone pruned checkpoint")?; + sync_directory(&sandbox_dir)?; + checkpoint_store_failpoint("checkpoint-prune-after-tombstone", &tombstone)?; + remove_directory(&tombstone, "remove pruned checkpoint tombstone")?; + sync_directory(&sandbox_dir)?; + removed.push(checkpoint_id); + } + Ok(removed) + } + + /// Atomically move HEAD to an already committed, verified checkpoint. + pub fn set_head(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result<()> { + self.verify(sandbox_id, checkpoint_id)?; + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} disappeared")))?; + validate_existing_head_type(&sandbox_dir)?; + + let temporary = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + let outcome = (|| { + let mut file = open_new_file(&temporary, "create temporary HEAD")?; + write_all(&mut file, &temporary, checkpoint_id.as_bytes())?; + write_all(&mut file, &temporary, b"\n")?; + sync_open_file(&file, &temporary)?; + rename_path( + &temporary, + &sandbox_dir.join(HEAD_FILE), + "publish checkpoint HEAD", + )?; + checkpoint_store_failpoint( + "checkpoint-store-head-after-rename", + &sandbox_dir.join(HEAD_FILE), + )?; + sync_directory(&sandbox_dir) + })(); + if outcome.is_err() { + let _ = remove_file_if_exists(&temporary, "remove temporary HEAD"); + } + outcome + } + + /// Return the persisted HEAD, if present. + pub fn read_head(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(None); + }; + let path = sandbox_dir.join(HEAD_FILE); + let Some(metadata) = optional_symlink_metadata(&path, "inspect checkpoint HEAD")? else { + return Ok(None); + }; + require_plain_file_metadata(&path, &metadata, "checkpoint HEAD")?; + require_direct_child(&sandbox_dir, &path, "checkpoint HEAD")?; + let bytes = read_file(&path, "read checkpoint HEAD")?; + let raw = std::str::from_utf8(&bytes) + .map_err(|error| invariant(format!("checkpoint HEAD is not UTF-8: {error}")))?; + let checkpoint_id = raw + .strip_suffix('\n') + .filter(|value| !value.contains('\n') && !value.contains('\r')) + .ok_or_else(|| invariant("checkpoint HEAD is not one canonical line"))?; + validate_checkpoint_id(checkpoint_id)?; + self.committed_dir(sandbox_id, checkpoint_id)?; + Ok(Some(checkpoint_id.to_string())) + } + + /// Return whether a checkpoint transaction left cleanup artifacts. + /// + /// Missing sandbox directories report `false`. Recognized names with an + /// unexpected file type return an error instead of being silently ignored, + /// matching [`Self::cleanup_transaction_artifacts`] so the caller can + /// report the unsafe layout without deleting an unrelated entry. + pub(crate) fn has_transaction_artifacts(&self, sandbox_id: Uuid) -> Result { + Ok(!self.transaction_artifacts(sandbox_id)?.is_empty()) + } + + /// Remove incomplete stages, temporary HEAD files, and cleanup tombstones. + /// + /// Committed checkpoint directories and the published HEAD are retained. + pub fn cleanup_transaction_artifacts(&self, sandbox_id: Uuid) -> Result> { + let scratch = self.transaction_artifacts(sandbox_id)?; + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + + let mut removed = Vec::with_capacity(scratch.len()); + for (path, kind) in scratch { + match kind { + ScratchKind::Directory => { + remove_directory(&path, "remove checkpoint scratch directory")? + } + ScratchKind::File => remove_file(&path, "remove checkpoint scratch file")?, + } + removed.push(path); + } + if !removed.is_empty() { + sync_directory(&sandbox_dir)?; + } + Ok(removed) + } + + fn transaction_artifacts(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + let mut scratch = Vec::new(); + let entries = read_directory(&sandbox_dir, "scan checkpoint scratch")?; + for entry in entries { + let entry = entry + .map_err(|source| io_error("read checkpoint scratch", &sandbox_dir, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(kind) = classify_scratch_name(name)? else { + continue; + }; + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint scratch", &path, source))?; + match kind { + ScratchKind::Directory if file_type.is_dir() && !file_type.is_symlink() => { + scratch.push((path, ScratchKind::Directory)); + } + ScratchKind::File if file_type.is_file() && !file_type.is_symlink() => { + scratch.push((path, ScratchKind::File)); + } + _ => { + return Err(invariant(format!( + "checkpoint scratch {} has an unexpected file type", + path.display() + ))); + } + } + } + scratch.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(scratch) + } + + fn validate_stage(&self, stage: &CheckpointStage) -> Result<()> { + validate_checkpoint_id(&stage.id)?; + let sandbox_dir = self + .optional_sandbox_dir(stage.sandbox_id)? + .ok_or_else(|| invariant("checkpoint staging sandbox does not exist"))?; + let expected_path = sandbox_dir.join(format!(".{}{STAGING_SUFFIX}", stage.id)); + let expected_final_path = sandbox_dir.join(&stage.id); + if stage.path != expected_path || stage.final_path != expected_final_path { + return Err(invariant( + "checkpoint stage paths do not match its frozen identity", + )); + } + require_plain_directory(&stage.path, "checkpoint staging directory")?; + require_direct_child(&sandbox_dir, &stage.path, "checkpoint staging directory") + } + + fn validated_chain_from(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result> { + validate_checkpoint_id(checkpoint_id)?; + let mut current = checkpoint_id.to_string(); + let mut lineage = Vec::new(); + let mut seen = HashSet::new(); + loop { + if !seen.insert(current.clone()) { + return Err(invariant(format!( + "checkpoint parent cycle reaches {current}" + ))); + } + let metadata = self.verify(sandbox_id, ¤t)?; + lineage.push(current); + let Some(parent) = metadata.parent else { + break; + }; + current = parent; + } + Ok(lineage) + } + + fn load_catalog(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(HashMap::new()); + }; + let mut catalog = HashMap::new(); + let entries = read_directory(&sandbox_dir, "scan checkpoint catalog")?; + for entry in entries { + let entry = entry + .map_err(|source| io_error("read checkpoint catalog", &sandbox_dir, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !name.starts_with("ckpt-") { + continue; + } + validate_checkpoint_id(name)?; + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint entry", entry.path(), source))?; + if !file_type.is_dir() || file_type.is_symlink() { + return Err(invariant(format!( + "checkpoint entry {} is not a plain directory", + entry.path().display() + ))); + } + let metadata = self.verify(sandbox_id, name)?; + catalog.insert(name.to_string(), metadata); + } + Ok(catalog) + } + + fn ensure_sandbox_dir(&self, sandbox_id: Uuid) -> Result { + let root_was_missing = + optional_symlink_metadata(&self.root, "inspect checkpoint root")?.is_none(); + create_directories(&self.root, "create checkpoint root")?; + require_plain_directory(&self.root, "checkpoint root")?; + if root_was_missing + && let Some(parent) = self + .root + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + sync_directory(parent)?; + } + let sandbox_dir = self.sandbox_dir(sandbox_id); + match fs::create_dir(&sandbox_dir) { + Ok(()) => { + sync_directory(&self.root)?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(io_error( + "create checkpoint sandbox directory", + &sandbox_dir, + source, + )); + } + } + require_plain_directory(&sandbox_dir, "checkpoint sandbox directory")?; + require_direct_child(&self.root, &sandbox_dir, "checkpoint sandbox directory")?; + Ok(sandbox_dir) + } + + fn optional_sandbox_dir(&self, sandbox_id: Uuid) -> Result> { + let Some(root_metadata) = optional_symlink_metadata(&self.root, "inspect checkpoint root")? + else { + return Ok(None); + }; + require_plain_directory_metadata(&self.root, &root_metadata, "checkpoint root")?; + let sandbox_dir = self.sandbox_dir(sandbox_id); + let Some(metadata) = + optional_symlink_metadata(&sandbox_dir, "inspect checkpoint sandbox directory")? + else { + return Ok(None); + }; + require_plain_directory_metadata(&sandbox_dir, &metadata, "checkpoint sandbox directory")?; + require_direct_child(&self.root, &sandbox_dir, "checkpoint sandbox directory")?; + Ok(Some(sandbox_dir)) + } + + fn committed_dir(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result { + validate_checkpoint_id(checkpoint_id)?; + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} does not exist")))?; + let path = sandbox_dir.join(checkpoint_id); + require_plain_directory(&path, "committed checkpoint directory")?; + require_direct_child(&sandbox_dir, &path, "committed checkpoint directory")?; + Ok(path) + } + + fn sandbox_dir(&self, sandbox_id: Uuid) -> PathBuf { + self.root.join(sandbox_id.to_string()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScratchKind { + Directory, + File, +} + +fn lineage_from( + catalog: &HashMap, + checkpoint_id: &str, +) -> Result> { + validate_checkpoint_id(checkpoint_id)?; + let mut current = checkpoint_id; + let mut lineage = HashSet::new(); + loop { + if !lineage.insert(current.to_string()) { + return Err(invariant(format!( + "checkpoint parent cycle reaches {current}" + ))); + } + let metadata = catalog.get(current).ok_or_else(|| { + invariant(format!( + "checkpoint lineage references missing checkpoint {current}" + )) + })?; + let Some(parent) = metadata.parent.as_deref() else { + break; + }; + current = parent; + } + Ok(lineage) +} + +fn validate_exact_entries(directory: &Path, expected: &[&str]) -> Result<()> { + let expected = expected.iter().copied().collect::>(); + let mut observed = HashSet::new(); + let entries = read_directory(directory, "scan checkpoint directory")?; + for entry in entries { + let entry = + entry.map_err(|source| io_error("read checkpoint directory", directory, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + return Err(invariant(format!( + "checkpoint directory {} contains a non-UTF-8 entry", + directory.display() + ))); + }; + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint entry", entry.path(), source))?; + if !expected.contains(name) || !file_type.is_file() || file_type.is_symlink() { + return Err(invariant(format!( + "checkpoint directory {} contains unexpected entry {name:?}", + directory.display() + ))); + } + observed.insert(name.to_string()); + } + if observed.len() != expected.len() || expected.iter().any(|name| !observed.contains(*name)) { + return Err(invariant(format!( + "checkpoint directory {} does not contain the exact required file set", + directory.display() + ))); + } + Ok(()) +} + +fn hash_artifact(path: &Path, name: &str) -> Result { + let mut file = open_file(path, "open checkpoint artifact")?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 128 * 1024]; + loop { + let read = file + .read(&mut buffer) + .map_err(|source| io_error("read checkpoint artifact", path, source))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let size_bytes = file + .metadata() + .map_err(|source| io_error("inspect checkpoint artifact", path, source))? + .len(); + Ok(CheckpointArtifact { + name: name.to_string(), + size_bytes, + sha256: format!("{:x}", hasher.finalize()), + }) +} + +fn classify_scratch_name(name: &str) -> Result> { + if let Some(checkpoint_id) = name + .strip_prefix('.') + .and_then(|name| name.strip_suffix(STAGING_SUFFIX)) + .filter(|name| name.starts_with("ckpt-")) + { + validate_checkpoint_id(checkpoint_id)?; + return Ok(Some(ScratchKind::Directory)); + } + if let Some(nonce) = name + .strip_prefix(".HEAD.") + .and_then(|name| name.strip_suffix(STAGING_SUFFIX)) + { + parse_uuid_component(nonce, "temporary HEAD")?; + return Ok(Some(ScratchKind::File)); + } + for prefix in [PRUNE_TOMBSTONE_PREFIX, ABORT_TOMBSTONE_PREFIX] { + if let Some(body) = name + .strip_prefix(prefix) + .and_then(|name| name.strip_suffix(TOMBSTONE_SUFFIX)) + { + let (checkpoint_id, nonce) = body + .rsplit_once('.') + .ok_or_else(|| invariant(format!("invalid checkpoint tombstone {name:?}")))?; + validate_checkpoint_id(checkpoint_id)?; + parse_uuid_component(nonce, "checkpoint tombstone")?; + return Ok(Some(ScratchKind::Directory)); + } + } + Ok(None) +} + +fn parse_uuid_component(value: &str, label: &str) -> Result { + let uuid = Uuid::parse_str(value) + .map_err(|error| invariant(format!("invalid {label} identifier {value:?}: {error}")))?; + if value != uuid.to_string() { + return Err(invariant(format!( + "{label} identifier {value:?} is not canonical" + ))); + } + Ok(uuid) +} + +fn tombstone_path(directory: &Path, prefix: &str, checkpoint_id: &str) -> PathBuf { + directory.join(format!( + "{prefix}{checkpoint_id}.{}{TOMBSTONE_SUFFIX}", + Uuid::new_v4() + )) +} + +fn validate_existing_head_type(sandbox_dir: &Path) -> Result<()> { + let path = sandbox_dir.join(HEAD_FILE); + let Some(metadata) = optional_symlink_metadata(&path, "inspect existing checkpoint HEAD")? + else { + return Ok(()); + }; + require_plain_file_metadata(&path, &metadata, "existing checkpoint HEAD") +} + +fn require_plain_directory(path: &Path, label: &str) -> Result<()> { + let metadata = symlink_metadata(path, "inspect directory")?; + require_plain_directory_metadata(path, &metadata, label) +} + +fn require_plain_directory_metadata( + path: &Path, + metadata: &fs::Metadata, + label: &str, +) -> Result<()> { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(invariant(format!( + "{label} {} is not a plain directory", + path.display() + ))); + } + Ok(()) +} + +fn require_plain_file_metadata(path: &Path, metadata: &fs::Metadata, label: &str) -> Result<()> { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(invariant(format!( + "{label} {} is not a plain file", + path.display() + ))); + } + Ok(()) +} + +fn require_contained_file(directory: &Path, name: &str) -> Result { + if name != METADATA_FILE { + validate_artifact_name(name)?; + } + let path = directory.join(name); + let metadata = symlink_metadata(&path, "inspect checkpoint file")?; + require_plain_file_metadata(&path, &metadata, "checkpoint file")?; + require_direct_child(directory, &path, "checkpoint file")?; + Ok(path) +} + +fn require_direct_child(parent: &Path, child: &Path, label: &str) -> Result<()> { + let canonical_parent = canonicalize(parent, "canonicalize checkpoint parent")?; + let canonical_child = canonicalize(child, "canonicalize checkpoint child")?; + if canonical_child.parent() != Some(canonical_parent.as_path()) { + return Err(invariant(format!( + "{label} {} is not directly contained by {}", + child.display(), + parent.display() + ))); + } + Ok(()) +} + +fn ensure_missing(path: &Path, operation: &'static str) -> Result<()> { + match fs::symlink_metadata(path) { + Ok(_) => Err(invariant(format!( + "checkpoint publication target {} already exists", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn optional_symlink_metadata(path: &Path, operation: &'static str) -> Result> { + match fs::symlink_metadata(path) { + Ok(metadata) => Ok(Some(metadata)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn symlink_metadata(path: &Path, operation: &'static str) -> Result { + fs::symlink_metadata(path).map_err(|source| io_error(operation, path, source)) +} + +fn create_directories(path: &Path, operation: &'static str) -> Result<()> { + fs::create_dir_all(path).map_err(|source| io_error(operation, path, source)) +} + +fn create_directory(path: &Path, operation: &'static str) -> Result<()> { + fs::create_dir(path).map_err(|source| io_error(operation, path, source)) +} + +fn open_file(path: &Path, operation: &'static str) -> Result { + File::open(path).map_err(|source| io_error(operation, path, source)) +} + +fn open_new_file(path: &Path, operation: &'static str) -> Result { + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| io_error(operation, path, source)) +} + +fn write_all(file: &mut File, path: &Path, bytes: &[u8]) -> Result<()> { + file.write_all(bytes) + .map_err(|source| io_error("write file", path, source)) +} + +fn sync_open_file(file: &File, path: &Path) -> Result<()> { + file.sync_all() + .map_err(|source| io_error("sync file", path, source)) +} + +fn write_json_new(path: &Path, value: &T) -> Result<()> { + let bytes = serde_json::to_vec_pretty(value).map_err(|source| CheckpointStoreError::Json { + path: path.to_path_buf(), + source, + })?; + let mut file = open_new_file(path, "create checkpoint metadata")?; + write_all(&mut file, path, &bytes)?; + write_all(&mut file, path, b"\n")?; + sync_open_file(&file, path) +} + +fn read_file(path: &Path, operation: &'static str) -> Result> { + fs::read(path).map_err(|source| io_error(operation, path, source)) +} + +fn read_directory(path: &Path, operation: &'static str) -> Result { + fs::read_dir(path).map_err(|source| io_error(operation, path, source)) +} + +fn canonicalize(path: &Path, operation: &'static str) -> Result { + fs::canonicalize(path).map_err(|source| io_error(operation, path, source)) +} + +fn rename_path(source: &Path, target: &Path, operation: &'static str) -> Result<()> { + fs::rename(source, target).map_err(|error| { + io_error( + operation, + PathBuf::from(format!("{} -> {}", source_path(source), target.display())), + error, + ) + }) +} + +fn source_path(path: &Path) -> String { + path.display().to_string() +} + +fn sync_file(path: &Path) -> Result<()> { + let file = open_file(path, "open file for sync")?; + sync_open_file(&file, path) +} + +fn sync_directory(path: &Path) -> Result<()> { + let directory = open_file(path, "open directory for sync")?; + directory + .sync_all() + .map_err(|source| io_error("sync directory", path, source)) +} + +fn remove_directory(path: &Path, operation: &'static str) -> Result<()> { + fs::remove_dir_all(path).map_err(|source| io_error(operation, path, source)) +} + +fn remove_file(path: &Path, operation: &'static str) -> Result<()> { + fs::remove_file(path).map_err(|source| io_error(operation, path, source)) +} + +fn remove_file_if_exists(path: &Path, operation: &'static str) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn io_error( + operation: &'static str, + path: impl AsRef, + source: std::io::Error, +) -> CheckpointStoreError { + CheckpointStoreError::Io { + operation, + path: path.as_ref().to_path_buf(), + source, + } +} + +fn invariant(message: impl Into) -> CheckpointStoreError { + CheckpointStoreError::Invariant(message.into()) +} + +fn checkpoint_store_failpoint(name: &'static str, path: &Path) -> Result<()> { + crate::failpoint::storage(name).map_err(|error| { + io_error( + "run checkpoint store failpoint", + path, + std::io::Error::other(error.to_string()), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use blaze_core::backend::{BackendKind, SnapshotKind}; + + fn commit_input(parent: Option) -> CommitCheckpoint { + CommitCheckpoint { + parent, + policy_name: "default".to_string(), + image_digest: "sha256:test".to_string(), + backend: BackendKind::Mock, + backend_version: Some("mock-v1".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, + } + } + + fn populate(stage: &CheckpointStage, suffix: &str) { + for name in REQUIRED_ARTIFACTS { + fs::write(stage.path.join(name), format!("{name}-{suffix}")).expect("write artifact"); + } + } + + fn publish( + store: &CheckpointStore, + sandbox_id: Uuid, + parent: Option, + move_head: bool, + ) -> String { + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let id = stage.id().to_string(); + populate(&stage, &id); + store + .publish(&stage, commit_input(parent)) + .expect("publish checkpoint"); + if move_head { + store.set_head(sandbox_id, &id).expect("move HEAD"); + } + id + } + + #[test] + fn publish_verify_and_list_preserve_the_head_boundary() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + store + .verify(sandbox_id, &unreachable) + .expect("published checkpoint"); + let listed = store.list(sandbox_id).expect("list checkpoints"); + assert_eq!(listed.len(), 2); + assert_eq!(listed.iter().filter(|info| info.is_head).count(), 1); + assert!( + listed + .iter() + .any(|info| info.id == unreachable && !info.on_head_chain) + ); + } + + #[test] + fn verify_rejects_corrupted_artifact_content() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, true); + fs::write( + store + .root + .join(sandbox_id.to_string()) + .join(&checkpoint_id) + .join("memory.snap"), + b"corrupt", + ) + .expect("corrupt artifact"); + + assert!(store.verify(sandbox_id, &checkpoint_id).is_err()); + } + + #[test] + fn publish_rejects_an_unexpected_stage_entry() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + populate(&stage, "candidate"); + fs::write(stage.path.join("unexpected"), b"unexpected").expect("write extra entry"); + + let error = store + .publish(&stage, commit_input(None)) + .expect_err("unexpected entry must fail"); + assert!(error.to_string().contains("unexpected entry")); + store + .abort_staging(sandbox_id, stage.id()) + .expect("abort stage"); + } + + #[test] + fn prune_retains_head_and_explicit_lineages() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let protected = publish(&store, sandbox_id, Some(root.clone()), false); + let abandoned = publish(&store, sandbox_id, Some(root.clone()), false); + + let removed = store + .prune_preserving(sandbox_id, std::slice::from_ref(&protected)) + .expect("prune checkpoints"); + + assert_eq!(removed, vec![abandoned]); + assert!(store.verify(sandbox_id, &root).is_ok()); + assert!(store.verify(sandbox_id, &protected).is_ok()); + } + + #[test] + fn prune_can_remove_all_unreachable_checkpoints_before_head_exists() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let unreachable = publish(&store, sandbox_id, None, false); + + let removed = store + .prune_preserving(sandbox_id, &[]) + .expect("prune unreachable checkpoint"); + + assert_eq!(removed, vec![unreachable]); + assert!(store.list(sandbox_id).expect("list checkpoints").is_empty()); + } + + #[test] + fn cleanup_removes_transaction_scratch_but_retains_committed_history() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let committed = publish(&store, sandbox_id, None, true); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let sandbox_dir = store.root.join(sandbox_id.to_string()); + let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); + let tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, &committed); + fs::create_dir(&tombstone).expect("create prune tombstone"); + + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup scratch"); + + assert_eq!(removed.len(), 3); + assert!(!stage.path.exists()); + assert!(!temporary_head.exists()); + assert!(!tombstone.exists()); + assert!(store.verify(sandbox_id, &committed).is_ok()); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(committed)); + } + + #[test] + fn transaction_artifact_detection_matches_cleanup_boundaries() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let missing_sandbox = Uuid::new_v4(); + assert!( + !store + .has_transaction_artifacts(missing_sandbox) + .expect("inspect missing sandbox") + ); + + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let sandbox_dir = store.root.join(sandbox_id.to_string()); + let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); + let prune_tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, stage.id()); + fs::create_dir(&prune_tombstone).expect("create prune tombstone"); + let abort_tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, stage.id()); + fs::create_dir(&abort_tombstone).expect("create abort tombstone"); + + assert!( + store + .has_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + ); + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup transaction artifacts"); + assert_eq!(removed.len(), 4); + assert!(!stage.path.exists()); + assert!(!temporary_head.exists()); + assert!(!prune_tombstone.exists()); + assert!(!abort_tombstone.exists()); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect cleaned sandbox") + ); + } + + #[test] + fn transaction_artifact_detection_reports_unsafe_layout_without_deleting_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + fs::remove_dir(&stage.path).expect("remove staging directory"); + fs::write(&stage.path, b"not a directory").expect("replace stage with file"); + + let inspect_error = store + .has_transaction_artifacts(sandbox_id) + .expect_err("unsafe layout must be reported"); + assert!(inspect_error.to_string().contains("unexpected file type")); + let cleanup_error = store + .cleanup_transaction_artifacts(sandbox_id) + .expect_err("cleanup must not delete an entry with the wrong type"); + assert!(cleanup_error.to_string().contains("unexpected file type")); + assert!(stage.path.is_file()); + } + + #[test] + fn abort_staging_is_idempotent() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let checkpoint_id = stage.id().to_string(); + + store + .abort_staging(sandbox_id, &checkpoint_id) + .expect("abort stage"); + store + .abort_staging(sandbox_id, &checkpoint_id) + .expect("repeat abort"); + assert!(!stage.path.exists()); + } + + #[test] + fn missing_catalog_has_no_head_or_entries() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("missing")); + let sandbox_id = Uuid::new_v4(); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); + assert!(store.list(sandbox_id).expect("list").is_empty()); + assert!( + store + .prune_preserving(sandbox_id, &[]) + .expect("prune") + .is_empty() + ); + assert!( + store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup") + .is_empty() + ); + } + + #[cfg(unix)] + #[test] + fn verify_rejects_artifact_symlinks() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, true); + let artifact = store + .root + .join(sandbox_id.to_string()) + .join(&checkpoint_id) + .join("rootfs.snap"); + fs::remove_file(&artifact).expect("remove artifact"); + let outside = temp.path().join("outside"); + fs::write(&outside, b"outside").expect("write outside file"); + symlink(&outside, &artifact).expect("link artifact"); + + assert!(store.verify(sandbox_id, &checkpoint_id).is_err()); + } + + #[cfg(unix)] + #[test] + fn begin_rejects_a_symlinked_catalog_root() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let actual = temp.path().join("actual"); + fs::create_dir(&actual).expect("create actual root"); + let linked = temp.path().join("linked"); + symlink(&actual, &linked).expect("link root"); + let store = CheckpointStore::new(linked); + + assert!(store.begin(Uuid::new_v4()).is_err()); + } + + #[cfg(not(feature = "test-failpoints"))] + #[test] + fn production_checkpoint_store_boundary_hooks_are_inert() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + + assert_eq!( + store + .prune_preserving(sandbox_id, &[]) + .expect("prune unreachable checkpoint"), + vec![unreachable] + ); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn publish_boundary_error_leaves_a_committed_unreachable_checkpoint() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let checkpoint_id = stage.id().to_string(); + let final_path = stage.final_path.clone(); + populate(&stage, "publish-boundary"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-publish-after-rename"]); + + let error = hook + .run(async { store.publish(&stage, commit_input(None)) }) + .await + .expect_err("publish boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-store-publish-after-rename") + ); + assert!(!stage.path.exists()); + assert!(final_path.is_dir()); + store + .verify(sandbox_id, &checkpoint_id) + .expect("renamed checkpoint remains committed"); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn head_boundary_error_leaves_the_new_head_visible() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, false); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-head-after-rename"]); + + let error = hook + .run(async { store.set_head(sandbox_id, &checkpoint_id) }) + .await + .expect_err("HEAD boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-store-head-after-rename") + ); + assert_eq!( + store.read_head(sandbox_id).expect("HEAD"), + Some(checkpoint_id) + ); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn prune_boundary_error_leaves_a_cleanup_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + + let error = hook + .run(async { store.prune_preserving(sandbox_id, &[]) }) + .await + .expect_err("prune boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-prune-after-tombstone") + ); + assert!(store.verify(sandbox_id, &unreachable).is_err()); + assert_eq!( + store + .list(sandbox_id) + .expect("list checkpoints after tombstone") + .into_iter() + .map(|checkpoint| checkpoint.id) + .collect::>(), + vec![root.clone()] + ); + assert!( + store + .has_transaction_artifacts(sandbox_id) + .expect("inspect prune tombstone") + ); + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("remove prune tombstone"); + assert_eq!(removed.len(), 1); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect cleaned sandbox") + ); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + } +} diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index a9616003f3..7b849b29ef 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -1,43 +1,173 @@ // SPDX-License-Identifier: Apache-2.0 //! Daemon runtime: bind UDS, accept connections, wire signal handlers. +use std::future::Future; use std::path::Path; use std::sync::Arc; +use std::time::Duration; use blaze_core::backend::BackendKind; -use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode}; +use blaze_core::config::{ + DaemonConfig, PolicyLoadErrorMode, StorageFlushSchedule, validate_runtime_storage_paths, +}; use blaze_core::kernel::HookRegistry; use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use blaze_core::template::TemplateRegistry; -use http_body_util::Full; -use hyper::body::Bytes; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; -use tokio::net::{TcpListener, UnixListener}; +use tokio::net::TcpListener; use tokio::signal::unix::{SignalKind, signal}; +use tokio::sync::watch; +use tokio::task::{JoinError, JoinSet}; use crate::api; +use crate::daemon_socket::{DaemonLock, DaemonSocket}; use crate::error::{BlazeDaemonError, Result}; +use crate::sandbox::FlushLoop; use crate::spawner::{ BubblewrapSpawner, DynSpawner, FirecrackerSpawner, MockSpawner, SpawnerRegistry, }; use crate::state::ServerState; +#[derive(Clone, Copy)] +struct ShutdownBudget { + connection_drain: Duration, + runtime_cleanup: Duration, +} + +const SHUTDOWN_BUDGET: ShutdownBudget = ShutdownBudget { + // Preserve the existing client grace period. The supervisor aborts and + // joins handlers that do not finish before this deadline. + connection_drain: Duration::from_secs(30), + // Runtime cleanup starts for all owners concurrently, so this is one + // shared work deadline. Timed-out tasks are joined before the stage ends. + runtime_cleanup: Duration::from_secs(30), +}; + +#[cfg(test)] +const SERVICE_MANAGER_MARGIN: Duration = Duration::from_secs(20); + +struct ConnectionSupervisor { + shutdown: watch::Sender, + tasks: JoinSet<()>, +} + +impl ConnectionSupervisor { + fn new() -> Self { + let (shutdown, _) = watch::channel(false); + Self { + shutdown, + tasks: JoinSet::new(), + } + } + + fn is_empty(&self) -> bool { + self.tasks.is_empty() + } + + fn spawn(&mut self, io: TokioIo, state: Arc) + where + I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, + { + let shutdown = self.shutdown.subscribe(); + self.spawn_task(serve_connection(io, state, shutdown)); + } + + fn spawn_task(&mut self, task: F) + where + F: Future + Send + 'static, + { + self.tasks.spawn(task); + } + + async fn join_next(&mut self) -> Option> { + self.tasks.join_next().await + } + + async fn shutdown(mut self, grace: Duration) -> Result<()> { + self.shutdown.send_replace(true); + let mut task_failed = false; + let completed = tokio::time::timeout(grace, async { + while let Some(result) = self.tasks.join_next().await { + task_failed |= report_connection_result(result); + } + }) + .await; + + if completed.is_err() { + let remaining = self.tasks.len(); + tracing::warn!( + remaining, + timeout_secs = grace.as_secs(), + "connection drain timed out; aborting remaining tasks" + ); + self.tasks.abort_all(); + while let Some(result) = self.tasks.join_next().await { + if let Err(error) = result + && !error.is_cancelled() + { + report_connection_result(Err(error)); + } + } + return Err(BlazeDaemonError::Internal(format!( + "connection drain timed out after {} seconds; aborted {remaining} task(s)", + grace.as_secs() + ))); + } + + if task_failed { + return Err(BlazeDaemonError::Internal( + "one or more connection tasks failed during shutdown".to_string(), + )); + } + Ok(()) + } +} + +fn report_connection_result(result: std::result::Result<(), JoinError>) -> bool { + match result { + Ok(()) => false, + Err(error) if error.is_cancelled() => { + tracing::debug!("connection task cancelled"); + false + } + Err(error) => { + tracing::error!(%error, "connection task failed"); + true + } + } +} + /// Boot the daemon: load config + policies, prepare state directories, /// bind the API socket, and run the accept loop until SIGTERM/SIGINT. pub async fn run(config_path: &Path) -> Result<()> { let config = DaemonConfig::load(config_path)?; + let flush_schedule = config.storage.flush_schedule()?; + let flush_timeout = config.storage.flush_timeout_duration()?; tracing::info!(?config_path, "loaded daemon config"); ensure_dirs(&config)?; + let socket_path = config.daemon.socket.clone(); + let daemon_lock = DaemonLock::acquire(&socket_path)?; let policy = load_policy_engine(&config)?; let pool = PoolManager::new(); let template = TemplateRegistry::new(); let hook = HookRegistry::new(); - let (spawners, active_backend) = build_spawners(&config).await; + let network_required = policy.policies().iter().any(|policy| { + policy + .backend + .firecracker + .as_ref() + .is_some_and(|config| config.enable_network) + && policy + .select + .backend_priority + .contains(&BackendKind::Firecracker) + }); + let (spawners, active_backend) = build_spawners(&config, network_required).await; // Build storage provider if config.storage.provider != "file" && config.storage.provider != "auto" { @@ -66,7 +196,6 @@ pub async fn run(config_path: &Path) -> Result<()> { Arc::new(fp) }; - let socket_path = config.daemon.socket.clone(); let http_addr = config.listen.http_addr.clone(); let state = Arc::new(ServerState::build( config, @@ -77,15 +206,30 @@ pub async fn run(config_path: &Path) -> Result<()> { spawners, active_backend, storage, - )); - - if socket_path.exists() { - std::fs::remove_file(&socket_path)?; + )?); + let reconciled_slots = state.manager.reconcile_runtime_pool_startup().await?; + if reconciled_slots > 0 { + tracing::warn!( + runtimes = reconciled_slots, + "reconciled unclaimed runtime slots" + ); } - if let Some(parent) = socket_path.parent() { - std::fs::create_dir_all(parent)?; + let reconciliation = state.manager.reconcile_startup().await; + tracing::info!( + attempted = reconciliation.attempted, + completed = reconciliation.completed, + failed = reconciliation.failures.len(), + "startup sandbox reconciliation completed" + ); + for failure in reconciliation.failures { + tracing::warn!( + instance = %failure.instance_id, + error = %failure.error, + "sandbox remains recovery-required after startup reconciliation" + ); } - let listener = UnixListener::bind(&socket_path)?; + + let listener = DaemonSocket::bind(daemon_lock).await?; tracing::info!(socket = %socket_path.display(), "blaze UDS API listening"); // Optional TCP listener for remote platform API @@ -99,7 +243,7 @@ pub async fn run(config_path: &Path) -> Result<()> { None }; - serve(listener, tcp_listener, state).await + serve(listener, tcp_listener, state, flush_schedule, flush_timeout).await } fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { @@ -108,9 +252,14 @@ fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { std::fs::create_dir_all(&cfg.template.dir)?; std::fs::create_dir_all(&cfg.storage.images_dir)?; std::fs::create_dir_all(&cfg.storage.instances_dir)?; + let state_dir = std::fs::canonicalize(&cfg.daemon.state_dir)?; let images_dir = std::fs::canonicalize(&cfg.storage.images_dir)?; let instances_dir = std::fs::canonicalize(&cfg.storage.instances_dir)?; - blaze_core::config::validate_storage_paths(&images_dir, &instances_dir)?; + let runtime_root = state_dir.join("runtime-pool"); + validate_runtime_storage_paths(&runtime_root, &images_dir, &instances_dir)?; + std::fs::create_dir_all(&runtime_root)?; + let runtime_root = std::fs::canonicalize(runtime_root)?; + validate_runtime_storage_paths(&runtime_root, &images_dir, &instances_dir)?; if let Some(parent) = cfg.daemon.socket.parent() { std::fs::create_dir_all(parent)?; } @@ -123,8 +272,14 @@ fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { /// 1. `firecracker` → [`FirecrackerSpawner`] /// 2. `bubblewrap` → [`BubblewrapSpawner`] /// 3. fallback → [`MockSpawner`] -async fn build_spawners(cfg: &DaemonConfig) -> (SpawnerRegistry, BackendKind) { - let firecracker: DynSpawner = Arc::new(FirecrackerSpawner::new(cfg.storage.images_dir.clone())); +async fn build_spawners( + cfg: &DaemonConfig, + network_required: bool, +) -> (SpawnerRegistry, BackendKind) { + let firecracker: DynSpawner = Arc::new(FirecrackerSpawner::with_network_requirement( + cfg.storage.images_dir.clone(), + network_required, + )); let bubblewrap: DynSpawner = Arc::new(BubblewrapSpawner); let mock: DynSpawner = Arc::new(MockSpawner); let mut spawners = SpawnerRegistry::new(); @@ -216,16 +371,42 @@ fn load_policy_engine(cfg: &DaemonConfig) -> Result { } } -async fn serve(uds: UnixListener, tcp: Option, state: Arc) -> Result<()> { +async fn serve( + mut uds: DaemonSocket, + tcp: Option, + state: Arc, + flush_schedule: StorageFlushSchedule, + flush_timeout: Duration, +) -> Result<()> { let mut sighup = signal(SignalKind::hangup()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGHUP handler: {e}")))?; let mut sigterm = signal(SignalKind::terminate()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGTERM handler: {e}")))?; let mut sigint = signal(SignalKind::interrupt()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGINT handler: {e}")))?; + let mut connections = ConnectionSupervisor::new(); + let mut flush_loop = match flush_schedule { + StorageFlushSchedule::Disabled => { + tracing::info!("periodic provider synchronization is disabled"); + None + } + StorageFlushSchedule::Every(interval) => { + Some(state.manager.start_flush_loop(interval, flush_timeout)) + } + }; + let mut service_result = Ok(()); loop { tokio::select! { + result = observe_flush_exit(&mut flush_loop), if flush_loop.is_some() => { + service_result = result; + break; + } + completed = connections.join_next(), if !connections.is_empty() => { + if let Some(completed) = completed { + report_connection_result(completed); + } + } res = uds.accept() => { let (stream, _peer) = match res { Ok(s) => s, @@ -234,7 +415,7 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc l.accept().await, None => std::future::pending().await }}, if tcp.is_some() => { let (stream, peer) = match res { @@ -245,7 +426,7 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc { tracing::info!("SIGHUP received: reloading policies"); @@ -264,24 +445,120 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc) -> Result<()> { + flush_loop + .as_mut() + .expect("flush loop exists while its select branch is enabled") + .observe_exit() + .await } -fn spawn_conn(io: TokioIo, state: Arc) +fn merge_stage_result(current: Result<()>, stage: &str, next: Result<()>) -> Result<()> { + match (current, next) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(previous), Err(next)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{previous}; {stage} also failed: {next}" + ))), + } +} + +async fn finish_shutdown( + drain: Drain, + imports: Imports, + cleanup: Cleanup, +) -> Result<()> where + Drain: Future>, + Imports: Future>, + Cleanup: Future>, +{ + let drain_result = drain.await; + let import_result = imports.await; + let cleanup_result = cleanup.await; + let mut failures = Vec::new(); + if let Err(error) = drain_result { + failures.push(("connection drain", error)); + } + if let Err(error) = import_result { + failures.push(("runtime template import shutdown", error)); + } + if let Err(error) = cleanup_result { + failures.push(("runtime cleanup", error)); + } + if failures.is_empty() { + return Ok(()); + } + if failures.len() == 1 { + return Err(failures.remove(0).1); + } + Err(BlazeDaemonError::RecoveryRequired( + failures + .into_iter() + .map(|(stage, error)| format!("{stage} failed: {error}")) + .collect::>() + .join("; "), + )) +} + +async fn serve_connection( + io: TokioIo, + state: Arc, + mut shutdown: watch::Receiver, +) where I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { - tokio::spawn(async move { - let svc = service_fn(move |req| { - let state = state.clone(); - async move { api::handle(req, state).await } - }); - if let Err(err) = http1::Builder::new().serve_connection(io, svc).await { - tracing::debug!(?err, "connection closed with error"); - } - let _: Option> = None; + let svc = service_fn(move |req| { + let state = state.clone(); + async move { api::handle(req, state).await } }); + let connection = http1::Builder::new().serve_connection(io, svc); + tokio::pin!(connection); + let result = tokio::select! { + result = &mut connection => result, + _ = wait_for_shutdown(&mut shutdown) => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + if let Err(error) = result { + tracing::debug!(%error, "connection closed with error"); + } +} + +async fn wait_for_shutdown(shutdown: &mut watch::Receiver) { + loop { + let stopping = *shutdown.borrow_and_update(); + if stopping { + return; + } + if shutdown.changed().await.is_err() { + return; + } + } } fn reload_policies(state: &Arc) -> Result<()> { @@ -304,3 +581,248 @@ fn reload_policies(state: &Arc) -> Result<()> { tracing::info!(policies = count, "policy engine reloaded via SIGHUP"); Ok(()) } + +#[cfg(test)] +mod tests { + use std::future; + use std::path::Path; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tokio::sync::oneshot; + + use super::*; + + struct ActiveTask(Arc); + + impl Drop for ActiveTask { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } + } + + fn directory_config(root: &Path) -> DaemonConfig { + let mut config = DaemonConfig::default(); + config.daemon.state_dir = root.join("state"); + config.daemon.socket = root.join("run").join("blazed.sock"); + config.storage.images_dir = root.join("images"); + config.storage.instances_dir = root.join("instances"); + config.template.dir = root.join("templates"); + config.policy.dir = root.join("policies"); + config + } + + #[test] + fn ensure_dirs_accepts_disjoint_runtime_and_storage_roots() { + let temp = tempfile::tempdir().expect("tempdir"); + let config = directory_config(temp.path()); + + ensure_dirs(&config).expect("disjoint ownership roots"); + + assert!(config.daemon.state_dir.join("runtime-pool").is_dir()); + assert!(config.storage.images_dir.is_dir()); + assert!(config.storage.instances_dir.is_dir()); + } + + #[cfg(unix)] + #[test] + fn ensure_dirs_rejects_canonical_runtime_storage_alias() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let config = directory_config(temp.path()); + std::fs::create_dir_all(&config.daemon.state_dir).expect("state dir"); + std::fs::create_dir_all(&config.storage.instances_dir).expect("instances dir"); + symlink( + &config.storage.instances_dir, + config.daemon.state_dir.join("runtime-pool"), + ) + .expect("runtime alias"); + + let error = ensure_dirs(&config).expect_err("canonical overlap must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::Core(blaze_core::BlazeError::ConfigError { .. }) + )); + assert!(error.to_string().contains("must be disjoint")); + } + + #[tokio::test] + async fn shutdown_waits_for_inflight_connection_task() { + let mut connections = ConnectionSupervisor::new(); + let mut shutdown = connections.shutdown.subscribe(); + let (entered_tx, entered_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + connections.spawn_task(async move { + entered_tx.send(()).expect("signal task entry"); + shutdown + .wait_for(|stopping| *stopping) + .await + .expect("shutdown sender"); + release_rx.await.expect("release task"); + }); + entered_rx.await.expect("task entered"); + + let drain = tokio::spawn(connections.shutdown(Duration::from_secs(1))); + tokio::task::yield_now().await; + assert!(!drain.is_finished(), "drain returned before task completed"); + + release_tx.send(()).expect("release connection task"); + drain + .await + .expect("drain task") + .expect("graceful connection drain"); + } + + #[tokio::test] + async fn shutdown_notifies_idle_connection_task() { + let mut connections = ConnectionSupervisor::new(); + let mut shutdown = connections.shutdown.subscribe(); + connections.spawn_task(async move { + shutdown + .wait_for(|stopping| *stopping) + .await + .expect("shutdown sender"); + }); + + connections + .shutdown(Duration::from_secs(1)) + .await + .expect("idle task drained"); + } + + #[tokio::test] + async fn shutdown_timeout_aborts_and_joins_stuck_task() { + let mut connections = ConnectionSupervisor::new(); + let active = Arc::new(AtomicBool::new(true)); + let task_active = active.clone(); + let (entered_tx, entered_rx) = oneshot::channel(); + connections.spawn_task(async move { + let _active = ActiveTask(task_active); + entered_tx.send(()).expect("signal task entry"); + future::pending::<()>().await; + }); + entered_rx.await.expect("task entered"); + + let error = connections + .shutdown(Duration::from_millis(10)) + .await + .expect_err("stuck task must time out"); + + assert!(error.to_string().contains("connection drain timed out")); + assert!( + !active.load(Ordering::Acquire), + "aborted task was not joined" + ); + } + + #[tokio::test] + async fn cleanup_starts_after_exhausted_connection_stage() { + let mut connections = ConnectionSupervisor::new(); + let connection_active = Arc::new(AtomicBool::new(true)); + let task_active = connection_active.clone(); + let (entered_tx, entered_rx) = oneshot::channel(); + connections.spawn_task(async move { + let _active = ActiveTask(task_active); + entered_tx.send(()).expect("signal task entry"); + future::pending::<()>().await; + }); + entered_rx.await.expect("task entered"); + + let cleanup_started = Arc::new(AtomicBool::new(false)); + let cleanup_observed = cleanup_started.clone(); + let active_observed = connection_active.clone(); + let cleanup = async move { + assert!( + !active_observed.load(Ordering::Acquire), + "runtime cleanup started before connection tasks stopped" + ); + cleanup_observed.store(true, Ordering::Release); + Ok(()) + }; + let error = finish_shutdown( + connections.shutdown(Duration::from_millis(10)), + future::ready(Ok(())), + cleanup, + ) + .await + .expect_err("connection timeout must be reported"); + + assert!(error.to_string().contains("connection drain timed out")); + assert!(cleanup_started.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn shutdown_preserves_import_and_runtime_failures() { + let error = finish_shutdown( + future::ready(Ok(())), + future::ready(Err(BlazeDaemonError::Internal( + "import did not stop".to_string(), + ))), + future::ready(Err(BlazeDaemonError::Internal( + "runtime cleanup failed".to_string(), + ))), + ) + .await + .expect_err("both failures must be reported"); + let message = error.to_string(); + + assert!(message.contains("runtime template import shutdown failed")); + assert!(message.contains("import did not stop")); + assert!(message.contains("runtime cleanup failed")); + } + + #[test] + fn service_and_shutdown_failures_are_both_retained() { + let service = Err(BlazeDaemonError::Internal( + "synchronization worker failed".to_string(), + )); + let shutdown = Err(BlazeDaemonError::Internal( + "runtime cleanup failed".to_string(), + )); + + let error = merge_stage_result(service, "coordinated shutdown", shutdown) + .expect_err("both failures must be reported"); + + assert!(error.to_string().contains("synchronization worker failed")); + assert!(error.to_string().contains("coordinated shutdown")); + assert!(error.to_string().contains("runtime cleanup failed")); + } + + #[test] + fn service_stop_timeout_covers_shutdown_stages() { + let unit = include_str!("../../../dist/blazed.service"); + let service_seconds = unit + .lines() + .find_map(|line| { + line.strip_prefix("TimeoutStopSec=") + .and_then(|value| value.strip_suffix('s')) + .and_then(|value| value.parse::().ok()) + }) + .expect("service stop timeout in seconds"); + let service_timeout = Duration::from_secs(service_seconds); + + assert!( + SHUTDOWN_BUDGET + .connection_drain + .saturating_add(SHUTDOWN_BUDGET.runtime_cleanup) + .saturating_add(SERVICE_MANAGER_MARGIN) + <= service_timeout, + "service manager must cover both stages and cancellation headroom" + ); + } + + #[tokio::test] + async fn completed_tasks_are_reaped_while_serving() { + let mut connections = ConnectionSupervisor::new(); + connections.spawn_task(async {}); + + let completed = connections + .join_next() + .await + .expect("one tracked connection"); + assert!(!report_connection_result(completed)); + assert!(connections.is_empty()); + } +} diff --git a/src/blaze/crates/blazed/src/daemon_socket.rs b/src/blaze/crates/blazed/src/daemon_socket.rs new file mode 100644 index 0000000000..ff88ac5956 --- /dev/null +++ b/src/blaze/crates/blazed/src/daemon_socket.rs @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Exclusive ownership and binding of the daemon Unix-domain socket. + +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use tokio::net::{UnixListener, UnixStream}; +use tokio::time::timeout; + +use crate::error::{BlazeDaemonError, Result}; + +const LOCK_MODE: u32 = 0o600; +const SOCKET_PROBE_TIMEOUT: Duration = Duration::from_secs(1); + +/// A bound API socket whose singleton lock remains owned for the same lifetime. +pub(super) struct DaemonSocket { + listener: Option, + _lock: DaemonLock, +} + +impl DaemonSocket { + /// Examines and replaces the socket only under exclusive ownership. + pub(super) async fn bind(lock: DaemonLock) -> Result { + let socket_path = &lock.socket_path; + prepare_socket_path(socket_path).await?; + let listener = + UnixListener::bind(socket_path).map_err(|source| BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + })?; + Ok(Self { + listener: Some(listener), + _lock: lock, + }) + } + + /// Accepts the next client while retaining exclusive socket ownership. + pub(super) async fn accept(&self) -> io::Result<(UnixStream, tokio::net::unix::SocketAddr)> { + let listener = self.listener.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotConnected, + "daemon socket is no longer accepting connections", + ) + })?; + listener.accept().await + } + + /// Closes the listener while retaining exclusive ownership for cleanup. + pub(super) fn stop_accepting(&mut self) { + self.listener.take(); + } +} + +/// Exclusive daemon ownership tied to one configured API socket. +pub(super) struct DaemonLock { + _file: File, + socket_path: PathBuf, +} + +impl DaemonLock { + /// Acquires ownership before daemon subsystems begin startup. + pub(super) fn acquire(socket_path: &Path) -> Result { + let lock_path = lock_path_for(socket_path); + let (file, created) = open_lock_file(&lock_path)?; + if created { + file.set_permissions(fs::Permissions::from_mode(LOCK_MODE)) + .map_err(|source| BlazeDaemonError::DaemonLockIo { + path: lock_path.clone(), + source, + })?; + } + + let opened_metadata = validate_opened_lock(&file, &lock_path).map_err(|reason| { + BlazeDaemonError::InvalidDaemonLock { + path: lock_path.clone(), + reason, + } + })?; + + // SAFETY: `file` owns a valid descriptor for the entire call. `flock` + // changes only the advisory lock associated with that open file. + let lock_result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if lock_result != 0 { + let source = io::Error::last_os_error(); + if source.kind() == io::ErrorKind::WouldBlock { + return Err(BlazeDaemonError::DaemonAlreadyRunning { + socket: socket_path.to_path_buf(), + }); + } + return Err(BlazeDaemonError::DaemonLockIo { + path: lock_path, + source, + }); + } + + validate_locked_path(&lock_path, &opened_metadata).map_err(|reason| { + BlazeDaemonError::InvalidDaemonLock { + path: lock_path, + reason, + } + })?; + + // The file is deliberately left on disk. Removing an advisory-lock + // file would let a new process lock a different inode during teardown. + Ok(Self { + _file: file, + socket_path: socket_path.to_path_buf(), + }) + } +} + +impl Drop for DaemonLock { + fn drop(&mut self) { + // SAFETY: `_file` remains open while its advisory lock is released. + // Process termination still closes the descriptor and releases the + // lock if this destructor cannot run. + unsafe { + libc::flock(self._file.as_raw_fd(), libc::LOCK_UN); + } + } +} + +fn lock_path_for(socket_path: &Path) -> PathBuf { + let mut lock_path = OsString::from(socket_path.as_os_str()); + lock_path.push(".lock"); + PathBuf::from(lock_path) +} + +fn open_lock_file(path: &Path) -> Result<(File, bool)> { + match lock_options(true).open(path) { + Ok(file) => Ok((file, true)), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { + let file = lock_options(false).open(path).map_err(|source| { + BlazeDaemonError::DaemonLockIo { + path: path.to_path_buf(), + source, + } + })?; + Ok((file, false)) + } + Err(source) => Err(BlazeDaemonError::DaemonLockIo { + path: path.to_path_buf(), + source, + }), + } +} + +fn lock_options(create_new: bool) -> OpenOptions { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create_new(create_new) + .mode(LOCK_MODE) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + options +} + +fn validate_opened_lock(file: &File, path: &Path) -> std::result::Result { + let metadata = file + .metadata() + .map_err(|error| format!("cannot inspect opened file: {error}"))?; + if !metadata.file_type().is_file() { + return Err("lock target is not a regular file".to_string()); + } + if metadata.mode() & 0o7777 != LOCK_MODE { + return Err(format!( + "mode must be {LOCK_MODE:#o}, found {:#o}", + metadata.mode() & 0o7777 + )); + } + // SAFETY: `geteuid` has no preconditions and does not modify memory. + let effective_uid = unsafe { libc::geteuid() }; + if metadata.uid() != effective_uid { + return Err(format!( + "owner uid {} does not match effective uid {effective_uid}", + metadata.uid() + )); + } + if metadata.nlink() != 1 { + return Err(format!( + "lock file must have one link, found {}", + metadata.nlink() + )); + } + + let path_metadata = + fs::symlink_metadata(path).map_err(|error| format!("cannot inspect path: {error}"))?; + if path_metadata.file_type().is_symlink() { + return Err("lock path is a symbolic link".to_string()); + } + if path_metadata.dev() != metadata.dev() || path_metadata.ino() != metadata.ino() { + return Err("lock path changed while it was opened".to_string()); + } + Ok(metadata) +} + +fn validate_locked_path( + path: &Path, + opened_metadata: &fs::Metadata, +) -> std::result::Result<(), String> { + let path_metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect locked path: {error}"))?; + if path_metadata.file_type().is_symlink() { + return Err("lock path became a symbolic link".to_string()); + } + if path_metadata.dev() != opened_metadata.dev() || path_metadata.ino() != opened_metadata.ino() + { + return Err("lock path changed while ownership was acquired".to_string()); + } + if path_metadata.mode() & 0o7777 != LOCK_MODE { + return Err(format!( + "mode changed while ownership was acquired: found {:#o}", + path_metadata.mode() & 0o7777 + )); + } + Ok(()) +} + +async fn prepare_socket_path(socket_path: &Path) -> Result<()> { + match fs::symlink_metadata(socket_path) { + Ok(metadata) if metadata.file_type().is_socket() => { + match timeout(SOCKET_PROBE_TIMEOUT, UnixStream::connect(socket_path)).await { + Ok(Ok(_stream)) => Err(BlazeDaemonError::DaemonAlreadyRunning { + socket: socket_path.to_path_buf(), + }), + Ok(Err(source)) if source.kind() == io::ErrorKind::ConnectionRefused => { + fs::remove_file(socket_path).map_err(|source| { + BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + } + }) + } + Ok(Err(source)) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(Err(source)) => Err(BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + }), + Err(_) => Err(BlazeDaemonError::InvalidDaemonSocket { + path: socket_path.to_path_buf(), + reason: "existing socket did not complete the ownership probe".to_string(), + }), + } + } + Ok(metadata) => { + let kind = if metadata.file_type().is_symlink() { + "symbolic link" + } else if metadata.is_dir() { + "directory" + } else { + "non-socket file" + }; + Err(BlazeDaemonError::InvalidDaemonSocket { + path: socket_path.to_path_buf(), + reason: format!("existing path is a {kind}"), + }) + } + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + use std::os::unix::net::UnixListener as StdUnixListener; + use std::process::{Command, Stdio}; + + const ABRUPT_EXIT_SOCKET_ENV: &str = "BLAZE_TEST_ABRUPT_EXIT_SOCKET"; + const ABRUPT_EXIT_READY_ENV: &str = "BLAZE_TEST_ABRUPT_EXIT_READY"; + static SOCKET_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + async fn serialize_socket_test() -> tokio::sync::MutexGuard<'static, ()> { + // The abrupt-exit test starts a helper process. Between fork and exec, + // that helper can inherit another concurrent test's listener and keep + // its endpoint reachable after the parent test drops the listener. + SOCKET_TEST_LOCK.lock().await + } + + fn socket_inode(path: &Path) -> u64 { + fs::symlink_metadata(path).expect("socket metadata").ino() + } + + async fn claim_and_bind(path: &Path) -> Result { + DaemonSocket::bind(DaemonLock::acquire(path)?).await + } + + async fn claim_and_bind_after_listener_handoff(path: &Path) -> Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(1); + loop { + // Lock release must be immediate. Only bind is retried because an + // unrelated child between fork and exec can briefly inherit the + // old CLOEXEC listener and keep its endpoint reachable. + let lock = DaemonLock::acquire(path)?; + match DaemonSocket::bind(lock).await { + Err(BlazeDaemonError::DaemonAlreadyRunning { .. }) + if tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(10)).await; + } + result => return result, + } + } + } + + #[tokio::test] + async fn second_daemon_cannot_replace_owned_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let first = claim_and_bind(&socket_path) + .await + .expect("first daemon binds"); + let first_inode = socket_inode(&socket_path); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("second daemon must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), first_inode); + drop(first); + } + + #[tokio::test] + async fn released_lock_can_be_reacquired_and_stale_socket_replaced() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let first = claim_and_bind(&socket_path) + .await + .expect("first daemon binds"); + drop(first); + + let second = claim_and_bind_after_listener_handoff(&socket_path) + .await + .expect("released lock is reusable"); + + assert!( + fs::symlink_metadata(&socket_path) + .expect("replacement socket metadata") + .file_type() + .is_socket() + ); + drop(second); + } + + #[tokio::test] + async fn closing_listener_retains_daemon_ownership() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let mut daemon = claim_and_bind(&socket_path).await.expect("daemon binds"); + + daemon.stop_accepting(); + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("closed listener must retain daemon ownership"); + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + + drop(daemon); + let recovered = DaemonLock::acquire(&socket_path).expect("ownership releases with daemon"); + drop(recovered); + } + + #[tokio::test] + async fn lock_is_released_after_owner_process_exits_abruptly() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let ready_path = temp.path().join("lock-ready"); + let status = Command::new(std::env::current_exe().expect("current test binary")) + .arg("--exact") + .arg("daemon_socket::tests::abrupt_exit_lock_helper") + .arg("--nocapture") + .env(ABRUPT_EXIT_SOCKET_ENV, &socket_path) + .env(ABRUPT_EXIT_READY_ENV, &ready_path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("run abrupt-exit helper"); + + assert_eq!(status.code(), Some(73)); + assert_eq!( + fs::read(&ready_path).expect("helper acquired lock"), + b"locked" + ); + + let recovered = DaemonLock::acquire(&socket_path).expect("kernel released process lock"); + drop(recovered); + } + + #[test] + fn abrupt_exit_lock_helper() { + let Some(socket_path) = std::env::var_os(ABRUPT_EXIT_SOCKET_ENV) else { + return; + }; + let Some(ready_path) = std::env::var_os(ABRUPT_EXIT_READY_ENV) else { + return; + }; + let _lock = + DaemonLock::acquire(Path::new(&socket_path)).expect("helper acquires daemon lock"); + fs::write(ready_path, b"locked").expect("publish helper readiness"); + + // SAFETY: `_exit` terminates only this dedicated helper process. It + // intentionally skips Rust destructors to exercise kernel lock release. + unsafe { + libc::_exit(73); + } + } + + #[tokio::test] + async fn stale_socket_is_untouched_when_lock_is_held() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let lock = DaemonLock::acquire(&socket_path).expect("hold daemon lock"); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("competing daemon must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), stale_inode); + drop(lock); + + let daemon = claim_and_bind_after_listener_handoff(&socket_path) + .await + .expect("owner may replace stale socket"); + assert!( + fs::symlink_metadata(&socket_path) + .expect("replacement socket metadata") + .file_type() + .is_socket() + ); + drop(daemon); + } + + #[tokio::test] + async fn symlinked_lock_is_rejected_without_touching_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let target = temp.path().join("lock-target"); + File::create(&target).expect("create target"); + symlink(&target, lock_path_for(&socket_path)).expect("create lock symlink"); + + assert!(DaemonLock::acquire(&socket_path).is_err()); + assert_eq!(socket_inode(&socket_path), stale_inode); + } + + #[tokio::test] + async fn live_socket_without_lock_is_preserved() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let legacy_listener = + StdUnixListener::bind(&socket_path).expect("bind legacy daemon socket"); + let original_inode = socket_inode(&socket_path); + let lock = DaemonLock::acquire(&socket_path).expect("acquire new daemon lock"); + + let error = DaemonSocket::bind(lock) + .await + .err() + .expect("live socket must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), original_inode); + drop(legacy_listener); + } + + #[tokio::test] + async fn insecure_lock_mode_is_rejected_without_touching_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let lock_path = lock_path_for(&socket_path); + let lock = File::create(&lock_path).expect("create lock"); + lock.set_permissions(fs::Permissions::from_mode(0o644)) + .expect("set insecure mode"); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("insecure lock must be rejected"); + + assert!(matches!(error, BlazeDaemonError::InvalidDaemonLock { .. })); + assert_eq!(socket_inode(&socket_path), stale_inode); + } + + #[tokio::test] + async fn non_socket_endpoint_is_rejected_without_removal() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + fs::write(&socket_path, b"do not remove").expect("write endpoint sentinel"); + + let lock = DaemonLock::acquire(&socket_path).expect("acquire daemon lock"); + let error = DaemonSocket::bind(lock) + .await + .err() + .expect("non-socket endpoint must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::InvalidDaemonSocket { .. } + )); + assert_eq!( + fs::read(&socket_path).expect("sentinel remains"), + b"do not remove" + ); + } +} diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index 6c0358eb2b..d076d67441 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//! Local errors for the blazed binary (daemon + CLI client). +//! Local errors for the daemon binary and HTTP API. //! //! Wraps [`blaze_core::BlazeError`] so the daemon can additionally //! surface I/O, hyper, and CLI-side failures without expanding the @@ -16,6 +16,9 @@ pub enum BlazeDaemonError { #[error("core error: {0}")] Core(#[from] blaze_core::BlazeError), + #[error(transparent)] + Guest(#[from] crate::guest::GuestError), + #[error("io error: {0}")] Io(#[from] std::io::Error), @@ -51,24 +54,104 @@ pub enum BlazeDaemonError { #[error("not found: {0}")] NotFound(String), + #[error("conflict: {0}")] + Conflict(String), + + #[error("service unavailable: {0}")] + ServiceUnavailable(String), + + #[error("unsupported operation: {0}")] + UnsupportedOperation(String), + + #[error("request body read failed: {0}")] + RequestBody(String), + + #[error("request body too large: {actual} bytes exceeds {limit}")] + PayloadTooLarge { actual: u64, limit: usize }, + #[error("operation requires recovery: {0}")] RecoveryRequired(String), + #[error("another blaze daemon already owns API socket {socket}")] + DaemonAlreadyRunning { socket: PathBuf }, + + #[error("cannot access daemon lock {path}: {source}")] + DaemonLockIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + #[error("refusing daemon lock {path}: {reason}")] + InvalidDaemonLock { path: PathBuf, reason: String }, + + #[error("cannot prepare daemon API socket {path}: {source}")] + DaemonSocketIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + #[error("refusing daemon API socket {path}: {reason}")] + InvalidDaemonSocket { path: PathBuf, reason: String }, + #[error("internal error: {0}")] Internal(String), } impl BlazeDaemonError { + /// Stable machine-readable code for errors that callers must branch on. + pub fn api_code(&self) -> Option<&'static str> { + match self { + BlazeDaemonError::Guest(crate::guest::GuestError::Io(_)) => { + Some("guest_transport_error") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Json(_)) + | BlazeDaemonError::Guest(crate::guest::GuestError::Protocol(_)) => { + Some("guest_response_invalid") + } + BlazeDaemonError::Guest(crate::guest::GuestError::InvalidArgument(_)) => { + Some("guest_invalid_request") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Timeout(_)) => Some("guest_timeout"), + BlazeDaemonError::Guest(crate::guest::GuestError::OutcomeUnknown(_)) => { + Some("guest_outcome_unknown") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Rejected(_)) => { + Some("guest_rejected") + } + BlazeDaemonError::Guest(crate::guest::GuestError::PayloadTooLarge { .. }) => { + Some("guest_request_too_large") + } + BlazeDaemonError::Guest(crate::guest::GuestError::ResponseTooLarge { .. }) => { + Some("guest_response_too_large") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) => Some("guest_cancelled"), + _ => None, + } + } + /// HTTP status code that should accompany this error in API responses. pub fn status_code(&self) -> u16 { match self { - BlazeDaemonError::BadRequest(_) => 400, + BlazeDaemonError::BadRequest(_) | BlazeDaemonError::RequestBody(_) => 400, BlazeDaemonError::NotFound(_) => 404, + BlazeDaemonError::Conflict(_) => 409, + BlazeDaemonError::ServiceUnavailable(_) => 503, + BlazeDaemonError::PayloadTooLarge { .. } => 413, + BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::RecoveryRequired(_) => 500, BlazeDaemonError::HttpStatus { status, .. } => *status, BlazeDaemonError::Core(blaze_core::BlazeError::PolicyEvalError { .. }) | BlazeDaemonError::Core(blaze_core::BlazeError::InvalidStateTransition { .. }) => 422, + BlazeDaemonError::Core(blaze_core::BlazeError::OperationInProgress { .. }) => 409, BlazeDaemonError::Core(blaze_core::BlazeError::BackendUnavailable { .. }) => 503, + BlazeDaemonError::Guest(crate::guest::GuestError::InvalidArgument(_)) => 400, + BlazeDaemonError::Guest(crate::guest::GuestError::Timeout(_)) => 504, + BlazeDaemonError::Guest(crate::guest::GuestError::OutcomeUnknown(_)) => 504, + BlazeDaemonError::Guest(crate::guest::GuestError::PayloadTooLarge { .. }) => 413, + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) => 503, + BlazeDaemonError::Guest(_) => 502, _ => 500, } } diff --git a/src/blaze/crates/blazed/src/failpoint.rs b/src/blaze/crates/blazed/src/failpoint.rs index 8338d78660..15edb0bc92 100644 --- a/src/blaze/crates/blazed/src/failpoint.rs +++ b/src/blaze/crates/blazed/src/failpoint.rs @@ -122,6 +122,16 @@ pub(crate) fn storage(name: &str) -> blaze_core::Result<()> { Ok(()) } +/// Return a guest-domain error when `name` is currently armed. +pub(crate) fn guest(name: &str) -> crate::guest::Result<()> { + if hit(name) { + return Err(crate::guest::GuestError::Rejected(format!( + "test failpoint '{name}' triggered" + ))); + } + Ok(()) +} + /// Return a daemon state-commit error when `name` is currently armed. pub(crate) fn state(name: &str) -> crate::error::Result<()> { if hit(name) { diff --git a/src/blaze/crates/blazed/src/failpoint_disabled.rs b/src/blaze/crates/blazed/src/failpoint_disabled.rs index 049aa05ab9..a43b1ef25f 100644 --- a/src/blaze/crates/blazed/src/failpoint_disabled.rs +++ b/src/blaze/crates/blazed/src/failpoint_disabled.rs @@ -16,6 +16,11 @@ pub(crate) fn storage(_name: &str) -> blaze_core::Result<()> { Ok(()) } +/// Leave guest operations unchanged in production builds. +pub(crate) fn guest(_name: &str) -> crate::guest::Result<()> { + Ok(()) +} + /// Leave state commits unchanged in production builds. pub(crate) fn state(_name: &str) -> crate::error::Result<()> { Ok(()) @@ -31,6 +36,7 @@ mod tests { super::announce(); super::backend("any").expect("backend hook"); super::storage("any").expect("storage hook"); + super::guest("any").expect("guest hook"); super::state("any").expect("state hook"); super::pause("any").await; } diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index 0c9f096f16..424df8a666 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -3,15 +3,24 @@ //! rootfs and memory files on a local filesystem. Base images and mutable //! instance slots use separate roots; runtime pooling is owned by the daemon. -use std::path::{Path, PathBuf}; +use std::ffi::OsString; +use std::io::SeekFrom; +use std::path::{Component, Path, PathBuf}; use async_trait::async_trait; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use uuid::Uuid; use blaze_core::error::{BlazeError, Result}; use blaze_core::storage::{ - AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + AcquireOpts, PoolStatus, RuntimeTemplateArtifact, RuntimeTemplateStorage, + RuntimeTemplateStorageSlot, StorageAcquireError, StorageProvider, StorageRestoreTransaction, + StorageSlot, }; +mod restore; + /// A filesystem-based provider that copies base artifacts when available and /// otherwise creates sparse rootfs and memory files at configured sizes. pub struct FileStorageProvider { @@ -81,12 +90,56 @@ impl RequiredPathType { } } +/// Removes incomplete capture files if an error or cancellation interrupts +/// publication before the target directory is durably synchronized. +struct UnpublishedCheckpoint { + temporary: Option, + target: Option, +} + +impl UnpublishedCheckpoint { + fn new() -> Self { + Self { + temporary: None, + target: None, + } + } + + fn mark_temporary(&mut self, temporary: PathBuf) { + self.temporary = Some(temporary); + } + + fn mark_target(&mut self, target: PathBuf) { + self.target = Some(target); + } + + fn clear_temporary(&mut self) { + self.temporary = None; + } + + fn commit(&mut self) { + self.temporary = None; + self.target = None; + } +} + +impl Drop for UnpublishedCheckpoint { + fn drop(&mut self) { + if let Some(target) = self.target.take() { + let _ = std::fs::remove_file(target); + } + if let Some(temporary) = self.temporary.take() { + let _ = std::fs::remove_file(temporary); + } + } +} + async fn require_slot_path( instance_id: &str, path: &Path, required_type: RequiredPathType, ) -> Result<()> { - match tokio::fs::metadata(path).await { + match tokio::fs::symlink_metadata(path).await { Ok(metadata) if required_type.matches(&metadata) => Ok(()), Ok(_) => Err(BlazeError::StorageIncomplete { instance_id: instance_id.to_string(), @@ -195,17 +248,125 @@ impl StorageProvider for FileStorageProvider { Ok(slot) } + async fn acquire_runtime_template( + &self, + opts: &AcquireOpts, + source: RuntimeTemplateStorage, + ) -> std::result::Result { + crate::failpoint::storage("storage-acquire-runtime-template")?; + if opts.rootfs_size != source.rootfs.size_bytes || opts.mem_size != source.memory.size_bytes + { + return Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: format!( + "acquire runtime template '{}': requested sizes do not match the template", + opts.instance_id + ), + })); + } + + let slot = self.slot_for_id(&opts.instance_id)?; + let instance_dir = slot.instance_dir.clone(); + match tokio::fs::create_dir(&instance_dir).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + return Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: format!( + "acquire runtime template '{}': instance directory already exists", + opts.instance_id + ), + })); + } + Err(error) => { + return Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: format!( + "acquire runtime template '{}': create dir: {error}", + opts.instance_id + ), + })); + } + } + + let snapshot_path = instance_dir.join("vmstate.snap"); + let result = async { + copy_runtime_template_artifact(source.rootfs, &slot.rootfs_path).await?; + copy_runtime_template_artifact(source.memory, &slot.mem_path).await?; + copy_runtime_template_artifact(source.vmstate, &snapshot_path).await?; + create_empty_durable_file(&slot.mem_diff_path).await?; + create_empty_durable_file(&slot.rootfs_diff_path).await?; + crate::failpoint::storage("storage-acquire-runtime-template-artifacts")?; + tokio::fs::File::open(&instance_dir) + .await? + .sync_all() + .await?; + Ok::<(), BlazeError>(()) + } + .await; + + if let Err(error) = result { + let rollback = match crate::failpoint::storage("storage-acquire-rollback") { + Ok(()) => tokio::fs::remove_dir_all(&instance_dir) + .await + .map_err(BlazeError::from), + Err(cleanup) => Err(cleanup), + }; + return match rollback { + Ok(()) => Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: format!( + "acquire runtime template '{}': artifact setup failed, rolled back: {error}", + opts.instance_id + ), + })), + Err(cleanup) => Err(StorageAcquireError::with_residual( + BlazeError::StorageError { + msg: format!( + "acquire runtime template '{}': artifact setup failed ({error}); rollback failed for {}: {cleanup}", + opts.instance_id, + instance_dir.display() + ), + }, + slot, + )), + }; + } + + Ok(RuntimeTemplateStorageSlot { + storage: slot, + snapshot_path, + }) + } + + fn supports_runtime_templates(&self) -> bool { + true + } + async fn release(&self, slot: StorageSlot) -> Result<()> { crate::failpoint::storage("storage-release")?; // Re-derive the canonical path from instances_dir + slot.id. Do not // trust path strings carried in a persisted or externally built slot. let canonical_dir = self.slot_for_id(&slot.id)?.instance_dir; - if canonical_dir.exists() { - tokio::fs::remove_dir_all(&canonical_dir) - .await - .map_err(|e| BlazeError::StorageError { - msg: format!("release '{}': {}", slot.id, e), - })?; + match tokio::fs::symlink_metadata(&canonical_dir).await { + Ok(metadata) if metadata.file_type().is_dir() => { + tokio::fs::remove_dir_all(&canonical_dir) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("release '{}': {error}", slot.id), + })?; + } + Ok(_) => { + return Err(BlazeError::StorageError { + msg: format!( + "release '{}': refusing non-directory slot {}", + slot.id, + canonical_dir.display() + ), + }); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(BlazeError::StorageError { + msg: format!("release '{}': inspect: {error}", slot.id), + }); + } } Ok(()) } @@ -215,6 +376,53 @@ impl StorageProvider for FileStorageProvider { self.release(slot).await } + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> Result> { + let mut entries = tokio::fs::read_dir(&self.instances_dir) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: {error}", self.instances_dir.display()), + })?; + let mut ids = Vec::new(); + while let Some(entry) = + entries + .next_entry() + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: {error}", self.instances_dir.display()), + })? + { + let path = entry.path(); + let file_type = entry + .file_type() + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: inspect: {error}", path.display()), + })?; + if !file_type.is_dir() { + return Err(BlazeError::StorageError { + msg: format!( + "inventory {}: unexpected non-directory entry", + path.display() + ), + }); + } + let id = entry + .file_name() + .into_string() + .map_err(|_| BlazeError::StorageError { + msg: format!("inventory {}: slot name is not UTF-8", path.display()), + })?; + validate_instance_id(&id)?; + ids.push(id); + } + ids.sort(); + Ok(ids) + } + async fn reconstruct(&self, instance_id: &str) -> Result { let slot = self.slot_for_id(instance_id)?; require_slot_path(instance_id, &slot.instance_dir, RequiredPathType::Directory).await?; @@ -234,12 +442,19 @@ impl StorageProvider for FileStorageProvider { // Never trust paths carried by a runtime or persisted slot. Rebuild // the complete provider-owned artifact set from the validated ID. let canonical = self.slot_for_id(&slot.id)?; + require_slot_path( + &slot.id, + &canonical.instance_dir, + RequiredPathType::Directory, + ) + .await?; for path in [ &canonical.rootfs_path, &canonical.mem_path, &canonical.mem_diff_path, &canonical.rootfs_diff_path, ] { + require_slot_path(&slot.id, path, RequiredPathType::File).await?; let file = tokio::fs::OpenOptions::new() .read(true) .write(true) @@ -276,6 +491,66 @@ impl StorageProvider for FileStorageProvider { Ok(()) } + fn supports_checkpoint_capture(&self) -> bool { + true + } + + async fn capture_checkpoint(&self, slot: &StorageSlot, target: &Path) -> Result<()> { + let source = self.checkpoint_source(slot).await?; + let (target_parent, target) = checkpoint_target(target).await?; + ensure_checkpoint_target_absent(&target).await?; + + let temporary = checkpoint_temporary_path(&target_parent, &target); + let mut cleanup = UnpublishedCheckpoint::new(); + let result = + capture_rootfs(&source, &temporary, &target_parent, &target, &mut cleanup).await; + result.map_err(|error| BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': copy {} to {}: {error}", + slot.id, + source.display(), + target.display() + ), + }) + } + + fn supports_checkpoint_restore(&self) -> bool { + true + } + + async fn stage_checkpoint_restore( + &self, + slot: &StorageSlot, + source: &Path, + ) -> Result { + restore::stage(self, slot, source).await + } + + async fn activate_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::activate(self, transaction).await + } + + async fn commit_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::commit(self, transaction).await + } + + async fn abort_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::abort(self, transaction).await + } + + async fn reconcile_checkpoint_restore(&self, instance_id: &str) -> Result<()> { + restore::reconcile(self, instance_id).await + } + fn pool_status(&self) -> PoolStatus { PoolStatus::default() } @@ -285,6 +560,41 @@ impl StorageProvider for FileStorageProvider { } } +impl FileStorageProvider { + async fn checkpoint_source(&self, slot: &StorageSlot) -> Result { + let canonical = self.slot_for_id(&slot.id)?; + let instances_dir = + canonical_plain_path(&self.instances_dir, RequiredPathType::Directory).await?; + let instance_dir = + canonical_plain_path(&canonical.instance_dir, RequiredPathType::Directory).await?; + if instance_dir.parent() != Some(instances_dir.as_path()) { + return Err(BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': slot {} is outside instances directory {}", + slot.id, + instance_dir.display(), + instances_dir.display() + ), + }); + } + + let source = canonical_plain_path(&canonical.rootfs_path, RequiredPathType::File).await?; + if source.parent() != Some(instance_dir.as_path()) + || source.file_name() != canonical.rootfs_path.file_name() + { + return Err(BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': rootfs {} is outside slot {}", + slot.id, + source.display(), + instance_dir.display() + ), + }); + } + Ok(source) + } +} + async fn create_or_copy( source: &std::path::Path, target: &std::path::Path, @@ -301,6 +611,194 @@ async fn create_or_copy( Ok(()) } +async fn copy_runtime_template_artifact( + source: RuntimeTemplateArtifact, + target: &Path, +) -> Result<()> { + let metadata = source + .file + .metadata() + .map_err(|error| BlazeError::StorageError { + msg: format!("inspect runtime template artifact: {error}"), + })?; + if !metadata.is_file() || metadata.len() != source.size_bytes { + return Err(BlazeError::StorageError { + msg: format!( + "runtime template artifact has size {}; expected {}", + metadata.len(), + source.size_bytes + ), + }); + } + + let mut source_file = tokio::fs::File::from_std(source.file); + source_file.seek(SeekFrom::Start(0)).await?; + let mut destination = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target) + .await?; + let mut digest = Sha256::new(); + let mut copied = 0_u64; + let mut buffer = vec![0_u8; 1024 * 1024]; + loop { + let read = source_file.read(&mut buffer).await?; + if read == 0 { + break; + } + copied = copied + .checked_add(u64::try_from(read).unwrap_or(u64::MAX)) + .ok_or_else(|| BlazeError::StorageError { + msg: "runtime template artifact size overflow".to_string(), + })?; + if copied > source.size_bytes { + return Err(BlazeError::StorageError { + msg: format!( + "runtime template artifact exceeds declared size {}", + source.size_bytes + ), + }); + } + digest.update(&buffer[..read]); + destination.write_all(&buffer[..read]).await?; + } + if copied != source.size_bytes { + return Err(BlazeError::StorageError { + msg: format!( + "runtime template artifact has {copied} bytes; expected {}", + source.size_bytes + ), + }); + } + let actual = format!("{:x}", digest.finalize()); + if actual != source.sha256 { + return Err(BlazeError::StorageError { + msg: format!( + "runtime template artifact digest mismatch: expected {}, got {actual}", + source.sha256 + ), + }); + } + destination.sync_all().await?; + Ok(()) +} + +async fn create_empty_durable_file(path: &Path) -> Result<()> { + tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await? + .sync_all() + .await?; + Ok(()) +} + +async fn canonical_plain_path(path: &Path, required_type: RequiredPathType) -> Result { + let metadata = + tokio::fs::symlink_metadata(path) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inspect checkpoint path {}: {error}", path.display()), + })?; + if !required_type.matches(&metadata) || metadata.file_type().is_symlink() { + return Err(BlazeError::StorageError { + msg: format!( + "checkpoint path {} is not a plain {}", + path.display(), + required_type.description() + ), + }); + } + tokio::fs::canonicalize(path) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("canonicalize checkpoint path {}: {error}", path.display()), + }) +} + +async fn checkpoint_target(target: &Path) -> Result<(PathBuf, PathBuf)> { + if !matches!(target.components().next_back(), Some(Component::Normal(_))) { + return Err(BlazeError::StorageError { + msg: format!( + "checkpoint target {} must end in a file name", + target.display() + ), + }); + } + let parent = target.parent().ok_or_else(|| BlazeError::StorageError { + msg: format!( + "checkpoint target {} has no parent directory", + target.display() + ), + })?; + let parent = canonical_plain_path(parent, RequiredPathType::Directory).await?; + let file_name = target.file_name().ok_or_else(|| BlazeError::StorageError { + msg: format!("checkpoint target {} has no file name", target.display()), + })?; + let target = parent.join(file_name); + Ok((parent, target)) +} + +async fn ensure_checkpoint_target_absent(target: &Path) -> Result<()> { + match tokio::fs::symlink_metadata(target).await { + Ok(_) => Err(BlazeError::StorageError { + msg: format!("checkpoint target {} already exists", target.display()), + }), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(BlazeError::StorageError { + msg: format!("inspect checkpoint target {}: {error}", target.display()), + }), + } +} + +fn checkpoint_temporary_path(parent: &Path, target: &Path) -> PathBuf { + let mut name = OsString::from("."); + name.push(target.file_name().expect("validated checkpoint target")); + name.push(format!(".capture-{}.tmp", Uuid::new_v4())); + parent.join(name) +} + +async fn capture_rootfs( + source: &Path, + temporary: &Path, + parent: &Path, + target: &Path, + cleanup: &mut UnpublishedCheckpoint, +) -> std::io::Result<()> { + let mut source_options = tokio::fs::OpenOptions::new(); + source_options.read(true); + #[cfg(unix)] + source_options.custom_flags(libc::O_NOFOLLOW); + let mut source_file = source_options.open(source).await?; + if !source_file.metadata().await?.is_file() { + return Err(std::io::Error::other(format!( + "checkpoint source {} is not a regular file", + source.display() + ))); + } + let mut temporary_file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(temporary) + .await?; + cleanup.mark_temporary(temporary.to_path_buf()); + tokio::io::copy(&mut source_file, &mut temporary_file).await?; + temporary_file.sync_all().await?; + drop(temporary_file); + drop(source_file); + + tokio::fs::hard_link(temporary, target).await?; + cleanup.mark_target(target.to_path_buf()); + crate::failpoint::storage("storage-capture-after-link") + .map_err(|error| std::io::Error::other(error.to_string()))?; + tokio::fs::remove_file(temporary).await?; + cleanup.clear_temporary(); + tokio::fs::File::open(parent).await?.sync_all().await?; + cleanup.commit(); + Ok(()) +} + fn validate_instance_id(instance_id: &str) -> Result<()> { if instance_id.is_empty() || instance_id.contains('/') @@ -319,6 +817,45 @@ fn validate_instance_id(instance_id: &str) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use uuid::Uuid; + + fn runtime_template_artifact(root: &Path, name: &str, bytes: &[u8]) -> RuntimeTemplateArtifact { + let path = root.join(name); + std::fs::write(&path, bytes).expect("template artifact"); + RuntimeTemplateArtifact { + file: std::fs::File::open(path).expect("open template artifact"), + size_bytes: u64::try_from(bytes.len()).expect("artifact length"), + sha256: format!("{:x}", Sha256::digest(bytes)), + } + } + + fn runtime_template_storage(root: &Path) -> RuntimeTemplateStorage { + RuntimeTemplateStorage { + vmstate: runtime_template_artifact(root, "source-vmstate", b"snapshot"), + memory: runtime_template_artifact(root, "source-memory", b"memory"), + rootfs: runtime_template_artifact(root, "source-rootfs", b"rootfs"), + } + } + + async fn checkpoint_fixture( + instance_id: &str, + ) -> (tempfile::TempDir, FileStorageProvider, StorageSlot, PathBuf) { + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let checkpoints = temp.path().join("checkpoints"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&checkpoints).await.unwrap(); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + (temp, provider, slot, checkpoints) + } #[tokio::test] async fn probe_existing_dir_returns_true() { @@ -359,6 +896,109 @@ mod tests { ); } + #[tokio::test] + async fn runtime_template_acquire_owns_independent_artifacts() { + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let source = temp.path().join("source"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&source).await.unwrap(); + let provider = FileStorageProvider::new(instances); + let materialized = provider + .acquire_runtime_template( + &AcquireOpts { + instance_id: "template-instance".to_string(), + rootfs_size: 6, + mem_size: 6, + }, + runtime_template_storage(&source), + ) + .await + .expect("materialize template"); + + std::fs::write(source.join("source-rootfs"), b"changed").unwrap(); + std::fs::write(source.join("source-memory"), b"changed").unwrap(); + std::fs::write(source.join("source-vmstate"), b"changed").unwrap(); + + assert_eq!( + tokio::fs::read(&materialized.storage.rootfs_path) + .await + .unwrap(), + b"rootfs" + ); + assert_eq!( + tokio::fs::read(&materialized.storage.mem_path) + .await + .unwrap(), + b"memory" + ); + assert_eq!( + tokio::fs::read(&materialized.snapshot_path).await.unwrap(), + b"snapshot" + ); + assert!(materialized.storage.mem_diff_path.is_file()); + assert!(materialized.storage.rootfs_diff_path.is_file()); + } + + #[tokio::test] + async fn runtime_template_acquire_rolls_back_digest_mismatch() { + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let source = temp.path().join("source"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&source).await.unwrap(); + let provider = FileStorageProvider::new(instances.clone()); + let mut storage = runtime_template_storage(&source); + storage.rootfs.sha256 = "0".repeat(64); + + let error = provider + .acquire_runtime_template( + &AcquireOpts { + instance_id: "bad-template".to_string(), + rootfs_size: 6, + mem_size: 6, + }, + storage, + ) + .await + .expect_err("digest mismatch"); + let (_, residual) = error.into_parts(); + + assert!(residual.is_none()); + assert!(!instances.join("bad-template").exists()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn runtime_template_acquire_retains_failed_rollback() { + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let source = temp.path().join("source"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&source).await.unwrap(); + let provider = FileStorageProvider::new(instances.clone()); + let hook = crate::failpoint::TestFailpoint::new(&[ + "storage-acquire-runtime-template-artifacts", + "storage-acquire-rollback", + ]); + + let error = hook + .run(provider.acquire_runtime_template( + &AcquireOpts { + instance_id: "residual-template".to_string(), + rootfs_size: 6, + mem_size: 6, + }, + runtime_template_storage(&source), + )) + .await + .expect_err("rollback failure"); + let (_, residual) = error.into_parts(); + + assert_eq!(residual.expect("residual owner").id, "residual-template"); + assert!(instances.join("residual-template").is_dir()); + } + #[tokio::test] async fn release_removes_instance_dir() { let tmp = tempfile::TempDir::new().unwrap(); @@ -375,6 +1015,85 @@ mod tests { assert!(!dir.exists()); } + #[tokio::test] + async fn release_by_id_recovers_missing_and_partial_slots() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let id = Uuid::new_v4().to_string(); + let missing_id = Uuid::new_v4().to_string(); + provider.release_by_id(&missing_id).await.unwrap(); + provider.release_by_id(&missing_id).await.unwrap(); + let partial = tmp.path().join(&id); + tokio::fs::create_dir(&partial).await.unwrap(); + tokio::fs::write(partial.join("rootfs.ext4"), b"partial") + .await + .unwrap(); + + provider.release_by_id(&id).await.unwrap(); + provider.release_by_id(&id).await.unwrap(); + + assert!(!partial.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn release_by_id_rejects_non_directory_and_symlink_slots() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let id = Uuid::new_v4().to_string(); + let slot_path = tmp.path().join(&id); + tokio::fs::write(&slot_path, b"not a directory") + .await + .unwrap(); + + let file_error = provider.release_by_id(&id).await.unwrap_err(); + assert!(file_error.to_string().contains("refusing non-directory")); + assert!(slot_path.is_file()); + + tokio::fs::remove_file(&slot_path).await.unwrap(); + let target = tempfile::TempDir::new().unwrap(); + symlink(target.path(), &slot_path).unwrap(); + + let symlink_error = provider.release_by_id(&id).await.unwrap_err(); + assert!(symlink_error.to_string().contains("refusing non-directory")); + assert!(std::fs::symlink_metadata(&slot_path).unwrap().is_symlink()); + assert!(target.path().is_dir()); + } + + #[tokio::test] + async fn owned_slot_inventory_returns_stable_ids() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let first = Uuid::new_v4().to_string(); + let second = Uuid::new_v4().to_string(); + tokio::fs::create_dir(tmp.path().join(&second)) + .await + .unwrap(); + tokio::fs::create_dir(tmp.path().join(&first)) + .await + .unwrap(); + let mut expected = vec![first, second]; + expected.sort(); + + assert!(provider.supports_runtime_pool_recovery()); + assert_eq!(provider.list_owned_ids().await.unwrap(), expected); + } + + #[tokio::test] + async fn owned_slot_inventory_rejects_unknown_entry_types() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + tokio::fs::write(tmp.path().join("unexpected"), b"not a slot") + .await + .unwrap(); + + let error = provider.list_owned_ids().await.unwrap_err(); + + assert!(error.to_string().contains("unexpected non-directory")); + } + #[tokio::test] async fn pool_status_returns_defaults() { let tmp = tempfile::TempDir::new().unwrap(); @@ -521,6 +1240,153 @@ mod tests { )); } + #[cfg(unix)] + #[tokio::test] + async fn reconstruct_rejects_a_linked_slot_root() { + use std::os::unix::fs::symlink; + + let storage = tempfile::TempDir::new().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + for artifact in ["rootfs.ext4", "mem.bin", "mem.diff", "rootfs.diff"] { + tokio::fs::write(target.path().join(artifact), b"external") + .await + .unwrap(); + } + symlink(target.path(), storage.path().join("linked-slot")).unwrap(); + let provider = FileStorageProvider::new(storage.path().to_path_buf()); + + let error = provider + .reconstruct("linked-slot") + .await + .expect_err("linked slot root must be rejected"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "directory", + } if instance_id == "linked-slot" && path == &storage.path().join("linked-slot") + )); + assert!( + std::fs::symlink_metadata(storage.path().join("linked-slot")) + .unwrap() + .file_type() + .is_symlink() + ); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconstruct_rejects_a_linked_slot_artifact() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(temp.path().to_path_buf()); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "linked-artifact".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + tokio::fs::remove_file(&slot.mem_diff_path).await.unwrap(); + let external = temp.path().join("external-memory-diff"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.mem_diff_path).unwrap(); + + let error = provider + .reconstruct("linked-artifact") + .await + .expect_err("linked artifact must be rejected"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "file", + } if instance_id == "linked-artifact" && path == &slot.mem_diff_path + )); + assert!( + std::fs::symlink_metadata(&slot.mem_diff_path) + .unwrap() + .file_type() + .is_symlink() + ); + assert!(external.is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn flush_rejects_a_linked_slot_root() { + use std::os::unix::fs::symlink; + + let storage = tempfile::TempDir::new().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + for artifact in ["rootfs.ext4", "mem.bin", "mem.diff", "rootfs.diff"] { + tokio::fs::write(target.path().join(artifact), b"external") + .await + .unwrap(); + } + symlink(target.path(), storage.path().join("linked-flush")).unwrap(); + let provider = FileStorageProvider::new(storage.path().to_path_buf()); + let slot = provider.slot_for_id("linked-flush").unwrap(); + + let error = provider + .flush_dirty(&slot) + .await + .expect_err("linked slot root must not be flushed"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "directory", + } if instance_id == "linked-flush" && path == &storage.path().join("linked-flush") + )); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn flush_rejects_a_linked_slot_artifact() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(temp.path().to_path_buf()); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "linked-flush-artifact".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + tokio::fs::remove_file(&slot.mem_diff_path).await.unwrap(); + let external = temp.path().join("external-memory-diff"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.mem_diff_path).unwrap(); + + let error = provider + .flush_dirty(&slot) + .await + .expect_err("linked artifact must not be flushed"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "file", + } if instance_id == "linked-flush-artifact" && path == &slot.mem_diff_path + )); + assert_eq!(tokio::fs::read(&external).await.unwrap(), b"external"); + } + #[tokio::test] async fn flush_rederives_canonical_paths_from_slot_id() { let temp = tempfile::TempDir::new().unwrap(); @@ -573,4 +1439,191 @@ mod tests { .expect_err("missing artifact must fail the sweep item"); assert!(error.to_string().contains("mem.diff"), "{error}"); } + + #[tokio::test] + async fn checkpoint_capture_is_explicit_and_independent() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-independent").await; + tokio::fs::write(&slot.rootfs_path, b"captured-rootfs") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + + assert!(provider.supports_checkpoint_capture()); + provider.capture_checkpoint(&slot, &target).await.unwrap(); + tokio::fs::write(&slot.rootfs_path, b"changed-live-rootfs") + .await + .unwrap(); + + assert_eq!(tokio::fs::read(&target).await.unwrap(), b"captured-rootfs"); + } + + #[tokio::test] + async fn checkpoint_capture_does_not_replace_the_live_rootfs() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-read-only").await; + tokio::fs::write(&slot.rootfs_path, b"live-rootfs") + .await + .unwrap(); + + provider + .capture_checkpoint(&slot, &checkpoints.join("rootfs.snap")) + .await + .unwrap(); + + assert_eq!( + tokio::fs::read(&slot.rootfs_path).await.unwrap(), + b"live-rootfs" + ); + } + + #[tokio::test] + async fn checkpoint_capture_ignores_forged_slot_paths() { + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-canonical").await; + tokio::fs::write(&slot.rootfs_path, b"canonical-rootfs") + .await + .unwrap(); + let forged_source = temp.path().join("forged-rootfs"); + tokio::fs::write(&forged_source, b"forged-rootfs") + .await + .unwrap(); + let mut forged = slot.clone(); + forged.rootfs_path = forged_source; + forged.mem_path = temp.path().join("forged-memory"); + forged.mem_diff_path = temp.path().join("forged-memory-diff"); + forged.rootfs_diff_path = temp.path().join("forged-rootfs-diff"); + forged.instance_dir = temp.path().to_path_buf(); + let target = checkpoints.join("rootfs.snap"); + + provider.capture_checkpoint(&forged, &target).await.unwrap(); + + assert_eq!(tokio::fs::read(&target).await.unwrap(), b"canonical-rootfs"); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_rootfs() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-linked-source").await; + tokio::fs::remove_file(&slot.rootfs_path).await.unwrap(); + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.rootfs_path).unwrap(); + let target = checkpoints.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked rootfs must not be captured"); + + assert!(!target.exists()); + assert_eq!(tokio::fs::read(external).await.unwrap(), b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_slot_directory() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-linked-slot").await; + tokio::fs::remove_dir_all(&slot.instance_dir).await.unwrap(); + let external = temp.path().join("external-slot"); + tokio::fs::create_dir(&external).await.unwrap(); + tokio::fs::write(external.join("rootfs.ext4"), b"external") + .await + .unwrap(); + symlink(&external, &slot.instance_dir).unwrap(); + let target = checkpoints.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked slot directory must be rejected"); + + assert!(!target.exists()); + assert_eq!( + tokio::fs::read(external.join("rootfs.ext4")).await.unwrap(), + b"external" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_target_parent() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let external = temp.path().join("external-checkpoints"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&external).await.unwrap(); + let linked_parent = temp.path().join("linked-checkpoints"); + symlink(&external, &linked_parent).unwrap(); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "capture-linked-parent".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + let target = linked_parent.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked target parent must be rejected"); + + assert!(!external.join("rootfs.snap").exists()); + } + + #[tokio::test] + async fn checkpoint_capture_preserves_an_existing_target() { + let (_temp, provider, slot, checkpoints) = + checkpoint_fixture("capture-existing-target").await; + tokio::fs::write(&slot.rootfs_path, b"new-checkpoint") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + tokio::fs::write(&target, b"existing-checkpoint") + .await + .unwrap(); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("capture must never replace an existing target"); + + assert_eq!( + tokio::fs::read(&target).await.unwrap(), + b"existing-checkpoint" + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_capture_cleans_temporary_data_after_failure() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-cleanup").await; + tokio::fs::write(&slot.rootfs_path, b"complete-temporary-copy") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-capture-after-link"]); + + hook.run(provider.capture_checkpoint(&slot, &target)) + .await + .expect_err("armed capture must roll back its unpublished target"); + + assert!(!target.exists()); + assert!( + tokio::fs::read_dir(&checkpoints) + .await + .unwrap() + .next_entry() + .await + .unwrap() + .is_none(), + "capture failure must remove its temporary file" + ); + } } diff --git a/src/blaze/crates/blazed/src/file_provider/restore.rs b/src/blaze/crates/blazed/src/file_provider/restore.rs new file mode 100644 index 0000000000..fcb0c72452 --- /dev/null +++ b/src/blaze/crates/blazed/src/file_provider/restore.rs @@ -0,0 +1,1541 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable rootfs replacement for the file storage provider. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use uuid::Uuid; + +use blaze_core::error::{BlazeError, Result}; +use blaze_core::storage::{StorageRestoreTransaction, StorageSlot}; + +use super::{FileStorageProvider, RequiredPathType}; + +const JOURNAL_VERSION: u32 = 1; +const MAX_JOURNAL_SIZE: u64 = 16 * 1024; +const COPY_BUFFER_SIZE: usize = 64 * 1024; + +#[derive(Debug)] +struct RestorePaths { + instance_id: String, + instance_dir: PathBuf, + rootfs: PathBuf, + copying: PathBuf, + staged: PathBuf, + backup: PathBuf, + discard: PathBuf, + journal: PathBuf, + journal_temporary: PathBuf, +} + +impl RestorePaths { + fn transaction_artifacts(&self) -> [&Path; 6] { + [ + &self.copying, + &self.staged, + &self.backup, + &self.discard, + &self.journal, + &self.journal_temporary, + ] + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum RestoreState { + Staged, + Activated, + Aborting, + Committing, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct RestoreJournal { + version: u32, + instance_id: String, + transaction_id: Uuid, + state: RestoreState, +} + +impl RestoreJournal { + fn transaction(&self) -> StorageRestoreTransaction { + StorageRestoreTransaction { + instance_id: self.instance_id.clone(), + transaction_id: self.transaction_id, + } + } +} + +/// Removes files that have not yet become part of a durable transaction. +struct UnpublishedFiles { + paths: Vec, +} + +impl UnpublishedFiles { + fn new() -> Self { + Self { paths: Vec::new() } + } + + fn track(&mut self, path: &Path) { + self.paths.push(path.to_path_buf()); + } + + fn untrack(&mut self, path: &Path) { + self.paths.retain(|tracked| tracked != path); + } + + fn commit(&mut self) { + self.paths.clear(); + } +} + +impl Drop for UnpublishedFiles { + fn drop(&mut self) { + for path in self.paths.drain(..) { + let _ = std::fs::remove_file(path); + } + } +} + +pub(super) async fn stage( + provider: &FileStorageProvider, + slot: &StorageSlot, + source: &Path, +) -> Result { + let paths = restore_paths(provider, &slot.id).await?; + require_plain_file(&paths.rootfs, "live rootfs").await?; + ensure_no_transaction(&paths).await?; + + let source = canonical_plain_file(source, "restore source").await?; + let rootfs = tokio::fs::canonicalize(&paths.rootfs) + .await + .map_err(|error| storage_error(format!("canonicalize live rootfs: {error}")))?; + if source == rootfs { + return Err(storage_error( + "restore source must be independent from the live rootfs", + )); + } + + let journal = RestoreJournal { + version: JOURNAL_VERSION, + instance_id: paths.instance_id.clone(), + transaction_id: Uuid::new_v4(), + state: RestoreState::Staged, + }; + let mut unpublished = UnpublishedFiles::new(); + + copy_for_restore(&source, &paths.copying, &mut unpublished).await?; + rename_new_plain_file(&paths.copying, &paths.staged).await?; + unpublished.untrack(&paths.copying); + unpublished.track(&paths.staged); + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-stage").await; + + publish_new_journal(&paths, &journal, &mut unpublished).await?; + unpublished.commit(); + sync_directory(&paths.instance_dir).await?; + Ok(journal.transaction()) +} + +pub(super) async fn activate( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let mut journal = require_journal(&paths).await?; + verify_transaction(&journal, transaction)?; + + match journal.state { + RestoreState::Activated => return ensure_activated_layout(&paths).await, + RestoreState::Staged => {} + RestoreState::Aborting => { + return Err(storage_error(format!( + "restore transaction {} is aborting", + transaction.transaction_id + ))); + } + RestoreState::Committing => { + return Err(storage_error(format!( + "restore transaction {} is committing", + transaction.transaction_id + ))); + } + } + + let (live, staged, backup, discard) = inspect_layout(&paths).await?; + if discard { + return Err(invalid_layout(&paths, journal.state)); + } + + if live && staged && !backup { + rename_new_plain_file(&paths.rootfs, &paths.backup).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-backup").await; + } else if !live && staged && backup { + // Resume after the predecessor was retained. + } else if live && !staged && backup { + // Resume after the staged rootfs was selected. + } else { + return Err(invalid_layout(&paths, journal.state)); + } + + let (live, staged, backup, discard) = inspect_layout(&paths).await?; + if !live && staged && backup && !discard { + if let Err(selection) = select_staged_rootfs(&paths).await { + let rollback = match crate::failpoint::storage("storage-restore-switch-rollback") { + Ok(()) => match rename_new_plain_file(&paths.backup, &paths.rootfs).await { + Ok(()) => sync_directory(&paths.instance_dir).await, + Err(error) => Err(error), + }, + Err(error) => Err(error), + }; + return match rollback { + Ok(()) => Err(storage_error(format!( + "select staged rootfs for '{}': {selection}; predecessor restored", + paths.instance_id + ))), + Err(rollback) => Err(storage_error(format!( + "select staged rootfs for '{}': {selection}; restoring predecessor failed: \ + {rollback}", + paths.instance_id + ))), + }; + } + crate::failpoint::pause("storage-restore-after-switch").await; + } + + ensure_activated_layout(&paths).await?; + journal.state = RestoreState::Activated; + replace_journal(&paths, &journal).await +} + +async fn select_staged_rootfs(paths: &RestorePaths) -> Result<()> { + crate::failpoint::storage("storage-restore-switch")?; + rename_new_plain_file(&paths.staged, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await +} + +pub(super) async fn commit( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let Some(mut journal) = read_journal(&paths).await? else { + return ensure_finalized_layout(&paths).await; + }; + verify_transaction(&journal, transaction)?; + + match journal.state { + RestoreState::Activated => { + ensure_activated_layout(&paths).await?; + journal.state = RestoreState::Committing; + replace_journal(&paths, &journal).await?; + crate::failpoint::pause("storage-restore-after-commit-intent").await; + } + RestoreState::Committing => {} + RestoreState::Staged => { + return Err(storage_error(format!( + "restore transaction {} is not activated", + transaction.transaction_id + ))); + } + RestoreState::Aborting => { + return Err(storage_error(format!( + "restore transaction {} is aborting", + transaction.transaction_id + ))); + } + } + finish_commit(&paths).await +} + +pub(super) async fn abort( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let Some(mut journal) = read_journal(&paths).await? else { + return ensure_finalized_layout(&paths).await; + }; + verify_transaction(&journal, transaction)?; + + if journal.state == RestoreState::Committing { + return Err(storage_error(format!( + "restore transaction {} has durable commit intent", + transaction.transaction_id + ))); + } + if journal.state != RestoreState::Aborting { + journal.state = RestoreState::Aborting; + replace_journal(&paths, &journal).await?; + } + finish_abort(&paths).await +} + +pub(super) async fn reconcile(provider: &FileStorageProvider, instance_id: &str) -> Result<()> { + let paths = restore_paths(provider, instance_id).await?; + remove_plain_file_if_present(&paths.copying, "restore copying file").await?; + remove_plain_file_if_present(&paths.journal_temporary, "restore journal temporary").await?; + + let Some(mut journal) = read_journal(&paths).await? else { + return reconcile_without_journal(&paths).await; + }; + if journal.instance_id != paths.instance_id { + return Err(storage_error(format!( + "restore journal instance '{}' does not match slot '{}'", + journal.instance_id, paths.instance_id + ))); + } + + match journal.state { + RestoreState::Committing => finish_commit(&paths).await, + RestoreState::Staged | RestoreState::Activated => { + journal.state = RestoreState::Aborting; + replace_journal(&paths, &journal).await?; + finish_abort(&paths).await + } + RestoreState::Aborting => finish_abort(&paths).await, + } +} + +async fn restore_paths(provider: &FileStorageProvider, instance_id: &str) -> Result { + let slot = provider.slot_for_id(instance_id)?; + let instances_dir = canonical_plain_path( + &provider.instances_dir, + RequiredPathType::Directory, + "instances directory", + ) + .await?; + let instance_dir = canonical_plain_path( + &slot.instance_dir, + RequiredPathType::Directory, + "slot directory", + ) + .await?; + if instance_dir.parent() != Some(instances_dir.as_path()) + || instance_dir.file_name() != Some(std::ffi::OsStr::new(instance_id)) + { + return Err(storage_error(format!( + "restore slot {} is not the direct '{}' child of instances directory {}", + instance_dir.display(), + instance_id, + instances_dir.display() + ))); + } + + Ok(RestorePaths { + instance_id: instance_id.to_string(), + rootfs: instance_dir.join("rootfs.ext4"), + copying: instance_dir.join(".rootfs.restore-copying"), + staged: instance_dir.join(".rootfs.restore-staged"), + backup: instance_dir.join(".rootfs.restore-backup"), + discard: instance_dir.join(".rootfs.restore-discard"), + journal: instance_dir.join(".rootfs.restore.json"), + journal_temporary: instance_dir.join(".rootfs.restore-journal.tmp"), + instance_dir, + }) +} + +async fn canonical_plain_path( + path: &Path, + required_type: RequiredPathType, + description: &str, +) -> Result { + let metadata = tokio::fs::symlink_metadata(path).await.map_err(|error| { + storage_error(format!("inspect {description} {}: {error}", path.display())) + })?; + if !required_type.matches(&metadata) || metadata.file_type().is_symlink() { + return Err(storage_error(format!( + "{description} {} is not a plain {}", + path.display(), + required_type.description() + ))); + } + tokio::fs::canonicalize(path).await.map_err(|error| { + storage_error(format!( + "canonicalize {description} {}: {error}", + path.display() + )) + }) +} + +async fn canonical_plain_file(path: &Path, description: &str) -> Result { + canonical_plain_path(path, RequiredPathType::File, description).await +} + +async fn require_plain_file(path: &Path, description: &str) -> Result<()> { + if plain_file_exists(path, description).await? { + Ok(()) + } else { + Err(storage_error(format!( + "{description} {} does not exist", + path.display() + ))) + } +} + +async fn plain_file_exists(path: &Path, description: &str) -> Result { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(true), + Ok(_) => Err(storage_error(format!( + "{description} {} is not a plain file", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(storage_error(format!( + "inspect {description} {}: {error}", + path.display() + ))), + } +} + +async fn entry_exists(path: &Path) -> Result { + match tokio::fs::symlink_metadata(path).await { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(storage_error(format!( + "inspect restore artifact {}: {error}", + path.display() + ))), + } +} + +async fn ensure_no_transaction(paths: &RestorePaths) -> Result<()> { + for path in paths.transaction_artifacts() { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has unfinished restore artifact {}; reconcile it first", + paths.instance_id, + path.display() + ))); + } + } + Ok(()) +} + +async fn ensure_no_transient_files(paths: &RestorePaths) -> Result<()> { + for (path, description) in [ + (&paths.copying, "restore copying file"), + (&paths.journal_temporary, "restore journal temporary"), + ] { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has unfinished {description}; reconcile it first", + paths.instance_id + ))); + } + } + Ok(()) +} + +async fn copy_for_restore( + source: &Path, + destination: &Path, + unpublished: &mut UnpublishedFiles, +) -> Result<()> { + let mut source_options = tokio::fs::OpenOptions::new(); + source_options.read(true); + #[cfg(unix)] + source_options.custom_flags(libc::O_NOFOLLOW); + let mut source_file = source_options.open(source).await.map_err(|error| { + storage_error(format!("open restore source {}: {error}", source.display())) + })?; + if !source_file + .metadata() + .await + .map_err(|error| storage_error(format!("inspect restore source: {error}")))? + .is_file() + { + return Err(storage_error(format!( + "restore source {} is not a regular file", + source.display() + ))); + } + + let mut destination_options = tokio::fs::OpenOptions::new(); + destination_options.write(true).create_new(true); + #[cfg(unix)] + destination_options.custom_flags(libc::O_NOFOLLOW); + let mut destination_file = destination_options + .open(destination) + .await + .map_err(|error| { + storage_error(format!( + "create restore stage {}: {error}", + destination.display() + )) + })?; + unpublished.track(destination); + + let mut buffer = vec![0_u8; COPY_BUFFER_SIZE]; + let mut first_chunk = true; + loop { + let count = source_file + .read(&mut buffer) + .await + .map_err(|error| storage_error(format!("read restore source: {error}")))?; + if count == 0 { + break; + } + destination_file + .write_all(&buffer[..count]) + .await + .map_err(|error| storage_error(format!("write restore stage: {error}")))?; + if first_chunk { + first_chunk = false; + crate::failpoint::pause("storage-restore-copy-after-chunk").await; + } + } + destination_file + .sync_all() + .await + .map_err(|error| storage_error(format!("sync restore stage: {error}"))) +} + +async fn publish_new_journal( + paths: &RestorePaths, + journal: &RestoreJournal, + unpublished: &mut UnpublishedFiles, +) -> Result<()> { + let bytes = encode_journal(journal)?; + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let mut file = options + .open(&paths.journal_temporary) + .await + .map_err(|error| storage_error(format!("create restore journal: {error}")))?; + unpublished.track(&paths.journal_temporary); + file.write_all(&bytes) + .await + .map_err(|error| storage_error(format!("write restore journal: {error}")))?; + file.sync_all() + .await + .map_err(|error| storage_error(format!("sync restore journal: {error}")))?; + drop(file); + rename_new_plain_file(&paths.journal_temporary, &paths.journal).await?; + unpublished.untrack(&paths.journal_temporary); + // The journal now owns the staged rootfs, even if the directory sync fails. + unpublished.commit(); + Ok(()) +} + +async fn replace_journal(paths: &RestorePaths, journal: &RestoreJournal) -> Result<()> { + require_plain_file(&paths.journal, "restore journal").await?; + if entry_exists(&paths.journal_temporary).await? { + return Err(storage_error(format!( + "slot '{}' has an unfinished journal update; reconcile it first", + paths.instance_id + ))); + } + + let bytes = encode_journal(journal)?; + let mut cleanup = UnpublishedFiles::new(); + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let mut file = options + .open(&paths.journal_temporary) + .await + .map_err(|error| storage_error(format!("create restore journal update: {error}")))?; + cleanup.track(&paths.journal_temporary); + file.write_all(&bytes) + .await + .map_err(|error| storage_error(format!("write restore journal update: {error}")))?; + file.sync_all() + .await + .map_err(|error| storage_error(format!("sync restore journal update: {error}")))?; + drop(file); + tokio::fs::rename(&paths.journal_temporary, &paths.journal) + .await + .map_err(|error| storage_error(format!("replace restore journal: {error}")))?; + cleanup.untrack(&paths.journal_temporary); + sync_directory(&paths.instance_dir).await +} + +fn encode_journal(journal: &RestoreJournal) -> Result> { + serde_json::to_vec(journal) + .map_err(|error| storage_error(format!("encode restore journal: {error}"))) +} + +async fn read_journal(paths: &RestorePaths) -> Result> { + if !plain_file_exists(&paths.journal, "restore journal").await? { + return Ok(None); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let file = options + .open(&paths.journal) + .await + .map_err(|error| storage_error(format!("open restore journal: {error}")))?; + let mut bytes = Vec::new(); + file.take(MAX_JOURNAL_SIZE + 1) + .read_to_end(&mut bytes) + .await + .map_err(|error| storage_error(format!("read restore journal: {error}")))?; + if bytes.len() as u64 > MAX_JOURNAL_SIZE { + return Err(storage_error("restore journal exceeds the size limit")); + } + let journal: RestoreJournal = serde_json::from_slice(&bytes) + .map_err(|error| storage_error(format!("parse restore journal: {error}")))?; + if journal.version != JOURNAL_VERSION { + return Err(storage_error(format!( + "unsupported restore journal version {}", + journal.version + ))); + } + Ok(Some(journal)) +} + +async fn require_journal(paths: &RestorePaths) -> Result { + read_journal(paths).await?.ok_or_else(|| { + storage_error(format!( + "slot '{}' has no restore transaction", + paths.instance_id + )) + }) +} + +fn verify_transaction( + journal: &RestoreJournal, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + if journal.instance_id != transaction.instance_id + || journal.transaction_id != transaction.transaction_id + { + return Err(storage_error(format!( + "restore transaction {} does not own slot '{}'", + transaction.transaction_id, transaction.instance_id + ))); + } + Ok(()) +} + +async fn inspect_layout(paths: &RestorePaths) -> Result<(bool, bool, bool, bool)> { + Ok(( + plain_file_exists(&paths.rootfs, "live rootfs").await?, + plain_file_exists(&paths.staged, "staged rootfs").await?, + plain_file_exists(&paths.backup, "retained rootfs").await?, + plain_file_exists(&paths.discard, "discarded rootfs").await?, + )) +} + +async fn ensure_activated_layout(paths: &RestorePaths) -> Result<()> { + if inspect_layout(paths).await? != (true, false, true, false) { + return Err(invalid_layout(paths, RestoreState::Activated)); + } + Ok(()) +} + +async fn ensure_finalized_layout(paths: &RestorePaths) -> Result<()> { + require_plain_file(&paths.rootfs, "live rootfs").await?; + for path in paths.transaction_artifacts() { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has restore artifact {}; reconcile it first", + paths.instance_id, + path.display() + ))); + } + } + Ok(()) +} + +async fn finish_abort(paths: &RestorePaths) -> Result<()> { + for _ in 0..6 { + let layout = inspect_layout(paths).await?; + match layout { + (false, true, true, false) => { + rename_new_plain_file(&paths.backup, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-rollback-rootfs").await; + } + (true, true, false, false) => { + remove_plain_file(&paths.staged, "staged rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + (true, false, true, false) => { + rename_new_plain_file(&paths.rootfs, &paths.discard).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-discard").await; + } + (false, false, true, true) => { + rename_new_plain_file(&paths.backup, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-rollback-rootfs").await; + } + (true, false, false, true) => { + remove_plain_file(&paths.discard, "discarded rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + (true, false, false, false) => { + remove_plain_file(&paths.journal, "restore journal").await?; + sync_directory(&paths.instance_dir).await?; + return Ok(()); + } + _ => return Err(invalid_layout(paths, RestoreState::Aborting)), + } + } + Err(storage_error(format!( + "restore abort for '{}' did not converge", + paths.instance_id + ))) +} + +async fn finish_commit(paths: &RestorePaths) -> Result<()> { + let (live, staged, _backup, discard) = inspect_layout(paths).await?; + if !live || staged || discard { + return Err(invalid_layout(paths, RestoreState::Committing)); + } + remove_plain_file_if_present(&paths.backup, "retained rootfs").await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-backup-release").await; + remove_plain_file(&paths.journal, "restore journal").await?; + sync_directory(&paths.instance_dir).await +} + +async fn reconcile_without_journal(paths: &RestorePaths) -> Result<()> { + let (live, staged, backup, discard) = inspect_layout(paths).await?; + if backup || discard || !live { + return Err(storage_error(format!( + "slot '{}' has ambiguous restore artifacts without a journal", + paths.instance_id + ))); + } + if staged { + remove_plain_file(&paths.staged, "unpublished staged rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + Ok(()) +} + +async fn rename_new_plain_file(source: &Path, target: &Path) -> Result<()> { + require_plain_file(source, "restore rename source").await?; + if entry_exists(target).await? { + return Err(storage_error(format!( + "restore rename target {} already exists", + target.display() + ))); + } + tokio::fs::rename(source, target).await.map_err(|error| { + storage_error(format!( + "rename restore file {} to {}: {error}", + source.display(), + target.display() + )) + }) +} + +async fn remove_plain_file(path: &Path, description: &str) -> Result<()> { + require_plain_file(path, description).await?; + tokio::fs::remove_file(path) + .await + .map_err(|error| storage_error(format!("remove {description} {}: {error}", path.display()))) +} + +async fn remove_plain_file_if_present(path: &Path, description: &str) -> Result<()> { + if plain_file_exists(path, description).await? { + remove_plain_file(path, description).await?; + } + Ok(()) +} + +async fn sync_directory(path: &Path) -> Result<()> { + tokio::fs::File::open(path) + .await + .map_err(|error| { + storage_error(format!( + "open restore directory {}: {error}", + path.display() + )) + })? + .sync_all() + .await + .map_err(|error| { + storage_error(format!( + "sync restore directory {}: {error}", + path.display() + )) + }) +} + +fn invalid_layout(paths: &RestorePaths, state: RestoreState) -> BlazeError { + storage_error(format!( + "slot '{}' has an invalid {:?} restore layout", + paths.instance_id, state + )) +} + +fn storage_error(message: impl Into) -> BlazeError { + BlazeError::StorageError { + msg: message.into(), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use async_trait::async_trait; + use blaze_core::storage::{ + AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + }; + + use super::*; + + struct UnsupportedStorage; + + #[async_trait] + impl StorageProvider for UnsupportedStorage { + async fn probe(&self) -> Result { + Ok(true) + } + + async fn acquire( + &self, + _opts: &AcquireOpts, + ) -> std::result::Result { + Err(StorageAcquireError::clean(storage_error( + "acquire unavailable", + ))) + } + + async fn release(&self, _slot: StorageSlot) -> Result<()> { + Ok(()) + } + + async fn reconstruct(&self, _instance_id: &str) -> Result { + Err(storage_error("reconstruct unavailable")) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> Result<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> Result { + Ok(0) + } + } + + async fn fixture( + instance_id: &str, + ) -> (tempfile::TempDir, FileStorageProvider, StorageSlot, PathBuf) { + let temp = tempfile::tempdir().expect("temporary storage"); + let instances = temp.path().join("instances"); + let checkpoints = temp.path().join("checkpoints"); + tokio::fs::create_dir(&instances) + .await + .expect("instances directory"); + tokio::fs::create_dir(&checkpoints) + .await + .expect("checkpoints directory"); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage slot"); + tokio::fs::write(&slot.rootfs_path, b"live-rootfs") + .await + .expect("live rootfs"); + let source = checkpoints.join("rootfs.snap"); + tokio::fs::write(&source, b"checkpoint-rootfs") + .await + .expect("checkpoint rootfs"); + (temp, provider, slot, source) + } + + async fn rootfs(path: &Path) -> Vec { + tokio::fs::read(path).await.expect("read rootfs") + } + + #[tokio::test] + async fn restore_contract_is_opt_in_and_fail_closed() { + let provider = UnsupportedStorage; + let slot = StorageSlot { + id: "unsupported".to_string(), + rootfs_path: PathBuf::from("rootfs"), + mem_path: PathBuf::from("memory"), + mem_diff_path: PathBuf::from("memory-diff"), + rootfs_diff_path: PathBuf::from("rootfs-diff"), + instance_dir: PathBuf::from("instance"), + }; + let transaction = StorageRestoreTransaction { + instance_id: slot.id.clone(), + transaction_id: Uuid::new_v4(), + }; + + assert!(!provider.supports_checkpoint_restore()); + assert!( + provider + .stage_checkpoint_restore(&slot, Path::new("checkpoint")) + .await + .is_err() + ); + assert!( + provider + .activate_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .commit_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .abort_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .is_err() + ); + } + + #[tokio::test] + async fn stage_keeps_the_live_rootfs_running_image_unchanged() { + let (_temp, provider, slot, source) = fixture("stage-independent").await; + + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage restore"); + + assert!(provider.supports_checkpoint_restore()); + assert_eq!(transaction.instance_id, slot.id); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("a second transaction must fail closed"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort staged restore"); + } + + #[tokio::test] + async fn activated_restore_can_be_aborted_to_the_predecessor() { + let (_temp, provider, slot, source) = fixture("activate-abort").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&provider, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn activation_retains_the_original_rootfs_inode_until_finalization() { + use std::os::unix::fs::MetadataExt; + + let (_temp, provider, slot, source) = fixture("retain-inode").await; + let original_inode = tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("live metadata") + .ino(); + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + assert_eq!( + tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("staged live metadata") + .ino(), + original_inode + ); + + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + assert_eq!( + tokio::fs::metadata(&paths.backup) + .await + .expect("backup metadata") + .ino(), + original_inode + ); + assert_ne!( + tokio::fs::metadata(&paths.rootfs) + .await + .expect("selected metadata") + .ino(), + original_inode + ); + + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + assert_eq!( + tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("restored metadata") + .ino(), + original_inode + ); + } + + #[tokio::test] + async fn committed_restore_releases_the_predecessor() { + let (_temp, provider, slot, source) = fixture("activate-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + provider + .commit_checkpoint_restore(&transaction) + .await + .expect("commit"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + ensure_finalized_layout(&restore_paths(&provider, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn stale_transaction_handle_cannot_select_a_rootfs() { + let (_temp, provider, slot, source) = fixture("stale-handle").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let stale = StorageRestoreTransaction { + instance_id: transaction.instance_id.clone(), + transaction_id: Uuid::new_v4(), + }; + + provider + .activate_checkpoint_restore(&stale) + .await + .expect_err("stale transaction must be rejected"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + } + + #[tokio::test] + async fn staging_rederives_provider_paths_from_the_slot_id() { + let (temp, provider, slot, source) = fixture("canonical-slot").await; + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external") + .await + .expect("external rootfs"); + let mut forged = slot.clone(); + forged.rootfs_path = external.clone(); + forged.instance_dir = temp.path().to_path_buf(); + + let transaction = provider + .stage_checkpoint_restore(&forged, &source) + .await + .expect("stage through canonical slot"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + assert_eq!(rootfs(&external).await, b"external"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_linked_sources_and_slot_paths() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-paths").await; + let linked_source = temp.path().join("linked-source"); + symlink(&source, &linked_source).expect("source link"); + provider + .stage_checkpoint_restore(&slot, &linked_source) + .await + .expect_err("linked source must be rejected"); + + tokio::fs::remove_file(&slot.rootfs_path) + .await + .expect("remove live rootfs"); + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external") + .await + .expect("external rootfs"); + symlink(&external, &slot.rootfs_path).expect("rootfs link"); + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked live rootfs must be rejected"); + assert_eq!(rootfs(&external).await, b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_a_linked_slot_directory() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-slot").await; + tokio::fs::remove_dir_all(&slot.instance_dir) + .await + .expect("remove slot"); + let external = temp.path().join("external-slot"); + tokio::fs::create_dir(&external) + .await + .expect("external slot"); + tokio::fs::write(external.join("rootfs.ext4"), b"external") + .await + .expect("external rootfs"); + symlink(&external, &slot.instance_dir).expect("slot link"); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked slot directory must be rejected"); + + assert_eq!(rootfs(&external.join("rootfs.ext4")).await, b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_linked_transaction_artifacts() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-artifact").await; + let external = temp.path().join("external"); + tokio::fs::write(&external, b"external") + .await + .expect("external"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + symlink(&external, &paths.staged).expect("stage link"); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked transaction artifact must fail closed"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert_eq!(rootfs(&external).await, b"external"); + } + + #[tokio::test] + async fn staging_rejects_instance_id_path_components() { + let (_temp, provider, mut slot, source) = fixture("valid-id").await; + slot.id = "../escape".to_string(); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("path component must be rejected"); + } + + #[tokio::test] + async fn restart_aborts_a_staged_restore() { + let (_temp, provider, slot, source) = fixture("restart-staged").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn restart_aborts_an_activated_restore() { + let (_temp, provider, slot, source) = fixture("restart-activated").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[tokio::test] + async fn restart_finishes_a_durable_commit_intent() { + let (_temp, provider, slot, source) = fixture("restart-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + let mut journal = require_journal(&paths).await.expect("journal"); + journal.state = RestoreState::Committing; + replace_journal(&paths, &journal) + .await + .expect("commit intent"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn restart_cleans_a_partial_copy_without_touching_the_live_rootfs() { + let (_temp, provider, slot, _source) = fixture("restart-copying").await; + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + tokio::fs::write(&paths.copying, b"partial") + .await + .expect("partial copy"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert!(!paths.copying.exists()); + } + + #[tokio::test] + async fn restart_recovers_after_retaining_the_predecessor() { + let (_temp, provider, slot, source) = fixture("restart-after-backup").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + rename_new_plain_file(&paths.rootfs, &paths.backup) + .await + .expect("retain predecessor"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert!(!paths.staged.exists()); + assert!(!paths.backup.exists()); + } + + #[tokio::test] + async fn restart_recovers_after_switching_before_journal_update() { + let (_temp, provider, slot, source) = fixture("restart-after-switch").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + rename_new_plain_file(&paths.rootfs, &paths.backup) + .await + .expect("retain predecessor"); + rename_new_plain_file(&paths.staged, &paths.rootfs) + .await + .expect("switch rootfs"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[tokio::test] + async fn corrupt_journal_preserves_both_rootfs_versions() { + let (_temp, provider, slot, source) = fixture("corrupt-journal").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + tokio::fs::write(&paths.journal, b"not-json") + .await + .expect("corrupt journal"); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect_err("corrupt journal must fail closed"); + + assert_eq!(rootfs(&paths.rootfs).await, b"checkpoint-rootfs"); + assert_eq!(rootfs(&paths.backup).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + async fn cancel_at( + provider: FileStorageProvider, + slot: StorageSlot, + source: PathBuf, + failpoint: &'static str, + ) { + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let operation_hook = hook.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(provider.stage_checkpoint_restore(&slot, &source)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("operation must be cancelled") + .is_cancelled() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_partial_copy_is_removed() { + let (_temp, provider, slot, source) = fixture("cancel-copy").await; + tokio::fs::write(&source, vec![42_u8; COPY_BUFFER_SIZE * 4]) + .await + .expect("large checkpoint"); + let instances = provider.instances_dir.clone(); + let id = slot.id.clone(); + let rootfs_path = slot.rootfs_path.clone(); + + cancel_at(provider, slot, source, "storage-restore-copy-after-chunk").await; + + let restarted = FileStorageProvider::new(instances); + restarted + .reconcile_checkpoint_restore(&id) + .await + .expect("reconcile cancellation"); + assert_eq!(rootfs(&rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &id).await.unwrap()) + .await + .unwrap(); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn failed_second_rename_immediately_restores_the_predecessor() { + let (_temp, provider, slot, source) = fixture("failed-switch").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-switch"]); + + hook.run(provider.activate_checkpoint_restore(&transaction)) + .await + .expect_err("selecting the staged rootfs must fail"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort retained stage"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn failed_switch_compensation_remains_reconcilable() { + let (_temp, provider, slot, source) = fixture("failed-switch-rollback").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&[ + "storage-restore-switch", + "storage-restore-switch-rollback", + ]); + + hook.run(provider.activate_checkpoint_restore(&transaction)) + .await + .expect_err("selection and immediate compensation must fail"); + + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + assert!(!paths.rootfs.exists()); + assert_eq!(rootfs(&paths.backup).await, b"live-rootfs"); + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile retained predecessor"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_activation_after_backup_is_reconciled() { + let (_temp, provider, slot, source) = fixture("cancel-activation").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-backup"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.activate_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("activation must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_commit_intent_is_completed_on_restart() { + let (_temp, provider, slot, source) = fixture("cancel-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-commit-intent"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.commit_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("commit must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_abort_is_completed_on_restart() { + let (_temp, provider, slot, source) = fixture("cancel-abort").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-discard"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.abort_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("abort must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } +} diff --git a/src/blaze/crates/blazed/src/guest.rs b/src/blaze/crates/blazed/src/guest.rs new file mode 100644 index 0000000000..2b92cc1b89 --- /dev/null +++ b/src/blaze/crates/blazed/src/guest.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Firecracker-vsock guest agent client. + +pub mod client; + +pub use client::GuestClient; +pub use client::GuestExecResult; + +use thiserror::Error; + +/// Maximum decoded file payload accepted by guest read and write operations. +pub(crate) const MAX_GUEST_FILE_BYTES: usize = 16 * 1024 * 1024; + +/// Guest protocol and transport failures. +#[derive(Debug, Error)] +pub enum GuestError { + /// Unix socket I/O failed. + #[error("guest transport error: {0}")] + Io(#[from] std::io::Error), + /// JSON encoding or decoding failed. + #[error("guest JSON error: {0}")] + Json(#[from] serde_json::Error), + /// Firecracker vsock or guest framing was invalid. + #[error("guest protocol error: {0}")] + Protocol(String), + /// Caller supplied an invalid guest operation argument. + #[error("invalid guest request: {0}")] + InvalidArgument(String), + /// A bounded guest operation timed out. + #[error("guest operation timed out: {0}")] + Timeout(String), + /// A state-changing request timed out after it may have reached the guest. + #[error("guest operation outcome is unknown: {0}")] + OutcomeUnknown(String), + /// The guest returned an application error. + #[error("guest operation failed: {0}")] + Rejected(String), + /// Caller-supplied decoded file data exceeded the guest file hard limit. + #[error("guest payload too large: {actual} bytes exceeds {limit}")] + PayloadTooLarge { + /// Decoded or framed byte count. + actual: usize, + /// Configured limit. + limit: usize, + }, + /// A guest response exceeded the bounded frame or decoded output limit. + #[error("guest response too large: {actual} bytes exceeds {limit}")] + ResponseTooLarge { + /// Decoded or framed byte count. + actual: usize, + /// Configured limit. + limit: usize, + }, + /// Readiness polling was cancelled during daemon shutdown. + #[error("guest readiness wait cancelled")] + Cancelled, +} + +/// Result alias for guest operations. +pub type Result = std::result::Result; diff --git a/src/blaze/crates/blazed/src/guest/client.rs b/src/blaze/crates/blazed/src/guest/client.rs new file mode 100644 index 0000000000..13dd2e91db --- /dev/null +++ b/src/blaze/crates/blazed/src/guest/client.rs @@ -0,0 +1,1051 @@ +// SPDX-License-Identifier: Apache-2.0 +//! JSON-line client layered over Firecracker's vsock Unix socket proxy. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use blaze_core::guest_protocol::{ + DEFAULT_GUEST_PORT, DEFAULT_MAX_RESPONSE_BYTES, GuestOp, GuestRequest, GuestResponse, +}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixStream; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use super::{GuestError, Result}; + +const READY_ATTEMPT_TIMEOUT: Duration = Duration::from_millis(250); +const PROTOCOL_GRACE: Duration = Duration::from_secs(10); +const MAX_EXEC_COMMAND_BYTES: usize = 64 * 1024; +const MAX_EXEC_CWD_BYTES: usize = 4096; +const MAX_EXEC_ENV_ENTRIES: usize = 256; +const MAX_EXEC_ENV_BYTES: usize = 64 * 1024; + +/// Result of one command executed by the guest agent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GuestExecResult { + /// Guest process exit status. + pub exit_code: i32, + /// Decoded stdout bytes. + pub stdout: Vec, + /// Decoded stderr bytes. + pub stderr: Vec, +} + +/// Client for one Firecracker guest agent. +#[derive(Debug, Clone)] +pub struct GuestClient { + vsock_path: PathBuf, + port: u32, + io_timeout: Duration, + max_response_bytes: usize, + max_file_bytes: usize, +} + +impl GuestClient { + /// Create a client with production protocol defaults. + pub fn new(vsock_path: PathBuf, io_timeout: Duration, max_file_bytes: usize) -> Self { + Self { + vsock_path, + port: DEFAULT_GUEST_PORT, + io_timeout, + max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, + max_file_bytes, + } + } + + /// Override protocol limits for focused tests. + #[cfg(test)] + fn with_response_limit(mut self, max_response_bytes: usize) -> Self { + self.max_response_bytes = max_response_bytes; + self + } + + /// Check whether the guest agent is responsive. + pub async fn ping(&self) -> Result<()> { + let request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Ping); + self.send_recv(&request).await?; + Ok(()) + } + + /// Poll readiness with bounded exponential backoff. + pub async fn wait_ready( + &self, + deadline: Duration, + cancellation: &CancellationToken, + ) -> Result<()> { + let started = Instant::now(); + let mut backoff = Duration::from_millis(10); + let mut last_error = None; + while started.elapsed() < deadline { + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + break; + } + let attempt_timeout = READY_ATTEMPT_TIMEOUT.min(remaining); + tokio::select! { + _ = cancellation.cancelled() => return Err(GuestError::Cancelled), + result = tokio::time::timeout(attempt_timeout, self.ping()) => { + match result { + Ok(Ok(())) => return Ok(()), + Ok(Err(error)) => last_error = Some(error), + Err(_) => { + last_error = Some(GuestError::Timeout(format!( + "readiness ping exceeded {attempt_timeout:?}" + ))); + } + } + } + } + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + break; + } + tokio::select! { + _ = cancellation.cancelled() => return Err(GuestError::Cancelled), + _ = tokio::time::sleep(backoff.min(remaining)) => {} + } + backoff = (backoff * 2).min(Duration::from_millis(250)); + } + Err(GuestError::Timeout(format!( + "guest at {} was not ready within {:?}: {}", + self.vsock_path.display(), + deadline, + last_error + .map(|error| error.to_string()) + .unwrap_or_else(|| "no attempt completed".to_string()) + ))) + } + + /// Execute one shell command in the guest. + pub async fn exec( + &self, + command: String, + cwd: Option, + env: Option>, + timeout_secs: u32, + ) -> Result { + validate_exec_inputs(&command, cwd.as_deref(), env.as_ref())?; + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Exec); + request.cmd = Some(command); + request.cwd = Some(cwd.unwrap_or_else(|| "/".to_string())); + request.env = env; + request.timeout = Some(timeout_secs); + let timeout = operation_timeout(timeout_secs); + if timeout > self.io_timeout { + return Err(GuestError::InvalidArgument(format!( + "exec timeout plus protocol grace ({timeout:?}) exceeds configured limit {:?}", + self.io_timeout + ))); + } + let response = self.send_recv_with_timeout(&request, timeout).await?; + let exit_code = response + .rc + .ok_or_else(|| { + GuestError::Protocol("successful exec response is missing rc".to_string()) + }) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + let stdout = decode_limited( + response.stdout_b64.as_deref().unwrap_or_default(), + self.max_response_bytes, + ) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + let stderr = decode_limited( + response.stderr_b64.as_deref().unwrap_or_default(), + self.max_response_bytes, + ) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + Ok(GuestExecResult { + exit_code, + stdout, + stderr, + }) + } + + /// Read one guest file. + pub async fn read_file(&self, path: String) -> Result> { + validate_guest_path(&path)?; + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Read); + request.path = Some(path); + let guest_timeout = request_timeout_secs(self.io_timeout); + request.timeout = Some(guest_timeout); + let response = self + .send_recv_with_timeout( + &request, + operation_timeout(guest_timeout).min(self.io_timeout), + ) + .await?; + let data = response.data_b64.as_deref().ok_or_else(|| { + GuestError::Protocol("successful read response is missing data_b64".to_string()) + })?; + decode_limited(data, self.max_file_bytes) + } + + /// Replace one guest file. + pub async fn write_file(&self, path: String, data: &[u8]) -> Result<()> { + validate_guest_path(&path)?; + if data.len() > self.max_file_bytes { + return Err(GuestError::PayloadTooLarge { + actual: data.len(), + limit: self.max_file_bytes, + }); + } + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Write); + request.path = Some(path); + request.data_b64 = Some(BASE64.encode(data)); + let guest_timeout = request_timeout_secs(self.io_timeout); + request.timeout = Some(guest_timeout); + self.send_recv_with_timeout( + &request, + operation_timeout(guest_timeout).min(self.io_timeout), + ) + .await?; + Ok(()) + } + + async fn send_recv(&self, request: &GuestRequest) -> Result { + self.send_recv_with_timeout(request, self.io_timeout).await + } + + async fn send_recv_with_timeout( + &self, + request: &GuestRequest, + timeout: Duration, + ) -> Result { + let mut encoded = serde_json::to_vec(request)?; + encoded.push(b'\n'); + + let started = Instant::now(); + let mut stream = match tokio::time::timeout(timeout, self.connect_guest()).await { + Ok(result) => result?, + Err(_) => return Err(self.timeout_error(request, timeout, "before request delivery")), + }; + let remaining = timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return Err(self.timeout_error(request, timeout, "before request delivery")); + } + + match tokio::time::timeout( + remaining, + self.exchange_request(&mut stream, request, &encoded), + ) + .await + { + Ok(Ok(response)) => Ok(response), + Ok(Err(error)) => Err(classify_after_request(request.op, error)), + Err(_) => Err(classify_after_request( + request.op, + self.timeout_error(request, timeout, "after request delivery began"), + )), + } + } + + async fn connect_guest(&self) -> Result { + let mut stream = UnixStream::connect(&self.vsock_path).await?; + stream + .write_all(format!("CONNECT {}\n", self.port).as_bytes()) + .await?; + let handshake = read_line(&mut stream, 128).await?; + let handshake = std::str::from_utf8(&handshake).map_err(|error| { + GuestError::Protocol(format!("CONNECT response is not UTF-8: {error}")) + })?; + let peer_cid = handshake + .strip_prefix("OK ") + .and_then(|value| value.parse::().ok()); + if peer_cid.is_none() { + return Err(GuestError::Protocol(format!( + "unexpected CONNECT {} response: expected \"OK \", received {handshake:?}", + self.port, + ))); + } + Ok(stream) + } + + async fn exchange_request( + &self, + stream: &mut UnixStream, + request: &GuestRequest, + encoded: &[u8], + ) -> Result { + stream.write_all(encoded).await?; + stream.flush().await?; + let line = read_line(stream, self.max_response_bytes).await?; + let response: GuestResponse = serde_json::from_slice(&line)?; + if response.id != request.id { + return Err(GuestError::Protocol(format!( + "response id mismatch: sent {}, received {}", + request.id, response.id + ))); + } + if !response.ok { + return Err(GuestError::Rejected(response.err.unwrap_or_else(|| { + "guest rejected request without an error".to_string() + }))); + } + Ok(response) + } + + fn timeout_error(&self, request: &GuestRequest, timeout: Duration, phase: &str) -> GuestError { + GuestError::Timeout(format!( + "{:?} request to {} exceeded {:?} {phase}", + request.op, + self.vsock_path.display(), + timeout + )) + } +} + +fn classify_after_request(operation: GuestOp, error: GuestError) -> GuestError { + if matches!(operation, GuestOp::Exec | GuestOp::Write) + && !matches!( + error, + GuestError::Rejected(_) | GuestError::OutcomeUnknown(_) + ) + { + GuestError::OutcomeUnknown(error.to_string()) + } else { + error + } +} + +fn operation_timeout(guest_timeout_secs: u32) -> Duration { + Duration::from_secs(u64::from(guest_timeout_secs)).saturating_add(PROTOCOL_GRACE) +} + +fn request_timeout_secs(io_timeout: Duration) -> u32 { + io_timeout + .saturating_sub(PROTOCOL_GRACE) + .as_secs() + .clamp(1, u64::from(u32::MAX)) as u32 +} + +async fn read_line(stream: &mut R, limit: usize) -> Result> +where + R: AsyncRead + Unpin, +{ + let bounded = limit.saturating_add(1); + let mut reader = BufReader::new(stream).take(bounded as u64); + let mut output = Vec::with_capacity(limit.min(8192)); + let count = reader.read_until(b'\n', &mut output).await?; + if output.last() == Some(&b'\n') { + output.pop(); + if output.len() <= limit { + return Ok(output); + } + } + if output.len() > limit { + return Err(GuestError::ResponseTooLarge { + actual: output.len(), + limit, + }); + } + debug_assert_eq!(count, output.len()); + Err(GuestError::Protocol( + "connection closed before newline delimiter".to_string(), + )) +} + +fn decode_limited(encoded: &str, limit: usize) -> Result> { + let encoded_limit = limit.div_ceil(3).saturating_mul(4); + if encoded.len() > encoded_limit { + return Err(GuestError::ResponseTooLarge { + actual: encoded.len(), + limit: encoded_limit, + }); + } + let decoded = BASE64 + .decode(encoded) + .map_err(|error| GuestError::Protocol(format!("invalid base64 payload: {error}")))?; + if decoded.len() > limit { + return Err(GuestError::ResponseTooLarge { + actual: decoded.len(), + limit, + }); + } + Ok(decoded) +} + +fn validate_exec_inputs( + command: &str, + cwd: Option<&str>, + env: Option<&HashMap>, +) -> Result<()> { + if command.is_empty() { + return Err(GuestError::InvalidArgument( + "exec command is empty".to_string(), + )); + } + if command.len() > MAX_EXEC_COMMAND_BYTES || command.contains('\0') { + return Err(GuestError::InvalidArgument(format!( + "exec command must be NUL-free and at most {MAX_EXEC_COMMAND_BYTES} bytes" + ))); + } + if let Some(cwd) = cwd + && (cwd.len() > MAX_EXEC_CWD_BYTES || cwd.contains('\0')) + { + return Err(GuestError::InvalidArgument(format!( + "exec cwd must be NUL-free and at most {MAX_EXEC_CWD_BYTES} bytes" + ))); + } + if let Some(env) = env { + if env.len() > MAX_EXEC_ENV_ENTRIES { + return Err(GuestError::InvalidArgument(format!( + "exec environment has {} entries; limit is {MAX_EXEC_ENV_ENTRIES}", + env.len() + ))); + } + let mut total = 0_usize; + for (key, value) in env { + if key.is_empty() || key.contains('=') || key.contains('\0') || value.contains('\0') { + return Err(GuestError::InvalidArgument( + "exec environment keys must be non-empty and contain no '=', and keys and \ + values must be NUL-free" + .to_string(), + )); + } + total = total + .checked_add(key.len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .ok_or_else(|| { + GuestError::InvalidArgument("exec environment size overflow".to_string()) + })?; + if total > MAX_EXEC_ENV_BYTES { + return Err(GuestError::InvalidArgument(format!( + "exec environment is {total} bytes; limit is {MAX_EXEC_ENV_BYTES}" + ))); + } + } + } + Ok(()) +} + +fn validate_guest_path(path: &str) -> Result<()> { + if path.is_empty() || !path.starts_with('/') || path.len() > 4096 || path.contains('\0') { + return Err(GuestError::InvalidArgument( + "guest file path must be absolute, NUL-free, and at most 4096 bytes".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use serde_json::json; + use tokio::net::UnixListener; + + use super::*; + + async fn spawn_server( + socket: PathBuf, + response: Arc serde_json::Value + Send + Sync>, + ) { + let listener = UnixListener::bind(socket).expect("bind"); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let response = response.clone(); + tokio::spawn(async move { + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 1073742006\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request: serde_json::Value = + serde_json::from_slice(&request).expect("json"); + let response = response(request); + let mut bytes = serde_json::to_vec(&response).expect("encode"); + bytes.push(b'\n'); + stream.write_all(&bytes).await.expect("write"); + }); + } + }); + } + + async fn accept_request(listener: &UnixListener) -> (UnixStream, serde_json::Value) { + let (mut stream, _) = listener.accept().await.expect("accept"); + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 1073742006\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request = serde_json::from_slice(&request).expect("json"); + (stream, request) + } + + #[test] + fn side_effect_errors_become_unknown_only_after_delivery_starts() { + for operation in [GuestOp::Exec, GuestOp::Write] { + assert!(matches!( + classify_after_request(operation, GuestError::Protocol("EOF".into())), + GuestError::OutcomeUnknown(_) + )); + assert!(matches!( + classify_after_request( + operation, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + ), + GuestError::OutcomeUnknown(_) + )); + assert!(matches!( + classify_after_request(operation, GuestError::Rejected("denied".into())), + GuestError::Rejected(_) + )); + } + + assert!(matches!( + classify_after_request(GuestOp::Read, GuestError::Protocol("EOF".into())), + GuestError::Protocol(_) + )); + assert!(matches!( + classify_after_request( + GuestOp::Read, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + ), + GuestError::ResponseTooLarge { .. } + )); + } + + #[tokio::test] + async fn ping_exec_read_and_write_follow_existing_protocol() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let requests = Arc::new(AtomicUsize::new(0)); + let server_requests = requests.clone(); + spawn_server( + socket.clone(), + Arc::new(move |request| { + server_requests.fetch_add(1, Ordering::Relaxed); + match request["op"].as_str().expect("op") { + "exec" => json!({ + "id": request["id"], + "ok": true, + "rc": 7, + "stdout_b64": BASE64.encode(b"out"), + "stderr_b64": BASE64.encode(b"err") + }), + "read" => { + assert_eq!(request["timeout"], 5); + json!({ + "id": request["id"], + "ok": true, + "data_b64": BASE64.encode(b"data") + }) + } + "write" => { + assert_eq!(request["timeout"], 5); + json!({"id": request["id"], "ok": true}) + } + _ => json!({"id": request["id"], "ok": true}), + } + }), + ) + .await; + let client = GuestClient::new(socket, Duration::from_secs(15), 1024); + client.ping().await.expect("ping"); + let exec = client + .exec("exit 7".into(), None, None, 1) + .await + .expect("exec"); + assert_eq!(exec.exit_code, 7); + assert_eq!(exec.stdout, b"out"); + assert_eq!( + client.read_file("/tmp/x".into()).await.expect("read"), + b"data" + ); + client + .write_file("/tmp/x".into(), b"replacement") + .await + .expect("write"); + assert_eq!(requests.load(Ordering::Relaxed), 4); + } + + #[tokio::test] + async fn mismatched_response_id_is_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|_| json!({"id": "wrong", "ok": true})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("mismatch"); + assert!(matches!(error, GuestError::Protocol(_))); + } + + #[tokio::test] + async fn malformed_json_is_rejected_without_poisoning_the_next_call() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (mut first, _) = accept_request(&listener).await; + first.write_all(b"{not-json}\n").await.expect("malformed"); + + let (mut second, request) = accept_request(&listener).await; + let mut response = + serde_json::to_vec(&json!({"id": request["id"], "ok": true})).expect("response"); + response.push(b'\n'); + second.write_all(&response).await.expect("valid"); + }); + let client = GuestClient::new(socket, Duration::from_secs(1), 1024); + assert!(matches!(client.ping().await, Err(GuestError::Json(_)))); + client.ping().await.expect("subsequent request"); + server.await.expect("server task"); + } + + #[tokio::test] + async fn missing_socket_is_reported_as_connection_failure() { + let temp = tempfile::tempdir().expect("temp"); + let client = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_millis(100), + 1024, + ); + let error = client.ping().await.expect_err("connection failure"); + assert!(matches!(error, GuestError::Io(_))); + + let write_error = client + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("write did not reach the guest"); + assert!(matches!(write_error, GuestError::Io(_))); + } + + #[tokio::test] + async fn invalid_exec_timeout_is_a_caller_error() { + let temp = tempfile::tempdir().expect("temp"); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(15), + 1024, + ) + .exec("true".into(), None, None, 6) + .await + .expect_err("timeout exceeds request budget"); + assert!(matches!(error, GuestError::InvalidArgument(_))); + assert_eq!( + crate::error::BlazeDaemonError::from(error).status_code(), + 400 + ); + } + + #[tokio::test] + async fn connect_response_requires_a_numeric_peer_cid() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream + .write_all(b"OK not-a-cid\n") + .await + .expect("invalid peer cid"); + }); + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("invalid peer cid"); + assert!(matches!(error, GuestError::Protocol(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn guest_rejection_is_returned_without_panicking() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": false, "err": "denied by guest"})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("rejected"); + assert!(matches!(error, GuestError::Rejected(message) if message == "denied by guest")); + } + + #[tokio::test] + async fn read_and_write_response_timeouts_are_bounded() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (stream, _) = accept_request(&listener).await; + tokio::spawn(async move { + let _stream = stream; + tokio::time::sleep(Duration::from_millis(250)).await; + }); + } + }); + let client = GuestClient::new(socket, Duration::from_millis(30), 1024); + let read_error = client + .read_file("/tmp/x".into()) + .await + .expect_err("read timeout"); + assert!(matches!(read_error, GuestError::Timeout(_))); + let write_error = client + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("write timeout"); + assert!(matches!(write_error, GuestError::OutcomeUnknown(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn mutating_handshake_timeout_is_retryable() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("accept"); + tokio::time::sleep(Duration::from_millis(100)).await; + }); + + let error = GuestClient::new(socket, Duration::from_millis(20), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("handshake timeout"); + assert!(matches!(error, GuestError::Timeout(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn mutating_response_corruption_has_unknown_outcome() { + let temp = tempfile::tempdir().expect("temp"); + + let malformed_socket = temp.path().join("malformed.uds"); + let malformed_listener = UnixListener::bind(&malformed_socket).expect("bind"); + tokio::spawn(async move { + let (mut stream, _) = accept_request(&malformed_listener).await; + stream.write_all(b"{not-json}\n").await.expect("response"); + }); + let malformed = GuestClient::new(malformed_socket, Duration::from_secs(15), 1024) + .exec("true".into(), None, None, 1) + .await + .expect_err("malformed response"); + assert!(matches!(malformed, GuestError::OutcomeUnknown(_))); + + let mismatch_socket = temp.path().join("mismatch.uds"); + spawn_server( + mismatch_socket.clone(), + Arc::new(|_| json!({"id": "wrong", "ok": true})), + ) + .await; + let mismatch = GuestClient::new(mismatch_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("mismatched response"); + assert!(matches!(mismatch, GuestError::OutcomeUnknown(_))); + + let missing_ok_socket = temp.path().join("missing-ok.uds"); + spawn_server( + missing_ok_socket.clone(), + Arc::new(|request| json!({"id": request["id"]})), + ) + .await; + let missing_ok = GuestClient::new(missing_ok_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("missing outcome flag"); + assert!(matches!(missing_ok, GuestError::OutcomeUnknown(_))); + + let eof_socket = temp.path().join("eof.uds"); + let eof_listener = UnixListener::bind(&eof_socket).expect("bind"); + tokio::spawn(async move { + let (_stream, _) = accept_request(&eof_listener).await; + }); + let eof = GuestClient::new(eof_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("response EOF"); + assert!(matches!(eof, GuestError::OutcomeUnknown(_))); + + let oversized_socket = temp.path().join("oversized-write.uds"); + spawn_server( + oversized_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "padding": "xxxxxxxx"})), + ) + .await; + let oversized = GuestClient::new(oversized_socket, Duration::from_secs(1), 1024) + .with_response_limit(4) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("oversized response"); + assert!(matches!(oversized, GuestError::OutcomeUnknown(_))); + } + + #[tokio::test] + async fn explicit_mutating_rejection_has_known_outcome() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("rejected.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": false, "err": "write denied"})), + ) + .await; + + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("guest rejection"); + assert!(matches!(error, GuestError::Rejected(message) if message == "write denied")); + } + + #[tokio::test] + async fn successful_responses_require_operation_fields() { + let temp = tempfile::tempdir().expect("temp"); + let exec_socket = temp.path().join("exec.uds"); + spawn_server( + exec_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true})), + ) + .await; + let exec = GuestClient::new(exec_socket, Duration::from_secs(15), 1024) + .exec("true".into(), None, None, 1) + .await + .expect_err("missing rc"); + assert!( + matches!(exec, GuestError::OutcomeUnknown(message) if message.contains("missing rc")) + ); + + let read_socket = temp.path().join("read.uds"); + spawn_server( + read_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true})), + ) + .await; + let read = GuestClient::new(read_socket, Duration::from_secs(1), 1024) + .read_file("/tmp/x".into()) + .await + .expect_err("missing data"); + assert!( + matches!(read, GuestError::Protocol(message) if message.contains("missing data_b64")) + ); + } + + #[tokio::test] + async fn exec_inputs_are_bounded_before_connecting() { + let temp = tempfile::tempdir().expect("temp"); + let client = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(15), + 1024, + ); + let command = "x".repeat(MAX_EXEC_COMMAND_BYTES + 1); + assert!(matches!( + client.exec(command, None, None, 1).await, + Err(GuestError::InvalidArgument(_)) + )); + + let mut env = HashMap::new(); + env.insert("KEY".to_string(), "x".repeat(MAX_EXEC_ENV_BYTES)); + assert!(matches!( + client.exec("true".into(), None, Some(env), 1).await, + Err(GuestError::InvalidArgument(_)) + )); + } + + #[tokio::test] + async fn oversized_response_is_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "data_b64": "AAAAAAAA"})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .with_response_limit(4) + .read_file("/tmp/x".into()) + .await + .expect_err("oversized line"); + assert!(matches!(error, GuestError::ResponseTooLarge { .. })); + } + + #[tokio::test] + async fn line_reader_accepts_the_limit_and_rejects_one_more_byte() { + let (mut exact_reader, mut exact_writer) = tokio::io::duplex(64); + tokio::spawn(async move { + exact_writer + .write_all(b"1234\n") + .await + .expect("write exact"); + }); + assert_eq!( + read_line(&mut exact_reader, 4).await.expect("exact limit"), + b"1234" + ); + + let (mut oversized_reader, mut oversized_writer) = tokio::io::duplex(64); + tokio::spawn(async move { + oversized_writer + .write_all(b"12345\n") + .await + .expect("write oversized"); + }); + assert!(matches!( + read_line(&mut oversized_reader, 4) + .await + .expect_err("one byte over"), + GuestError::ResponseTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn invalid_and_decoded_oversized_base64_are_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let invalid_socket = temp.path().join("invalid.uds"); + spawn_server( + invalid_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "data_b64": "not/base64!"})), + ) + .await; + let invalid = GuestClient::new(invalid_socket, Duration::from_secs(1), 16) + .read_file("/tmp/x".into()) + .await + .expect_err("invalid base64"); + assert!(matches!(invalid, GuestError::Protocol(_))); + + let oversized_socket = temp.path().join("oversized.uds"); + spawn_server( + oversized_socket.clone(), + Arc::new(|request| { + json!({ + "id": request["id"], + "ok": true, + "data_b64": BASE64.encode(b"12345") + }) + }), + ) + .await; + let oversized = GuestClient::new(oversized_socket, Duration::from_secs(1), 4) + .read_file("/tmp/x".into()) + .await + .expect_err("decoded limit"); + assert!(matches!( + oversized, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn oversized_write_is_rejected_before_connecting() { + let temp = tempfile::tempdir().expect("temp"); + let error = GuestClient::new(temp.path().join("missing.uds"), Duration::from_secs(1), 4) + .write_file("/tmp/x".into(), b"12345") + .await + .expect_err("write limit"); + assert!(matches!( + error, + GuestError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn wait_ready_honors_cancellation() { + let temp = tempfile::tempdir().expect("temp"); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_millis(10), + 1024, + ) + .wait_ready(Duration::from_secs(1), &cancellation) + .await + .expect_err("cancelled"); + assert!(matches!(error, GuestError::Cancelled)); + } + + #[tokio::test] + async fn wait_ready_stops_at_its_deadline() { + let temp = tempfile::tempdir().expect("temp"); + let started = Instant::now(); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(1), + 1024, + ) + .wait_ready(Duration::from_millis(60), &CancellationToken::new()) + .await + .expect_err("deadline"); + assert!(matches!(error, GuestError::Timeout(_))); + assert!(started.elapsed() < Duration::from_millis(500)); + } + + #[tokio::test] + async fn wait_ready_retries_after_one_stalled_connection() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let attempts = Arc::new(AtomicUsize::new(0)); + let server_attempts = attempts.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let attempt = server_attempts.fetch_add(1, Ordering::Relaxed); + tokio::spawn(async move { + if attempt == 0 { + tokio::time::sleep(Duration::from_secs(1)).await; + return; + } + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 5000\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request: serde_json::Value = + serde_json::from_slice(&request).expect("json"); + let response = json!({"id": request["id"], "ok": true}); + let mut bytes = serde_json::to_vec(&response).expect("encode"); + bytes.push(b'\n'); + stream.write_all(&bytes).await.expect("write"); + }); + } + }); + + GuestClient::new(socket, Duration::from_secs(5), 1024) + .wait_ready(Duration::from_secs(1), &CancellationToken::new()) + .await + .expect("second readiness attempt"); + assert!(attempts.load(Ordering::Relaxed) >= 2); + } + + #[test] + fn guest_path_validation_does_not_treat_guest_paths_as_host_paths() { + assert!(validate_guest_path("/tmp/../etc/hosts").is_ok()); + assert!(matches!( + validate_guest_path("relative"), + Err(GuestError::InvalidArgument(_)) + )); + } +} diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index 2760d59c19..606ac2c252 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -5,8 +5,10 @@ //! exposed via the HTTP API; this binary only handles daemon lifecycle. mod api; +mod checkpoint_store; mod cli; mod daemon; +mod daemon_socket; mod error; #[cfg(feature = "test-failpoints")] mod failpoint; @@ -14,7 +16,11 @@ mod failpoint; #[path = "failpoint_disabled.rs"] mod failpoint; mod file_provider; +mod guest; mod metrics; +mod request_body; +mod runtime_pool; +mod sandbox; mod spawner; mod state; diff --git a/src/blaze/crates/blazed/src/request_body.rs b/src/blaze/crates/blazed/src/request_body.rs new file mode 100644 index 0000000000..2a35ed8be1 --- /dev/null +++ b/src/blaze/crates/blazed/src/request_body.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Bounded HTTP request-body collection for every daemon API route. + +use http_body_util::BodyExt; +use hyper::Request; +use hyper::body::{Body, Bytes}; +use hyper::header::CONTENT_LENGTH; + +use crate::error::{BlazeDaemonError, Result}; + +/// Collect a request body without buffering more than `limit` bytes. +pub(crate) async fn collect(req: Request, limit: usize) -> Result> +where + B: Body + Unpin, + B::Error: std::fmt::Display, +{ + if let Some(declared) = declared_body_length(&req)? + && declared > limit as u64 + { + return Err(BlazeDaemonError::PayloadTooLarge { + actual: declared, + limit, + }); + } + + let mut body = req.into_body(); + let mut collected = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|error| BlazeDaemonError::RequestBody(error.to_string()))?; + if let Ok(data) = frame.into_data() { + let actual = collected.len().checked_add(data.len()).ok_or( + BlazeDaemonError::PayloadTooLarge { + actual: u64::MAX, + limit, + }, + )?; + if actual > limit { + return Err(BlazeDaemonError::PayloadTooLarge { + actual: actual as u64, + limit, + }); + } + collected.extend_from_slice(&data); + } + } + Ok(collected) +} + +fn declared_body_length(req: &Request) -> Result> { + let mut declared = None; + for value in req.headers().get_all(CONTENT_LENGTH) { + let value = value + .to_str() + .map_err(|_| BlazeDaemonError::BadRequest("invalid Content-Length".into()))?; + for item in value.split(',') { + let length = item + .trim() + .parse::() + .map_err(|_| BlazeDaemonError::BadRequest("invalid Content-Length".into()))?; + match declared { + Some(previous) if previous != length => { + return Err(BlazeDaemonError::BadRequest( + "conflicting Content-Length values".into(), + )); + } + None => declared = Some(length), + _ => {} + } + } + } + Ok(declared) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::fmt; + use std::pin::Pin; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::task::{Context, Poll}; + + use hyper::body::Frame; + use hyper::header::{CONTENT_LENGTH, TRANSFER_ENCODING}; + + use super::*; + + #[derive(Debug)] + struct TestBodyError; + + impl fmt::Display for TestBodyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("test body failed") + } + } + + impl std::error::Error for TestBodyError {} + + struct TestBody { + frames: VecDeque, TestBodyError>>, + polls: Arc, + panic_when_exhausted: bool, + } + + impl TestBody { + fn new( + frames: impl IntoIterator, TestBodyError>>, + polls: Arc, + ) -> Self { + Self { + frames: frames.into_iter().collect(), + polls, + panic_when_exhausted: false, + } + } + + fn panic_when_exhausted(mut self) -> Self { + self.panic_when_exhausted = true; + self + } + } + + impl Body for TestBody { + type Data = Bytes; + type Error = TestBodyError; + + fn poll_frame( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + this.polls.fetch_add(1, Ordering::AcqRel); + match this.frames.pop_front() { + Some(frame) => Poll::Ready(Some(frame)), + None if this.panic_when_exhausted => { + panic!("collector polled after the limit had already been exceeded") + } + None => Poll::Ready(None), + } + } + } + + #[tokio::test] + async fn accepts_body_at_declared_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"ab"))), + Ok(Frame::data(Bytes::from_static(b"cd"))), + ], + polls.clone(), + ); + let request = Request::builder() + .header(CONTENT_LENGTH, "4") + .body(body) + .expect("request"); + + assert_eq!(collect(request, 4).await.expect("body"), b"abcd"); + assert_eq!(polls.load(Ordering::Acquire), 3); + } + + #[tokio::test] + async fn rejects_large_content_length_before_polling_body() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [Ok(Frame::data(Bytes::from_static(b"body")))], + polls.clone(), + ) + .panic_when_exhausted(); + let request = Request::builder() + .header(CONTENT_LENGTH, "5") + .body(body) + .expect("request"); + + let error = collect(request, 4).await.expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(error.status_code(), 413); + assert_eq!(polls.load(Ordering::Acquire), 0); + } + + #[tokio::test] + async fn stops_collecting_chunked_body_at_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"abcd"))), + Ok(Frame::data(Bytes::from_static(b"e"))), + ], + polls.clone(), + ) + .panic_when_exhausted(); + let request = Request::builder() + .header(TRANSFER_ENCODING, "chunked") + .body(body) + .expect("request"); + + let error = collect(request, 4).await.expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(polls.load(Ordering::Acquire), 2); + } + + #[tokio::test] + async fn stops_collecting_undelimited_body_at_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"ab"))), + Ok(Frame::data(Bytes::from_static(b"cde"))), + ], + polls.clone(), + ) + .panic_when_exhausted(); + + let error = collect(Request::new(body), 4) + .await + .expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(polls.load(Ordering::Acquire), 2); + } + + #[tokio::test] + async fn maps_body_read_failures_to_bad_request() { + let body = TestBody::new([Err(TestBodyError)], Arc::new(AtomicUsize::new(0))); + let error = collect(Request::new(body), 4) + .await + .expect_err("body read must fail"); + + assert!(matches!(error, BlazeDaemonError::RequestBody(_))); + assert_eq!(error.status_code(), 400); + } +} diff --git a/src/blaze/crates/blazed/src/runtime_pool.rs b/src/blaze/crates/blazed/src/runtime_pool.rs new file mode 100644 index 0000000000..77aeee3bc4 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Runtime slot ownership, recovery, and capacity management. + +mod pool; +mod recovery; + +#[cfg(test)] +pub(crate) use pool::RuntimePoolStatus; +pub(crate) use pool::{PoolPrototype, RuntimePoolLease, RuntimeWarmPool}; +pub(crate) use recovery::{ + DurableRuntimeOwner, begin_lifecycle_cleanup, reconcile_runtime_slots, + remove_lifecycle_tombstone, runtime_dir, tombstone_lifecycle_slot, +}; diff --git a/src/blaze/crates/blazed/src/runtime_pool/pool.rs b/src/blaze/crates/blazed/src/runtime_pool/pool.rs new file mode 100644 index 0000000000..09553357e6 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool/pool.rs @@ -0,0 +1,2451 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Bounded background construction and ownership transfer for warm runtimes. + +use std::collections::{BTreeSet, VecDeque}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::lifecycle::BackendOwnership; +use blaze_core::policy::{BackendConfigs, VmConfig, WorkloadClass}; +use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; +use blaze_core::{BlazeError, Result}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; +use tokio::task::JoinHandle; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use super::recovery::{ + RuntimeSlotOwnership, RuntimeSlotPhase, finish_ownership_handoff, read_ownership, + remove_pool_tombstone, tombstone_pool_slot, write_ownership, +}; +use crate::guest::{GuestClient, MAX_GUEST_FILE_BYTES}; +use crate::spawner::{DynBackendInstance, SpawnerRegistry}; + +const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +const RETRY_MAX_DELAY: Duration = Duration::from_secs(30); +const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Immutable inputs shared by every slot in one runtime pool generation. +#[derive(Debug, Clone)] +pub(crate) struct PoolPrototype { + pub(crate) image_digest: String, + pub(crate) policy_name: String, + pub(crate) workload_class: WorkloadClass, + pub(crate) templates: Vec, + pub(crate) kernel_hooks: Vec, + pub(crate) binary_path: PathBuf, + pub(crate) runtime_backend: BackendKind, + pub(crate) backend: BackendConfigs, + pub(crate) vm: Option, + pub(crate) warm_ttl: Duration, +} + +impl PoolPrototype { + fn fingerprint(&self) -> Result> { + serde_json::to_vec(&( + &self.image_digest, + &self.policy_name, + self.workload_class, + &self.templates, + &self.kernel_hooks, + &self.binary_path, + self.runtime_backend, + &self.backend, + &self.vm, + self.warm_ttl.as_nanos(), + )) + .map_err(|error| pool_error(format!("serialize runtime pool prototype: {error}"))) + } +} + +/// Resources transferred from the pool into lifecycle ownership. +pub(crate) struct RuntimePoolSlot { + pub(crate) instance_id: Uuid, + pub(crate) storage: StorageSlot, + pub(crate) backend: Option, + pub(crate) run_dir: PathBuf, + pub(crate) runtime_backend: BackendKind, + pub(crate) backend_ownership: BackendOwnership, + ready_at: Instant, +} + +struct UnresolvedHandoff { + slot: RuntimePoolSlot, + token: Uuid, + reason: String, +} + +struct CleanupSlot { + ownership: RuntimeSlotOwnership, + backend: Option, + run_dir: PathBuf, + authority: PoolCleanupAuthority, +} + +impl CleanupSlot { + fn from_runtime(slot: RuntimePoolSlot) -> Self { + Self::from_runtime_with_authority(slot, PoolCleanupAuthority::Ready) + } + + fn from_runtime_with_authority(slot: RuntimePoolSlot, authority: PoolCleanupAuthority) -> Self { + let mut ownership = RuntimeSlotOwnership::new(slot.instance_id, slot.runtime_backend); + ownership.backend_ownership = slot.backend_ownership; + ownership.storage_owned = true; + ownership.phase = RuntimeSlotPhase::Ready; + Self { + ownership, + backend: slot.backend, + run_dir: slot.run_dir, + authority, + } + } +} + +#[derive(Clone, Copy)] +enum PoolCleanupAuthority { + Build, + Ready, + Handoff(Uuid), +} + +#[derive(Default)] +struct PoolState { + ready: VecDeque, + quarantined: VecDeque, + unresolved: VecDeque, + building: BTreeSet, + leased: BTreeSet, + cleanup_pending: usize, + prototype: Option, + prototype_fingerprint: Option>, + generation: u64, + consecutive_build_failures: u32, + consecutive_cleanup_failures: u32, + shutting_down: bool, +} + +impl PoolState { + fn physical_count(&self) -> usize { + self.ready + .len() + .saturating_add(self.building.len()) + .saturating_add(self.leased.len()) + .saturating_add(self.quarantined.len()) + .saturating_add(self.unresolved.len()) + .saturating_add(self.cleanup_pending) + } +} + +/// Bounded worker that retains every incomplete resource for retry. +pub(crate) struct RuntimeWarmPool { + target: usize, + prefork: bool, + rootfs_size: u64, + mem_size: u64, + runtime_root: PathBuf, + storage: Arc, + spawners: Arc, + default_warm_ttl: Duration, + gc_interval: Duration, + state: Mutex, + maintenance: AsyncMutex<()>, + shutdown: AsyncMutex<()>, + cancellation: CancellationToken, + wake: Notify, + worker: Mutex>>, +} + +struct WorkerJoinGuard<'a> { + worker_slot: &'a Mutex>>, + handle: Option>, +} + +impl<'a> WorkerJoinGuard<'a> { + fn new(worker_slot: &'a Mutex>>, handle: Option>) -> Self { + Self { + worker_slot, + handle, + } + } + + fn disarm(&mut self) { + self.handle.take(); + } +} + +impl Drop for WorkerJoinGuard<'_> { + fn drop(&mut self) { + let Some(handle) = self.handle.take() else { + return; + }; + handle.abort(); + let mut worker = match self.worker_slot.lock() { + Ok(worker) => worker, + Err(poisoned) => poisoned.into_inner(), + }; + if worker.is_none() { + *worker = Some(handle); + } else { + tracing::error!( + "runtime pool worker slot was occupied while retaining an aborted worker" + ); + } + } +} + +impl RuntimeWarmPool { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + target: usize, + prefork: bool, + rootfs_size: u64, + mem_size: u64, + runtime_root: PathBuf, + storage: Arc, + spawners: Arc, + default_warm_ttl: Duration, + gc_interval: Duration, + cancellation: CancellationToken, + ) -> Result> { + if target > 0 && !storage.supports_runtime_pool_recovery() { + return Err(pool_error( + "configured storage provider does not expose runtime slot cleanup inventory", + )); + } + Ok(Arc::new(Self { + target, + prefork, + rootfs_size, + mem_size, + runtime_root, + storage, + spawners, + default_warm_ttl, + gc_interval, + state: Mutex::new(PoolState::default()), + maintenance: AsyncMutex::new(()), + shutdown: AsyncMutex::new(()), + cancellation, + wake: Notify::new(), + worker: Mutex::new(None), + })) + } + + /// Fix the first compatible build shape and start maintenance on demand. + pub(crate) fn configure(self: &Arc, prototype: PoolPrototype) -> Result { + if self.target == 0 { + return Ok(false); + } + let fingerprint = prototype.fingerprint()?; + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down { + return Err(pool_error("runtime pool is shutting down")); + } + match state.prototype_fingerprint.as_deref() { + None => { + state.prototype = Some(prototype); + state.prototype_fingerprint = Some(fingerprint); + } + Some(existing) if existing == fingerprint.as_slice() => {} + Some(_) => return Ok(false), + } + drop(state); + self.ensure_worker()?; + self.wake.notify_one(); + Ok(true) + } + + /// Lease one ready slot while a synchronous guard protects cancellation. + pub(crate) async fn acquire(self: &Arc) -> Result> { + loop { + let (slot, warm_ttl) = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + let warm_ttl = state + .prototype + .as_ref() + .map(|prototype| prototype.warm_ttl) + .unwrap_or(self.default_warm_ttl); + let slot = state.ready.pop_front(); + if let Some(slot) = &slot { + state.leased.insert(slot.instance_id); + } + (slot, warm_ttl) + }; + self.wake.notify_one(); + let Some(slot) = slot else { + return Ok(None); + }; + let lease = RuntimePoolLease::new(self.clone(), slot); + if lease.slot()?.ready_at.elapsed() >= warm_ttl { + tracing::info!( + instance = %lease.slot()?.instance_id, + "discarding expired runtime slot before claim" + ); + lease.quarantine(); + continue; + } + let Some(backend) = lease.slot()?.backend.as_ref().cloned() else { + return Ok(Some(lease)); + }; + let live = tokio::time::timeout(GUEST_REQUEST_TIMEOUT, backend.try_wait()).await; + match live { + Ok(Ok(None)) => return Ok(Some(lease)), + Ok(Ok(Some(result))) => tracing::warn!( + instance = %lease.slot()?.instance_id, + exit_code = ?result.exit_code, + signal = ?result.signal, + "discarding exited prefork runtime" + ), + Ok(Err(error)) => tracing::warn!( + instance = %lease.slot()?.instance_id, + %error, + "discarding runtime after liveness check failed" + ), + Err(_) => tracing::warn!( + instance = %lease.slot()?.instance_id, + "discarding runtime after liveness check timed out" + ), + } + lease.quarantine(); + } + } + + pub(crate) fn begin_shutdown(&self) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.shutting_down = true; + state.generation = state.generation.wrapping_add(1); + drop(state); + self.cancellation.cancel(); + self.wake.notify_waiters(); + } + + pub(crate) fn default_warm_ttl(&self) -> Duration { + self.default_warm_ttl + } + + /// Stop maintenance and release pool-owned slots before one deadline. + pub(crate) async fn shutdown_until( + self: &Arc, + deadline: tokio::time::Instant, + ) -> Result<()> { + self.begin_shutdown(); + let _shutdown = tokio::time::timeout_at(deadline, self.shutdown.lock()) + .await + .map_err(|_| { + pool_error("runtime pool shutdown coordination exceeded the shared deadline") + })?; + let worker = self + .worker + .lock() + .map_err(|_| pool_error("runtime pool worker lock poisoned"))? + .take(); + let mut worker = WorkerJoinGuard::new(&self.worker, worker); + let mut errors = Vec::new(); + if let Some(handle) = worker.handle.as_mut() { + match tokio::time::timeout_at(deadline, &mut *handle).await { + Ok(Ok(())) => { + worker.disarm(); + } + Ok(Err(error)) if error.is_cancelled() => { + worker.disarm(); + } + Ok(Err(error)) => { + worker.disarm(); + errors.push(format!( + "runtime pool worker failed while stopping: {error}" + )); + } + Err(_) => { + handle.abort(); + let result = (&mut *handle).await; + worker.disarm(); + if let Err(error) = result + && !error.is_cancelled() + { + errors.push(format!( + "runtime pool worker failed while stopping: {error}" + )); + } + errors.push( + "runtime pool worker exceeded the shared shutdown deadline".to_string(), + ); + } + } + } + + loop { + let (building, leased) = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + (state.building.len(), state.leased.len()) + }; + if building == 0 && leased == 0 { + break; + } + if tokio::time::timeout_at(deadline, self.wake.notified()) + .await + .is_err() + { + if building != 0 { + errors.push(format!( + "{building} runtime pool build(s) exceeded the shared shutdown deadline" + )); + } + if leased != 0 { + errors.push(format!( + "{leased} runtime pool lease(s) exceeded the shared shutdown deadline" + )); + } + break; + } + } + + let unresolved = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state + .unresolved + .iter() + .map(|handoff| { + format!( + "{} (token {}, {})", + handoff.slot.instance_id, handoff.token, handoff.reason + ) + }) + .collect::>() + }; + if !unresolved.is_empty() { + errors.push(format!( + "{} runtime pool owner(s) have unresolved lifecycle publication: {}", + unresolved.len(), + unresolved.join(", ") + )); + } + + let maintenance = match tokio::time::timeout_at(deadline, self.maintenance.lock()).await { + Ok(maintenance) => Some(maintenance), + Err(_) => { + errors.push( + "runtime pool maintenance lock exceeded the shared shutdown deadline" + .to_string(), + ); + None + } + }; + if let Some(_maintenance) = maintenance { + let attempts = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.ready.len().saturating_add(state.quarantined.len()) + }; + for _ in 0..attempts { + let cleanup = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + let cleanup = state + .quarantined + .pop_front() + .or_else(|| state.ready.pop_front().map(CleanupSlot::from_runtime)); + if cleanup.is_some() { + state.cleanup_pending = state.cleanup_pending.saturating_add(1); + } + cleanup + }; + let Some(cleanup) = cleanup else { + break; + }; + let instance_id = cleanup.ownership.instance_id; + let mut cleanup = CleanupGuard::new(self.clone(), cleanup); + match tokio::time::timeout_at(deadline, self.cleanup_slot(cleanup.slot_mut()?)) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + errors.push(format!("{instance_id}: {error}")); + cleanup.retry(); + continue; + } + Err(_) => { + errors.push(format!( + "{instance_id}: cleanup exceeded the shared shutdown deadline" + )); + cleanup.retry(); + break; + } + } + cleanup.complete(); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(pool_error(errors.join("; "))) + } + } + + #[cfg(test)] + pub(crate) fn status(&self) -> RuntimePoolStatus { + match self.state.lock() { + Ok(state) => RuntimePoolStatus::from_state(self.target, &state), + Err(poisoned) => RuntimePoolStatus::from_state(self.target, &poisoned.into_inner()), + } + } + + #[cfg(test)] + pub(crate) fn has_tracked_worker(&self) -> bool { + match self.worker.lock() { + Ok(worker) => worker.is_some(), + Err(poisoned) => poisoned.into_inner().is_some(), + } + } + + fn ensure_worker(self: &Arc) -> Result<()> { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down { + return Err(pool_error("runtime pool is shutting down")); + } + let mut worker = self + .worker + .lock() + .map_err(|_| pool_error("runtime pool worker lock poisoned"))?; + if worker.as_ref().is_some_and(JoinHandle::is_finished) { + worker.take(); + } + if worker.is_none() { + let pool = self.clone(); + *worker = Some(tokio::spawn(async move { + pool.run_worker().await; + })); + } + Ok(()) + } + + async fn run_worker(self: Arc) { + let mut gc = tokio::time::interval(self.gc_interval); + gc.set_missed_tick_behavior(MissedTickBehavior::Skip); + gc.tick().await; + loop { + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = self.wake.notified() => {} + _ = gc.tick() => {} + } + loop { + if self.cancellation.is_cancelled() { + return; + } + match self.maintain_once().await { + Ok(Maintenance::Progress) => continue, + Ok(Maintenance::Idle) => break, + Ok(Maintenance::RetryAfter(delay)) => { + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = tokio::time::sleep(delay) => {} + } + } + Err(error) => { + tracing::warn!(%error, "runtime pool maintenance failed"); + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = tokio::time::sleep(RETRY_BASE_DELAY) => {} + } + } + } + } + } + } + + async fn maintain_once(self: &Arc) -> Result { + let _maintenance = self.maintenance.lock().await; + let action = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down || state.prototype.is_none() { + return Ok(Maintenance::Idle); + } + let warm_ttl = state + .prototype + .as_ref() + .map(|prototype| prototype.warm_ttl) + .unwrap_or(self.default_warm_ttl); + let now = Instant::now(); + let mut retained = VecDeque::new(); + while let Some(slot) = state.ready.pop_front() { + if now.duration_since(slot.ready_at) >= warm_ttl { + state.quarantined.push_back(CleanupSlot::from_runtime(slot)); + } else { + retained.push_back(slot); + } + } + state.ready = retained; + if let Some(cleanup) = state.quarantined.pop_front() { + state.cleanup_pending = state.cleanup_pending.saturating_add(1); + PoolAction::Cleanup(cleanup) + } else if state.physical_count() < self.target { + PoolAction::Build(state.generation) + } else { + PoolAction::Idle + } + }; + + match action { + PoolAction::Idle => Ok(Maintenance::Idle), + PoolAction::Build(generation) => match self.build_slot(generation).await { + Ok(()) => { + if let Ok(mut state) = self.state.lock() { + state.consecutive_build_failures = 0; + } + Ok(Maintenance::Progress) + } + Err(error) => { + let delay = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.consecutive_build_failures = + state.consecutive_build_failures.saturating_add(1); + build_retry_delay(state.consecutive_build_failures) + }; + tracing::warn!( + %error, + retry_delay_ms = delay.as_millis(), + "runtime slot build failed" + ); + Ok(Maintenance::RetryAfter(delay)) + } + }, + PoolAction::Cleanup(cleanup) => { + let mut cleanup = CleanupGuard::new(self.clone(), cleanup); + match self.cleanup_slot(cleanup.slot_mut()?).await { + Ok(()) => { + cleanup.complete(); + Ok(Maintenance::Progress) + } + Err(error) => { + let delay = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.consecutive_cleanup_failures = + state.consecutive_cleanup_failures.saturating_add(1); + build_retry_delay(state.consecutive_cleanup_failures) + }; + cleanup.retry(); + tracing::warn!( + %error, + retry_delay_ms = delay.as_millis(), + "runtime slot cleanup will be retried" + ); + Ok(Maintenance::RetryAfter(delay)) + } + } + } + } + } + + async fn build_slot(self: &Arc, generation: u64) -> Result<()> { + crate::failpoint::storage("pool-build")?; + let prototype = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))? + .prototype + .clone() + .ok_or_else(|| pool_error("runtime pool has no build prototype"))?; + let instance_id = Uuid::new_v4(); + let run_dir = self.runtime_root.join(instance_id.to_string()); + std::fs::create_dir(&run_dir)?; + let ownership = RuntimeSlotOwnership::new(instance_id, prototype.runtime_backend); + let mut build = BuildGuard::new(self.clone(), run_dir, ownership); + std::fs::File::open(&self.runtime_root)?.sync_all()?; + build.persist().await?; + + // Acquire may be cancelled after the provider has created artifacts + // but before it can return a residual slot. Claim cleanup authority + // first; recovery-capable providers make release-by-ID safe when no + // artifact was created. + build.ownership.storage_owned = true; + build.persist().await?; + let acquire_opts = AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: self.rootfs_size, + mem_size: self.mem_size, + }; + let acquire = self.storage.acquire(&acquire_opts); + let storage = match tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error("runtime slot build cancelled during storage acquire")); + } + result = acquire => result, + } { + Ok(storage) => storage, + Err(error) => { + let (source, residual) = error.into_parts(); + if let Some(residual) = residual { + build.storage = Some(residual); + if let Err(persist) = build.persist().await { + return Err(pool_error(format!( + "{source}; retain residual storage journal: {persist}" + ))); + } + } + return Err(source); + } + }; + build.storage = Some(storage); + build.persist().await?; + + if self.prefork { + let spawner = self + .spawners + .get(prototype.runtime_backend) + .ok_or_else(|| { + pool_error(format!( + "no spawner registered for runtime backend {}", + prototype.runtime_backend + )) + })?; + build.ownership.backend_ownership = BackendOwnership::Starting; + build.persist().await?; + tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error( + "runtime slot build cancelled during backend preparation" + )); + } + result = spawner.prepare_spawn(&build.run_dir) => result?, + } + let spawn_request = SpawnRequest { + instance_id, + run_dir: build.run_dir.clone(), + binary_path: prototype.binary_path, + storage: build + .storage + .as_ref() + .ok_or_else(|| pool_error("runtime slot lost storage ownership"))? + .clone(), + backend: prototype.backend, + vm: prototype.vm, + }; + let spawn = tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error( + "runtime slot build cancelled during backend spawn" + )); + } + result = spawner.spawn(spawn_request) => result, + }; + let backend = match spawn { + Ok(backend) => backend, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + build.backend = Some(owner); + build.ownership.backend_ownership = BackendOwnership::Running; + } else { + build.ownership.backend_ownership = BackendOwnership::Stopped; + } + if let Err(persist) = build.persist().await { + return Err(pool_error(format!( + "{source}; retain failed backend journal: {persist}" + ))); + } + return Err(source); + } + }; + build.ownership.backend_ownership = BackendOwnership::Running; + build.backend = Some(backend.clone()); + build.persist().await?; + if backend.backend() != prototype.runtime_backend { + return Err(pool_error(format!( + "runtime slot requested {} but spawner returned {}", + prototype.runtime_backend, + backend.backend() + ))); + } + let socket = backend.guest_socket_path(); + if !socket.as_os_str().is_empty() { + GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + ) + .wait_ready(GUEST_REQUEST_TIMEOUT, &self.cancellation) + .await + .map_err(|error| pool_error(format!("prefork guest readiness failed: {error}")))?; + } + } + + build.ownership.phase = RuntimeSlotPhase::Ready; + build.persist().await?; + build.publish(generation) + } + + async fn cleanup_slot(&self, slot: &mut CleanupSlot) -> Result<()> { + let persisted = match read_ownership(&slot.run_dir, slot.ownership.instance_id).await { + Ok(persisted) => Some(persisted), + Err(error) => { + let ownership_path = slot.run_dir.join("ownership.json"); + let journal_missing = matches!( + tokio::fs::symlink_metadata(&ownership_path).await, + Err(io_error) if io_error.kind() == std::io::ErrorKind::NotFound + ); + if matches!(slot.authority, PoolCleanupAuthority::Build) && journal_missing { + None + } else { + return Err(pool_error(error)); + } + } + }; + let cleanup_committed = persisted + .as_ref() + .is_some_and(|ownership| ownership.phase == RuntimeSlotPhase::PoolCleanup); + if let Some(persisted) = persisted { + if persisted.backend != slot.ownership.backend { + return Err(pool_error(format!( + "runtime slot {} changed backend from {} to {} before pool cleanup", + slot.ownership.instance_id, slot.ownership.backend, persisted.backend + ))); + } + let phase_allowed = match (slot.authority, persisted.phase) { + (PoolCleanupAuthority::Build, RuntimeSlotPhase::Building) + | (PoolCleanupAuthority::Build, RuntimeSlotPhase::Ready) + | (PoolCleanupAuthority::Ready, RuntimeSlotPhase::Ready) + | (_, RuntimeSlotPhase::PoolCleanup) => true, + (PoolCleanupAuthority::Handoff(expected), RuntimeSlotPhase::Handoff { token }) => { + expected == token + } + _ => false, + }; + if !phase_allowed { + return Err(pool_error(format!( + "refusing pool cleanup for runtime slot {} with persisted phase {:?}", + slot.ownership.instance_id, persisted.phase + ))); + } + slot.ownership.backend_ownership = strongest_backend_ownership( + slot.ownership.backend_ownership, + persisted.backend_ownership, + ); + slot.ownership.storage_owned |= persisted.storage_owned; + } + slot.ownership.phase = RuntimeSlotPhase::PoolCleanup; + if !cleanup_committed { + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + + if let Some(backend) = slot.backend.as_ref() { + backend.kill().await?; + slot.backend = None; + slot.ownership.backend_ownership = BackendOwnership::Stopped; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } else if matches!( + slot.ownership.backend_ownership, + BackendOwnership::Unknown | BackendOwnership::Starting | BackendOwnership::Running + ) { + let spawner = self.spawners.get(slot.ownership.backend).ok_or_else(|| { + pool_error(format!( + "no cleanup spawner registered for {}", + slot.ownership.backend + )) + })?; + spawner + .cleanup_orphan(slot.ownership.instance_id, &slot.run_dir) + .await?; + slot.ownership.backend_ownership = BackendOwnership::Stopped; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + + if slot.ownership.storage_owned { + self.storage + .release_by_id(&slot.ownership.instance_id.to_string()) + .await?; + slot.ownership.storage_owned = false; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + tombstone_pool_slot(&self.runtime_root, slot.ownership.instance_id) + .await + .map_err(pool_error)?; + remove_pool_tombstone(&self.runtime_root, slot.ownership.instance_id) + .await + .map_err(pool_error) + } + + fn abandon_lease(&self, slot: RuntimePoolSlot, authority: PoolCleanupAuthority) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&slot.instance_id); + state + .quarantined + .push_back(CleanupSlot::from_runtime_with_authority(slot, authority)); + drop(state); + self.wake.notify_one(); + } + + fn retain_unresolved(&self, handoff: UnresolvedHandoff) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&handoff.slot.instance_id); + state.unresolved.push_back(handoff); + drop(state); + self.wake.notify_one(); + } + + fn complete_lease(&self, instance_id: Uuid) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&instance_id); + drop(state); + self.wake.notify_one(); + } +} + +/// Claim guard whose drop path never loses pool or lifecycle ownership. +pub(crate) struct RuntimePoolLease { + pool: Arc, + slot: Option, + owner: LeaseOwner, + cleanup_authority: PoolCleanupAuthority, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LeaseOwner { + Pool, + Lifecycle, +} + +impl RuntimePoolLease { + fn new(pool: Arc, slot: RuntimePoolSlot) -> Self { + Self { + pool, + slot: Some(slot), + owner: LeaseOwner::Pool, + cleanup_authority: PoolCleanupAuthority::Ready, + } + } + + pub(crate) fn slot(&self) -> Result<&RuntimePoolSlot> { + self.slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot")) + } + + pub(crate) async fn begin_handoff(&mut self, token: Uuid) -> Result<()> { + let slot = self + .slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + let mut ownership = read_ownership(&slot.run_dir, slot.instance_id) + .await + .map_err(pool_error)?; + if ownership.phase != RuntimeSlotPhase::Ready { + return Err(pool_error(format!( + "runtime slot {} is not ready for handoff", + slot.instance_id + ))); + } + ownership.phase = RuntimeSlotPhase::Handoff { token }; + match write_ownership(&slot.run_dir, &ownership).await { + Ok(()) => { + self.cleanup_authority = PoolCleanupAuthority::Handoff(token); + Ok(()) + } + Err(error) => { + if read_ownership(&slot.run_dir, slot.instance_id) + .await + .is_ok_and(|persisted| persisted == ownership) + { + self.cleanup_authority = PoolCleanupAuthority::Handoff(token); + } + Err(pool_error(error)) + } + } + } + + /// Switch the synchronous cancellation fallback after lifecycle is durable. + pub(crate) fn transfer_to_lifecycle(&mut self) { + self.owner = LeaseOwner::Lifecycle; + } + + pub(crate) async fn finish_handoff(&mut self, token: Uuid) -> Result<()> { + let slot = self + .slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + finish_ownership_handoff( + &slot.run_dir, + slot.instance_id, + slot.runtime_backend, + slot.backend_ownership, + token, + ) + .await + .map_err(pool_error) + } + + pub(crate) fn into_slot(mut self) -> Result { + let slot = self + .slot + .take() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + self.pool.complete_lease(slot.instance_id); + Ok(slot) + } + + /// Keep an ambiguous handoff visible and counted without choosing a + /// cleanup owner in the current process. + pub(crate) fn retain_unresolved(mut self, token: Uuid, reason: String) -> Result<()> { + if !matches!( + self.cleanup_authority, + PoolCleanupAuthority::Handoff(expected) if expected == token + ) { + return Err(pool_error( + "runtime pool lease has no matching handoff authority", + )); + } + let slot = self + .slot + .take() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + self.pool.retain_unresolved(UnresolvedHandoff { + slot, + token, + reason, + }); + Ok(()) + } + + fn quarantine(mut self) { + if let Some(slot) = self.slot.take() { + self.pool.abandon_lease(slot, self.cleanup_authority); + } + } +} + +impl Drop for RuntimePoolLease { + fn drop(&mut self) { + let Some(slot) = self.slot.take() else { + return; + }; + match self.owner { + LeaseOwner::Pool => self.pool.abandon_lease(slot, self.cleanup_authority), + LeaseOwner::Lifecycle => self.pool.complete_lease(slot.instance_id), + } + } +} + +struct BuildGuard { + pool: Arc, + ownership: RuntimeSlotOwnership, + storage: Option, + backend: Option, + run_dir: PathBuf, + armed: bool, +} + +impl BuildGuard { + fn new(pool: Arc, run_dir: PathBuf, ownership: RuntimeSlotOwnership) -> Self { + let mut state = match pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.building.insert(ownership.instance_id); + drop(state); + Self { + pool, + ownership, + storage: None, + backend: None, + run_dir, + armed: true, + } + } + + async fn persist(&self) -> Result<()> { + write_ownership(&self.run_dir, &self.ownership) + .await + .map_err(pool_error) + } + + fn publish(mut self, generation: u64) -> Result<()> { + let storage = self + .storage + .take() + .ok_or_else(|| pool_error("completed runtime slot has no storage owner"))?; + let slot = RuntimePoolSlot { + instance_id: self.ownership.instance_id, + storage, + backend: self.backend.take(), + run_dir: self.run_dir.clone(), + runtime_backend: self.ownership.backend, + backend_ownership: self.ownership.backend_ownership, + ready_at: Instant::now(), + }; + let mut state = self + .pool + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.building.remove(&slot.instance_id); + if !state.shutting_down + && state.generation == generation + && state.physical_count() < self.pool.target + { + state.ready.push_back(slot); + } else { + state.quarantined.push_back(CleanupSlot::from_runtime(slot)); + } + self.armed = false; + drop(state); + self.pool.wake.notify_one(); + Ok(()) + } +} + +impl Drop for BuildGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + let cleanup = CleanupSlot { + ownership: RuntimeSlotOwnership { + version: self.ownership.version, + instance_id: self.ownership.instance_id, + backend: self.ownership.backend, + backend_ownership: self.ownership.backend_ownership, + storage_owned: self.ownership.storage_owned, + phase: self.ownership.phase, + }, + backend: self.backend.take(), + run_dir: self.run_dir.clone(), + authority: PoolCleanupAuthority::Build, + }; + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.building.remove(&cleanup.ownership.instance_id); + state.quarantined.push_back(cleanup); + drop(state); + self.pool.wake.notify_one(); + } +} + +struct CleanupGuard { + pool: Arc, + slot: Option, +} + +impl CleanupGuard { + fn new(pool: Arc, slot: CleanupSlot) -> Self { + Self { + pool, + slot: Some(slot), + } + } + + fn slot_mut(&mut self) -> Result<&mut CleanupSlot> { + self.slot + .as_mut() + .ok_or_else(|| pool_error("runtime pool cleanup guard lost its slot")) + } + + fn complete(mut self) { + self.slot.take(); + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.cleanup_pending = state.cleanup_pending.saturating_sub(1); + state.consecutive_cleanup_failures = 0; + drop(state); + self.pool.wake.notify_one(); + } + + fn retry(mut self) { + self.retain(); + } + + fn retain(&mut self) { + let Some(slot) = self.slot.take() else { + return; + }; + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.cleanup_pending = state.cleanup_pending.saturating_sub(1); + state.quarantined.push_back(slot); + drop(state); + self.pool.wake.notify_one(); + } +} + +impl Drop for CleanupGuard { + fn drop(&mut self) { + self.retain(); + } +} + +enum PoolAction { + Build(u64), + Cleanup(CleanupSlot), + Idle, +} + +enum Maintenance { + Progress, + Idle, + RetryAfter(Duration), +} + +fn build_retry_delay(consecutive_failures: u32) -> Duration { + let exponent = consecutive_failures.saturating_sub(1).min(9); + RETRY_BASE_DELAY + .saturating_mul(1_u32 << exponent) + .min(RETRY_MAX_DELAY) +} + +fn strongest_backend_ownership( + left: BackendOwnership, + right: BackendOwnership, +) -> BackendOwnership { + use BackendOwnership::{NotStarted, Running, Starting, Stopped, Unknown}; + + if matches!(left, Unknown) || matches!(right, Unknown) { + Unknown + } else if matches!(left, Running) || matches!(right, Running) { + Running + } else if matches!(left, Starting) || matches!(right, Starting) { + Starting + } else if matches!(left, Stopped) || matches!(right, Stopped) { + Stopped + } else { + NotStarted + } +} + +fn pool_error(message: impl Into) -> BlazeError { + BlazeError::StorageError { + msg: message.into(), + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RuntimePoolStatus { + pub(crate) ready: usize, + pub(crate) building: usize, + pub(crate) leased: usize, + pub(crate) quarantined: usize, + pub(crate) unresolved: usize, + pub(crate) cleanup_pending: usize, + pub(crate) capacity: usize, + pub(crate) deficit: usize, +} + +#[cfg(test)] +impl RuntimePoolStatus { + fn from_state(capacity: usize, state: &PoolState) -> Self { + Self { + ready: state.ready.len(), + building: state.building.len(), + leased: state.leased.len(), + quarantined: state.quarantined.len(), + unresolved: state.unresolved.len(), + cleanup_pending: state.cleanup_pending, + capacity, + deficit: capacity.saturating_sub(state.physical_count()), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::Path; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use async_trait::async_trait; + use blaze_core::storage::{PoolStatus, StorageAcquireError}; + + use crate::spawner::{ + BackendInstance, BackendSpawner, SpawnFailure, SpawnResult, SpawnerRegistry, + }; + + use super::*; + + struct RecordingStorage { + root: PathBuf, + acquire_count: AtomicUsize, + release_started: AtomicUsize, + release_completed: AtomicUsize, + release_delay: Duration, + fail_with_residual_once: AtomicBool, + pending_after_ownership_once: AtomicBool, + owned: Mutex>, + } + + struct RetainedBackend { + kill_count: Arc, + } + + #[async_trait] + impl BackendInstance for RetainedBackend { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> Result> { + Ok(None) + } + + async fn kill(&self) -> Result<()> { + self.kill_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + struct ResidualSpawner { + kill_count: Arc, + } + + #[async_trait] + impl BackendSpawner for ResidualSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::with_owner( + pool_error("test spawn retained backend owner"), + Arc::new(RetainedBackend { + kill_count: self.kill_count.clone(), + }), + )) + } + + async fn probe(&self, _binary_path: &Path) -> Result { + Ok(true) + } + + async fn cleanup_orphan(&self, _instance_id: Uuid, _run_dir: &Path) -> Result<()> { + Err(pool_error( + "test retained backend must be cleaned through its handle", + )) + } + } + + impl RecordingStorage { + fn new(root: PathBuf) -> Self { + Self { + root, + acquire_count: AtomicUsize::new(0), + release_started: AtomicUsize::new(0), + release_completed: AtomicUsize::new(0), + release_delay: Duration::ZERO, + fail_with_residual_once: AtomicBool::new(false), + pending_after_ownership_once: AtomicBool::new(false), + owned: Mutex::new(BTreeSet::new()), + } + } + + fn with_release_delay(root: PathBuf, release_delay: Duration) -> Self { + Self { + release_delay, + ..Self::new(root) + } + } + + fn with_residual_failure(root: PathBuf) -> Self { + Self { + fail_with_residual_once: AtomicBool::new(true), + ..Self::new(root) + } + } + + fn with_pending_acquire(root: PathBuf) -> Self { + Self { + pending_after_ownership_once: AtomicBool::new(true), + ..Self::new(root) + } + } + + fn slot(&self, instance_id: &str) -> StorageSlot { + let instance_dir = self.root.join(instance_id); + StorageSlot { + id: instance_id.to_string(), + rootfs_path: instance_dir.join("rootfs"), + mem_path: instance_dir.join("memory"), + mem_diff_path: instance_dir.join("memory.diff"), + rootfs_diff_path: instance_dir.join("rootfs.diff"), + instance_dir, + } + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> Result { + Ok(true) + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.acquire_count.fetch_add(1, Ordering::SeqCst); + let slot = self.slot(&opts.instance_id); + tokio::fs::create_dir_all(&slot.instance_dir) + .await + .map_err(BlazeError::from)?; + self.owned + .lock() + .map_err(|_| StorageAcquireError::clean(pool_error("test storage lock poisoned")))? + .insert(opts.instance_id.clone()); + if self + .pending_after_ownership_once + .swap(false, Ordering::SeqCst) + { + std::future::pending::<()>().await; + } + if self.fail_with_residual_once.swap(false, Ordering::SeqCst) { + return Err(StorageAcquireError::with_residual( + pool_error("test acquire retained residual storage"), + slot, + )); + } + Ok(slot) + } + + async fn release(&self, slot: StorageSlot) -> Result<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> Result<()> { + self.release_started.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(self.release_delay).await; + self.owned + .lock() + .map_err(|_| pool_error("test storage lock poisoned"))? + .remove(instance_id); + self.release_completed.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> Result> { + Ok(self + .owned + .lock() + .map_err(|_| pool_error("test storage lock poisoned"))? + .iter() + .cloned() + .collect()) + } + + async fn reconstruct(&self, instance_id: &str) -> Result { + Ok(self.slot(instance_id)) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> Result<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> Result { + Ok(0) + } + } + + fn prototype(warm_ttl: Duration) -> PoolPrototype { + PoolPrototype { + image_digest: "sha256:test".to_string(), + policy_name: "test-policy".to_string(), + workload_class: WorkloadClass::AgentTool, + templates: Vec::new(), + kernel_hooks: Vec::new(), + binary_path: PathBuf::from("/unused"), + runtime_backend: BackendKind::Mock, + backend: BackendConfigs::default(), + vm: None, + warm_ttl, + } + } + + fn make_pool( + target: usize, + runtime_root: &Path, + storage: Arc, + cancellation: CancellationToken, + ) -> Arc { + std::fs::create_dir_all(runtime_root).expect("runtime root"); + RuntimeWarmPool::new( + target, + false, + 1024, + 1024, + runtime_root.to_path_buf(), + storage, + Arc::new(SpawnerRegistry::new()), + Duration::from_secs(60), + Duration::from_secs(3600), + cancellation, + ) + .expect("runtime pool") + } + + fn slot(storage: &RecordingStorage, runtime_root: &Path, age: Duration) -> RuntimePoolSlot { + let instance_id = Uuid::new_v4(); + RuntimePoolSlot { + instance_id, + storage: storage.slot(&instance_id.to_string()), + backend: None, + run_dir: runtime_root.join(instance_id.to_string()), + runtime_backend: BackendKind::Mock, + backend_ownership: BackendOwnership::NotStarted, + ready_at: Instant::now() + .checked_sub(age) + .expect("slot age must fit in test clock"), + } + } + + async fn persist_ready_slot(slot: &RuntimePoolSlot) { + std::fs::create_dir_all(&slot.run_dir).expect("slot run directory"); + let mut ownership = RuntimeSlotOwnership::new(slot.instance_id, slot.runtime_backend); + ownership.storage_owned = true; + ownership.backend_ownership = slot.backend_ownership; + ownership.phase = RuntimeSlotPhase::Ready; + write_ownership(&slot.run_dir, &ownership) + .await + .expect("ready ownership"); + } + + async fn wait_for_ready(pool: &RuntimeWarmPool) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if pool.status().ready != 0 { + return; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("runtime slot becomes ready"); + } + + #[tokio::test] + async fn zero_capacity_never_starts_worker_or_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 0, + &temp.path().join("runtime"), + storage.clone(), + CancellationToken::new(), + ); + + assert!( + !pool + .configure(prototype(Duration::from_secs(60))) + .expect("configure disabled pool") + ); + tokio::task::yield_now().await; + + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!( + pool.status(), + RuntimePoolStatus { + ready: 0, + building: 0, + leased: 0, + quarantined: 0, + unresolved: 0, + cleanup_pending: 0, + capacity: 0, + deficit: 0, + } + ); + } + + #[tokio::test] + async fn non_prefork_build_persists_ready_slot_before_claim() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage.clone(), + CancellationToken::new(), + ); + + assert!( + pool.configure(prototype(Duration::from_secs(60))) + .expect("configure pool") + ); + wait_for_ready(&pool).await; + + let (instance_id, run_dir) = { + let state = pool.state.lock().expect("pool state"); + let slot = state.ready.front().expect("ready slot"); + (slot.instance_id, slot.run_dir.clone()) + }; + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("read ready ownership"); + assert_eq!(ownership.phase, RuntimeSlotPhase::Ready); + assert!(ownership.storage_owned); + assert_eq!(ownership.backend_ownership, BackendOwnership::NotStarted); + + let lease = pool + .acquire() + .await + .expect("claim ready slot") + .expect("available ready slot"); + assert_eq!(lease.slot().expect("leased slot").instance_id, instance_id); + assert_eq!(pool.status().leased, 1); + + pool.begin_shutdown(); + drop(lease); + pool.shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect("shutdown pool"); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn cleanup_does_not_overwrite_a_transferred_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + let owner_token = Uuid::new_v4(); + let mut transferred = read_ownership(&runtime.run_dir, runtime.instance_id) + .await + .expect("ready ownership"); + transferred.phase = RuntimeSlotPhase::LifecycleOwned { token: owner_token }; + write_ownership(&runtime.run_dir, &transferred) + .await + .expect("transferred ownership"); + let mut cleanup = CleanupSlot::from_runtime(runtime); + + let error = pool + .cleanup_slot(&mut cleanup) + .await + .expect_err("stale pool owner must not clean lifecycle resources"); + + assert!(error.to_string().contains("refusing pool cleanup")); + assert_eq!( + read_ownership(&cleanup.run_dir, cleanup.ownership.instance_id) + .await + .expect("ownership remains") + .phase, + RuntimeSlotPhase::LifecycleOwned { token: owner_token } + ); + assert_eq!(storage.release_started.load(Ordering::SeqCst), 0); + assert!( + storage + .owned + .lock() + .expect("owned storage") + .contains(&cleanup.ownership.instance_id.to_string()) + ); + } + + #[tokio::test] + async fn matching_handoff_can_return_to_pool_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + let instance_id = runtime.instance_id; + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(instance_id.to_string()); + pool.state + .lock() + .expect("pool state") + .ready + .push_back(runtime); + let mut lease = pool + .acquire() + .await + .expect("acquire slot") + .expect("ready slot"); + let owner_token = Uuid::new_v4(); + lease + .begin_handoff(owner_token) + .await + .expect("begin handoff"); + + drop(lease); + let mut cleanup = pool + .state + .lock() + .expect("pool state") + .quarantined + .pop_front() + .expect("abandoned handoff"); + pool.cleanup_slot(&mut cleanup) + .await + .expect("matching handoff cleanup"); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!( + !storage + .owned + .lock() + .expect("owned storage") + .contains(&instance_id.to_string()) + ); + assert!(!runtime_root.join(instance_id.to_string()).exists()); + } + + #[tokio::test] + async fn cleanup_retry_recommits_intent_before_release() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + let mut cleanup = CleanupSlot::from_runtime(runtime); + cleanup.ownership.phase = RuntimeSlotPhase::PoolCleanup; + let instance_id = cleanup.ownership.instance_id; + let run_dir = cleanup.run_dir.clone(); + + let cleanup_pool = pool.clone(); + let cleanup_task = + tokio::spawn(async move { cleanup_pool.cleanup_slot(&mut cleanup).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("release starts after cleanup intent is durable"); + cleanup_task.abort(); + assert!( + cleanup_task + .await + .expect_err("cleanup is cancelled during release") + .is_cancelled() + ); + + assert_eq!(storage.release_started.load(Ordering::SeqCst), 1); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("cleanup journal") + .phase, + RuntimeSlotPhase::PoolCleanup + ); + } + + #[tokio::test] + async fn shutdown_reports_a_panicked_worker() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage, CancellationToken::new()); + *pool.worker.lock().expect("worker lock") = Some(tokio::spawn(async { + panic!("injected runtime pool worker panic"); + })); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect_err("worker panic must fail shutdown"); + + assert!( + error + .to_string() + .contains("runtime pool worker failed while stopping") + ); + } + + #[tokio::test] + async fn every_owned_state_consumes_physical_capacity() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(5, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let quarantined = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&quarantined).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(quarantined.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state.ready.push_back(ready); + state.building.insert(Uuid::new_v4()); + state.leased.insert(Uuid::new_v4()); + state + .quarantined + .push_back(CleanupSlot::from_runtime(quarantined)); + state.cleanup_pending = 1; + } + + let status = pool.status(); + assert_eq!(status.ready, 1); + assert_eq!(status.building, 1); + assert_eq!(status.leased, 1); + assert_eq!(status.quarantined, 1); + assert_eq!(status.cleanup_pending, 1); + assert_eq!(status.capacity, 5); + assert_eq!(status.deficit, 0); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + assert!(matches!( + pool.maintain_once().await.expect("maintain full pool"), + Maintenance::Progress + )); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn worker_start_rechecks_shutdown_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage, + CancellationToken::new(), + ); + + pool.begin_shutdown(); + let error = pool + .ensure_worker() + .expect_err("worker must not start after shutdown wins the race"); + + assert!(error.to_string().contains("shutting down")); + assert!(!pool.has_tracked_worker()); + } + + #[tokio::test] + async fn expired_shutdown_wait_still_stops_new_work() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage, + CancellationToken::new(), + ); + let _held_shutdown = pool.shutdown.lock().await; + + let error = pool + .shutdown_until(Instant::now()) + .await + .expect_err("expired coordination wait must fail"); + + assert!(error.to_string().contains("coordination")); + assert!(pool.cancellation.is_cancelled()); + assert!(pool.state.lock().expect("pool state").shutting_down); + assert!( + pool.ensure_worker().is_err(), + "expired shutdown still prevents later worker creation" + ); + } + + #[tokio::test] + async fn expired_slot_is_quarantined_instead_of_claimed() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + { + let mut state = pool.state.lock().expect("pool state"); + state + .ready + .push_back(slot(&storage, &runtime_root, Duration::from_secs(61))); + } + + assert!(pool.acquire().await.expect("claim expired slot").is_none()); + let status = pool.status(); + assert_eq!(status.ready, 0); + assert_eq!(status.leased, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + } + + #[tokio::test] + async fn residual_acquire_failure_remains_owned_until_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_residual_failure( + temp.path().join("storage"), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + pool.state.lock().expect("pool state").prototype = Some(prototype(Duration::from_secs(60))); + + pool.build_slot(0) + .await + .expect_err("acquire reports residual owner"); + + let status = pool.status(); + assert_eq!(status.building, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + assert_eq!(storage.owned.lock().expect("owned storage").len(), 1); + + assert!(matches!( + pool.maintain_once().await.expect("cleanup residual"), + Maintenance::Progress + )); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().quarantined, 0); + } + + #[tokio::test] + async fn shutdown_cleans_storage_from_a_cancelled_acquire() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_pending_acquire( + temp.path().join("storage"), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + pool.configure(prototype(Duration::from_secs(60))) + .expect("configure pool"); + let instance_id = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Some(instance_id) = storage + .owned + .lock() + .expect("owned storage") + .iter() + .next() + .cloned() + { + return Uuid::parse_str(&instance_id).expect("owned UUID"); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("provider creates an owner before acquire returns"); + let ownership = read_ownership(&runtime_root.join(instance_id.to_string()), instance_id) + .await + .expect("conservative storage journal"); + assert!(ownership.storage_owned); + assert_eq!(ownership.phase, RuntimeSlotPhase::Building); + + pool.shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect("shutdown cancels acquire and cleans the owner"); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().building, 0); + assert_eq!(pool.status().quarantined, 0); + assert!(!runtime_root.join(instance_id.to_string()).exists()); + } + + #[tokio::test] + async fn residual_prefork_failure_remains_owned_until_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let kill_count = Arc::new(AtomicUsize::new(0)); + let mut registry = SpawnerRegistry::new(); + registry.insert( + BackendKind::Mock, + Arc::new(ResidualSpawner { + kill_count: kill_count.clone(), + }), + ); + let pool = RuntimeWarmPool::new( + 1, + true, + 1024, + 1024, + runtime_root, + storage.clone(), + Arc::new(registry), + Duration::from_secs(60), + Duration::from_secs(3600), + CancellationToken::new(), + ) + .expect("runtime pool"); + pool.state.lock().expect("pool state").prototype = Some(prototype(Duration::from_secs(60))); + + pool.build_slot(0) + .await + .expect_err("prefork spawn reports residual owner"); + + let status = pool.status(); + assert_eq!(status.building, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + assert_eq!(kill_count.load(Ordering::SeqCst), 0); + + assert!(matches!( + pool.maintain_once().await.expect("cleanup residual"), + Maintenance::Progress + )); + assert_eq!(kill_count.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().quarantined, 0); + } + + #[tokio::test] + async fn unresolved_handoff_counts_capacity_and_blocks_shutdown() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let instance_id = ready.instance_id; + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state.ready.push_back(ready); + } + + let mut lease = pool + .acquire() + .await + .expect("acquire slot") + .expect("ready slot"); + let token = Uuid::new_v4(); + lease.begin_handoff(token).await.expect("begin handoff"); + lease + .retain_unresolved(token, "test publication is ambiguous".to_string()) + .expect("retain ambiguous handoff"); + + let status = pool.status(); + assert_eq!(status.leased, 0); + assert_eq!(status.unresolved, 1); + assert_eq!(status.deficit, 0); + assert!(matches!( + pool.maintain_once().await.expect("maintain bounded pool"), + Maintenance::Idle + )); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + let ownership = read_ownership(&runtime_root.join(instance_id.to_string()), instance_id) + .await + .expect("retained handoff journal"); + assert_eq!(ownership.phase, RuntimeSlotPhase::Handoff { token }); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect_err("unresolved owner must be reported"); + assert!(error.to_string().contains(&instance_id.to_string())); + assert!( + error + .to_string() + .contains("unresolved lifecycle publication") + ); + assert!(error.to_string().contains("test publication is ambiguous")); + assert_eq!(pool.status().unresolved, 1); + assert!( + storage + .owned + .lock() + .expect("owned storage") + .contains(&instance_id.to_string()) + ); + } + + #[tokio::test] + async fn cancelled_shutdown_retains_active_and_unstarted_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_millis(500), + )); + let pool = make_pool(2, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let quarantined = slot(&storage, &runtime_root, Duration::ZERO); + let expected = BTreeSet::from([ready.instance_id, quarantined.instance_id]); + for runtime in [&ready, &quarantined] { + persist_ready_slot(runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + } + { + let mut state = pool.state.lock().expect("pool state"); + state.ready.push_back(ready); + state + .quarantined + .push_back(CleanupSlot::from_runtime(quarantined)); + } + + let shutdown_pool = pool.clone(); + let shutdown = tokio::spawn(async move { + shutdown_pool + .shutdown_until(Instant::now() + Duration::from_secs(30)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown starts one cleanup"); + + shutdown.abort(); + assert!( + shutdown + .await + .expect_err("shutdown task is cancelled") + .is_cancelled() + ); + + let status = pool.status(); + assert_eq!(status.ready, 1); + assert_eq!(status.quarantined, 1); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.deficit, 0); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert_eq!(storage.owned.lock().expect("owned storage").len(), 2); + let retained = { + let state = pool.state.lock().expect("pool state"); + state + .ready + .iter() + .map(|slot| slot.instance_id) + .chain( + state + .quarantined + .iter() + .map(|slot| slot.ownership.instance_id), + ) + .collect::>() + }; + assert_eq!(retained, expected); + + pool.shutdown_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry retained shutdown cleanup"); + let status = pool.status(); + assert_eq!(status.ready, 0); + assert_eq!(status.quarantined, 0); + assert_eq!(status.cleanup_pending, 0); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_joins_worker_and_shares_one_deadline() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_millis(40), + )); + let cancellation = CancellationToken::new(); + let pool = make_pool(2, &runtime_root, storage.clone(), cancellation.clone()); + for _ in 0..2 { + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + pool.state + .lock() + .expect("pool state") + .ready + .push_back(ready); + } + + let worker_joined = Arc::new(AtomicBool::new(false)); + let joined = worker_joined.clone(); + *pool.worker.lock().expect("worker lock") = Some(tokio::spawn(async move { + cancellation.cancelled().await; + joined.store(true, Ordering::SeqCst); + })); + + let start = Instant::now(); + let deadline = start + Duration::from_millis(60); + let error = pool + .shutdown_until(deadline) + .await + .expect_err("second cleanup must share the first deadline"); + + assert!(error.to_string().contains("shared shutdown deadline")); + assert_eq!(Instant::now(), deadline); + assert!(worker_joined.load(Ordering::SeqCst)); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!(storage.release_started.load(Ordering::SeqCst), 2); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert_eq!(pool.status().quarantined, 1); + } + + #[tokio::test] + async fn aborted_worker_retains_its_active_cleanup_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_millis(10)) + .await + .expect_err("active cleanup exceeds shutdown deadline"); + + assert!(error.to_string().contains("shared shutdown deadline")); + let status = pool.status(); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + } + + #[tokio::test] + async fn cancelled_shutdown_retains_worker_for_a_joining_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let shutdown_pool = pool.clone(); + let shutdown = tokio::spawn(async move { + shutdown_pool + .shutdown_until(Instant::now() + Duration::from_secs(10)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while pool.worker.lock().expect("worker lock").is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown takes worker handle"); + shutdown.abort(); + assert!( + shutdown + .await + .expect_err("outer shutdown is cancelled") + .is_cancelled() + ); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert!( + pool.worker.lock().expect("worker lock").is_some(), + "cancelled shutdown must retain the aborted worker handle" + ); + + pool.shutdown_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry joins the worker and releases the retained owner"); + let status = pool.status(); + assert_eq!(status.quarantined, 0); + assert_eq!(status.cleanup_pending, 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } + + #[tokio::test] + async fn concurrent_shutdown_waiter_joins_a_worker_retained_by_cancellation() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let first_pool = pool.clone(); + let first = tokio::spawn(async move { + first_pool + .shutdown_until(Instant::now() + Duration::from_secs(10)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while pool.has_tracked_worker() { + tokio::task::yield_now().await; + } + }) + .await + .expect("first shutdown takes worker handle"); + let second_pool = pool.clone(); + let second = tokio::spawn(async move { + second_pool + .shutdown_until(Instant::now() + Duration::from_secs(3)) + .await + }); + tokio::task::yield_now().await; + + first.abort(); + assert!( + first + .await + .expect_err("first shutdown is cancelled") + .is_cancelled() + ); + tokio::time::timeout(Duration::from_secs(3), second) + .await + .expect("second shutdown finishes") + .expect("second shutdown task") + .expect("second shutdown joins and cleans the retained worker"); + + assert!(!pool.has_tracked_worker()); + assert_eq!(pool.status().quarantined, 0); + assert_eq!(pool.status().cleanup_pending, 0); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } +} diff --git a/src/blaze/crates/blazed/src/runtime_pool/recovery.rs b/src/blaze/crates/blazed/src/runtime_pool/recovery.rs new file mode 100644 index 0000000000..daf8421c04 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool/recovery.rs @@ -0,0 +1,3660 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable ownership records and startup recovery for unclaimed runtime slots. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use blaze_core::backend::BackendKind; +use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, RuntimeLocation, SandboxInstance, SandboxState, +}; +use blaze_core::storage::StorageProvider; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::SpawnerRegistry; + +const OWNERSHIP_FILE: &str = "ownership.json"; +const OWNERSHIP_VERSION: u32 = 1; +const CLEANUP_NAMESPACE: &str = ".cleanup"; +const DELETION_PROOF_NAMESPACE: &str = ".deletion-proofs"; +const STARTUP_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case", tag = "kind")] +pub(super) enum RuntimeSlotPhase { + Building, + Ready, + Handoff { token: Uuid }, + LifecycleOwned { token: Uuid }, + PoolCleanup, + LifecycleCleanup { token: Uuid }, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RuntimeSlotOwnership { + pub(super) version: u32, + pub(super) instance_id: Uuid, + pub(super) backend: BackendKind, + pub(super) backend_ownership: BackendOwnership, + pub(super) storage_owned: bool, + pub(super) phase: RuntimeSlotPhase, +} + +impl RuntimeSlotOwnership { + pub(super) fn new(instance_id: Uuid, backend: BackendKind) -> Self { + Self { + version: OWNERSHIP_VERSION, + instance_id, + backend, + backend_ownership: BackendOwnership::NotStarted, + storage_owned: false, + phase: RuntimeSlotPhase::Building, + } + } +} + +/// Lifecycle identity used to keep transferred warm resources out of cleanup. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DurableRuntimeOwner { + instance_id: Uuid, + runtime_location: RuntimeLocation, + backend: BackendKind, + backend_ownership: BackendOwnership, + state: SandboxState, + operation: Option, + runtime_owner_token: Option, + clean_terminal: bool, +} + +impl From<&SandboxInstance> for DurableRuntimeOwner { + fn from(instance: &SandboxInstance) -> Self { + Self { + instance_id: instance.id, + runtime_location: instance.runtime_location, + backend: instance.backend, + backend_ownership: instance.backend_ownership, + state: instance.state, + operation: instance.operation.as_ref().map(|operation| operation.kind), + runtime_owner_token: instance.runtime_owner_token, + clean_terminal: instance.is_clean_terminal(), + } + } +} + +impl DurableRuntimeOwner { + fn is_clean_terminal(&self) -> bool { + self.clean_terminal + } +} + +/// Derive the only accepted backend runtime directory for an owned instance. +pub(crate) fn runtime_dir( + state_dir: &Path, + location: RuntimeLocation, + instance_id: Uuid, +) -> PathBuf { + match location { + RuntimeLocation::Sandbox => state_dir.join(instance_id.to_string()), + RuntimeLocation::WarmPool => state_dir.join("runtime-pool").join(instance_id.to_string()), + } +} + +/// Release runtime slots that were never adopted by durable lifecycle state. +/// +/// A provider-only slot is safe to release by stable ID. Once a runtime +/// directory exists, its ownership record is required so restart recovery can +/// select the original backend instead of guessing from current policy. +pub(crate) async fn reconcile_runtime_slots( + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + reconcile_runtime_slots_until( + tokio::time::Instant::now() + STARTUP_RECONCILE_TIMEOUT, + runtime_root, + durable_owners, + storage, + spawners, + ) + .await +} + +async fn reconcile_runtime_slots_until( + deadline: tokio::time::Instant, + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + match tokio::time::timeout_at( + deadline, + reconcile_runtime_slots_inner(runtime_root, durable_owners, storage, spawners), + ) + .await + { + Ok(result) => result, + Err(_) => Err(recovery_error( + "runtime slot reconciliation exceeded its shared startup deadline".to_string(), + )), + } +} + +async fn reconcile_runtime_slots_inner( + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + ensure_real_directory(runtime_root, "runtime slot root").await?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace").await?; + let deletion_proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &deletion_proof_root).await?; + restore_deletion_proofs(&cleanup_root, &deletion_proof_root).await?; + + let (runtime_dirs, tombstones, root_errors) = + scan_runtime_dirs(runtime_root, &cleanup_root, &deletion_proof_root).await?; + if !root_errors.is_empty() { + return Err(unresolved_items(root_errors)); + } + let mut errors = + validate_durable_owners(runtime_root, durable_owners, &runtime_dirs, &tombstones).await; + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + let mut provider_ids = BTreeSet::new(); + let mut provider_errors = Vec::new(); + if storage.supports_runtime_pool_recovery() { + match storage.list_owned_ids().await { + Ok(ids) => { + for owned_id in ids { + match parse_stable_id(&owned_id) { + Ok(id) => { + provider_ids.insert(id); + } + Err(error) => { + provider_errors + .push(format!("provider-owned slot {owned_id:?}: {error}")); + } + } + } + } + Err(error) => { + provider_errors.push(format!("read provider-owned slot inventory: {error}")); + } + } + } else if !runtime_dirs.is_empty() { + errors.push("storage provider cannot inventory slots for runtime recovery".to_string()); + return Err(unresolved_items(errors)); + } + if !provider_errors.is_empty() { + provider_errors.extend(errors); + return Err(unresolved_items(provider_errors)); + } + for instance_id in tombstones.keys() { + if provider_ids.contains(instance_id) { + errors.push(format!( + "{instance_id}: cleanup tombstone conflicts with provider-owned storage" + )); + } + } + for (instance_id, tombstone) in &tombstones { + if !durable_owners.contains_key(instance_id) { + match read_ownership(tombstone, *instance_id).await { + Ok(ownership) if ownership.phase == RuntimeSlotPhase::PoolCleanup => {} + Ok(_) => errors.push(format!( + "{instance_id}: cleanup tombstone has no durable lifecycle owner and is not \ + pool cleanup" + )), + Err(error) => errors.push(format!("{instance_id}: {error}")), + } + } + } + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + let candidates = runtime_dirs + .keys() + .chain(provider_ids.iter()) + .copied() + .collect::>(); + let mut cleaned = 0; + for (instance_id, tombstone) in tombstones { + if durable_owners.get(&instance_id).is_some_and(|owner| { + !owner.is_clean_terminal() + && owner.runtime_location == RuntimeLocation::WarmPool + && owner.state == SandboxState::Destroyed + && owner.backend_ownership == BackendOwnership::Stopped + && owner.operation == Some(OperationKind::Destroy) + }) { + continue; + } + let expected_owner = match durable_owners.get(&instance_id) { + Some(owner) => match owner.runtime_owner_token { + Some(token) => CleanupOwner::Lifecycle(token), + None => { + errors.push(format!( + "{instance_id}: durable lifecycle owner has no runtime ownership token" + )); + continue; + } + }, + None => CleanupOwner::Pool, + }; + match remove_owned_tombstone(runtime_root, instance_id, expected_owner).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: continue runtime tombstone cleanup {}: {error}", + tombstone.display() + )), + } + } + for instance_id in candidates { + let run_dir = runtime_dirs.get(&instance_id); + if durable_owners + .get(&instance_id) + .is_some_and(|owner| !owner.is_clean_terminal()) + { + continue; + } + + let Some(run_dir) = run_dir else { + match storage.release_by_id(&instance_id.to_string()).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: release provider-only slot: {error}" + )), + } + continue; + }; + + let ownership_path = run_dir.join(OWNERSHIP_FILE); + let ownership_missing = match tokio::fs::symlink_metadata(&ownership_path).await { + Ok(_) => false, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => true, + Err(error) => { + errors.push(format!( + "{instance_id}: inspect ownership journal {}: {error}", + ownership_path.display() + )); + continue; + } + }; + if ownership_missing && !provider_ids.contains(&instance_id) { + match tokio::fs::remove_dir(run_dir).await { + Ok(()) => match sync_directory(runtime_root).await { + Ok(()) => { + cleaned += 1; + continue; + } + Err(error) => { + errors.push(format!( + "{instance_id}: sync runtime slot root after removing an empty \ + unjournaled slot: {error}" + )); + continue; + } + }, + Err(error) => { + errors.push(format!( + "{instance_id}: ownership journal {} is missing and the runtime directory \ + is not safely removable as empty: {error}", + ownership_path.display() + )); + continue; + } + } + } + + let mut ownership = match read_ownership(run_dir, instance_id).await { + Ok(ownership) => ownership, + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + }; + if matches!( + ownership.phase, + RuntimeSlotPhase::LifecycleOwned { .. } | RuntimeSlotPhase::LifecycleCleanup { .. } + ) { + errors.push(format!( + "{instance_id}: runtime slot records lifecycle ownership but durable lifecycle \ + metadata is missing" + )); + continue; + } + if ownership.phase != RuntimeSlotPhase::PoolCleanup { + ownership.phase = RuntimeSlotPhase::PoolCleanup; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record pool cleanup ownership: {error}" + )); + continue; + } + } + + if matches!( + ownership.backend_ownership, + BackendOwnership::Unknown | BackendOwnership::Starting | BackendOwnership::Running + ) { + let Some(spawner) = spawners.get(ownership.backend) else { + errors.push(format!( + "{instance_id}: no recovery spawner registered for recorded backend {}", + ownership.backend + )); + continue; + }; + if let Err(error) = spawner.cleanup_orphan(instance_id, run_dir).await { + errors.push(format!( + "{instance_id}: clean recorded backend {}: {error}", + ownership.backend + )); + continue; + } + ownership.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record completed backend cleanup: {error}" + )); + continue; + } + } + + if ownership.storage_owned || provider_ids.contains(&instance_id) { + if let Err(error) = storage.release_by_id(&instance_id.to_string()).await { + errors.push(format!("{instance_id}: release recorded storage: {error}")); + continue; + } + ownership.storage_owned = false; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record completed storage cleanup: {error}" + )); + continue; + } + } + + match tombstone_pool_slot(runtime_root, instance_id).await { + Ok(()) => match remove_pool_tombstone(runtime_root, instance_id).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: remove runtime cleanup tombstone: {error}" + )), + }, + Err(error) => errors.push(format!( + "{instance_id}: tombstone runtime directory {}: {error}", + run_dir.display() + )), + } + } + + if errors.is_empty() { + Ok(cleaned) + } else { + Err(unresolved_items(errors)) + } +} + +async fn validate_durable_owners( + runtime_root: &Path, + durable_owners: &HashMap, + runtime_dirs: &BTreeMap, + tombstones: &BTreeMap, +) -> Vec { + let mut errors = Vec::new(); + for (instance_id, owner) in durable_owners { + if owner.is_clean_terminal() { + if let Some(run_dir) = runtime_dirs.get(instance_id) { + errors.push(format!( + "{instance_id}: clean terminal lifecycle state still has active runtime slot {}", + run_dir.display() + )); + } + if let Some(tombstone) = tombstones.get(instance_id) + && let Err(error) = + validate_transferred_run_dir(tombstone, *instance_id, owner, true).await + { + errors.push(format!("{instance_id}: {error}")); + } + continue; + } + if owner.instance_id != *instance_id { + errors.push(format!( + "{instance_id}: durable owner map records instance {}", + owner.instance_id + )); + continue; + } + if let Some(tombstone) = tombstones.get(instance_id) { + if owner.runtime_location != RuntimeLocation::WarmPool + || owner.state != SandboxState::Destroyed + || owner.backend_ownership != BackendOwnership::Stopped + || owner.operation != Some(OperationKind::Destroy) + { + errors.push(format!( + "{instance_id}: cleanup tombstone {} collides with durable lifecycle owner", + tombstone.display() + )); + } else if let Err(error) = + validate_transferred_run_dir(tombstone, *instance_id, owner, true).await + { + errors.push(format!("{instance_id}: {error}")); + } + continue; + } + match owner.runtime_location { + RuntimeLocation::Sandbox => { + if let Some(run_dir) = runtime_dirs.get(instance_id) { + errors.push(format!( + "{instance_id}: warm-slot directory {} collides with sandbox runtime owner \ + in {} state", + run_dir.display(), + owner.state + )); + } + } + RuntimeLocation::WarmPool => match runtime_dirs.get(instance_id) { + Some(run_dir) => { + if let Err(error) = + validate_transferred_run_dir(run_dir, *instance_id, owner, false).await + { + errors.push(format!("{instance_id}: {error}")); + } + } + None => errors.push(format!( + "{instance_id}: durable warm-slot owner in {} state is missing canonical \ + runtime directory {}", + owner.state, + runtime_root.join(instance_id.to_string()).display() + )), + }, + } + } + errors +} + +async fn validate_transferred_run_dir( + run_dir: &Path, + instance_id: Uuid, + owner: &DurableRuntimeOwner, + tombstoned: bool, +) -> std::result::Result<(), String> { + let ownership_path = run_dir.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&ownership_path).await { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(format!( + "durable warm-slot owner is missing ownership journal {}", + ownership_path.display() + )); + } + Err(error) => { + return Err(format!( + "inspect transferred ownership {}: {error}", + ownership_path.display() + )); + } + } + let ownership = read_ownership(run_dir, instance_id).await?; + if ownership.backend != owner.backend { + return Err(format!( + "{} records backend {} but durable lifecycle owns {}", + ownership_path.display(), + ownership.backend, + owner.backend + )); + } + let Some(owner_token) = owner.runtime_owner_token else { + return Err(format!( + "durable warm-slot owner has no runtime ownership token for {}", + ownership_path.display() + )); + }; + let current_phase = ownership.phase; + match current_phase { + RuntimeSlotPhase::Handoff { token } => { + if tombstoned { + return Err(format!( + "{} is tombstoned before its ownership handoff completed", + ownership_path.display() + )); + } + if token != owner_token { + return Err(format!( + "{} records handoff token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + if ownership.backend_ownership != owner.backend_ownership { + return Err(format!( + "{} records backend ownership {:?} but durable lifecycle records {:?}", + ownership_path.display(), + ownership.backend_ownership, + owner.backend_ownership + )); + } + if !ownership.storage_owned { + return Err(format!( + "{} records released storage during ownership handoff", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::LifecycleOwned { token } => { + if tombstoned { + return Err(format!( + "{} is tombstoned without a lifecycle cleanup phase", + ownership_path.display() + )); + } + if token != owner_token { + return Err(format!( + "{} records lifecycle token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::LifecycleCleanup { token } => { + if token != owner_token { + return Err(format!( + "{} records cleanup token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + if owner.operation != Some(OperationKind::Destroy) && !owner.is_clean_terminal() { + return Err(format!( + "{} records lifecycle cleanup without a durable destroy operation", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::Building | RuntimeSlotPhase::Ready | RuntimeSlotPhase::PoolCleanup => { + return Err(format!( + "{} remains pool-owned while durable lifecycle metadata exists", + ownership_path.display() + )); + } + } + if matches!( + current_phase, + RuntimeSlotPhase::Handoff { .. } | RuntimeSlotPhase::LifecycleOwned { .. } + ) && !ownership.storage_owned + { + return Err(format!( + "{} records lifecycle ownership without owned storage", + ownership_path.display() + )); + } + Ok(()) +} + +async fn scan_runtime_dirs( + runtime_root: &Path, + cleanup_root: &Path, + deletion_proof_root: &Path, +) -> Result<( + BTreeMap, + BTreeMap, + Vec, +)> { + let mut directories = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(runtime_root).await.map_err(|error| { + recovery_error(format!( + "read runtime slot root {}: {error}", + runtime_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime slot entry under {}: {error}", + runtime_root.display() + )) + })? { + let path = entry.path(); + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime slot name is not UTF-8", + path.display() + )); + continue; + }; + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!("{}: inspect runtime slot: {error}", path.display())); + continue; + } + }; + if name == CLEANUP_NAMESPACE { + if !file_type.is_dir() { + errors.push(format!( + "{}: runtime cleanup namespace is not a directory", + path.display() + )); + } + continue; + } + if !file_type.is_dir() { + errors.push(format!( + "{}: unexpected non-directory runtime slot entry", + path.display() + )); + continue; + } + match parse_stable_id(&name) { + Ok(id) => { + directories.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + let (tombstones, tombstone_errors) = + scan_cleanup_dirs(cleanup_root, deletion_proof_root).await?; + errors.extend(tombstone_errors); + for instance_id in directories.keys() { + if tombstones.contains_key(instance_id) { + errors.push(format!( + "{instance_id}: runtime slot exists in both active and cleanup namespaces" + )); + } + } + Ok((directories, tombstones, errors)) +} + +async fn scan_cleanup_dirs( + cleanup_root: &Path, + deletion_proof_root: &Path, +) -> Result<(BTreeMap, Vec)> { + let mut tombstones = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "read runtime cleanup namespace {}: {error}", + cleanup_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime cleanup entry under {}: {error}", + cleanup_root.display() + )) + })? { + let path = entry.path(); + if path == deletion_proof_root { + let file_type = entry.file_type().await.map_err(|error| { + recovery_error(format!( + "inspect runtime deletion proof namespace {}: {error}", + path.display() + )) + })?; + if !file_type.is_dir() { + errors.push(format!( + "{}: runtime deletion proof namespace is not a directory", + path.display() + )); + } + continue; + } + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!( + "{}: inspect runtime cleanup entry: {error}", + path.display() + )); + continue; + } + }; + if !file_type.is_dir() { + errors.push(format!( + "{}: unexpected non-directory runtime cleanup entry", + path.display() + )); + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime cleanup name is not UTF-8", + path.display() + )); + continue; + }; + match parse_stable_id(&name) { + Ok(id) => { + tombstones.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + Ok((tombstones, errors)) +} + +struct DeletionProofRepair { + instance_id: Uuid, + proof: PathBuf, + tombstone: PathBuf, + create_tombstone: bool, + restore_journal: bool, +} + +async fn ensure_deletion_proof_root(cleanup_root: &Path, proof_root: &Path) -> Result<()> { + ensure_real_directory(cleanup_root, "runtime cleanup namespace").await?; + ensure_real_directory(proof_root, "runtime deletion proof namespace").await?; + sync_directory(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "sync runtime cleanup namespace {} after ensuring deletion proofs: {error}", + cleanup_root.display() + )) + }) +} + +async fn restore_deletion_proofs(cleanup_root: &Path, proof_root: &Path) -> Result<()> { + let (proofs, mut errors) = scan_deletion_proofs(proof_root).await?; + let mut repairs = Vec::new(); + for (instance_id, proof) in proofs { + let proof_ownership = match read_ownership_file(&proof, instance_id).await { + Ok(ownership) + if matches!( + ownership.phase, + RuntimeSlotPhase::PoolCleanup | RuntimeSlotPhase::LifecycleCleanup { .. } + ) => + { + ownership + } + Ok(ownership) => { + errors.push(format!( + "{instance_id}: deletion proof {} records non-cleanup phase {:?}", + proof.display(), + ownership.phase + )); + continue; + } + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + }; + let tombstone = cleanup_root.join(instance_id.to_string()); + let (create_tombstone, restore_journal) = + match tokio::fs::symlink_metadata(&tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => { + let journal = tombstone.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&journal).await { + Ok(metadata) if metadata.file_type().is_file() => { + match read_ownership_file(&journal, instance_id).await { + Ok(ownership) if ownership == proof_ownership => (false, false), + Ok(_) => { + errors.push(format!( + "{instance_id}: deletion proof {} does not match {}", + proof.display(), + journal.display() + )); + continue; + } + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + } + } + Ok(_) => { + errors.push(format!( + "{instance_id}: cleanup journal {} is not a regular file", + journal.display() + )); + continue; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (false, true), + Err(error) => { + errors.push(format!( + "{instance_id}: inspect cleanup journal {}: {error}", + journal.display() + )); + continue; + } + } + } + Ok(_) => { + errors.push(format!( + "{instance_id}: cleanup tombstone {} is not a real directory", + tombstone.display() + )); + continue; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (true, true), + Err(error) => { + errors.push(format!( + "{instance_id}: inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + continue; + } + }; + repairs.push(DeletionProofRepair { + instance_id, + proof, + tombstone, + create_tombstone, + restore_journal, + }); + } + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + + for repair in repairs { + if repair.create_tombstone { + tokio::fs::create_dir(&repair.tombstone) + .await + .map_err(|error| { + recovery_error(format!( + "{}: recreate cleanup tombstone {} from deletion proof: {error}", + repair.instance_id, + repair.tombstone.display() + )) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "{}: sync runtime cleanup namespace after recreating tombstone: {error}", + repair.instance_id + )) + })?; + } + if repair.restore_journal { + let journal = repair.tombstone.join(OWNERSHIP_FILE); + tokio::fs::hard_link(&repair.proof, &journal) + .await + .map_err(|error| { + recovery_error(format!( + "{}: restore cleanup journal {} from deletion proof {}: {error}", + repair.instance_id, + journal.display(), + repair.proof.display() + )) + })?; + sync_directory(&repair.tombstone).await.map_err(|error| { + recovery_error(format!( + "{}: sync restored cleanup tombstone {}: {error}", + repair.instance_id, + repair.tombstone.display() + )) + })?; + } + } + Ok(()) +} + +async fn scan_deletion_proofs(proof_root: &Path) -> Result<(BTreeMap, Vec)> { + let mut proofs = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(proof_root).await.map_err(|error| { + recovery_error(format!( + "read runtime deletion proof namespace {}: {error}", + proof_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime deletion proof entry under {}: {error}", + proof_root.display() + )) + })? { + let path = entry.path(); + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!( + "{}: inspect runtime deletion proof: {error}", + path.display() + )); + continue; + } + }; + if !file_type.is_file() { + errors.push(format!( + "{}: runtime deletion proof is not a regular file", + path.display() + )); + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime deletion proof name is not UTF-8", + path.display() + )); + continue; + }; + match parse_stable_id(&name) { + Ok(id) => { + proofs.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + Ok((proofs, errors)) +} + +async fn ensure_real_directory(path: &Path, label: &str) -> Result<()> { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.file_type().is_dir() => return Ok(()), + Ok(_) => { + return Err(recovery_error(format!( + "{label} {} is not a real directory", + path.display() + ))); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(recovery_error(format!( + "inspect {label} {}: {error}", + path.display() + ))); + } + } + + tokio::fs::create_dir_all(path) + .await + .map_err(|error| recovery_error(format!("create {label} {}: {error}", path.display())))?; + let metadata = tokio::fs::symlink_metadata(path).await.map_err(|error| { + recovery_error(format!( + "inspect created {label} {}: {error}", + path.display() + )) + })?; + if !metadata.file_type().is_dir() { + return Err(recovery_error(format!( + "{label} {} is not a real directory", + path.display() + ))); + } + Ok(()) +} + +async fn rename_to_tombstone( + runtime_root: &Path, + cleanup_root: &Path, + run_dir: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + let tombstone = cleanup_root.join(instance_id.to_string()); + match tokio::fs::symlink_metadata(&tombstone).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(format!( + "cleanup tombstone {} already exists", + tombstone.display() + )); + } + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + } + + tokio::fs::rename(run_dir, &tombstone) + .await + .map_err(|error| { + format!( + "move {} to {}: {error}", + run_dir.display(), + tombstone.display() + ) + })?; + sync_directory(runtime_root).await.map_err(|error| { + format!( + "sync runtime slot root {} after rename: {error}", + runtime_root.display() + ) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "sync runtime cleanup namespace {} after rename: {error}", + cleanup_root.display() + ) + })?; + Ok(()) +} + +#[derive(Clone, Copy)] +enum CleanupOwner { + Pool, + Lifecycle(Uuid), +} + +impl CleanupOwner { + fn phase(self) -> RuntimeSlotPhase { + match self { + Self::Pool => RuntimeSlotPhase::PoolCleanup, + Self::Lifecycle(token) => RuntimeSlotPhase::LifecycleCleanup { token }, + } + } +} + +pub(super) async fn tombstone_pool_slot( + runtime_root: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + tombstone_owned_slot(runtime_root, instance_id, CleanupOwner::Pool).await +} + +pub(crate) async fn tombstone_lifecycle_slot( + runtime_root: &Path, + instance_id: Uuid, + owner_token: Uuid, +) -> std::result::Result<(), String> { + tombstone_owned_slot( + runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await +} + +async fn tombstone_owned_slot( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + ensure_real_directory(runtime_root, "runtime slot root") + .await + .map_err(|error| error.to_string())?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace") + .await + .map_err(|error| error.to_string())?; + let run_dir = runtime_root.join(instance_id.to_string()); + let tombstone = cleanup_root.join(instance_id.to_string()); + let run_dir_exists = real_directory_exists(&run_dir, "runtime slot").await?; + let tombstone_exists = real_directory_exists(&tombstone, "cleanup tombstone").await?; + match (run_dir_exists, tombstone_exists) { + (true, false) => { + require_cleanup_owner(&run_dir, instance_id, expected_owner).await?; + rename_to_tombstone(runtime_root, &cleanup_root, &run_dir, instance_id).await + } + (false, true) => { + require_cleanup_owner(&tombstone, instance_id, expected_owner).await?; + Ok(()) + } + (false, false) => match expected_owner { + CleanupOwner::Pool => sync_directory(&cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after prior pool removal: \ + {error}" + ) + }), + CleanupOwner::Lifecycle(_) => Err(format!( + "{instance_id}: runtime slot has neither an active directory nor a cleanup \ + tombstone" + )), + }, + (true, true) => Err(format!( + "{instance_id}: runtime slot exists in both active and cleanup namespaces" + )), + } +} + +async fn require_cleanup_owner( + directory: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result { + let ownership = read_ownership(directory, instance_id).await?; + if ownership.phase != expected_owner.phase() { + return Err(format!( + "{} records {:?} instead of the expected cleanup owner", + directory.join(OWNERSHIP_FILE).display(), + ownership.phase + )); + } + Ok(ownership) +} + +pub(super) async fn remove_pool_tombstone( + runtime_root: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + remove_owned_tombstone(runtime_root, instance_id, CleanupOwner::Pool).await +} + +pub(crate) async fn remove_lifecycle_tombstone( + runtime_root: &Path, + instance_id: Uuid, + owner_token: Uuid, +) -> std::result::Result<(), String> { + remove_owned_tombstone( + runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await +} + +async fn remove_owned_tombstone( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + // Pool maintenance, lifecycle operation locks, and startup isolation each + // serialize removal for one instance ID. The proof protocol relies on that + // per-owner exclusion while it repairs or removes the canonical paths. + let run_dir = runtime_root.join(instance_id.to_string()); + if real_directory_exists(&run_dir, "runtime slot").await? { + return Err(format!( + "{instance_id}: active runtime slot still exists before tombstone removal" + )); + } + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &proof_root) + .await + .map_err(|error| error.to_string())?; + let tombstone = cleanup_root.join(instance_id.to_string()); + let proof = proof_root.join(instance_id.to_string()); + + let proof_exists = match tokio::fs::symlink_metadata(&proof).await { + Ok(metadata) if metadata.file_type().is_file() => true, + Ok(_) => { + return Err(format!( + "runtime deletion proof {} is not a regular file", + proof.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "inspect runtime deletion proof {}: {error}", + proof.display() + )); + } + }; + + if proof_exists { + restore_one_deletion_proof( + &cleanup_root, + &proof_root, + &proof, + &tombstone, + instance_id, + expected_owner, + ) + .await?; + } else if real_directory_exists(&tombstone, "cleanup tombstone").await? { + arm_deletion_proof(&proof_root, &tombstone, &proof, instance_id, expected_owner).await?; + } else { + sync_directory(&cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after prior tombstone removal: \ + {error}" + ) + })?; + return Ok(()); + } + + remove_armed_tombstone(&cleanup_root, &proof_root, &tombstone, &proof, instance_id).await +} + +async fn arm_deletion_proof( + proof_root: &Path, + tombstone: &Path, + proof: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + let ownership = require_cleanup_owner(tombstone, instance_id, expected_owner).await?; + let journal = tombstone.join(OWNERSHIP_FILE); + tokio::fs::hard_link(&journal, proof) + .await + .map_err(|error| { + format!( + "{instance_id}: preserve cleanup ownership from {} in {}: {error}", + journal.display(), + proof.display() + ) + })?; + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {}: {error}", + proof_root.display() + ) + })?; + let proof_ownership = read_ownership_file(proof, instance_id).await?; + if proof_ownership != ownership { + return Err(format!( + "{instance_id}: deletion proof {} changed while it was persisted", + proof.display() + )); + } + Ok(()) +} + +async fn real_directory_exists(path: &Path, label: &str) -> std::result::Result { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.file_type().is_dir() => Ok(true), + Ok(_) => Err(format!( + "{label} {} is not a real directory", + path.display() + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(format!("inspect {label} {}: {error}", path.display())), + } +} + +async fn restore_one_deletion_proof( + cleanup_root: &Path, + proof_root: &Path, + proof: &Path, + tombstone: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + let proof_ownership = read_ownership_file(proof, instance_id).await?; + if proof_ownership.phase != expected_owner.phase() { + return Err(format!( + "{} records {:?} instead of the expected cleanup owner", + proof.display(), + proof_ownership.phase + )); + } + match tokio::fs::symlink_metadata(tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => { + return Err(format!( + "cleanup tombstone {} is not a real directory", + tombstone.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::create_dir(tombstone).await.map_err(|error| { + format!( + "{instance_id}: recreate cleanup tombstone {} from deletion proof: {error}", + tombstone.display() + ) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after recreating tombstone: \ + {error}" + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + } + let journal = tombstone.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&journal).await { + Ok(metadata) if metadata.file_type().is_file() => { + let ownership = read_ownership_file(&journal, instance_id).await?; + if ownership != proof_ownership { + return Err(format!( + "{instance_id}: deletion proof {} does not match {}", + proof.display(), + journal.display() + )); + } + } + Ok(_) => { + return Err(format!( + "{instance_id}: cleanup journal {} is not a regular file", + journal.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::hard_link(proof, &journal) + .await + .map_err(|error| { + format!( + "{instance_id}: restore cleanup journal {} from deletion proof {}: {error}", + journal.display(), + proof.display() + ) + })?; + sync_directory(tombstone).await.map_err(|error| { + format!( + "{instance_id}: sync restored cleanup tombstone {}: {error}", + tombstone.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "{instance_id}: inspect cleanup journal {}: {error}", + journal.display() + )); + } + } + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {}: {error}", + proof_root.display() + ) + }) +} + +async fn remove_armed_tombstone( + cleanup_root: &Path, + proof_root: &Path, + tombstone: &Path, + proof: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + let tombstone_exists = match tokio::fs::symlink_metadata(tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => true, + Ok(_) => { + return Err(format!( + "cleanup tombstone {} is not a real directory", + tombstone.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + }; + if tombstone_exists { + tokio::fs::remove_dir_all(tombstone) + .await + .map_err(|error| { + format!("remove cleanup tombstone {}: {error}", tombstone.display()) + })?; + } + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "sync runtime cleanup namespace {} after removal: {error}", + cleanup_root.display() + ) + })?; + match tokio::fs::remove_file(proof).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(format!( + "{instance_id}: remove runtime deletion proof {}: {error}", + proof.display() + )); + } + } + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {} after removal: {error}", + proof_root.display() + ) + })?; + Ok(()) +} + +async fn sync_directory(path: &Path) -> std::io::Result<()> { + tokio::fs::File::open(path).await?.sync_all().await +} + +pub(super) async fn finish_ownership_handoff( + run_dir: &Path, + expected_id: Uuid, + expected_backend: BackendKind, + expected_ownership: BackendOwnership, + expected_token: Uuid, +) -> std::result::Result<(), String> { + let mut ownership = read_ownership(run_dir, expected_id).await?; + let path = run_dir.join(OWNERSHIP_FILE); + if ownership.backend != expected_backend { + return Err(format!( + "{} records backend {} instead of {expected_backend}", + path.display(), + ownership.backend + )); + } + if ownership.backend_ownership != expected_ownership { + return Err(format!( + "{} records backend ownership {:?} instead of {:?}", + path.display(), + ownership.backend_ownership, + expected_ownership + )); + } + if !ownership.storage_owned { + return Err(format!( + "{} cannot transfer a slot without owned storage", + path.display() + )); + } + if ownership.phase + == (RuntimeSlotPhase::LifecycleOwned { + token: expected_token, + }) + { + return Ok(()); + } + if ownership.phase + != (RuntimeSlotPhase::Handoff { + token: expected_token, + }) + { + return Err(format!( + "{} does not record expected ownership handoff token {expected_token}", + path.display() + )); + } + ownership.phase = RuntimeSlotPhase::LifecycleOwned { + token: expected_token, + }; + write_ownership(run_dir, &ownership).await +} + +/// Durably transfer a claimed slot into lifecycle cleanup before mutating +/// backend or storage ownership. +pub(crate) async fn begin_lifecycle_cleanup( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + owner_token: Uuid, +) -> std::result::Result<(), String> { + ensure_real_directory(runtime_root, "runtime slot root") + .await + .map_err(|error| error.to_string())?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace") + .await + .map_err(|error| error.to_string())?; + let run_dir = runtime_root.join(instance_id.to_string()); + let tombstone = cleanup_root.join(instance_id.to_string()); + let run_dir_exists = real_directory_exists(&run_dir, "runtime slot").await?; + let tombstone_exists = real_directory_exists(&tombstone, "cleanup tombstone").await?; + let (directory, tombstoned) = match (run_dir_exists, tombstone_exists) { + (true, false) => (run_dir, false), + (false, true) => (tombstone, true), + (false, false) => { + return Err(format!( + "{instance_id}: lifecycle-owned runtime slot has neither an active directory nor \ + a cleanup tombstone" + )); + } + (true, true) => { + return Err(format!( + "{instance_id}: lifecycle-owned runtime slot exists in both active and cleanup \ + namespaces" + )); + } + }; + let mut ownership = read_ownership(&directory, instance_id).await?; + if ownership.backend != backend { + return Err(format!( + "{} records backend {} instead of lifecycle backend {backend}", + directory.join(OWNERSHIP_FILE).display(), + ownership.backend + )); + } + let current_phase = ownership.phase; + match current_phase { + RuntimeSlotPhase::Handoff { token } + | RuntimeSlotPhase::LifecycleOwned { token } + | RuntimeSlotPhase::LifecycleCleanup { token } + if token == owner_token => {} + RuntimeSlotPhase::Handoff { token } + | RuntimeSlotPhase::LifecycleOwned { token } + | RuntimeSlotPhase::LifecycleCleanup { token } => { + return Err(format!( + "{} records lifecycle token {token} instead of {owner_token}", + directory.join(OWNERSHIP_FILE).display() + )); + } + phase => { + return Err(format!( + "{} records pool phase {phase:?} instead of lifecycle ownership", + directory.join(OWNERSHIP_FILE).display() + )); + } + } + if matches!( + current_phase, + RuntimeSlotPhase::Handoff { .. } | RuntimeSlotPhase::LifecycleOwned { .. } + ) && !ownership.storage_owned + { + return Err(format!( + "{} records lifecycle ownership without owned storage", + directory.join(OWNERSHIP_FILE).display() + )); + } + if tombstoned { + if ownership.phase != (RuntimeSlotPhase::LifecycleCleanup { token: owner_token }) { + return Err(format!( + "{} was tombstoned before lifecycle cleanup was committed", + directory.join(OWNERSHIP_FILE).display() + )); + } + return Ok(()); + } + if ownership.phase == (RuntimeSlotPhase::LifecycleCleanup { token: owner_token }) { + return Ok(()); + } + ownership.phase = RuntimeSlotPhase::LifecycleCleanup { token: owner_token }; + write_ownership(&directory, &ownership).await +} + +pub(super) async fn read_ownership( + run_dir: &Path, + expected_id: Uuid, +) -> std::result::Result { + let path = run_dir.join(OWNERSHIP_FILE); + read_ownership_file(&path, expected_id).await +} + +async fn read_ownership_file( + path: &Path, + expected_id: Uuid, +) -> std::result::Result { + let metadata = tokio::fs::symlink_metadata(&path) + .await + .map_err(|error| format!("inspect {}: {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + let encoded = tokio::fs::read(&path) + .await + .map_err(|error| format!("read {}: {error}", path.display()))?; + let ownership: RuntimeSlotOwnership = serde_json::from_slice(&encoded) + .map_err(|error| format!("decode {}: {error}", path.display()))?; + validate_ownership(path, &ownership, expected_id)?; + Ok(ownership) +} + +fn validate_ownership( + path: &Path, + ownership: &RuntimeSlotOwnership, + expected_id: Uuid, +) -> std::result::Result<(), String> { + if ownership.version != OWNERSHIP_VERSION { + return Err(format!( + "{} has unsupported ownership version {}", + path.display(), + ownership.version + )); + } + if ownership.instance_id != expected_id { + return Err(format!( + "{} records instance {} instead of directory {expected_id}", + path.display(), + ownership.instance_id + )); + } + Ok(()) +} + +pub(super) async fn write_ownership( + run_dir: &Path, + ownership: &RuntimeSlotOwnership, +) -> std::result::Result<(), String> { + let encoded = serde_json::to_vec(ownership) + .map_err(|error| format!("encode ownership record: {error}"))?; + let path = run_dir.join(OWNERSHIP_FILE); + let temporary = run_dir.join(format!(".ownership-{}.tmp", Uuid::new_v4())); + let result = async { + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + .await?; + file.write_all(&encoded).await?; + file.sync_all().await?; + drop(file); + tokio::fs::rename(&temporary, &path).await?; + tokio::fs::File::open(run_dir).await?.sync_all().await + } + .await; + if let Err(error) = result { + let _ = tokio::fs::remove_file(&temporary).await; + return Err(format!("persist {}: {error}", path.display())); + } + Ok(()) +} + +fn parse_stable_id(value: &str) -> std::result::Result { + let id = Uuid::parse_str(value).map_err(|error| format!("slot ID is not a UUID: {error}"))?; + if value != id.to_string() { + return Err(format!("slot ID must use canonical UUID form {id}")); + } + Ok(id) +} + +fn recovery_error(message: String) -> BlazeDaemonError { + BlazeDaemonError::RecoveryRequired(message) +} + +fn unresolved_items(errors: Vec) -> BlazeDaemonError { + recovery_error(format!( + "runtime slot reconciliation found {} unresolved item(s): {}", + errors.len(), + errors.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::future; + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use blaze_core::backend::SpawnRequest; + use blaze_core::lifecycle::StartPath; + use blaze_core::storage::{AcquireOpts, PoolStatus, StorageAcquireError, StorageSlot}; + use blaze_core::{BlazeError, Result as CoreResult}; + + use super::*; + use crate::spawner::{BackendSpawner, DynBackendInstance, DynSpawner, SpawnFailure}; + + #[derive(Default)] + struct RecordingStorage { + owned: Mutex>, + fail_once: Mutex>, + events: Arc>>, + supports_recovery: bool, + inventory_error: Option, + inventory_override: Option>, + pending_inventory: bool, + pending_release: bool, + } + + impl RecordingStorage { + fn with_ids(ids: impl IntoIterator, events: Arc>>) -> Self { + Self { + owned: Mutex::new(ids.into_iter().collect()), + fail_once: Mutex::new(BTreeSet::new()), + events, + supports_recovery: true, + inventory_error: None, + inventory_override: None, + pending_inventory: false, + pending_release: false, + } + } + + fn fail_next_release(&self, instance_id: Uuid) { + self.fail_once + .lock() + .expect("release failures") + .insert(instance_id); + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> CoreResult { + Ok(true) + } + + async fn acquire( + &self, + _opts: &AcquireOpts, + ) -> std::result::Result { + Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: "test storage does not acquire slots".into(), + })) + } + + async fn release(&self, slot: StorageSlot) -> CoreResult<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> CoreResult<()> { + let id = Uuid::parse_str(instance_id).map_err(|error| BlazeError::StorageError { + msg: format!("invalid test slot id {instance_id}: {error}"), + })?; + self.events + .lock() + .expect("events") + .push(format!("storage:{id}")); + if self.pending_release { + future::pending::<()>().await; + } + if self.fail_once.lock().expect("release failures").remove(&id) { + return Err(BlazeError::StorageError { + msg: format!("injected release failure for {id}"), + }); + } + self.owned.lock().expect("owned slots").remove(&id); + Ok(()) + } + + async fn reconstruct(&self, instance_id: &str) -> CoreResult { + Err(BlazeError::StorageError { + msg: format!("test storage cannot reconstruct {instance_id}"), + }) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> CoreResult<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> CoreResult { + Ok(0) + } + + fn supports_runtime_pool_recovery(&self) -> bool { + self.supports_recovery + } + + async fn list_owned_ids(&self) -> CoreResult> { + if self.pending_inventory { + future::pending::<()>().await; + } + if let Some(message) = &self.inventory_error { + return Err(BlazeError::StorageError { + msg: message.clone(), + }); + } + if let Some(ids) = &self.inventory_override { + return Ok(ids.clone()); + } + Ok(self + .owned + .lock() + .expect("owned slots") + .iter() + .map(Uuid::to_string) + .collect()) + } + } + + struct RecordingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for RecordingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + Ok(()) + } + } + + struct FailingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for FailingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + Err(BlazeError::BackendError { + msg: format!("injected backend cleanup failure for {instance_id}"), + }) + } + } + + struct PendingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for PendingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + future::pending::>().await + } + } + + struct DelayedSpawner { + delay: Duration, + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for DelayedSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + tokio::time::sleep(self.delay).await; + Ok(()) + } + } + + fn registry(kind: BackendKind, spawner: DynSpawner) -> SpawnerRegistry { + let mut registry = SpawnerRegistry::new(); + registry.insert(kind, spawner); + registry + } + + fn write_ownership( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + backend_ownership: BackendOwnership, + storage_owned: bool, + ) -> PathBuf { + write_ownership_with_phase( + runtime_root, + instance_id, + backend, + backend_ownership, + storage_owned, + RuntimeSlotPhase::Building, + ) + } + + fn write_ownership_with_phase( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + backend_ownership: BackendOwnership, + storage_owned: bool, + phase: RuntimeSlotPhase, + ) -> PathBuf { + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend, + backend_ownership, + storage_owned, + phase, + }; + std::fs::write( + run_dir.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("serialize ownership"), + ) + .expect("write ownership"); + run_dir + } + + fn write_tombstone(runtime_root: &Path, instance_id: Uuid) -> PathBuf { + let tombstone = runtime_root + .join(CLEANUP_NAMESPACE) + .join(instance_id.to_string()); + std::fs::create_dir_all(&tombstone).expect("tombstone"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + storage_owned: false, + phase: RuntimeSlotPhase::PoolCleanup, + }; + std::fs::write( + tombstone.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("serialize tombstone ownership"), + ) + .expect("write tombstone ownership"); + tombstone + } + + async fn arm_test_deletion_proof( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, + ) -> PathBuf { + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &proof_root) + .await + .expect("proof root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + let proof = proof_root.join(instance_id.to_string()); + arm_deletion_proof(&proof_root, &tombstone, &proof, instance_id, expected_owner) + .await + .expect("arm deletion proof"); + proof + } + + #[tokio::test] + async fn reconcile_cleans_backend_before_storage_and_run_dir() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let cleaned = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect("reconcile"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_retains_storage_after_backend_cleanup_failure() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(FailingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("backend failure must retain later owners"); + + assert!( + error + .to_string() + .contains("injected backend cleanup failure") + ); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{instance_id}")] + ); + assert!(run_dir.exists()); + assert!( + storage + .owned + .lock() + .expect("owned slots") + .contains(&instance_id) + ); + } + + #[tokio::test] + async fn reconcile_releases_provider_only_slot_by_id() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("reconcile"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + } + + #[tokio::test] + async fn reconcile_protects_durable_lifecycle_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + RuntimeSlotPhase::LifecycleOwned { token: owner_token }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("durable owner must be excluded"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_checks_live_warm_owner_missing_from_inventory() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: None, + clean_terminal: false, + }; + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("live warm owner requires its canonical run dir"); + + let message = error.to_string(); + assert!(message.contains(&instance_id.to_string())); + assert!(message.contains("canonical runtime directory")); + assert!(message.contains("is missing")); + } + + #[tokio::test] + async fn reconcile_rejects_warm_owner_without_run_dir() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::NotStarted, + state: SandboxState::Creating, + operation: Some(OperationKind::Create), + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("durable warm owner must retain its canonical run dir"); + + assert!(error.to_string().contains("missing canonical")); + } + + #[test] + fn runtime_location_survives_warm_reclassification_and_reload() { + let temp = tempfile::tempdir().expect("tempdir"); + let mut sandbox = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:sandbox-location".into(), + StartPath::Cold, + "sandbox-location".into(), + ); + sandbox + .transition(SandboxState::Creating) + .expect("creating"); + sandbox.transition(SandboxState::Running).expect("running"); + sandbox.transition(SandboxState::Reset).expect("reset"); + sandbox.transition(SandboxState::Warm).expect("warm"); + sandbox + .transition(SandboxState::Creating) + .expect("warm claim"); + sandbox.persist(temp.path()).expect("persist sandbox"); + let sandbox = SandboxInstance::load(temp.path(), sandbox.id).expect("reload sandbox"); + assert_eq!(sandbox.start_path, StartPath::Warm); + assert_eq!( + runtime_dir(temp.path(), sandbox.runtime_location, sandbox.id), + temp.path().join(sandbox.id.to_string()) + ); + + let mut warm_pool = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:warm-location".into(), + StartPath::Warm, + "warm-location".into(), + ); + warm_pool.runtime_location = RuntimeLocation::WarmPool; + warm_pool.persist(temp.path()).expect("persist warm owner"); + let warm_pool = + SandboxInstance::load(temp.path(), warm_pool.id).expect("reload warm owner"); + assert_eq!( + runtime_dir(temp.path(), warm_pool.runtime_location, warm_pool.id), + temp.path() + .join("runtime-pool") + .join(warm_pool.id.to_string()) + ); + } + + #[tokio::test] + async fn reconcile_rejects_cold_lifecycle_run_dir_collision() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::Sandbox, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: None, + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("cold lifecycle must not adopt a pool run dir"); + + assert!(error.to_string().contains("sandbox runtime owner")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_transferred_backend_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Runc, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("backend mismatch must stop startup"); + + let message = error.to_string(); + assert!(message.contains("records backend mock")); + assert!(message.contains("durable lifecycle owns runc")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_does_not_protect_destroyed_lifecycle_record() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::Sandbox, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: None, + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("destroyed lifecycle record does not own runtime resources"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + } + + #[tokio::test] + async fn reconcile_protects_nonterminal_destroyed_owners() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let live_id = Uuid::new_v4(); + let journal_id = Uuid::new_v4(); + let mut live = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:live-destroyed".into(), + StartPath::Cold, + "live-destroyed".into(), + ); + live.id = live_id; + live.transition(SandboxState::Destroyed).expect("destroyed"); + live.backend_ownership = BackendOwnership::Running; + let mut journal = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:journal-destroyed".into(), + StartPath::Cold, + "journal-destroyed".into(), + ); + journal.id = journal_id; + journal + .begin_operation(blaze_core::lifecycle::OperationKind::Destroy) + .expect("destroy journal"); + journal + .transition(SandboxState::Destroyed) + .expect("destroyed"); + journal.backend_ownership = BackendOwnership::Stopped; + let owners = HashMap::from([ + (live_id, DurableRuntimeOwner::from(&live)), + (journal_id, DurableRuntimeOwner::from(&journal)), + ]); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([live_id, journal_id], events.clone()); + + let cleaned = + reconcile_runtime_slots(&runtime_root, &owners, &storage, &SpawnerRegistry::new()) + .await + .expect("nonterminal lifecycle records retain cleanup ownership"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + } + + #[tokio::test] + async fn reconcile_does_not_call_provider_without_recovery_contract() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage { + owned: Mutex::new(BTreeSet::from([instance_id])), + fail_once: Mutex::new(BTreeSet::new()), + events: events.clone(), + supports_recovery: false, + ..RecordingStorage::default() + }; + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("unsupported provider must stop before cleanup"); + + assert!( + error + .to_string() + .contains("cannot inventory slots for runtime recovery") + ); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_namespace_alias_before_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = + Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let alias = runtime_root.join(instance_id.to_string().to_uppercase()); + std::fs::create_dir_all(&alias).expect("alias run dir"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + storage_owned: true, + phase: RuntimeSlotPhase::Building, + }; + std::fs::write( + alias.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("ownership"), + ) + .expect("alias ownership"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("noncanonical alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(alias.exists()); + assert!( + storage + .owned + .lock() + .expect("owned slots") + .contains(&instance_id) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconcile_rejects_uuid_symlink_before_cleanup() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let instance_id = Uuid::new_v4(); + let target = tempfile::tempdir().expect("target"); + let alias = runtime_root.join(instance_id.to_string()); + symlink(target.path(), &alias).expect("runtime symlink"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("runtime symlink must stop preflight"); + + assert!(error.to_string().contains("non-directory runtime slot")); + assert!(events.lock().expect("events").is_empty()); + assert!( + std::fs::symlink_metadata(&alias) + .expect("alias") + .is_symlink() + ); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconcile_rejects_symlinked_runtime_roots() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_target = tempfile::tempdir().expect("runtime target"); + let runtime_root = temp.path().join("runtime-pool"); + symlink(runtime_target.path(), &runtime_root).expect("runtime root symlink"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let runtime_error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("runtime root symlink must be rejected"); + + assert!(runtime_error.to_string().contains("not a real directory")); + assert!(events.lock().expect("events").is_empty()); + + let real_runtime_root = temp.path().join("real-runtime-pool"); + std::fs::create_dir_all(&real_runtime_root).expect("real runtime root"); + let cleanup_target = tempfile::tempdir().expect("cleanup target"); + symlink( + cleanup_target.path(), + real_runtime_root.join(CLEANUP_NAMESPACE), + ) + .expect("cleanup root symlink"); + + let cleanup_error = reconcile_runtime_slots( + &real_runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("cleanup root symlink must be rejected"); + + assert!(cleanup_error.to_string().contains("not a real directory")); + assert!(events.lock().expect("events").is_empty()); + assert!(cleanup_target.path().is_dir()); + } + + #[tokio::test] + async fn reconcile_rejects_inventory_error_before_backend_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.inventory_error = Some("injected inventory failure".to_string()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("inventory failure must stop preflight"); + + assert!(error.to_string().contains("injected inventory failure")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_noncanonical_provider_id_before_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = + Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.inventory_override = Some(vec![instance_id.to_string().to_uppercase()]); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("provider alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_resumes_tombstone_without_external_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let cleaned = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect("resume tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_resumes_pool_tombstone_after_journal_unlink() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let nested = tombstone.join("partially-removed"); + std::fs::create_dir_all(&nested).expect("partial directory"); + std::fs::write(nested.join("leftover"), b"data").expect("partial file"); + let proof = arm_test_deletion_proof(&runtime_root, instance_id, CleanupOwner::Pool).await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("resume partial tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + assert_eq!( + reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new() + ) + .await + .expect("idempotent reconciliation"), + 0 + ); + } + + #[tokio::test] + async fn reconcile_resumes_lifecycle_tombstone_after_journal_unlink() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("resume partial lifecycle tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + } + + #[tokio::test] + async fn reconcile_finishes_deletion_when_only_the_proof_remains() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let proof = arm_test_deletion_proof(&runtime_root, instance_id, CleanupOwner::Pool).await; + std::fs::remove_dir_all(&tombstone).expect("simulate completed recursive deletion"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("finish deletion from proof"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_partial_tombstone_without_deletion_proof() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("missing deletion proof must stop recovery"); + + assert!(error.to_string().contains("ownership.json")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_mismatched_deletion_proof() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let other_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let other_tombstone = write_tombstone(&runtime_root, other_id); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + std::fs::create_dir_all(&proof_root).expect("proof root"); + let proof = proof_root.join(instance_id.to_string()); + std::fs::hard_link(other_tombstone.join(OWNERSHIP_FILE), &proof).expect("mismatched proof"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("mismatched deletion proof must stop recovery"); + + assert!(error.to_string().contains("records instance")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_deletion_proof_that_differs_from_its_journal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let proof_root = runtime_root + .join(CLEANUP_NAMESPACE) + .join(DELETION_PROOF_NAMESPACE); + std::fs::create_dir_all(&proof_root).expect("proof root"); + let proof = proof_root.join(instance_id.to_string()); + let mismatched = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Runc, + backend_ownership: BackendOwnership::Stopped, + storage_owned: false, + phase: RuntimeSlotPhase::PoolCleanup, + }; + std::fs::write( + &proof, + serde_json::to_vec(&mismatched).expect("proof ownership"), + ) + .expect("mismatched proof"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("proof and journal mismatch must stop recovery"); + + assert!(error.to_string().contains("does not match")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_partial_lifecycle_tombstone_with_wrong_token() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let proof_token = Uuid::new_v4(); + let durable_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: proof_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(proof_token), + ) + .await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(durable_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("wrong lifecycle proof token must stop recovery"); + + assert!(error.to_string().contains("cleanup token")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_restores_but_defers_nonterminal_lifecycle_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await; + let journal = tombstone.join(OWNERSHIP_FILE); + std::fs::remove_file(&journal).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: Some(OperationKind::Destroy), + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("nonterminal lifecycle cleanup remains lifecycle-owned"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + assert!(journal.is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn tombstone_removal_rejects_a_linked_cleanup_root() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let external = tempfile::tempdir().expect("external"); + let sentinel = external.path().join("keep"); + std::fs::write(&sentinel, b"keep").expect("sentinel"); + symlink(external.path(), runtime_root.join(CLEANUP_NAMESPACE)).expect("cleanup link"); + + let error = remove_pool_tombstone(&runtime_root, Uuid::new_v4()) + .await + .expect_err("linked cleanup root must be rejected"); + + assert!(error.contains("not a real directory")); + assert_eq!(std::fs::read(&sentinel).expect("sentinel remains"), b"keep"); + assert!(!external.path().join(DELETION_PROOF_NAMESPACE).exists()); + } + + #[tokio::test] + async fn reconcile_resumes_tombstone_for_clean_terminal_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("clean terminal owner no longer owns external resources"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_tombstone_alias_before_removal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let valid_id = Uuid::new_v4(); + let valid = write_tombstone(&runtime_root, valid_id); + let alias_id = Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let alias = cleanup_root.join(alias_id.to_string().to_uppercase()); + std::fs::create_dir_all(&alias).expect("alias tombstone"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("tombstone alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(valid.exists()); + assert!(alias.exists()); + } + + #[tokio::test] + async fn reconcile_preserves_tombstone_that_conflicts_with_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("storage and tombstone conflict must stop preflight"); + + assert!(error.to_string().contains("tombstone conflicts")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_preserves_tombstone_owned_by_lifecycle() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::RecoveryRequired, + operation: None, + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("lifecycle and tombstone conflict must stop preflight"); + + assert!( + error + .to_string() + .contains("collides with durable lifecycle") + ); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_bounds_pending_inventory() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([], events.clone()); + storage.pending_inventory = true; + let started = tokio::time::Instant::now(); + + let error = reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("pending inventory must time out"); + + assert!(error.to_string().contains("shared startup deadline")); + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert!(events.lock().expect("events").is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_retains_storage_while_backend_is_pending() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(PendingSpawner { + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("pending backend must time out"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{instance_id}")] + ); + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("ownership retained"); + assert_eq!(ownership.backend_ownership, BackendOwnership::Running); + assert!(ownership.storage_owned); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_persists_backend_phase_before_pending_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.pending_release = true; + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("pending storage must time out"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("ownership retained"); + assert_eq!(ownership.backend_ownership, BackendOwnership::Stopped); + assert!(ownership.storage_owned); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_uses_one_deadline_for_all_candidates() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let first = Uuid::parse_str("11111111-1111-4111-8111-111111111111").expect("first UUID"); + let second = Uuid::parse_str("22222222-2222-4222-8222-222222222222").expect("second UUID"); + let first_dir = write_ownership( + &runtime_root, + first, + BackendKind::Mock, + BackendOwnership::Running, + false, + ); + let second_dir = write_ownership( + &runtime_root, + second, + BackendKind::Mock, + BackendOwnership::Running, + false, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(DelayedSpawner { + delay: Duration::from_secs(6), + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("the second cleanup must share the first deadline"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{first}"), format!("backend:{second}")] + ); + assert!(!first_dir.exists()); + let second_ownership = read_ownership(&second_dir, second) + .await + .expect("second ownership retained"); + assert_eq!( + second_ownership.backend_ownership, + BackendOwnership::Running + ); + } + + #[tokio::test] + async fn reconcile_rejects_run_dir_without_ownership() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("missing ownership must stop startup"); + + assert!(error.to_string().contains(&instance_id.to_string())); + assert!(error.to_string().contains(OWNERSHIP_FILE)); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_removes_empty_slot_left_before_first_journal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("empty pre-journal slot has no external owner"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_reports_corrupt_and_unregistered_owners() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let corrupt_id = Uuid::new_v4(); + let corrupt_dir = runtime_root.join(corrupt_id.to_string()); + std::fs::create_dir_all(&corrupt_dir).expect("corrupt run dir"); + std::fs::write(corrupt_dir.join(OWNERSHIP_FILE), b"{not-json").expect("corrupt ownership"); + let unregistered_id = Uuid::new_v4(); + write_ownership( + &runtime_root, + unregistered_id, + BackendKind::Runc, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([corrupt_id, unregistered_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("ambiguous owners must stop startup"); + let message = error.to_string(); + + assert!(message.contains(&corrupt_id.to_string())); + assert!(message.contains("decode")); + assert!(message.contains(&unregistered_id.to_string())); + assert!(message.contains("no recovery spawner")); + assert!(events.lock().expect("events").is_empty()); + assert!(corrupt_dir.exists()); + } + + #[tokio::test] + async fn reconcile_persists_backend_cleanup_before_storage_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.fail_next_release(instance_id); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("storage failure must retain the journal"); + + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + let persisted = read_ownership(&run_dir, instance_id) + .await + .expect("persisted cleanup phase"); + assert_eq!(persisted.phase, RuntimeSlotPhase::PoolCleanup); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert!(persisted.storage_owned); + + events.lock().expect("events").clear(); + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("stopped backend needs no recovery spawner"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_continues_safe_cleanup_before_returning_error() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let good_id = Uuid::new_v4(); + let bad_id = Uuid::new_v4(); + let good_dir = write_ownership( + &runtime_root, + good_id, + BackendKind::Mock, + BackendOwnership::Stopped, + true, + ); + std::fs::create_dir_all(runtime_root.join(bad_id.to_string())).expect("bad run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([good_id, bad_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("one ambiguous slot must stop startup"); + + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{good_id}")] + ); + assert!(!good_dir.exists()); + assert!(runtime_root.join(bad_id.to_string()).exists()); + } + + #[tokio::test] + async fn reconcile_rejects_unknown_runtime_root_entry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + std::fs::write(runtime_root.join("unexpected"), b"not a slot").expect("entry"); + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("unknown entry must stop startup"); + + assert!(error.to_string().contains("unexpected non-directory")); + } + + #[tokio::test] + async fn handoff_and_cleanup_transitions_are_idempotent_and_token_bound() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::Handoff { token: owner_token }, + ); + + for _ in 0..2 { + finish_ownership_handoff( + &run_dir, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + owner_token, + ) + .await + .expect("idempotent lifecycle handoff"); + } + let wrong_token = Uuid::new_v4(); + begin_lifecycle_cleanup(&runtime_root, instance_id, BackendKind::Mock, wrong_token) + .await + .expect_err("wrong cleanup token"); + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("ownership after rejected token") + .phase, + RuntimeSlotPhase::LifecycleOwned { token: owner_token } + ); + + for _ in 0..2 { + begin_lifecycle_cleanup(&runtime_root, instance_id, BackendKind::Mock, owner_token) + .await + .expect("idempotent lifecycle cleanup"); + } + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("cleanup ownership") + .phase, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token } + ); + } + + #[tokio::test] + async fn reconcile_cleans_an_uncommitted_handoff_as_pool_owned() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::Handoff { + token: Uuid::new_v4(), + }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("uncommitted handoff remains pool-owned"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_lifecycle_marker_without_lifecycle_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::LifecycleOwned { + token: Uuid::new_v4(), + }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("lifecycle marker without state is ambiguous"); + + assert!(error.to_string().contains("lifecycle ownership")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_defers_a_valid_lifecycle_cleanup_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(run_dir, &tombstone).expect("tombstone runtime"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: Some(OperationKind::Destroy), + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("lifecycle must finish its own cleanup"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs new file mode 100644 index 0000000000..3fd811fea0 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Managed sandbox lifecycle and runtime ownership. + +mod checkpoint; +mod flush; +mod hibernate; +mod manager; +mod restore; +pub(crate) mod template; + +pub(crate) use flush::FlushLoop; +pub use hibernate::{HibernateSandbox, ResumeSandbox}; +pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; +pub use restore::{RestoreSandbox, RestoreSandboxResult}; diff --git a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs new file mode 100644 index 0000000000..1e1348351c --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable checkpoint capture, listing, and pruning. + +use blaze_core::backend::{BackendKind, SnapshotKind, SnapshotRequest}; +use blaze_core::checkpoint::{CheckpointInfo, CheckpointMetadata, CommitCheckpoint}; +use blaze_core::lifecycle::{OperationPhase, SandboxInstance, SandboxState}; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +impl SandboxManager { + /// Capture a self-contained checkpoint and resume the existing backend. + pub async fn checkpoint(&self, id: Uuid) -> Result { + let operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != SandboxState::Running { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected running", + instance.state + ))); + } + + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if !backend.supports_checkpoint_capture() || !self.storage.supports_checkpoint_capture() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} and configured storage do not support checkpoint capture", + backend.backend() + ))); + } + if backend.instance_id() != id || backend.backend() != instance.backend { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + let backend_version = backend.version().map(str::to_string); + if backend_version + .as_deref() + .is_some_and(|version| version.trim().is_empty()) + || (backend.backend() == BackendKind::Firecracker && backend_version.is_none()) + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not report a usable checkpoint version", + backend.backend() + ))); + } + self.require_live_backend(id, &backend).await?; + let storage = self.storage.reconstruct(&id.to_string()).await?; + let expose_guest_socket = !backend.guest_socket_path().as_os_str().is_empty(); + let network_slot = backend.network_slot(); + + let stage = self.checkpoints.begin(id).map_err(checkpoint_store_error)?; + let checkpoint_id = stage.id().to_string(); + let snapshot_path = match stage.artifact_path("vmstate.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + let memory_path = match stage.artifact_path("memory.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + let rootfs_path = match stage.artifact_path("rootfs.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + if let Err(error) = crate::failpoint::state("checkpoint-begin-state") { + let _ = self.checkpoints.abort(stage); + return Err(error); + } + if let Err(error) = instance.begin_checkpoint_operation(checkpoint_id.clone()) { + let _ = self.checkpoints.abort(stage); + return Err(error.into()); + } + if let Err(error) = self.persist_and_retain(instance.clone()) { + let _ = self.checkpoints.abort(stage); + return Err(error); + } + crate::failpoint::pause("checkpoint-after-begin").await; + + let paused = match crate::failpoint::backend("checkpoint-pause") { + Ok(()) => backend.pause().await, + Err(error) => Err(error), + }; + if let Err(error) = paused { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + if let Err(error) = instance + .transition(SandboxState::Paused) + .and_then(|_| instance.advance_checkpoint_phase(OperationPhase::CheckpointPaused)) + { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-paused-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error) + .await; + } + crate::failpoint::pause("checkpoint-after-pause").await; + + let snapshot = SnapshotRequest { + snapshot_path, + mem_path: memory_path, + kind: SnapshotKind::Full, + }; + let snapshot_result = match crate::failpoint::backend("checkpoint-snapshot") { + Ok(()) => backend.snapshot(snapshot).await, + Err(error) => Err(error), + }; + if let Err(error) = snapshot_result { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let flushed = match crate::failpoint::storage("checkpoint-storage-flush") { + Ok(()) => self.storage.flush_dirty(&storage).await, + Err(error) => Err(error), + }; + if let Err(error) = flushed { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let captured = match crate::failpoint::storage("checkpoint-rootfs-capture") { + Ok(()) => { + self.storage + .capture_checkpoint(&storage, &rootfs_path) + .await + } + Err(error) => Err(error), + }; + if let Err(error) = captured { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let parent = match self.checkpoints.read_head(id) { + Ok(parent) => parent, + Err(error) => { + return self + .finish_failed_unpublished_checkpoint( + id, + &backend, + &checkpoint_id, + checkpoint_store_error(error), + ) + .await; + } + }; + if let Err(error) = crate::failpoint::storage("checkpoint-publish") { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + let published = self + .checkpoints + .publish( + &stage, + CommitCheckpoint { + parent, + policy_name: instance.policy_name.clone(), + image_digest: instance.image_digest.clone(), + backend: instance.backend, + backend_version, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket, + network_slot, + }, + ) + .map_err(checkpoint_store_error); + let metadata = match published { + Ok(metadata) => metadata, + Err(error) => { + return self + .fail_published_checkpoint( + &backend, + &instance, + error, + "publication with uncertain outcome", + ) + .await; + } + }; + + if let Err(error) = instance.advance_checkpoint_phase(OperationPhase::CheckpointPublished) { + return self + .fail_published_checkpoint( + &backend, + &instance, + error.into(), + "published journal update", + ) + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-published-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .fail_published_checkpoint(&backend, &instance, error, "published state commit") + .await; + } + crate::failpoint::pause("checkpoint-after-publish-before-head").await; + + if let Err(error) = crate::failpoint::storage("checkpoint-head-update") { + return self + .finish_failed_published_checkpoint(id, &backend, error.into()) + .await; + } + if let Err(error) = self.checkpoints.set_head(id, &checkpoint_id) { + return self + .fail_published_checkpoint( + &backend, + &instance, + checkpoint_store_error(error), + "HEAD update with uncertain outcome", + ) + .await; + } + + if let Err(error) = instance.advance_checkpoint_phase(OperationPhase::CheckpointHeadUpdated) + { + return self + .fail_published_checkpoint(&backend, &instance, error.into(), "HEAD journal update") + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-head-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .fail_published_checkpoint(&backend, &instance, error, "HEAD state commit") + .await; + } + crate::failpoint::pause("checkpoint-after-head").await; + + let resumed = match crate::failpoint::backend("checkpoint-resume") { + Ok(()) => backend.resume().await, + Err(error) => Err(error), + }; + if let Err(error) = resumed { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint {checkpoint_id} became HEAD, but backend resume failed: {error}" + ))); + } + if let Err(error) = self.verify_backend_ready(id, &backend).await { + self.mark_recovery(id)?; + return Err(error); + } + + if let Err(error) = instance + .transition(SandboxState::Checkpointed) + .and_then(|_| instance.transition(SandboxState::Running)) + { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint runtime resumed, but lifecycle transition failed: {error}" + ))); + } + instance.last_checkpoint = Some(checkpoint_id); + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("checkpoint-final-state") + .and_then(|_| self.persist_and_retain(instance)) + { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint completed, but final lifecycle state could not be committed: {error}" + ))); + } + drop(operation); + Ok(metadata) + } + + /// List every committed checkpoint and its HEAD reachability. + pub async fn list_checkpoints(&self, id: Uuid) -> Result> { + let _operation = self.operation_lock(id).lock_owned().await; + self.get(id)?; + self.checkpoints.list(id).map_err(checkpoint_store_error) + } + + /// Remove branches not retained by HEAD or durable lifecycle references. + pub async fn prune_checkpoints(&self, id: Uuid) -> Result> { + let _operation = self.operation_lock(id).lock_owned().await; + let instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + self.checkpoints + .cleanup_transaction_artifacts(id) + .map_err(checkpoint_store_error)?; + let protected = instance.last_checkpoint.into_iter().collect::>(); + self.checkpoints + .prune_preserving(id, &protected) + .map_err(checkpoint_store_error) + } + + async fn finish_failed_unpublished_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + checkpoint_id: &str, + original: BlazeDaemonError, + ) -> Result { + let compensation = self + .resume_and_clear_checkpoint(id, backend, Some(checkpoint_id)) + .await; + match compensation { + Ok(()) => Err(original), + Err(compensation) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{original}; checkpoint compensation failed: {compensation}" + ))), + } + } + + async fn finish_failed_published_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + original: BlazeDaemonError, + ) -> Result { + let compensation = self.resume_and_clear_checkpoint(id, backend, None).await; + match compensation { + Ok(()) => Err(original), + Err(compensation) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{original}; checkpoint compensation failed: {compensation}" + ))), + } + } + + async fn fail_published_checkpoint( + &self, + backend: &DynBackendInstance, + instance: &SandboxInstance, + original: BlazeDaemonError, + boundary: &str, + ) -> Result { + let resume = self.resume_backend(backend).await; + let recovery = self.mark_instance_recovery(instance.clone()); + Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint {boundary} failed: {original}{}{}", + resume + .err() + .map(|error| format!("; backend resume failed: {error}")) + .unwrap_or_default(), + recovery + .err() + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))) + } + + async fn resume_and_clear_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + staging_checkpoint_id: Option<&str>, + ) -> Result<()> { + if let Err(error) = self.resume_backend(backend).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "backend resume failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(checkpoint_id) = staging_checkpoint_id + && let Err(error) = self.checkpoints.abort_staging(id, checkpoint_id) + { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint staging cleanup failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + let mut instance = self.get(id)?; + if instance.state == SandboxState::Paused { + instance.transition(SandboxState::Running)?; + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint compensation state commit failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + Ok(()) + } + + async fn resume_backend(&self, backend: &DynBackendInstance) -> Result<()> { + match crate::failpoint::backend("checkpoint-compensation-resume") { + Ok(()) => backend.resume().await?, + Err(error) => return Err(error.into()), + } + self.verify_backend_ready(backend.instance_id(), backend) + .await + } + + async fn require_live_backend(&self, id: Uuid, backend: &DynBackendInstance) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited before checkpoint capture \ + (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))) + } + Err(error) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))) + } + } + } + + async fn verify_backend_ready(&self, id: Uuid, backend: &DynBackendInstance) -> Result<()> { + self.require_live_backend(id, backend).await?; + self.wait_for_guest_ready(backend, "checkpoint-guest-ready") + .await?; + self.require_live_backend(id, backend).await + } +} + +fn checkpoint_store_error(error: impl std::fmt::Display) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("checkpoint store: {error}")) +} diff --git a/src/blaze/crates/blazed/src/sandbox/flush.rs b/src/blaze/crates/blazed/src/sandbox/flush.rs new file mode 100644 index 0000000000..fc29269725 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/flush.rs @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Periodic synchronization of provider-owned sandbox storage. + +use std::sync::Arc; +use std::time::Duration; + +use blaze_core::lifecycle::{BackendOwnership, SandboxState}; +use tokio::task::JoinHandle; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +#[cfg(test)] +use tokio::sync::Notify; + +use crate::error::{BlazeDaemonError, Result}; + +use super::manager::SandboxManager; + +const FLUSH_LOOP_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Supervised periodic synchronization task. +pub(crate) struct FlushLoop { + cancellation: CancellationToken, + task: Option>, + #[cfg(test)] + started: Arc, +} + +impl FlushLoop { + /// Wait for an early worker exit and report it as a daemon failure. + pub(crate) async fn observe_exit(&mut self) -> Result<()> { + self.join().await?; + Err(BlazeDaemonError::Internal( + "provider synchronization task exited unexpectedly".to_string(), + )) + } + + /// Request cooperative shutdown and join the worker before returning. + pub(crate) async fn shutdown(&mut self) -> Result<()> { + self.cancellation.cancel(); + let Some(task) = self.task.as_mut() else { + return Ok(()); + }; + match tokio::time::timeout(FLUSH_LOOP_SHUTDOWN_TIMEOUT, task).await { + Ok(result) => { + self.task.take(); + result.map_err(join_error) + } + Err(_) => { + let task = self.task.as_mut().expect("flush task is present"); + task.abort(); + let result = task.await; + self.task.take(); + match result { + Err(error) if error.is_cancelled() => Err(BlazeDaemonError::Internal( + "provider synchronization task exceeded its shutdown deadline".to_string(), + )), + Err(error) => Err(join_error(error)), + Ok(()) => Err(BlazeDaemonError::Internal( + "provider synchronization task ignored cancellation".to_string(), + )), + } + } + } + } + + async fn join(&mut self) -> Result<()> { + let Some(task) = self.task.as_mut() else { + return Ok(()); + }; + let result = task.await; + self.task.take(); + result.map_err(join_error) + } + + #[cfg(test)] + async fn wait_started(&self) { + self.started.notified().await; + } +} + +impl Drop for FlushLoop { + fn drop(&mut self) { + self.cancellation.cancel(); + if let Some(task) = self.task.as_ref() { + task.abort(); + } + } +} + +fn join_error(error: tokio::task::JoinError) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!( + "provider synchronization task join failed: {error}" + )) +} + +/// Counters emitted for one synchronization sweep. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FlushSummary { + /// Running records selected before any provider await. + pub(crate) selected: usize, + /// Provider calls that completed successfully. + pub(crate) flushed: usize, + /// Records that stopped being Running before their operation lock won. + pub(crate) skipped: usize, + /// Invalid owners and provider failures isolated from the rest of the sweep. + pub(crate) failed: usize, +} + +enum FlushAttempt { + Flushed, + Skipped, + Cancelled, +} + +impl SandboxManager { + /// Start a cancellable periodic storage synchronization worker. + /// + /// The first sweep starts after one complete interval. Missed ticks are + /// skipped instead of being queued behind a slow sweep. + pub(crate) fn start_flush_loop( + self: &Arc, + interval: Duration, + attempt_timeout: Duration, + ) -> FlushLoop { + let manager = self.clone(); + let cancellation = CancellationToken::new(); + let worker_cancellation = cancellation.clone(); + #[cfg(test)] + let started = Arc::new(Notify::new()); + #[cfg(test)] + let worker_started = started.clone(); + tracing::info!( + interval_secs = interval.as_secs_f64(), + attempt_timeout_secs = attempt_timeout.as_secs_f64(), + "starting provider synchronization task" + ); + let task = tokio::spawn(async move { + let first_tick = Instant::now() + interval; + let mut ticker = tokio::time::interval_at(first_tick, interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + #[cfg(test)] + worker_started.notify_one(); + loop { + tokio::select! { + biased; + _ = worker_cancellation.cancelled() => break, + _ = ticker.tick() => { + let summary = manager + .flush_all_until(&worker_cancellation, attempt_timeout) + .await; + let Some(summary) = summary else { + break; + }; + tracing::debug!( + selected = summary.selected, + flushed = summary.flushed, + skipped = summary.skipped, + failed = summary.failed, + "provider synchronization sweep completed" + ); + } + } + } + tracing::info!("provider synchronization task stopped"); + }); + FlushLoop { + cancellation, + task: Some(task), + #[cfg(test)] + started, + } + } + + #[cfg(test)] + async fn flush_all(&self, attempt_timeout: Duration) -> FlushSummary { + self.flush_all_until(&CancellationToken::new(), attempt_timeout) + .await + .expect("uncancelled sweep") + } + + async fn flush_all_until( + &self, + cancellation: &CancellationToken, + attempt_timeout: Duration, + ) -> Option { + let running_ids = match self.list() { + Ok(instances) => instances + .into_iter() + .filter_map(|instance| { + (instance.state == SandboxState::Running).then_some(instance.id) + }) + .collect::>(), + Err(error) => { + tracing::error!(%error, "cannot select provider synchronization candidates"); + return Some(FlushSummary { + failed: 1, + ..FlushSummary::default() + }); + } + }; + let mut summary = FlushSummary { + selected: running_ids.len(), + ..FlushSummary::default() + }; + for id in running_ids { + match self + .flush_if_running(id, cancellation, attempt_timeout) + .await + { + Ok(FlushAttempt::Flushed) => summary.flushed += 1, + Ok(FlushAttempt::Skipped) => summary.skipped += 1, + Ok(FlushAttempt::Cancelled) => return None, + Err(error) => { + summary.failed += 1; + tracing::warn!( + sandbox_id = %id, + %error, + "sandbox provider synchronization failed" + ); + } + } + } + Some(summary) + } + + async fn flush_if_running( + &self, + id: Uuid, + cancellation: &CancellationToken, + attempt_timeout: Duration, + ) -> Result { + let operation_lock = self.operation_lock(id); + let _operation = tokio::select! { + biased; + _ = cancellation.cancelled() => return Ok(FlushAttempt::Cancelled), + operation = operation_lock.lock() => operation, + }; + let instance = self.get(id)?; + if instance.state != SandboxState::Running { + return Ok(FlushAttempt::Skipped); + } + if let Some(operation) = instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running with unfinished {} operation", + operation.kind + ))); + } + if instance.backend_ownership != BackendOwnership::Running { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running with {} backend ownership", + format!("{:?}", instance.backend_ownership).to_lowercase() + ))); + } + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running without a backend owner" + )) + })?; + match backend.try_wait().await? { + None => {} + Some(status) => { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend already exited: {status:?}" + ))); + } + } + let storage = self.reconstruct_storage(id).await.map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has no complete storage owner: {error}" + )) + })?; + tokio::select! { + biased; + _ = cancellation.cancelled() => Ok(FlushAttempt::Cancelled), + result = tokio::time::timeout(attempt_timeout, self.flush_storage(&storage)) => { + match result { + Ok(result) => { + result?; + Ok(FlushAttempt::Flushed) + } + Err(_) => Err(BlazeDaemonError::Internal(format!( + "provider synchronization for {id} timed out after {:.3} seconds", + attempt_timeout.as_secs_f64() + ))), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use blaze_core::backend::{BackendKind, SpawnRequest}; + use blaze_core::config::RuntimeTemplateSection; + use blaze_core::error::{BlazeError, Result as CoreResult}; + use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, + }; + use blaze_core::policy::{BackendConfigs, WorkloadClass}; + use blaze_core::pool::PoolManager; + use blaze_core::storage::{ + AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + }; + use tokio::sync::Notify; + + use crate::file_provider::FileStorageProvider; + use crate::sandbox::manager::{SandboxManagerInit, SandboxManagerResources}; + use crate::sandbox::template::RuntimeTemplateCatalog; + use crate::spawner::{BackendSpawner, MockSpawner, SpawnerRegistry}; + + use super::*; + + struct RecordingStorage { + inner: FileStorageProvider, + instances: PathBuf, + calls: Mutex>, + call_recorded: Notify, + failures: Mutex>, + block_next: AtomicBool, + started: Notify, + } + + impl RecordingStorage { + fn new(images: PathBuf, instances: PathBuf) -> Self { + Self { + inner: FileStorageProvider::with_images(images, instances.clone()), + instances, + calls: Mutex::new(Vec::new()), + call_recorded: Notify::new(), + failures: Mutex::new(HashSet::new()), + block_next: AtomicBool::new(false), + started: Notify::new(), + } + } + + fn calls(&self) -> Vec { + self.calls.lock().expect("calls").clone() + } + + async fn wait_for_calls(&self, expected: usize) { + loop { + let call_recorded = self.call_recorded.notified(); + if self.calls.lock().expect("calls").len() >= expected { + return; + } + call_recorded.await; + } + } + + fn fail(&self, id: Uuid) { + self.failures + .lock() + .expect("failures") + .insert(id.to_string()); + } + + fn block_once(&self) { + self.block_next.store(true, Ordering::Release); + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> CoreResult { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> CoreResult<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> CoreResult<()> { + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> CoreResult { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> CoreResult<()> { + self.calls.lock().expect("calls").push(slot.id.clone()); + self.call_recorded.notify_waiters(); + if self.block_next.swap(false, Ordering::AcqRel) { + self.started.notify_one(); + std::future::pending::<()>().await; + } + if self.failures.lock().expect("failures").contains(&slot.id) { + return Err(BlazeError::StorageError { + msg: format!("injected provider synchronization failure for {}", slot.id), + }); + } + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> CoreResult { + self.inner.drain_pool().await + } + } + + fn manager( + temp: &Path, + storage: Arc, + ) -> (Arc, SandboxManagerResources) { + let state_dir = temp.join("state"); + let images = temp.join("images"); + let instances = temp.join("instances"); + for directory in [&state_dir, &images, &instances] { + std::fs::create_dir_all(directory).expect("test directory"); + } + let mut spawners = SpawnerRegistry::new(); + spawners.insert(BackendKind::Mock, Arc::new(MockSpawner)); + let runtime_templates = RuntimeTemplateCatalog::open(&RuntimeTemplateSection { + dir: temp.join("runtime-templates"), + ..RuntimeTemplateSection::default() + }) + .expect("runtime template catalog"); + let (manager, resources) = SandboxManager::new(SandboxManagerInit { + instances: HashMap::new(), + pool: PoolManager::new(), + spawners, + active_backend: BackendKind::Mock, + storage, + state_dir, + rootfs_size: 64, + mem_size: 32, + pool_size: 0, + prefork: false, + default_warm_ttl: "30m".to_string(), + gc_interval: "5m".to_string(), + runtime_templates, + }) + .expect("manager"); + (Arc::new(manager), resources) + } + + async fn insert_running( + manager: &SandboxManager, + resources: &SandboxManagerResources, + storage: &RecordingStorage, + id: Uuid, + acquire_storage: bool, + insert_backend: bool, + active_operation: bool, + ) { + let slot = if acquire_storage { + Some( + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("slot"), + ) + } else { + None + }; + let mut metadata = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:flush-test".into(), + StartPath::Cold, + "flush-test".into(), + ); + metadata.id = id; + metadata + .transition(SandboxState::Creating) + .expect("pending to creating"); + metadata + .transition(SandboxState::Running) + .expect("creating to running"); + metadata.backend_ownership = BackendOwnership::Running; + if active_operation { + metadata + .begin_operation(OperationKind::Create) + .expect("active operation"); + } + resources + .instances + .lock() + .expect("instances") + .insert(id, metadata); + if insert_backend { + let slot = slot.clone().unwrap_or_else(|| StorageSlot { + id: id.to_string(), + rootfs_path: PathBuf::new(), + mem_path: PathBuf::new(), + mem_diff_path: PathBuf::new(), + rootfs_diff_path: PathBuf::new(), + instance_dir: PathBuf::new(), + }); + let owner = MockSpawner + .spawn(SpawnRequest { + instance_id: id, + run_dir: storage.instances.join(id.to_string()).join("runtime"), + binary_path: PathBuf::new(), + storage: slot, + backend: BackendConfigs::default(), + vm: None, + }) + .await + .expect("mock owner"); + manager + .insert_backend_owner(id, owner) + .expect("register owner"); + } + } + + async fn settle() { + for _ in 0..8 { + tokio::task::yield_now().await; + } + } + + #[tokio::test] + async fn sweep_flushes_running_records_and_isolates_failures() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let failing = Uuid::new_v4(); + let succeeding = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, failing, true, true, false).await; + insert_running( + &manager, &resources, &storage, succeeding, true, true, false, + ) + .await; + storage.fail(failing); + + let summary = manager.flush_all(Duration::from_secs(1)).await; + + assert_eq!( + summary, + FlushSummary { + selected: 2, + flushed: 1, + skipped: 0, + failed: 1, + } + ); + assert_eq!( + storage.calls().into_iter().collect::>(), + HashSet::from([failing.to_string(), succeeding.to_string()]) + ); + } + + #[tokio::test] + async fn sweep_reports_incomplete_running_owners_without_flushing() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + true, + false, + false, + ) + .await; + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + false, + true, + false, + ) + .await; + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + true, + true, + true, + ) + .await; + + assert_eq!( + manager.flush_all(Duration::from_secs(1)).await, + FlushSummary { + selected: 3, + flushed: 0, + skipped: 0, + failed: 3, + } + ); + assert!(storage.calls().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn timed_out_provider_call_keeps_slot_retryable() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + storage.block_once(); + + let first = { + let manager = manager.clone(); + tokio::spawn(async move { manager.flush_all(Duration::from_secs(5)).await }) + }; + storage.started.notified().await; + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(first.await.expect("sweep").failed, 1); + + let second = manager.flush_all(Duration::from_secs(5)).await; + assert_eq!(second.flushed, 1); + assert_eq!(storage.calls(), vec![id.to_string(), id.to_string()]); + } + + #[tokio::test] + async fn sweep_waits_for_operation_lock_and_rechecks_state() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + let lock = manager.operation_lock(id); + let guard = lock.lock().await; + let sweep = { + let manager = manager.clone(); + tokio::spawn(async move { manager.flush_all(Duration::from_secs(1)).await }) + }; + settle().await; + resources + .instances + .lock() + .expect("instances") + .get_mut(&id) + .expect("metadata") + .transition(SandboxState::RecoveryRequired) + .expect("running to recovery-required"); + drop(guard); + + assert_eq!(sweep.await.expect("sweep").skipped, 1); + assert!(storage.calls().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn periodic_worker_delays_first_tick_and_stops_before_next_sweep() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + + let mut worker = manager.start_flush_loop(Duration::from_secs(10), Duration::from_secs(5)); + worker.wait_started().await; + settle().await; + assert!(storage.calls().is_empty()); + tokio::time::advance(Duration::from_secs(10)).await; + storage.wait_for_calls(1).await; + assert_eq!(storage.calls(), vec![id.to_string()]); + + worker.shutdown().await.expect("worker shutdown"); + tokio::time::advance(Duration::from_secs(30)).await; + settle().await; + assert_eq!(storage.calls(), vec![id.to_string()]); + } + + #[tokio::test(start_paused = true)] + async fn worker_shutdown_cancels_a_sweep_waiting_for_operation_lock() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + let lock = manager.operation_lock(id); + let guard = lock.lock().await; + let mut worker = manager.start_flush_loop(Duration::from_secs(10), Duration::from_secs(5)); + worker.wait_started().await; + tokio::time::advance(Duration::from_secs(10)).await; + settle().await; + + worker.shutdown().await.expect("worker shutdown"); + drop(guard); + assert!(storage.calls().is_empty()); + } + + #[tokio::test] + async fn supervisor_reports_an_unexpected_worker_exit() { + let cancellation = CancellationToken::new(); + let mut worker = FlushLoop { + cancellation, + task: Some(tokio::spawn(async {})), + started: Arc::new(Notify::new()), + }; + + let error = worker + .observe_exit() + .await + .expect_err("early worker exit must stop the daemon"); + + assert!(error.to_string().contains("exited unexpectedly")); + worker + .shutdown() + .await + .expect("finished worker is already joined"); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/hibernate.rs b/src/blaze/crates/blazed/src/sandbox/hibernate.rs new file mode 100644 index 0000000000..4da0878b20 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/hibernate.rs @@ -0,0 +1,1062 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable hibernation and restartable resume for managed sandboxes. + +use std::collections::BTreeSet; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use blaze_core::backend::{BackendKind, RestoreRequest, SnapshotKind, SnapshotRequest}; +use blaze_core::checkpoint::CheckpointArtifact; +use blaze_core::lifecycle::{BackendOwnership, OperationPhase, SandboxInstance, SandboxState}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +const HIBERNATE_FORMAT_VERSION: u32 = 1; +const HIBERNATE_DIRECTORY: &str = "hibernate"; +const MANIFEST_ARTIFACT: &str = "manifest.json"; +const MEMORY_ARTIFACT: &str = "memory.snap"; +const VMSTATE_ARTIFACT: &str = "vmstate.snap"; +const REQUIRED_ARTIFACTS: [&str; 2] = [VMSTATE_ARTIFACT, MEMORY_ARTIFACT]; + +/// Inputs resolved from the current daemon configuration before hibernation. +#[derive(Debug, Clone)] +pub struct HibernateSandbox { + /// Current executable for the sandbox backend. + pub binary_path: PathBuf, +} + +/// Inputs resolved from the current daemon configuration before resume. +#[derive(Debug, Clone)] +pub struct ResumeSandbox { + /// Current executable for the sandbox backend. + pub binary_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct HibernateManifest { + format_version: u32, + sandbox_id: Uuid, + policy_name: String, + image_digest: String, + backend: BackendKind, + backend_version: Option, + snapshot_kind: SnapshotKind, + expose_guest_socket: bool, + network_slot: Option, + artifacts: Vec, +} + +impl SandboxManager { + /// Stop a running backend after publishing durable hibernation artifacts. + pub fn hibernate( + &self, + id: Uuid, + request: HibernateSandbox, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + require_quiescent_state(&instance, SandboxState::Running)?; + + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if backend.instance_id() != id || backend.backend() != instance.backend { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + require_backend_live(id, &backend).await.map_err(|error| { + let recovery = self.mark_instance_recovery(instance.clone()).err(); + with_recovery_error(error, recovery) + })?; + if !backend.supports_checkpoint_capture() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support hibernation", + backend.backend() + ))); + } + + let spawner = self.spawner(instance.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no resume adapter for {}", + instance.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support resume", + instance.backend + )) + })?; + let backend_version = backend.version().map(str::to_string); + if capability.backend != instance.backend + || capability.version != backend_version + || capability.snapshot_kind != SnapshotKind::Full + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend capture identity does not match its resume adapter" + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + let expose_guest_socket = !backend.guest_socket_path().as_os_str().is_empty(); + let network_slot = backend.network_slot(); + let hibernate_dir = self.hibernate_dir(id); + if let Err(error) = prepare_hibernate_directory(&hibernate_dir).await { + let recovery = self.mark_instance_recovery(instance).err(); + return Err(with_recovery_error(error, recovery)); + } + + instance.begin_hibernate_operation()?; + instance.transition(SandboxState::Hibernating)?; + crate::failpoint::state("hibernate-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("hibernate-after-begin").await; + + let paused = match crate::failpoint::backend("hibernate-pause") { + Ok(()) => backend.pause().await, + Err(error) => Err(error), + }; + if let Err(error) = paused { + return Err(self + .compensate_hibernate( + instance, + &backend, + None, + format!("backend pause failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernatePaused) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-paused-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .compensate_hibernate( + instance, + &backend, + None, + format!("paused-state commit failed: {error}"), + ) + .await); + } + + let staging_dir = self.hibernate_staging_dir(id); + if let Err(error) = tokio::fs::create_dir(&staging_dir).await { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("staging directory creation failed: {error}"), + ) + .await); + } + let snapshot_path = staging_dir.join(VMSTATE_ARTIFACT); + let memory_path = staging_dir.join(MEMORY_ARTIFACT); + let snapshot = match crate::failpoint::backend("hibernate-snapshot") { + Ok(()) => { + backend + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: memory_path.clone(), + kind: SnapshotKind::Full, + }) + .await + } + Err(error) => Err(error), + }; + match snapshot { + Ok(snapshot) + if snapshot.snapshot_path == snapshot_path + && snapshot.mem_path == memory_path => {} + Ok(snapshot) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!( + "backend returned unexpected hibernation artifacts ({}, {})", + snapshot.snapshot_path.display(), + snapshot.mem_path.display() + ), + ) + .await); + } + Err(error) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("snapshot capture failed: {error}"), + ) + .await); + } + } + let flushed = match crate::failpoint::storage("hibernate-storage-flush") { + Ok(()) => self.storage.flush_dirty(&storage).await, + Err(error) => Err(error), + }; + if let Err(error) = flushed { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("storage flush failed: {error}"), + ) + .await); + } + + let manifest = match build_hibernate_manifest( + &staging_dir, + &instance, + capability.version, + expose_guest_socket, + network_slot, + ) + .await + { + Ok(manifest) => manifest, + Err(error) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact hashing failed: {error}"), + ) + .await); + } + }; + if let Err(error) = write_and_sync_manifest(&staging_dir, &manifest).await { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact publication failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernateArtifactsSynced) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-artifacts-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact-state commit failed: {error}"), + ) + .await); + } + + let stopped = match crate::failpoint::backend("hibernate-backend-stop") { + Ok(()) => backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + return Err(self.fail_hibernate_after_stop( + instance, + format!("backend termination failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernateBackendStopped) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-stopped-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_hibernate_after_stop( + instance, + format!("backend stopped but lifecycle commit failed: {error}"), + )); + } + self.remove_backend_owner(id); + crate::failpoint::pause("hibernate-after-stop").await; + + let backup_dir = self.hibernate_backup_dir(id); + if hibernate_dir.exists() { + if let Err(error) = tokio::fs::rename(&hibernate_dir, &backup_dir).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("previous hibernation backup failed: {error}"), + )); + } + if let Err(error) = sync_directory(self.instance_dir(id)).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("previous hibernation backup sync failed: {error}"), + )); + } + } + let published = match crate::failpoint::storage("hibernate-publish") { + Ok(()) => tokio::fs::rename(&staging_dir, &hibernate_dir) + .await + .map_err(BlazeDaemonError::from), + Err(error) => Err(error.into()), + }; + if let Err(error) = published { + return Err(self.fail_hibernate_after_stop( + instance, + format!("hibernate directory publication failed: {error}"), + )); + } + if let Err(error) = sync_directory(self.instance_dir(id)).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("hibernate directory sync failed: {error}"), + )); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernatePublished) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-published-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_hibernate_after_stop( + instance, + format!("published-state commit failed: {error}"), + )); + } + + let recovery = instance.clone(); + instance.transition(SandboxState::Hibernated)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("hibernate-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_hibernate_after_stop( + recovery, + format!("final hibernated-state commit failed: {error}"), + )); + } + if backup_dir.exists() + && let Err(error) = remove_directory_and_sync(&backup_dir).await + { + tracing::warn!( + instance = %id, + %error, + "obsolete hibernation backup retained for later cleanup" + ); + } + Ok(instance) + }) + } + + /// Start a backend from verified hibernation artifacts. + pub fn resume( + &self, + id: Uuid, + request: ResumeSandbox, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + require_quiescent_state(&instance, SandboxState::Hibernated)?; + if instance.backend_ownership != BackendOwnership::Stopped { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is hibernated with unresolved backend ownership" + ))); + } + if self.backend_owner(id).is_some() { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is hibernated but still retains a backend owner" + ))); + } + + let hibernate_dir = self.hibernate_dir(id); + let manifest = load_and_verify_manifest(&hibernate_dir) + .await + .map_err(|error| { + let recovery = self.mark_instance_recovery(instance.clone()).err(); + with_recovery_error( + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} hibernation artifacts are invalid: {error}" + )), + recovery, + ) + })?; + if let Err(error) = validate_manifest_identity(&manifest, &instance) { + let recovery = self.mark_instance_recovery(instance).err(); + return Err(with_recovery_error(error, recovery)); + } + let spawner = self.spawner(instance.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no resume adapter for {}", + instance.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support resume", + instance.backend + )) + })?; + if capability.backend != manifest.backend + || capability.version != manifest.backend_version + || capability.snapshot_kind != manifest.snapshot_kind + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} hibernation image is incompatible with the current resume adapter" + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + + instance.begin_resume_operation()?; + instance.transition(SandboxState::Resuming)?; + crate::failpoint::state("resume-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("resume-after-begin").await; + + let run_dir = self.instance_dir(id); + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self.fail_resume_without_owner( + instance, + format!("resume ownership preparation failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendStarting) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-starting-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_without_owner( + instance, + format!("resume ownership intent commit failed: {error}"), + )); + } + + let restored = match crate::failpoint::backend("resume-backend-start") { + Ok(()) => { + spawner + .restore(RestoreRequest { + instance_id: id, + run_dir, + binary_path: request.binary_path, + storage, + snapshot_path: hibernate_dir.join(VMSTATE_ARTIFACT), + mem_path: hibernate_dir.join(MEMORY_ARTIFACT), + checkpoint_backend: manifest.backend, + expected_version: manifest.backend_version.clone(), + snapshot_kind: manifest.snapshot_kind, + expose_guest_socket: manifest.expose_guest_socket, + network_slot: manifest.network_slot, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let restored = match restored { + Ok(owner) => owner, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + let _ = self.retain_backend(id, owner); + instance.backend_ownership = BackendOwnership::Running; + return Err(self.fail_resume_with_owner( + instance, + format!("resume backend start failed: {source}"), + )); + } + return Err(self.fail_resume_without_owner( + instance, + format!("resume backend start failed: {source}"), + )); + } + }; + instance.backend_ownership = BackendOwnership::Running; + if let Some(error) = self.retain_backend(id, restored.clone()) { + return Err(self.fail_resume_with_owner(instance, error)); + } + if restored.instance_id() != id + || restored.backend() != manifest.backend + || restored.version().map(str::to_string) != manifest.backend_version + { + return Err(self + .abort_resumed_backend( + instance, + &restored, + "restored backend identity does not match the hibernation manifest" + .to_string(), + ) + .await); + } + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendStarted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-started-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_with_owner( + instance, + format!("restored backend ownership commit failed: {error}"), + )); + } + if let Err(error) = self + .verify_resumed_backend(id, &restored, manifest.expose_guest_socket) + .await + { + return Err(self + .abort_resumed_backend( + instance, + &restored, + format!("restored backend readiness failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendReady) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-ready-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_with_owner( + instance, + format!("restored backend readiness commit failed: {error}"), + )); + } + + let recovery = instance.clone(); + instance.transition(SandboxState::Running)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("resume-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_resume_with_owner( + recovery, + format!("final running-state commit failed: {error}"), + )); + } + Ok(instance) + }) + } + + async fn compensate_hibernate( + &self, + mut instance: SandboxInstance, + backend: &DynBackendInstance, + staging_dir: Option<&Path>, + cause: String, + ) -> BlazeDaemonError { + let resumed = match crate::failpoint::backend("hibernate-compensation-resume") { + Ok(()) => backend.resume().await, + Err(error) => Err(error), + }; + if let Err(error) = resumed { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; backend resume compensation failed: {error}"), + ); + } + if let Err(error) = self + .verify_resumed_backend( + instance.id, + backend, + !backend.guest_socket_path().as_os_str().is_empty(), + ) + .await + { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; resumed backend readiness failed: {error}"), + ); + } + if let Some(staging_dir) = staging_dir + && let Err(error) = remove_directory_and_sync(staging_dir).await + { + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; staging cleanup failed: {error}"), + ); + } + let recovery = instance.clone(); + instance.backend_ownership = BackendOwnership::Running; + if let Err(error) = instance.transition(SandboxState::Running) { + return self.fail_hibernate_after_stop( + recovery, + format!("{cause}; running-state compensation failed: {error}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + return self.fail_hibernate_after_stop( + recovery, + format!("{cause}; running-state compensation commit failed: {error}"), + ); + } + BlazeDaemonError::Internal(cause) + } + + async fn verify_resumed_backend( + &self, + id: Uuid, + backend: &DynBackendInstance, + expose_guest_socket: bool, + ) -> Result<()> { + require_backend_live(id, backend).await?; + if expose_guest_socket { + self.wait_for_guest_ready(backend, "resume-guest-ready") + .await?; + } + require_backend_live(id, backend).await + } + + async fn abort_resumed_backend( + &self, + mut instance: SandboxInstance, + backend: &DynBackendInstance, + cause: String, + ) -> BlazeDaemonError { + let stopped = match crate::failpoint::backend("resume-backend-stop") { + Ok(()) => backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_resume_with_owner( + instance, + format!("{cause}; restored backend termination failed: {error}"), + ); + } + self.remove_backend_owner(instance.id); + instance.backend_ownership = BackendOwnership::Stopped; + self.fail_resume_without_owner(instance, cause) + } + + fn fail_resume_without_owner( + &self, + mut instance: SandboxInstance, + cause: String, + ) -> BlazeDaemonError { + let recovery = instance.clone(); + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Hibernated) { + return self.fail_resume_with_owner( + recovery, + format!("{cause}; hibernated-state compensation failed: {error}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + return self.fail_resume_with_owner( + recovery, + format!("{cause}; hibernated-state compensation commit failed: {error}"), + ); + } + BlazeDaemonError::Internal(cause) + } + + fn fail_hibernate_after_stop( + &self, + instance: SandboxInstance, + cause: String, + ) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "hibernate {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } + + fn fail_resume_with_owner(&self, instance: SandboxInstance, cause: String) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "resume {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } + + pub(super) async fn cleanup_hibernate_artifacts(&self, id: Uuid) -> Result<()> { + let instance_dir = self.instance_dir(id); + let mut entries = match tokio::fs::read_dir(&instance_dir).await { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name == HIBERNATE_DIRECTORY + || (name.starts_with(".hibernate.") + && (name.ends_with(".tmp") || name.ends_with(".bak"))) + { + remove_directory_and_sync(&entry.path()).await?; + } + } + Ok(()) + } + + fn instance_dir(&self, id: Uuid) -> PathBuf { + self.state_dir.join(id.to_string()) + } + + fn hibernate_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id).join(HIBERNATE_DIRECTORY) + } + + fn hibernate_staging_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id) + .join(format!(".hibernate.{}.tmp", Uuid::new_v4())) + } + + fn hibernate_backup_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id) + .join(format!(".hibernate.{}.bak", Uuid::new_v4())) + } +} + +fn require_quiescent_state(instance: &SandboxInstance, expected: SandboxState) -> Result<()> { + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {} has unfinished {} operation", + instance.id, journal.kind + ))); + } + if instance.state != expected { + return Err(BlazeDaemonError::Conflict(format!( + "instance {} is {}, expected {expected}", + instance.id, instance.state + ))); + } + Ok(()) +} + +async fn require_backend_live(id: Uuid, backend: &DynBackendInstance) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))), + Err(error) => Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))), + } +} + +async fn build_hibernate_manifest( + directory: &Path, + instance: &SandboxInstance, + backend_version: Option, + expose_guest_socket: bool, + network_slot: Option, +) -> Result { + let mut artifacts = Vec::with_capacity(REQUIRED_ARTIFACTS.len()); + for name in REQUIRED_ARTIFACTS { + artifacts.push(hash_artifact(&directory.join(name), name).await?); + } + Ok(HibernateManifest { + format_version: HIBERNATE_FORMAT_VERSION, + sandbox_id: instance.id, + policy_name: instance.policy_name.clone(), + image_digest: instance.image_digest.clone(), + backend: instance.backend, + backend_version, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket, + network_slot, + artifacts, + }) +} + +async fn write_and_sync_manifest(directory: &Path, manifest: &HibernateManifest) -> Result<()> { + for name in REQUIRED_ARTIFACTS { + tokio::fs::File::open(directory.join(name)) + .await? + .sync_all() + .await?; + } + let mut encoded = serde_json::to_vec_pretty(manifest)?; + encoded.push(b'\n'); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join(MANIFEST_ARTIFACT)) + .await?; + file.write_all(&encoded).await?; + file.sync_all().await?; + drop(file); + sync_directory(directory.to_path_buf()).await +} + +async fn load_and_verify_manifest(directory: &Path) -> Result { + let manifest_path = directory.join(MANIFEST_ARTIFACT); + let manifest_metadata = tokio::fs::symlink_metadata(&manifest_path).await?; + if manifest_metadata.file_type().is_symlink() || !manifest_metadata.is_file() { + return Err(BlazeDaemonError::Internal(format!( + "hibernate manifest {} is not a regular file", + manifest_path.display() + ))); + } + let manifest: HibernateManifest = + serde_json::from_slice(&tokio::fs::read(&manifest_path).await?)?; + if manifest.format_version != HIBERNATE_FORMAT_VERSION { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "unsupported hibernation format {}", + manifest.format_version + ))); + } + if manifest.snapshot_kind != SnapshotKind::Full { + return Err(BlazeDaemonError::UnsupportedOperation( + "hibernation image is not self-contained".to_string(), + )); + } + let expected_names = REQUIRED_ARTIFACTS + .into_iter() + .map(str::to_string) + .collect::>(); + let observed_names = manifest + .artifacts + .iter() + .map(|artifact| artifact.name.clone()) + .collect::>(); + if observed_names != expected_names || manifest.artifacts.len() != REQUIRED_ARTIFACTS.len() { + return Err(BlazeDaemonError::Internal( + "hibernation manifest has an invalid artifact set".to_string(), + )); + } + let directory_names = read_directory_names(directory).await?; + let expected_directory_names = [ + MANIFEST_ARTIFACT.to_string(), + MEMORY_ARTIFACT.to_string(), + VMSTATE_ARTIFACT.to_string(), + ] + .into_iter() + .collect::>(); + if directory_names != expected_directory_names { + return Err(BlazeDaemonError::Internal( + "hibernation directory has an unexpected file set".to_string(), + )); + } + for artifact in &manifest.artifacts { + let path = directory.join(&artifact.name); + let metadata = tokio::fs::symlink_metadata(&path).await?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(BlazeDaemonError::Internal(format!( + "hibernate artifact {} is not a regular file", + path.display() + ))); + } + let observed = hash_artifact(&path, &artifact.name).await?; + if &observed != artifact { + return Err(BlazeDaemonError::Internal(format!( + "hibernate artifact {} failed integrity verification", + artifact.name + ))); + } + } + Ok(manifest) +} + +fn validate_manifest_identity( + manifest: &HibernateManifest, + instance: &SandboxInstance, +) -> Result<()> { + if manifest.sandbox_id != instance.id + || manifest.policy_name != instance.policy_name + || manifest.image_digest != instance.image_digest + || manifest.backend != instance.backend + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {} hibernation identity does not match durable lifecycle state", + instance.id + ))); + } + Ok(()) +} + +async fn hash_artifact(path: &Path, name: &str) -> Result { + let mut file = tokio::fs::File::open(path).await?; + let mut hasher = Sha256::new(); + let mut size_bytes = 0_u64; + let mut buffer = [0_u8; 128 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + size_bytes = size_bytes + .checked_add(read as u64) + .ok_or_else(|| BlazeDaemonError::Internal("artifact size overflow".to_string()))?; + hasher.update(&buffer[..read]); + } + Ok(CheckpointArtifact { + name: name.to_string(), + size_bytes, + sha256: format!("{:x}", hasher.finalize()), + }) +} + +async fn read_directory_names(directory: &Path) -> Result> { + let mut names = BTreeSet::new(); + let mut entries = tokio::fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().into_string().map_err(|_| { + BlazeDaemonError::Internal(format!( + "hibernate directory {} contains a non-UTF-8 name", + directory.display() + )) + })?; + names.insert(name); + } + Ok(names) +} + +async fn prepare_hibernate_directory(final_dir: &Path) -> Result<()> { + let parent = final_dir.parent().ok_or_else(|| { + BlazeDaemonError::Internal(format!( + "hibernate directory {} has no parent", + final_dir.display() + )) + })?; + let mut entries = tokio::fs::read_dir(parent).await?; + let mut obsolete_backups = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(".hibernate.") && name.ends_with(".tmp") { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance directory {} contains unfinished hibernation artifacts", + parent.display() + ))); + } + if name.starts_with(".hibernate.") && name.ends_with(".bak") { + if !final_dir.is_dir() { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance directory {} contains an unpaired hibernation backup", + parent.display() + ))); + } + obsolete_backups.push(entry.path()); + } + } + for backup in obsolete_backups { + remove_directory_and_sync(&backup).await?; + } + Ok(()) +} + +async fn sync_directory(directory: PathBuf) -> Result<()> { + tokio::fs::File::open(directory).await?.sync_all().await?; + Ok(()) +} + +async fn remove_directory_and_sync(directory: &Path) -> Result<()> { + match tokio::fs::remove_dir_all(directory).await { + Ok(()) => { + let parent = directory.parent().ok_or_else(|| { + BlazeDaemonError::Internal(format!( + "removed hibernation path {} has no parent", + directory.display() + )) + })?; + sync_directory(parent.to_path_buf()).await + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn with_recovery_error( + error: BlazeDaemonError, + recovery: Option, +) -> BlazeDaemonError { + match recovery { + Some(recovery) => BlazeDaemonError::RecoveryRequired(format!( + "{error}; recovery state persistence failed: {recovery}" + )), + None => error, + } +} + +#[cfg(test)] +mod tests { + use blaze_core::backend::BackendKind; + use blaze_core::lifecycle::StartPath; + use blaze_core::policy::WorkloadClass; + + use super::*; + + #[tokio::test] + async fn manifest_preserves_the_network_slot_for_resume() { + let temp = tempfile::tempdir().expect("temp"); + tokio::fs::write(temp.path().join(VMSTATE_ARTIFACT), b"vmstate") + .await + .expect("VM state"); + tokio::fs::write(temp.path().join(MEMORY_ARTIFACT), b"memory") + .await + .expect("memory"); + let instance = SandboxInstance::new( + BackendKind::Firecracker, + WorkloadClass::AgentTool, + "sha256:image".to_string(), + StartPath::Cold, + "default".to_string(), + ); + + let manifest = build_hibernate_manifest( + temp.path(), + &instance, + Some("Firecracker v1.16.0".to_string()), + true, + Some(7), + ) + .await + .expect("manifest"); + + assert!(manifest.expose_guest_socket); + assert_eq!(manifest.network_slot, Some(7)); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs new file mode 100644 index 0000000000..34a336b3df --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -0,0 +1,2159 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable sandbox create, warm activation, destroy, and startup cleanup. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use blaze_core::BlazeError; +use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, RuntimeLocation, SandboxInstance, SandboxState, StartPath, +}; +use blaze_core::policy::{RuntimeDecision, parse_duration}; +use blaze_core::pool::{PoolKey, PoolManager}; +use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::checkpoint_store::CheckpointStore; +use crate::error::{BlazeDaemonError, Result}; +use crate::guest::{GuestClient, GuestExecResult, MAX_GUEST_FILE_BYTES}; +use crate::metrics::Metrics; +#[cfg(test)] +use crate::runtime_pool::RuntimePoolStatus; +use crate::runtime_pool::{ + DurableRuntimeOwner, PoolPrototype, RuntimePoolLease, RuntimeWarmPool, begin_lifecycle_cleanup, + reconcile_runtime_slots, remove_lifecycle_tombstone, runtime_dir as derive_runtime_dir, + tombstone_lifecycle_slot, +}; +use crate::sandbox::template::RuntimeTemplateCatalog; +use crate::spawner::{DynBackendInstance, DynSpawner, SpawnerRegistry}; + +const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); +const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Inputs already parsed and policy-evaluated by the API. +#[derive(Debug, Clone)] +pub struct CreateSandbox { + /// Policy decision for this request. + pub decision: RuntimeDecision, + /// Image identity used by storage and warm-pool matching. + pub image_digest: String, + /// Concrete backend selected from the policy and daemon availability. + pub runtime_backend: BackendKind, + /// Executable selected during daemon startup. + pub binary_path: PathBuf, +} + +/// Result of one managed create request. +#[derive(Debug, Clone)] +pub struct CreateSandboxResult { + /// Persisted sandbox metadata. + pub instance: SandboxInstance, + /// Backend implementation that owns the runtime. + pub selected_backend: BackendKind, +} + +/// One startup cleanup failure. Other records continue to be reconciled. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReconcileFailure { + /// Sandbox whose cleanup remains incomplete. + pub instance_id: Uuid, + /// Actionable failure description. + pub error: String, +} + +/// Aggregate startup cleanup outcome. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReconcileReport { + /// Number of records selected for runtime or transaction cleanup. + pub attempted: usize, + /// Number of selected records whose cleanup completed. + pub completed: usize, + /// Records that remain recoverable. + pub failures: Vec, +} + +/// Owns durable lifecycle metadata and non-serializable runtime handles. +/// +/// The maps are shared with read-only and non-lifecycle API paths. All +/// create, warm activation, destroy, and restart cleanup mutations enter +/// through this type and are serialized by a per-sandbox async lock. +pub struct SandboxManager { + instances: Arc>>, + backend_instances: Arc>>, + operation_locks: Mutex>>>, + pool: Arc>, + runtime_pool: Arc, + spawners: Arc, + active_backend: BackendKind, + pub(super) storage: Arc, + pub(super) state_dir: PathBuf, + pub(super) checkpoints: CheckpointStore, + rootfs_size: u64, + mem_size: u64, + metrics: Arc, + cancellation: CancellationToken, + pub(super) runtime_templates: RuntimeTemplateCatalog, +} + +/// Construction inputs grouped to keep daemon wiring explicit. +pub struct SandboxManagerInit { + pub instances: HashMap, + pub pool: PoolManager, + pub spawners: SpawnerRegistry, + pub active_backend: BackendKind, + pub storage: Arc, + pub state_dir: PathBuf, + pub rootfs_size: u64, + pub mem_size: u64, + pub pool_size: usize, + pub prefork: bool, + pub default_warm_ttl: String, + pub gc_interval: String, + pub runtime_templates: RuntimeTemplateCatalog, +} + +/// Shared read-only resources retained by [`crate::state::ServerState`]. +pub struct SandboxManagerResources { + #[cfg(test)] + pub instances: Arc>>, + pub pool: Arc>, + pub metrics: Arc, +} + +impl SandboxManager { + /// Build a manager around state loaded from the durable state directory. + pub fn new(init: SandboxManagerInit) -> Result<(Self, SandboxManagerResources)> { + let SandboxManagerInit { + instances, + pool, + spawners, + active_backend, + storage, + state_dir, + rootfs_size, + mem_size, + pool_size, + prefork, + default_warm_ttl, + gc_interval, + runtime_templates, + } = init; + let instances = Arc::new(Mutex::new(instances)); + let backend_instances = Arc::new(Mutex::new(HashMap::new())); + let pool = Arc::new(Mutex::new(pool)); + let metrics = Arc::new(Metrics::new()); + let spawners = Arc::new(spawners); + let cancellation = CancellationToken::new(); + let default_warm_ttl = parse_duration(&default_warm_ttl).ok_or_else(|| { + BlazeDaemonError::Internal( + "pool.default_warm_ttl passed validation but could not be parsed".to_string(), + ) + })?; + let gc_interval = parse_duration(&gc_interval).ok_or_else(|| { + BlazeDaemonError::Internal( + "pool.gc_interval passed validation but could not be parsed".to_string(), + ) + })?; + let runtime_pool = RuntimeWarmPool::new( + pool_size, + prefork, + rootfs_size, + mem_size, + state_dir.join("runtime-pool"), + storage.clone(), + spawners.clone(), + default_warm_ttl, + gc_interval, + cancellation.clone(), + )?; + let checkpoints = CheckpointStore::new(state_dir.join("checkpoints")); + let resources = SandboxManagerResources { + #[cfg(test)] + instances: instances.clone(), + pool: pool.clone(), + metrics: metrics.clone(), + }; + Ok(( + Self { + instances, + backend_instances, + operation_locks: Mutex::new(HashMap::new()), + pool, + runtime_pool, + spawners, + active_backend, + storage, + state_dir, + checkpoints, + rootfs_size, + mem_size, + metrics, + cancellation, + runtime_templates, + }, + resources, + )) + } + + /// Return the async operation lock that serializes one sandbox mutation. + pub fn operation_lock(&self, id: Uuid) -> Arc> { + match self.operation_locks.lock() { + Ok(mut locks) => operation_lock(&mut locks, id), + Err(poisoned) => operation_lock(&mut poisoned.into_inner(), id), + } + } + + #[cfg(test)] + pub(crate) fn runtime_pool_status(&self) -> RuntimePoolStatus { + self.runtime_pool.status() + } + + #[cfg(test)] + pub(crate) fn runtime_pool_has_tracked_worker(&self) -> bool { + self.runtime_pool.has_tracked_worker() + } + + #[cfg(test)] + pub(crate) fn configure_runtime_pool_for_test(&self, prototype: PoolPrototype) -> Result { + self.runtime_pool + .configure(prototype) + .map_err(BlazeDaemonError::from) + } + + /// Hold the per-sandbox mutation lock after confirming that the persisted + /// record is in `expected_state` and has no unfinished operation. + /// + /// Future lifecycle operations must retain the returned guard until their + /// final state and journal update have been persisted. + pub async fn lock_quiescent_state( + &self, + id: Uuid, + expected_state: SandboxState, + ) -> Result> { + let operation = self.operation_lock(id).lock_owned().await; + let instance = self.get(id)?; + if let Some(journal) = instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != expected_state { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected {}", + instance.state, expected_state + ))); + } + Ok(operation) + } + + pub(crate) fn backend_owner(&self, id: Uuid) -> Option { + match self.backend_instances.lock() { + Ok(instances) => instances.get(&id).cloned(), + Err(poisoned) => poisoned.into_inner().get(&id).cloned(), + } + } + + pub(super) fn spawner(&self, backend: BackendKind) -> Option { + self.spawners.get(backend) + } + + pub(super) fn runtime_dir(&self, id: Uuid) -> PathBuf { + self.state_dir.join(id.to_string()) + } + + pub(super) fn remove_backend_owner(&self, id: Uuid) -> Option { + match self.backend_instances.lock() { + Ok(mut instances) => instances.remove(&id), + Err(poisoned) => poisoned.into_inner().remove(&id), + } + } + + pub(super) async fn reconstruct_storage(&self, id: Uuid) -> Result { + self.storage + .reconstruct(&id.to_string()) + .await + .map_err(Into::into) + } + + pub(super) async fn flush_storage(&self, slot: &StorageSlot) -> Result<()> { + self.storage.flush_dirty(slot).await.map_err(Into::into) + } + + #[cfg(test)] + pub fn insert_backend_owner(&self, id: Uuid, backend: DynBackendInstance) -> Result<()> { + self.backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .insert(id, backend); + Ok(()) + } + + /// Return all persisted sandbox metadata. + pub fn list(&self) -> Result> { + Ok(self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .values() + .cloned() + .collect()) + } + + /// Return one persisted sandbox. + pub fn get(&self, id: Uuid) -> Result { + self.instances + .lock() + .map_err(|_| poisoned("instances"))? + .get(&id) + .cloned() + .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {id}"))) + } + + /// Return every sandbox for which lifecycle cleanup still owns resources. + /// + /// Shutdown uses this snapshot to start cleanup concurrently while all + /// mutations remain serialized by the manager's per-sandbox locks. + pub(crate) fn owned_instance_ids(&self) -> Result> { + let mut ids = self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .values() + .filter(|instance| requires_automatic_cleanup(instance)) + .map(|instance| instance.id) + .collect::>(); + ids.extend( + self.backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .keys() + .copied(), + ); + Ok(ids) + } + + /// Execute one command through the running sandbox guest. + pub async fn exec( + &self, + id: Uuid, + command: String, + cwd: Option, + env: Option>, + timeout_secs: u32, + ) -> Result { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .exec(command, cwd, env, timeout_secs) + .await + .map_err(BlazeDaemonError::from) + } + + /// Read one file through the running sandbox guest. + pub async fn read_file(&self, id: Uuid, path: String) -> Result> { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .read_file(path) + .await + .map_err(BlazeDaemonError::from) + } + + /// Replace one file through the running sandbox guest. + pub async fn write_file(&self, id: Uuid, path: String, data: &[u8]) -> Result<()> { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .write_file(path, data) + .await + .map_err(BlazeDaemonError::from) + } + + /// Create a cold sandbox or activate a compatible warm runtime. + pub async fn create(&self, request: CreateSandbox) -> Result { + let pool_key = PoolKey::new( + request.runtime_backend, + request.decision.workload_class, + request.image_digest.clone(), + ); + if request.decision.pool_eligible { + let warm_ttl = request + .decision + .pool + .as_ref() + .and_then(|pool| pool.warm_ttl.as_deref()) + .and_then(parse_duration) + .unwrap_or_else(|| self.runtime_pool.default_warm_ttl()); + let configured = self.runtime_pool.configure(PoolPrototype { + image_digest: request.image_digest.clone(), + policy_name: request.decision.policy_name.clone(), + workload_class: request.decision.workload_class, + templates: request.decision.templates.clone(), + kernel_hooks: request.decision.kernel_hooks.clone(), + binary_path: request.binary_path.clone(), + runtime_backend: request.runtime_backend, + backend: request.decision.backend.clone(), + vm: request.decision.vm.clone(), + warm_ttl, + })?; + if configured && let Some(lease) = self.runtime_pool.acquire().await? { + let result = self.activate_runtime_slot(lease, request).await?; + self.metrics.inc(&self.metrics.pool_hits); + return Ok(result); + } + if let Some(result) = self.activate_warm(&pool_key).await? { + self.metrics.inc(&self.metrics.pool_hits); + return Ok(result); + } + self.metrics.inc(&self.metrics.pool_misses); + } + + let mut instance = SandboxInstance::new( + request.runtime_backend, + request.decision.workload_class, + request.image_digest, + StartPath::Cold, + request.decision.policy_name.clone(), + ); + let operation_lock = self.operation_lock(instance.id); + let _operation = operation_lock.lock().await; + instance.transition(SandboxState::Creating)?; + instance.begin_operation(OperationKind::Create)?; + + // Publish the stable identity and create intent before allocation. + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "create {}: {error}", + instance.id + ))); + } + + let storage = match self + .storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: self.rootfs_size, + mem_size: self.mem_size, + }) + .await + { + Ok(storage) => storage, + Err(error) => { + let (source, residual) = error.into_parts(); + return Err(self.retain_failed_acquire(&mut instance, residual, source.into())); + } + }; + crate::failpoint::pause("create-after-storage-acquire").await; + + let run_dir = self.runtime_dir_for(&instance); + let spawner = match self.spawners.get(self.active_backend) { + Some(spawner) => spawner, + None => { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(format!( + "active backend {} has no registered spawner", + self.active_backend + )), + ) + .await); + } + }; + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, false, error.into()) + .await); + } + + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance.persist(&self.state_dir) { + instance.backend_ownership = BackendOwnership::NotStarted; + return Err(self + .cleanup_failed_create(&mut instance, storage, None, false, error.into()) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + instance.backend_ownership = BackendOwnership::NotStarted; + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(error), + ) + .await); + } + + let spawn = match crate::failpoint::backend("create-spawn") { + Ok(()) => { + spawner + .spawn(SpawnRequest { + instance_id: instance.id, + run_dir, + binary_path: request.binary_path, + storage: storage.clone(), + backend: request.decision.backend, + vm: request.decision.vm, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let actual_backend = match spawn { + Ok(backend_instance) => { + instance.backend_ownership = BackendOwnership::Running; + let actual_backend = backend_instance.backend(); + if let Err(error) = self + .wait_for_guest_ready(&backend_instance, "create-guest-ready") + .await + { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + Some(backend_instance), + false, + error.into(), + ) + .await); + } + let mut backend_instance = Some(backend_instance); + let registered = match self.backend_instances.lock() { + Ok(mut instances) => { + instances.insert( + instance.id, + backend_instance + .take() + .expect("backend instance is present"), + ); + true + } + Err(_) => false, + }; + if !registered { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + backend_instance, + false, + BlazeDaemonError::Internal( + "backend_instances lock poisoned".to_string(), + ), + ) + .await); + } + actual_backend + } + Err(error) => { + let (source, backend) = error.into_parts(); + instance.backend_ownership = if backend.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::Stopped + }; + return Err(self + .cleanup_failed_create(&mut instance, storage, backend, false, source.into()) + .await); + } + }; + + if let Err(error) = instance.transition(SandboxState::Running) { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, true, error.into()) + .await); + } + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("create-state-commit") + .and_then(|_| instance.persist(&self.state_dir).map_err(Into::into)) + { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, true, error) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + true, + BlazeDaemonError::Internal(error), + ) + .await); + } + self.metrics.inc(&self.metrics.instances_created); + Ok(CreateSandboxResult { + instance, + selected_backend: actual_backend, + }) + } + + async fn activate_runtime_slot( + &self, + mut lease: RuntimePoolLease, + request: CreateSandbox, + ) -> Result { + { + let slot = lease.slot()?; + if slot.runtime_backend != request.runtime_backend { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} records backend {} instead of requested {}", + slot.instance_id, slot.runtime_backend, request.runtime_backend + ))); + } + let expected_ownership = if slot.backend.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::NotStarted + }; + if slot.backend_ownership != expected_ownership { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} has backend ownership {:?} with backend handle present={}", + slot.instance_id, + slot.backend_ownership, + slot.backend.is_some() + ))); + } + } + + let owner_token = Uuid::new_v4(); + let instance_id = lease.slot()?.instance_id; + let operation_lock = self.operation_lock(instance_id); + let _operation = operation_lock.lock().await; + lease.begin_handoff(owner_token).await?; + let slot = lease.slot()?; + let reconstructed = self + .storage + .reconstruct(&slot.instance_id.to_string()) + .await?; + if reconstructed != slot.storage { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} storage paths changed before lifecycle handoff", + slot.instance_id + ))); + } + + let mut instance = SandboxInstance::new_warm_claim( + slot.instance_id, + slot.runtime_backend, + request.decision.workload_class, + request.image_digest, + request.decision.policy_name, + slot.backend_ownership, + owner_token, + )?; + let publication = instance + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + .and_then(|()| crate::failpoint::state("warm-runtime-lifecycle-publish-result")); + let publication_failure = match publication { + Ok(()) => None, + Err(error) => match inspect_warm_claim_publication(&self.state_dir, &instance) { + WarmClaimPublication::Absent => return Err(error), + WarmClaimPublication::Published(published) => { + instance = published; + Some(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} lifecycle publication reported an error after state \ + became visible: {error}", + instance.id + ))) + } + WarmClaimPublication::Ambiguous(reason) => { + let unresolved = + format!("lifecycle publication was ambiguous after {error}: {reason}"); + lease.retain_unresolved(owner_token, unresolved.clone())?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} {unresolved}; ownership remains counted in the runtime \ + pool and requires startup reconciliation", + instance.id + ))); + } + }, + }; + let retained_state = self.retain_instance(instance.clone()); + let retained_backend = slot + .backend + .as_ref() + .and_then(|backend| self.retain_backend(instance.id, backend.clone())); + let mut registered = slot.backend.is_some(); + lease.transfer_to_lifecycle(); + + let handoff = lease.finish_handoff(owner_token).await; + let mut slot = lease.into_slot()?; + let mut transfer_errors = [retained_state, retained_backend] + .into_iter() + .flatten() + .collect::>(); + if let Err(error) = handoff { + transfer_errors.push(format!("finish durable ownership handoff: {error}")); + } + if let Some(error) = publication_failure { + transfer_errors.push(error.to_string()); + } + if !transfer_errors.is_empty() { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} lifecycle handoff failed: {}", + instance.id, + transfer_errors.join("; ") + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + + let actual_backend = if let Some(backend) = slot.backend.as_ref() { + backend.backend() + } else { + let spawner = match self.spawners.get(instance.backend) { + Some(spawner) => spawner, + None => { + let failure = BlazeDaemonError::Internal(format!( + "runtime backend {} has no registered spawner", + instance.backend + )); + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, failure) + .await); + } + }; + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance.persist(&self.state_dir) { + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, error.into()) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + None, + false, + BlazeDaemonError::Internal(error), + ) + .await); + } + if let Err(error) = spawner.prepare_spawn(&slot.run_dir).await { + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, error.into()) + .await); + } + let spawn = spawner + .spawn(SpawnRequest { + instance_id: instance.id, + run_dir: slot.run_dir.clone(), + binary_path: request.binary_path, + storage: slot.storage.clone(), + backend: request.decision.backend, + vm: request.decision.vm, + }) + .await; + let backend = match spawn { + Ok(backend) => backend, + Err(error) => { + let (source, owner) = error.into_parts(); + instance.backend_ownership = if owner.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::Stopped + }; + let persist = instance.persist(&self.state_dir).err(); + let retained = self.retain_instance(instance.clone()); + let mut details = Vec::new(); + if let Some(error) = persist { + details.push(format!("backend failure state persistence failed: {error}")); + } + if let Some(error) = retained { + details.push(error); + } + let original = if details.is_empty() { + source.into() + } else { + BlazeDaemonError::RecoveryRequired(format!( + "{source}; {}", + details.join("; ") + )) + }; + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, owner, false, original) + .await); + } + }; + let actual_backend = backend.backend(); + instance.backend_ownership = BackendOwnership::Running; + let persist = instance.persist(&self.state_dir); + let retained_state = self.retain_instance(instance.clone()); + let retained_backend = self.retain_backend(instance.id, backend.clone()); + registered = true; + slot.backend = Some(backend.clone()); + if actual_backend != instance.backend { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} expected backend {} but spawner returned {}", + instance.id, instance.backend, actual_backend + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + if let Err(error) = persist { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + if let Some(error) = retained_state.or(retained_backend) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + BlazeDaemonError::Internal(error), + ) + .await); + } + if let Err(error) = self + .wait_for_guest_ready(&backend, "warm-runtime-guest-ready") + .await + { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + actual_backend + }; + + if actual_backend != instance.backend { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} expected backend {} but owner is {}", + instance.id, instance.backend, actual_backend + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + if let Err(error) = instance.transition(SandboxState::Running) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + instance.finish_operation(); + if let Err(error) = instance.persist(&self.state_dir) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + BlazeDaemonError::Internal(error), + ) + .await); + } + Ok(CreateSandboxResult { + instance, + selected_backend: actual_backend, + }) + } + + async fn activate_warm(&self, key: &PoolKey) -> Result> { + let candidate = self.pool.lock().map_err(|_| poisoned("pool"))?.lookup(key); + let Some(id) = candidate else { + return Ok(None); + }; + let operation_lock = self.operation_lock(id); + let _operation = operation_lock.lock().await; + + let instance = self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .get(&id) + .cloned(); + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + + let invalid_reason = match (&instance, &backend) { + (None, _) => Some("lifecycle metadata is missing".to_string()), + (_, None) => Some("backend owner is missing".to_string()), + (Some(instance), Some(_)) + if instance.state != SandboxState::Warm || instance.operation.is_some() => + { + Some(format!("lifecycle state is {}", instance.state)) + } + (Some(instance), Some(backend)) if backend.backend() != instance.backend => { + Some(format!( + "backend owner is {}, metadata is {}", + backend.backend(), + instance.backend + )) + } + (_, Some(backend)) => match backend.try_wait().await { + Ok(None) => None, + Ok(Some(status)) => Some(format!("backend exited: {status:?}")), + Err(error) => Some(format!("backend liveness check failed: {error}")), + }, + }; + if let Some(reason) = invalid_reason { + self.quarantine_warm(key, id, instance, backend, &reason) + .await; + return Ok(None); + } + + let original = instance.expect("validated warm metadata"); + match self.storage.reconstruct(&id.to_string()).await { + Ok(_) => {} + Err(error @ BlazeError::StorageIncomplete { .. }) => { + self.quarantine_warm( + key, + id, + Some(original), + backend, + &format!("storage validation failed: {error}"), + ) + .await; + return Ok(None); + } + Err(error) => { + return Err(self.restore_warm_claim(key, original, error.into())); + } + } + + let mut activating = original.clone(); + if let Err(error) = activating.begin_operation(OperationKind::Create) { + return Err(self.restore_warm_claim(key, original, error.into())); + } + if let Err(error) = crate::failpoint::state("warm-intent-state-commit").and_then(|_| { + activating + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + return Err(self.restore_warm_claim(key, original, error)); + } + if let Some(error) = self.retain_instance(activating.clone()) { + return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); + } + + crate::failpoint::pause("warm-before-state-commit").await; + let selected_backend = backend.expect("validated warm backend").backend(); + if let Err(error) = activating + .transition(SandboxState::Creating) + .and_then(|_| activating.transition(SandboxState::Running)) + { + return Err(self.restore_warm_claim(key, original, error.into())); + } + activating.finish_operation(); + if let Err(error) = crate::failpoint::state("warm-final-state-commit").and_then(|_| { + activating + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + return Err(self.restore_warm_claim(key, original, error)); + } + if let Some(error) = self.retain_instance(activating.clone()) { + return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); + } + Ok(Some(CreateSandboxResult { + instance: activating, + selected_backend, + })) + } + + fn restore_warm_claim( + &self, + key: &PoolKey, + instance: SandboxInstance, + cause: BlazeDaemonError, + ) -> BlazeDaemonError { + let id = instance.id; + let mut errors = Vec::new(); + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("restore warm state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance) { + errors.push(error); + } + match self.pool.lock() { + Ok(mut pool) => pool.restore_lookup(key.clone(), id), + Err(poisoned) => { + poisoned.into_inner().restore_lookup(key.clone(), id); + errors.push("pool lock poisoned while restoring warm claim".to_string()); + } + } + let details = if errors.is_empty() { + "warm claim restored for retry".to_string() + } else { + format!("warm claim restored with errors: {}", errors.join("; ")) + }; + BlazeDaemonError::RecoveryRequired(format!("{cause}; instance {id}: {details}")) + } + + async fn quarantine_warm( + &self, + key: &PoolKey, + id: Uuid, + mut instance: Option, + backend: Option, + reason: &str, + ) { + match self.pool.lock() { + Ok(mut pool) => pool.quarantine(key, id), + Err(poisoned) => poisoned.into_inner().quarantine(key, id), + } + tracing::warn!(instance = %id, reason, "warm instance validation failed"); + + let Some(metadata) = instance.as_mut() else { + if let Some(backend) = backend.as_ref() + && let Err(error) = backend.kill().await + { + tracing::error!( + instance = %id, + %error, + "quarantined backend cleanup failed" + ); + } + tracing::error!( + instance = %id, + "quarantined lifecycle metadata missing; retaining storage" + ); + return; + }; + metadata.begin_destroy_recovery(); + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantine intent commit failed"); + return; + } + if let Some(error) = self.retain_instance(metadata.clone()) { + tracing::error!(instance = %id, %error, "quarantine intent retention failed"); + return; + } + + let backend_stopped = match backend.as_ref() { + Some(backend) => match backend.kill().await { + Ok(()) => true, + Err(error) => { + tracing::error!(instance = %id, %error, "quarantined backend cleanup failed"); + false + } + }, + None if matches!( + metadata.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) => + { + true + } + None => match self.spawners.get(metadata.backend) { + Some(spawner) => match spawner + .cleanup_orphan(id, &self.runtime_dir_for(metadata)) + .await + { + Ok(()) => true, + Err(error) => { + tracing::error!( + instance = %id, + %error, + "quarantined orphan cleanup failed" + ); + false + } + }, + None => { + tracing::error!( + instance = %id, + backend = %metadata.backend, + "quarantined backend has no recovery spawner" + ); + false + } + }, + }; + if !backend_stopped { + let _ = self.mark_recovery(id); + return; + } + + metadata.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantined stop state commit failed"); + let _ = self.mark_recovery(id); + return; + } + if let Some(error) = self.retain_instance(metadata.clone()) { + tracing::error!(instance = %id, %error, "quarantined stop state retention failed"); + let _ = self.mark_recovery(id); + return; + } + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { + tracing::error!(instance = %id, %error, "quarantined storage cleanup failed"); + let _ = self.mark_recovery(id); + return; + } + + if metadata.state != SandboxState::Destroyed + && let Err(error) = metadata.transition(SandboxState::Destroyed) + { + tracing::error!(instance = %id, %error, "quarantined lifecycle cleanup failed"); + let _ = self.mark_recovery(id); + return; + } + metadata.finish_operation(); + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantined state commit failed"); + let _ = self.mark_recovery(id); + return; + } + let _ = self.retain_instance(metadata.clone()); + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.remove(&id); + } + Err(poisoned) => { + poisoned.into_inner().remove(&id); + } + } + } + + /// Idempotently destroy one sandbox and its owned runtime resources. + pub async fn destroy(&self, id: Uuid) -> Result { + let operation_lock = self.operation_lock(id); + let _operation = operation_lock.lock().await; + self.destroy_locked(id).await + } + + async fn destroy_locked(&self, id: Uuid) -> Result { + let mut original = self.get(id)?; + let has_retained_backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .contains_key(&id); + if is_clean_terminal(&original) && !has_retained_backend { + if original.runtime_location == RuntimeLocation::WarmPool { + let owner_token = required_runtime_owner_token(&original)?; + remove_lifecycle_tombstone(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: finish runtime tombstone cleanup: {error}" + )) + })?; + } + if let Err(error) = self.checkpoints.cleanup_transaction_artifacts(id) { + original.begin_destroy_recovery(); + let recovery = self.persist_and_retain(original).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: checkpoint transaction cleanup failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + return Ok(false); + } + if original.runtime_location == RuntimeLocation::WarmPool { + self.cleanup_warm_runtime_locked(&mut original).await?; + self.metrics.inc(&self.metrics.instances_destroyed); + return Ok(true); + } + + if original.operation.as_ref().map(|operation| operation.kind) + != Some(OperationKind::Destroy) + { + original.begin_destroy_recovery(); + } + if let Err(error) = crate::failpoint::state("destroy-intent-state-commit").and_then(|_| { + original + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: intent persistence failed: {error}; resources retained" + ))); + } + if let Some(error) = self.retain_instance(original.clone()) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: {error}; resources retained" + ))); + } + + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + let stop_result = match crate::failpoint::backend("destroy-kill") { + Ok(()) => { + if let Some(backend) = backend.as_ref() { + backend.kill().await + } else if matches!( + original.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) { + Ok(()) + } else { + match self.spawners.get(original.backend) { + Some(spawner) => { + spawner + .cleanup_orphan(id, &self.runtime_dir_for(&original)) + .await + } + None => Err(BlazeError::BackendError { + msg: format!( + "no recovery spawner registered for persisted backend {}", + original.backend + ), + }), + } + } + } + Err(error) => Err(error), + }; + if let Err(error) = stop_result { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend termination failed: {error}; owner and storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + original.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = crate::failpoint::state("destroy-stop-state-commit").and_then(|_| { + original + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let recovery = self.mark_instance_recovery(original.clone()).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but stop state persistence failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(error) = self.retain_instance(original.clone()) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but lifecycle retention failed: {error}; \ + storage retained" + ))); + } + self.forget_backend(id); + + if let Err(error) = self.checkpoints.cleanup_transaction_artifacts(id) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but checkpoint cleanup failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + if let Err(error) = self.cleanup_hibernate_artifacts(id).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but hibernation cleanup failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but storage release failed: {error}; \ + lifecycle retained for retry{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + let mut destroyed = original; + if destroyed.state != SandboxState::Destroyed { + destroyed.transition(SandboxState::Destroyed)?; + } + destroyed.finish_operation(); + if let Err(error) = crate::failpoint::state("destroy-final-state-commit").and_then(|_| { + destroyed + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: resources released but final state persistence failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(error) = self.retain_instance(destroyed) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: resources released but {error}" + ))); + } + self.metrics.inc(&self.metrics.instances_destroyed); + Ok(true) + } + + async fn cleanup_warm_runtime_locked(&self, instance: &mut SandboxInstance) -> Result<()> { + let id = instance.id; + instance.begin_destroy_recovery(); + crate::failpoint::state("destroy-intent-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: {error}; resources retained" + ))); + } + + let owner_token = required_runtime_owner_token(instance)?; + begin_lifecycle_cleanup( + &self.state_dir.join("runtime-pool"), + id, + instance.backend, + owner_token, + ) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: persist cleanup ownership: {error}" + )) + })?; + + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + crate::failpoint::backend("destroy-kill")?; + if let Some(backend) = backend.as_ref() { + backend.kill().await?; + } else if !matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) { + let spawner = self.spawners.get(instance.backend).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: no recovery spawner registered for {}", + instance.backend + )) + })?; + spawner + .cleanup_orphan(id, &self.runtime_dir_for(instance)) + .await?; + } + + instance.backend_ownership = BackendOwnership::Stopped; + crate::failpoint::state("destroy-stop-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: backend stopped but {error}; storage retained" + ))); + } + self.forget_backend(id); + + self.storage.release_by_id(&id.to_string()).await?; + if instance.state != SandboxState::Destroyed { + instance.transition(SandboxState::Destroyed)?; + } + crate::failpoint::state("destroy-released-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: resources released but {error}" + ))); + } + + tombstone_lifecycle_slot(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: tombstone runtime directory: {error}" + )) + })?; + instance.finish_operation(); + crate::failpoint::state("destroy-final-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: lifecycle is terminal but {error}" + ))); + } + remove_lifecycle_tombstone(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: remove runtime tombstone: {error}" + )) + }) + } + + /// Reconcile every record that is not cleanly terminal without aborting + /// on one failure. + pub async fn reconcile_startup(&self) -> ReconcileReport { + let mut classification_failures = self.classify_interrupted_hibernation(); + let mut report = self.cleanup_owned_instances().await; + report.failures.append(&mut classification_failures); + report + } + + /// Release unclaimed runtime slots before the daemon starts listening. + /// + /// Any ambiguous ownership record stops startup so the daemon never + /// serves an incomplete resource inventory. + pub async fn reconcile_runtime_pool_startup(&self) -> Result { + let durable_owners = match self.instances.lock() { + Ok(instances) => instances + .values() + .map(|instance| (instance.id, DurableRuntimeOwner::from(instance))) + .collect(), + Err(poisoned) => poisoned + .into_inner() + .values() + .map(|instance| (instance.id, DurableRuntimeOwner::from(instance))) + .collect(), + }; + reconcile_runtime_slots( + &self.state_dir.join("runtime-pool"), + &durable_owners, + self.storage.as_ref(), + self.spawners.as_ref(), + ) + .await + } + + /// Cancel readiness polling before the daemon drains active requests. + pub fn begin_shutdown(&self) { + self.runtime_pool.begin_shutdown(); + self.cancellation.cancel(); + } + + /// Stop background capacity work and release pool-owned slots within the + /// daemon's shared runtime cleanup deadline. + pub async fn shutdown_runtime_pool_until(&self, deadline: tokio::time::Instant) -> Result<()> { + self.runtime_pool + .shutdown_until(deadline) + .await + .map_err(BlazeDaemonError::from) + } + + /// Release every lifecycle record and retained backend owner, continuing + /// after individual failures. + pub async fn cleanup_owned_instances(&self) -> ReconcileReport { + self.cleanup_owned_instances_with_timeout(INSTANCE_CLEANUP_TIMEOUT) + .await + } + + fn classify_interrupted_hibernation(&self) -> Vec { + let interrupted = match self.instances.lock() { + Ok(instances) => instances + .values() + .filter(|instance| { + matches!( + instance.state, + SandboxState::Hibernating | SandboxState::Resuming + ) || matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ) + }) + .cloned() + .collect::>(), + Err(poisoned) => poisoned + .into_inner() + .values() + .filter(|instance| { + matches!( + instance.state, + SandboxState::Hibernating | SandboxState::Resuming + ) || matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ) + }) + .cloned() + .collect::>(), + }; + interrupted + .into_iter() + .filter_map(|instance| { + let id = instance.id; + self.mark_instance_recovery(instance) + .err() + .map(|error| ReconcileFailure { + instance_id: id, + error: format!("interrupted hibernation classification failed: {error}"), + }) + }) + .collect() + } + + fn guest_client(&self, id: Uuid) -> Result { + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned() + .ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + let socket = backend.guest_socket_path(); + if socket.as_os_str().is_empty() { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} has no guest transport" + ))); + } + Ok(GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + )) + } + + pub(super) async fn wait_for_guest_ready( + &self, + backend: &DynBackendInstance, + failpoint: &str, + ) -> crate::guest::Result<()> { + let socket = backend.guest_socket_path(); + if socket.as_os_str().is_empty() { + return Ok(()); + } + crate::failpoint::guest(failpoint)?; + GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + ) + .wait_ready(GUEST_REQUEST_TIMEOUT, &self.cancellation) + .await + } + + fn runtime_dir_for(&self, instance: &SandboxInstance) -> PathBuf { + derive_runtime_dir(&self.state_dir, instance.runtime_location, instance.id) + } + + /// Release owned instances with a caller-supplied per-instance deadline. + pub async fn cleanup_owned_instances_with_timeout( + &self, + item_timeout: Duration, + ) -> ReconcileReport { + let records = match self.instances.lock() { + Ok(instances) => instances.values().cloned().collect::>(), + Err(poisoned) => poisoned.into_inner().values().cloned().collect::>(), + }; + let mut ids = records + .into_iter() + .filter(|instance| { + requires_automatic_cleanup(instance) + || self + .checkpoints + .has_transaction_artifacts(instance.id) + .unwrap_or(true) + }) + .map(|instance| instance.id) + .collect::>(); + match self.backend_instances.lock() { + Ok(instances) => ids.extend(instances.keys().copied()), + Err(poisoned) => ids.extend(poisoned.into_inner().keys().copied()), + } + self.cleanup_instances_with_timeout(ids, item_timeout).await + } + + async fn cleanup_instances_with_timeout( + &self, + ids: BTreeSet, + item_timeout: Duration, + ) -> ReconcileReport { + let mut report = ReconcileReport { + attempted: ids.len(), + ..ReconcileReport::default() + }; + for id in ids { + let operation_lock = self.operation_lock(id); + let deadline = tokio::time::Instant::now() + item_timeout; + let _operation = match tokio::time::timeout_at(deadline, operation_lock.lock()).await { + Ok(operation) => operation, + Err(_) => { + report.failures.push(ReconcileFailure { + instance_id: id, + error: format!( + "instance cleanup exceeded {} ms per-record deadline while waiting \ + for ownership", + item_timeout.as_millis() + ), + }); + continue; + } + }; + match tokio::time::timeout_at(deadline, self.destroy_locked(id)).await { + Ok(Ok(_)) => report.completed += 1, + Err(_) => { + let error = format!( + "instance cleanup exceeded {} ms per-record deadline", + item_timeout.as_millis() + ); + let recovery = self.mark_recovery_if_nonterminal(id).err(); + report.failures.push(ReconcileFailure { + instance_id: id, + error: match recovery { + Some(recovery) => { + format!("{error}; recovery state persistence failed: {recovery}") + } + None => error, + }, + }); + } + Ok(Err(error)) => { + let recovery = self.mark_recovery_if_nonterminal(id).err(); + report.failures.push(ReconcileFailure { + instance_id: id, + error: match recovery { + Some(recovery) => { + format!("{error}; recovery state persistence failed: {recovery}") + } + None => error.to_string(), + }, + }); + } + } + } + report + } + + async fn cleanup_failed_create( + &self, + instance: &mut SandboxInstance, + storage: StorageSlot, + backend: Option, + registered: bool, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + if instance.runtime_location == RuntimeLocation::WarmPool { + return self + .cleanup_failed_warm_create(instance, storage, backend, registered, original) + .await; + } + if instance.operation.is_none() + && let Err(error) = instance.begin_operation(OperationKind::Create) + { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {}: cannot retain create journal: {error}", + instance.id + )); + } + let mut cleanup_errors = Vec::new(); + let backend = if registered { + match self.backend_instances.lock() { + Ok(mut instances) => instances.remove(&instance.id), + Err(poisoned) => poisoned.into_inner().remove(&instance.id), + } + } else { + backend + }; + let mut backend_stopped = matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ); + if registered && backend.is_none() { + backend_stopped = false; + cleanup_errors.push("registered backend owner is missing".to_string()); + } + if let Some(backend) = backend.as_ref() { + match backend.kill().await { + Ok(()) => { + backend_stopped = true; + instance.backend_ownership = BackendOwnership::Stopped; + } + Err(error) => { + backend_stopped = false; + cleanup_errors.push(format!("backend termination failed: {error}")); + } + } + } + + let mut storage_released = false; + if backend_stopped { + match self.storage.release(storage).await { + Ok(()) => storage_released = true, + Err(error) => cleanup_errors.push(format!("storage release failed: {error}")), + } + } else { + cleanup_errors.push("storage retained until backend termination succeeds".to_string()); + } + + if backend_stopped && storage_released { + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Destroyed) { + cleanup_errors.push(format!("lifecycle update failed: {error}")); + } else { + instance.finish_operation(); + } + if let Err(error) = instance.persist(&self.state_dir) { + cleanup_errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + cleanup_errors.push(error); + } + if cleanup_errors.is_empty() { + self.metrics.inc(&self.metrics.instances_destroyed); + return original; + } + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; cleanup completed but {}", + cleanup_errors.join("; ") + )); + } + + if let Some(backend) = backend + && let Some(error) = self.retain_backend(instance.id, backend) + { + cleanup_errors.push(error); + } + if instance.state != SandboxState::RecoveryRequired + && let Err(error) = instance.transition(SandboxState::RecoveryRequired) + { + cleanup_errors.push(format!("recovery state update failed: {error}")); + } + if let Err(error) = instance.persist(&self.state_dir) { + cleanup_errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + cleanup_errors.push(error); + } + BlazeDaemonError::RecoveryRequired(format!( + "{original}; cleanup incomplete: {}", + cleanup_errors.join("; ") + )) + } + + async fn cleanup_failed_warm_create( + &self, + instance: &mut SandboxInstance, + storage: StorageSlot, + backend: Option, + registered: bool, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + let id = instance.id; + let has_retained_backend = match self.backend_instances.lock() { + Ok(instances) => instances.contains_key(&id), + Err(poisoned) => poisoned.into_inner().contains_key(&id), + }; + if !has_retained_backend { + match backend { + Some(backend) => { + if let Some(error) = self.retain_backend(id, backend) { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: {error}" + )); + } + } + None if registered => { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: registered backend owner is missing" + )); + } + None => {} + } + } + if storage.id != id.to_string() { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: warm storage owner {} does not match lifecycle ID", + storage.id + )); + } + + match self.cleanup_warm_runtime_locked(instance).await { + Ok(()) => { + self.metrics.inc(&self.metrics.instances_destroyed); + original + } + Err(error) => BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: cleanup incomplete: {error}" + )), + } + } + + fn retain_failed_acquire( + &self, + instance: &mut SandboxInstance, + residual: Option, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + if residual.is_some() { + let mut errors = Vec::new(); + if instance.state != SandboxState::RecoveryRequired + && let Err(error) = instance.transition(SandboxState::RecoveryRequired) + { + errors.push(format!("recovery state update failed: {error}")); + } + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + errors.push(error); + } + let suffix = if errors.is_empty() { + "residual storage retained for destroy retry".to_string() + } else { + format!( + "residual storage retained with recovery errors: {}", + errors.join("; ") + ) + }; + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {}: {suffix}", + instance.id + )); + } + + let mut errors = Vec::new(); + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Destroyed) { + errors.push(format!("lifecycle update failed: {error}")); + } else { + instance.finish_operation(); + } + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + errors.push(error); + } + if errors.is_empty() { + original + } else { + BlazeDaemonError::RecoveryRequired(format!( + "{original}; acquire rollback completed but {}", + errors.join("; ") + )) + } + } + + pub(super) fn mark_recovery(&self, id: Uuid) -> Result<()> { + self.mark_instance_recovery(self.get(id)?) + } + + fn mark_recovery_if_nonterminal(&self, id: Uuid) -> Result<()> { + let instance = self.get(id)?; + if instance.is_clean_terminal() { + Ok(()) + } else { + self.mark_instance_recovery(instance) + } + } + + pub(super) fn persist_and_retain(&self, instance: SandboxInstance) -> Result<()> { + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance) { + return Err(BlazeDaemonError::RecoveryRequired(error)); + } + Ok(()) + } + + pub(super) fn mark_instance_recovery(&self, mut instance: SandboxInstance) -> Result<()> { + if instance.state == SandboxState::Destroyed { + if instance.operation.as_ref().map(|operation| operation.kind) + != Some(OperationKind::Destroy) + { + instance.begin_destroy_recovery(); + } + } else if instance.state != SandboxState::RecoveryRequired { + instance.transition(SandboxState::RecoveryRequired)?; + } + let persist = instance.persist(&self.state_dir); + let retained = self.retain_instance(instance); + match (persist, retained) { + (Ok(()), None) => Ok(()), + (Err(error), None) => Err(error.into()), + (Ok(()), Some(error)) => Err(BlazeDaemonError::Internal(error)), + (Err(persist), Some(retain)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "recovery state persistence failed: {persist}; {retain}" + ))), + } + } + + pub(super) fn retain_backend(&self, id: Uuid, backend: DynBackendInstance) -> Option { + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.insert(id, backend); + None + } + Err(poisoned) => { + poisoned.into_inner().insert(id, backend); + Some("backend owner retained in poisoned runtime map".to_string()) + } + } + } + + fn forget_backend(&self, id: Uuid) { + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.remove(&id); + } + Err(poisoned) => { + poisoned.into_inner().remove(&id); + } + } + } + + pub(super) fn retain_instance(&self, instance: SandboxInstance) -> Option { + match self.instances.lock() { + Ok(mut instances) => { + instances.insert(instance.id, instance); + None + } + Err(poisoned) => { + poisoned.into_inner().insert(instance.id, instance); + Some("instance state retained in poisoned lifecycle map".to_string()) + } + } + } +} + +fn poisoned(name: &str) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("{name} lock poisoned")) +} + +fn is_clean_terminal(instance: &SandboxInstance) -> bool { + instance.is_clean_terminal() +} + +enum WarmClaimPublication { + Absent, + Published(SandboxInstance), + Ambiguous(String), +} + +fn inspect_warm_claim_publication( + state_dir: &Path, + expected: &SandboxInstance, +) -> WarmClaimPublication { + let owner_dir = state_dir.join(expected.id.to_string()); + match std::fs::symlink_metadata(&owner_dir) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return WarmClaimPublication::Absent; + } + Ok(metadata) if !metadata.file_type().is_dir() => { + return WarmClaimPublication::Ambiguous(format!( + "{} is not a real lifecycle owner directory", + owner_dir.display() + )); + } + Ok(_) => {} + Err(error) => { + return WarmClaimPublication::Ambiguous(format!( + "inspect {}: {error}", + owner_dir.display() + )); + } + } + + let path = owner_dir.join("state.json"); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return WarmClaimPublication::Ambiguous(format!( + "{} has no lifecycle state file", + owner_dir.display() + )); + } + Ok(metadata) if !metadata.file_type().is_file() => { + return WarmClaimPublication::Ambiguous(format!( + "{} is not a regular lifecycle file", + path.display() + )); + } + Ok(_) => {} + Err(error) => { + return WarmClaimPublication::Ambiguous(format!("inspect {}: {error}", path.display())); + } + } + + let published = match SandboxInstance::load(state_dir, expected.id) { + Ok(published) => published, + Err(error) => { + return WarmClaimPublication::Ambiguous(format!("load {}: {error}", path.display())); + } + }; + let expected_value = serde_json::to_value(expected); + let published_value = serde_json::to_value(&published); + match (expected_value, published_value) { + (Ok(expected_value), Ok(published_value)) if expected_value == published_value => { + WarmClaimPublication::Published(published) + } + (Err(error), _) | (_, Err(error)) => WarmClaimPublication::Ambiguous(format!( + "compare visible lifecycle state {}: {error}", + path.display() + )), + _ => WarmClaimPublication::Ambiguous(format!( + "{} does not match the claimed runtime owner", + path.display() + )), + } +} + +fn required_runtime_owner_token(instance: &SandboxInstance) -> Result { + instance.runtime_owner_token.ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "warm runtime {} has no durable ownership token", + instance.id + )) + }) +} + +fn requires_automatic_cleanup(instance: &SandboxInstance) -> bool { + !(is_clean_terminal(instance) + || (instance.state == SandboxState::Hibernated + && instance.operation.is_none() + && instance.backend_ownership == BackendOwnership::Stopped) + || (instance.state == SandboxState::RecoveryRequired + && matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ))) +} + +fn operation_lock( + locks: &mut HashMap>>, + id: Uuid, +) -> Arc> { + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&id).and_then(Weak::upgrade) { + return lock; + } + + let lock = Arc::new(AsyncMutex::new(())); + locks.insert(id, Arc::downgrade(&lock)); + lock +} + +#[cfg(test)] +mod operation_lock_tests { + use super::*; + + #[test] + fn live_operation_locks_are_reused_and_dead_entries_are_pruned() { + let mut locks = HashMap::new(); + let id = Uuid::new_v4(); + let first = operation_lock(&mut locks, id); + let second = operation_lock(&mut locks, id); + + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(locks.len(), 1); + + drop(first); + drop(second); + for _ in 0..256 { + drop(operation_lock(&mut locks, Uuid::new_v4())); + } + + assert_eq!(locks.len(), 1); + } + + #[cfg(unix)] + #[test] + fn linked_owner_directory_is_an_ambiguous_publication() { + use std::os::unix::fs::symlink; + + let state = tempfile::tempdir().expect("state root"); + let external = tempfile::tempdir().expect("external owner"); + let expected = SandboxInstance::new_warm_claim( + Uuid::new_v4(), + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:test".to_string(), + "test-policy".to_string(), + BackendOwnership::NotStarted, + Uuid::new_v4(), + ) + .expect("warm claim"); + expected + .persist(external.path()) + .expect("external lifecycle state"); + symlink( + external.path().join(expected.id.to_string()), + state.path().join(expected.id.to_string()), + ) + .expect("linked owner directory"); + + assert!(matches!( + inspect_warm_claim_publication(state.path(), &expected), + WarmClaimPublication::Ambiguous(reason) + if reason.contains("not a real lifecycle owner directory") + )); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/restore.rs b/src/blaze/crates/blazed/src/sandbox/restore.rs new file mode 100644 index 0000000000..ceefb7669b --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/restore.rs @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable replacement of a running sandbox from a committed checkpoint. + +use std::path::PathBuf; + +use blaze_core::backend::{RestoreRequest, SnapshotKind}; +use blaze_core::checkpoint::validate_checkpoint_id; +use blaze_core::lifecycle::{BackendOwnership, OperationPhase, SandboxInstance, SandboxState}; +use blaze_core::storage::StorageRestoreTransaction; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +/// Inputs resolved from the current daemon configuration. +#[derive(Debug, Clone)] +pub struct RestoreSandbox { + /// Committed checkpoint selected by the caller. + pub checkpoint_id: String, + /// Current executable for the checkpoint's backend. + pub binary_path: PathBuf, +} + +/// Result of one completed checkpoint restore. +#[derive(Debug, Clone)] +pub struct RestoreSandboxResult { + /// Updated durable sandbox record. + pub instance: SandboxInstance, + /// Checkpoint now selected by the catalog HEAD. + pub checkpoint_id: String, +} + +impl SandboxManager { + /// Replace a running backend and rootfs from one verified checkpoint. + pub async fn restore(&self, id: Uuid, request: RestoreSandbox) -> Result { + validate_checkpoint_id(&request.checkpoint_id) + .map_err(|error| BlazeDaemonError::BadRequest(error.to_string()))?; + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != SandboxState::Running { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected running", + instance.state + ))); + } + + let target = self + .checkpoints + .verify_restore_target(id, &request.checkpoint_id) + .map_err(checkpoint_store_error)?; + if target.metadata.policy_name != instance.policy_name + || target.metadata.image_digest != instance.image_digest + || target.metadata.backend != instance.backend + { + return Err(BlazeDaemonError::Conflict(format!( + "checkpoint {} runtime identity does not match instance {id}", + request.checkpoint_id + ))); + } + if target.metadata.snapshot_kind != SnapshotKind::Full { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "checkpoint {} does not contain a full snapshot", + request.checkpoint_id + ))); + } + + let current_backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if current_backend.instance_id() != id || current_backend.backend() != instance.backend { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + self.require_restore_backend_live(id, ¤t_backend) + .await?; + if !self.storage.supports_checkpoint_restore() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} configured storage does not support checkpoint restore" + ))); + } + let spawner = self.spawner(target.metadata.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no restore adapter for {}", + target.metadata.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support checkpoint restore", + target.metadata.backend + )) + })?; + if capability.backend != target.metadata.backend + || capability.version != target.metadata.backend_version + || capability.snapshot_kind != target.metadata.snapshot_kind + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "checkpoint {} requires {} version {:?} {:?}, but the current adapter provides \ + {} version {:?} {:?}", + request.checkpoint_id, + target.metadata.backend, + target.metadata.backend_version, + target.metadata.snapshot_kind, + capability.backend, + capability.version, + capability.snapshot_kind + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + instance.begin_restore_operation(request.checkpoint_id.clone())?; + crate::failpoint::state("restore-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("restore-after-begin").await; + + let transaction = match crate::failpoint::storage("restore-storage-stage") { + Ok(()) => { + self.storage + .stage_checkpoint_restore(&storage, &target.rootfs_path) + .await + } + Err(error) => Err(error), + }; + let transaction = match transaction { + Ok(transaction) => transaction, + Err(error) => { + return Err(self + .fail_before_restore_stop(instance, None, error.into()) + .await); + } + }; + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-staged-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .fail_before_restore_stop(instance, Some(&transaction), error) + .await); + } + crate::failpoint::pause("restore-after-stage").await; + + let stopped = match crate::failpoint::backend("restore-backend-stop") { + Ok(()) => current_backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + let abort = self + .storage + .abort_checkpoint_restore(&transaction) + .await + .err(); + return Err(self.fail_after_restore_stop( + instance, + format!( + "current backend termination failed: {error}{}", + abort + .map(|error| format!("; staged storage cleanup failed: {error}")) + .unwrap_or_default() + ), + )); + } + + instance.backend_ownership = BackendOwnership::Stopped; + let stopped_state = instance + .advance_restore_phase(OperationPhase::RestoreBackendStopped) + .and_then(|_| instance.transition(SandboxState::Restoring)) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-stopped-state")?; + self.persist_and_retain(instance.clone()) + }); + if let Err(error) = stopped_state { + self.remove_backend_owner(id); + return Err(self.fail_after_restore_stop( + instance, + format!("backend stopped but lifecycle commit failed: {error}"), + )); + } + self.remove_backend_owner(id); + crate::failpoint::pause("restore-after-stop").await; + + let activated = match crate::failpoint::storage("restore-storage-activate") { + Ok(()) => self.storage.activate_checkpoint_restore(&transaction).await, + Err(error) => Err(error), + }; + if let Err(error) = activated { + let abort = self + .storage + .abort_checkpoint_restore(&transaction) + .await + .err(); + return Err(self.fail_after_restore_stop( + instance, + format!( + "replacement storage activation failed: {error}{}", + abort + .map(|error| format!("; predecessor restore failed: {error}")) + .unwrap_or_default() + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageActivated) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-activated-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage activated but lifecycle commit failed: {error}"), + )); + } + crate::failpoint::pause("restore-after-activate").await; + + let run_dir = self.runtime_dir(id); + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self.fail_after_restore_stop( + instance, + format!("prepare replacement backend ownership failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = crate::failpoint::state("restore-starting-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend intent commit failed: {error}"), + )); + } + + let restored = match crate::failpoint::backend("restore-backend-start") { + Ok(()) => { + spawner + .restore(RestoreRequest { + instance_id: id, + run_dir, + binary_path: request.binary_path, + storage, + snapshot_path: target.snapshot_path, + mem_path: target.memory_path, + checkpoint_backend: target.metadata.backend, + expected_version: target.metadata.backend_version.clone(), + snapshot_kind: target.metadata.snapshot_kind, + expose_guest_socket: target.metadata.expose_guest_socket, + network_slot: target.metadata.network_slot, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let restored = match restored { + Ok(owner) => owner, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + let _ = self.retain_backend(id, owner); + instance.backend_ownership = BackendOwnership::Running; + } else { + instance.backend_ownership = BackendOwnership::Stopped; + } + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend start failed: {source}"), + )); + } + }; + if let Some(error) = self.retain_backend(id, restored.clone()) { + instance.backend_ownership = BackendOwnership::Running; + return Err(self.fail_after_restore_stop(instance, error)); + } + instance.backend_ownership = BackendOwnership::Running; + + if restored.instance_id() != id + || restored.backend() != target.metadata.backend + || restored.version().map(str::to_string) != target.metadata.backend_version + { + return Err(self.fail_after_restore_stop( + instance, + format!( + "replacement backend identity ({}, {}, {:?}) does not match checkpoint \ + identity ({id}, {}, {:?})", + restored.instance_id(), + restored.backend(), + restored.version(), + target.metadata.backend, + target.metadata.backend_version + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreBackendStarted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-started-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend started but lifecycle commit failed: {error}"), + )); + } + if let Err(error) = self + .verify_restored_backend(id, &restored, target.metadata.expose_guest_socket) + .await + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend readiness failed: {error}"), + )); + } + + let head_updated = match crate::failpoint::storage("restore-head-update") { + Ok(()) => self + .checkpoints + .set_head(id, &request.checkpoint_id) + .map_err(checkpoint_store_error), + Err(error) => Err(error.into()), + }; + if let Err(error) = head_updated { + let observed = self.checkpoints.read_head(id); + return Err(self.fail_after_restore_stop( + instance, + format!( + "checkpoint HEAD update failed: {error}; observed HEAD after failure: \ + {observed:?}" + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreHeadUpdated) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-head-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("checkpoint HEAD changed but lifecycle commit failed: {error}"), + )); + } + crate::failpoint::pause("restore-after-head").await; + + let committed = match crate::failpoint::storage("restore-storage-commit") { + Ok(()) => self.storage.commit_checkpoint_restore(&transaction).await, + Err(error) => Err(error), + }; + if let Err(error) = committed { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage commit failed: {error}"), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageCommitted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-committed-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage committed but lifecycle commit failed: {error}"), + )); + } + + let recovery_instance = instance.clone(); + instance.transition(SandboxState::Running)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("restore-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_after_restore_stop( + recovery_instance, + format!("replacement is live but final lifecycle commit failed: {error}"), + )); + } + Ok(RestoreSandboxResult { + instance, + checkpoint_id: request.checkpoint_id, + }) + } + + async fn require_restore_backend_live( + &self, + id: Uuid, + backend: &DynBackendInstance, + ) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited before restore \ + (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))) + } + Err(error) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))) + } + } + } + + async fn verify_restored_backend( + &self, + id: Uuid, + backend: &DynBackendInstance, + expose_guest_socket: bool, + ) -> Result<()> { + self.require_restore_backend_live(id, backend).await?; + if expose_guest_socket { + self.wait_for_guest_ready(backend, "restore-guest-ready") + .await?; + } + self.require_restore_backend_live(id, backend).await + } + + async fn fail_before_restore_stop( + &self, + mut instance: SandboxInstance, + transaction: Option<&StorageRestoreTransaction>, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + let storage_cleanup = match transaction { + Some(transaction) => self + .storage + .abort_checkpoint_restore(transaction) + .await + .map_err(BlazeDaemonError::from), + None => self + .storage + .reconcile_checkpoint_restore(&instance.id.to_string()) + .await + .map_err(BlazeDaemonError::from), + }; + if let Err(cleanup) = storage_cleanup { + return self.fail_after_restore_stop( + instance, + format!("{original}; staged storage cleanup failed: {cleanup}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance.clone()) { + return self.fail_after_restore_stop( + instance, + format!("{original}; restore journal cleanup failed: {error}"), + ); + } + original + } + + fn fail_after_restore_stop( + &self, + instance: SandboxInstance, + cause: impl std::fmt::Display, + ) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "restore {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } +} + +fn checkpoint_store_error(error: impl std::fmt::Display) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("checkpoint store: {error}")) +} diff --git a/src/blaze/crates/blazed/src/sandbox/template.rs b/src/blaze/crates/blazed/src/sandbox/template.rs new file mode 100644 index 0000000000..7a30b55a29 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/template.rs @@ -0,0 +1,1565 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable runtime artifact publication and lookup. + +use std::collections::HashSet; +use std::ffi::{CStr, CString, OsStr, OsString}; +use std::fs::{DirBuilder, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::{AsRawFd, FromRawFd}; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; + +use blaze_core::config::RuntimeTemplateSection; +use serde_json::json; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; + +use super::manager::SandboxManager; + +const CATALOG_DIR_MODE: u32 = 0o700; +const CATALOG_FILE_MODE: u32 = 0o600; +const COPY_BUFFER_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Copy)] +struct ImportLimits { + max_files: usize, + max_bytes: u64, + max_metadata_bytes: u64, + max_total_bytes: u64, +} + +#[derive(Clone)] +pub(crate) struct RuntimeTemplateCatalog { + inner: Arc, +} + +struct CatalogInner { + root: PathBuf, + import_root: Option, + limits: ImportLimits, + state: Mutex, + active_count: watch::Sender, + cancellation: CancellationToken, + #[cfg(test)] + copy_gate: Mutex>>, +} + +struct CatalogState { + active_names: HashSet, + committed_bytes: u64, + reserved_bytes: u64, + stopping: bool, + blocked: Option, +} + +struct ImportClaim { + inner: Arc, + name: String, + reserved_bytes: u64, +} + +struct PreparedFile { + name: OsString, + file: File, + observed_bytes: u64, + observed_dev: u64, + observed_ino: u64, + observed_mtime: i64, + observed_mtime_nsec: i64, + observed_ctime: i64, + observed_ctime_nsec: i64, +} + +struct PreparedImport { + files: Vec, + metadata: serde_json::Value, + metadata_bytes: Vec, + reserved_bytes: u64, +} + +#[cfg(test)] +struct TestCopyGate { + entered: tokio::sync::mpsc::UnboundedSender<()>, + release: AtomicBool, +} + +impl RuntimeTemplateCatalog { + pub(crate) fn open(config: &RuntimeTemplateSection) -> Result { + create_catalog_root(&config.dir)?; + cleanup_staging(&config.dir)?; + let limits = ImportLimits { + max_files: config.max_files, + max_bytes: config.max_bytes, + max_metadata_bytes: config.max_metadata_bytes, + max_total_bytes: config.max_total_bytes, + }; + let committed_bytes = catalog_usage(&config.dir, limits)?; + if committed_bytes > limits.max_total_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog uses {committed_bytes} bytes; configured limit is {}", + limits.max_total_bytes + ))); + } + let (active_count, _) = watch::channel(0); + Ok(Self { + inner: Arc::new(CatalogInner { + root: config.dir.clone(), + import_root: config.import_root.clone(), + limits, + state: Mutex::new(CatalogState { + active_names: HashSet::new(), + committed_bytes, + reserved_bytes: 0, + stopping: false, + blocked: None, + }), + active_count, + cancellation: CancellationToken::new(), + #[cfg(test)] + copy_gate: Mutex::new(None), + }), + }) + } + + async fn list(&self) -> Result> { + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + list_published(&catalog.inner.root, catalog.inner.limits) + }) + .await + .map_err(join_error("runtime template list"))? + } + + async fn get(&self, name: String) -> Result { + validate_name(&name, "runtime template")?; + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + get_published(&catalog.inner.root, &name, catalog.inner.limits) + }) + .await + .map_err(join_error("runtime template read"))? + } + + async fn import( + &self, + name: String, + source: PathBuf, + description: String, + ) -> Result { + validate_name(&name, "runtime template")?; + validate_relative_source(&source)?; + if self.inner.import_root.is_none() { + return Err(BlazeDaemonError::Conflict( + "runtime template import is disabled; configure \ + runtime_templates.import_root" + .to_string(), + )); + } + + // Register before scheduling blocking work. Shutdown can therefore + // observe and wait for an import even when the blocking pool has not + // started its closure yet. + let claim = ImportClaim::begin(Arc::clone(&self.inner), name.clone())?; + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + catalog.import_blocking(claim, name, source, description) + }) + .await + .map_err(join_error("runtime template import"))? + } + + fn import_blocking( + &self, + mut claim: ImportClaim, + name: String, + source: PathBuf, + description: String, + ) -> Result { + check_cancelled(&self.inner.cancellation)?; + let import_root = + self.inner.import_root.as_deref().ok_or_else(|| { + BlazeDaemonError::Conflict("runtime template import disabled".into()) + })?; + let source = open_import_source(import_root, &source)?; + let prepared = prepare_import( + &source, + &name, + &description, + self.inner.limits, + &self.inner.cancellation, + )?; + claim.reserve(prepared.reserved_bytes)?; + publish_prepared( + &self.inner.root, + &name, + prepared, + &self.inner.cancellation, + &mut claim, + ) + } + + pub(super) fn cancel_imports(&self) { + let mut state = lock_catalog_state(&self.inner); + state.stopping = true; + drop(state); + self.inner.cancellation.cancel(); + } + + pub(super) async fn wait_for_imports(&self) -> Result<()> { + let mut active = self.inner.active_count.subscribe(); + loop { + if *active.borrow_and_update() == 0 { + return Ok(()); + } + active.changed().await.map_err(|_| { + BlazeDaemonError::Internal( + "runtime template import supervisor closed unexpectedly".to_string(), + ) + })?; + } + } + + #[cfg(test)] + fn active_imports(&self) -> usize { + *self.inner.active_count.borrow() + } + + #[cfg(test)] + fn install_copy_gate(&self) -> tokio::sync::mpsc::UnboundedReceiver<()> { + let (entered, receiver) = tokio::sync::mpsc::unbounded_channel(); + let gate = Arc::new(TestCopyGate { + entered, + release: AtomicBool::new(false), + }); + *self.inner.copy_gate.lock().expect("copy gate lock") = Some(gate); + receiver + } +} + +impl ImportClaim { + fn begin(inner: Arc, name: String) -> Result { + let mut state = lock_catalog_state(&inner); + if state.stopping { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template imports are stopping".to_string(), + )); + } + if let Some(error) = &state.blocked { + return Err(BlazeDaemonError::RecoveryRequired(error.clone())); + } + if !state.active_names.insert(name.clone()) { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} import is already in progress" + ))); + } + let count = state.active_names.len(); + inner.active_count.send_replace(count); + drop(state); + Ok(Self { + inner, + name, + reserved_bytes: 0, + }) + } + + fn reserve(&mut self, bytes: u64) -> Result<()> { + let mut state = lock_catalog_state(&self.inner); + if state.stopping || self.inner.cancellation.is_cancelled() { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template imports are stopping".to_string(), + )); + } + if let Some(error) = &state.blocked { + return Err(BlazeDaemonError::RecoveryRequired(error.clone())); + } + let used = state + .committed_bytes + .checked_add(state.reserved_bytes) + .and_then(|value| value.checked_add(bytes)) + .ok_or_else(|| payload_too_large(u64::MAX, self.inner.limits.max_total_bytes))?; + if used > self.inner.limits.max_total_bytes { + return Err(payload_too_large(used, self.inner.limits.max_total_bytes)); + } + state.reserved_bytes += bytes; + self.reserved_bytes = bytes; + Ok(()) + } + + fn publish(&mut self, actual_bytes: u64) -> Result<()> { + if actual_bytes > self.reserved_bytes { + let message = format!( + "runtime template {} wrote {actual_bytes} bytes beyond its {}-byte reservation", + self.name, self.reserved_bytes + ); + self.block_catalog(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + let mut state = lock_catalog_state(&self.inner); + let Some(remaining_reserved) = state.reserved_bytes.checked_sub(self.reserved_bytes) else { + let message = "runtime template reservation accounting underflow".to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + }; + let Some(committed_bytes) = state.committed_bytes.checked_add(actual_bytes) else { + let message = "runtime template catalog byte accounting overflow".to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + }; + if committed_bytes + .checked_add(remaining_reserved) + .is_none_or(|used| used > self.inner.limits.max_total_bytes) + { + let message = "runtime template catalog accounting exceeded the configured total limit" + .to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + state.reserved_bytes = remaining_reserved; + state.committed_bytes = committed_bytes; + self.reserved_bytes = 0; + Ok(()) + } + + fn block_catalog(&self, message: String) { + let mut state = lock_catalog_state(&self.inner); + state.blocked = Some(message); + } +} + +impl Drop for ImportClaim { + fn drop(&mut self) { + let mut state = lock_catalog_state(&self.inner); + if self.reserved_bytes > 0 { + state.reserved_bytes = state.reserved_bytes.saturating_sub(self.reserved_bytes); + } + state.active_names.remove(&self.name); + let count = state.active_names.len(); + self.inner.active_count.send_replace(count); + } +} + +impl SandboxManager { + /// List atomically published runtime artifact sets. + pub async fn list_runtime_templates(&self) -> Result> { + self.runtime_templates.list().await + } + + /// Read one published runtime artifact set by name. + pub async fn get_runtime_template(&self, name: String) -> Result { + self.runtime_templates.get(name).await + } + + /// Copy and atomically publish one operator-prepared artifact directory. + pub async fn import_runtime_template( + &self, + name: String, + source: PathBuf, + description: String, + ) -> Result { + self.runtime_templates + .import(name, source, description) + .await + } + + /// Reject new imports and request cancellation of every active import. + pub(crate) fn cancel_runtime_template_imports(&self) { + self.runtime_templates.cancel_imports(); + } + + /// Wait until every registered import has released its filesystem handles. + pub(crate) async fn wait_for_runtime_template_imports(&self) -> Result<()> { + self.runtime_templates.wait_for_imports().await + } +} + +fn prepare_import( + source: &File, + name: &str, + description: &str, + limits: ImportLimits, + cancellation: &CancellationToken, +) -> Result { + let names = source_entry_names(source)?; + let mut files = Vec::with_capacity(names.len()); + let mut metadata_file = None; + let mut artifact_bytes = 0_u64; + + for entry_name in names { + check_cancelled(cancellation)?; + validate_artifact_name(&entry_name)?; + let file = openat_regular(source, &entry_name)?; + let metadata = file.metadata()?; + validate_source_file(&metadata, &entry_name)?; + if entry_name == OsStr::new("template.json") { + if metadata.len() > limits.max_metadata_bytes { + return Err(payload_too_large(metadata.len(), limits.max_metadata_bytes)); + } + metadata_file = Some(file); + } else { + artifact_bytes = artifact_bytes + .checked_add(metadata.len()) + .ok_or_else(|| payload_too_large(u64::MAX, limits.max_bytes))?; + files.push(PreparedFile { + name: entry_name, + file, + observed_bytes: metadata.len(), + observed_dev: metadata.dev(), + observed_ino: metadata.ino(), + observed_mtime: metadata.mtime(), + observed_mtime_nsec: metadata.mtime_nsec(), + observed_ctime: metadata.ctime(), + observed_ctime_nsec: metadata.ctime_nsec(), + }); + } + } + + let published_files = files.len() + 1; + if published_files > limits.max_files { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template contains {published_files} files; limit is {}", + limits.max_files + ))); + } + let present = files + .iter() + .map(|file| file.name.as_os_str()) + .collect::>(); + for required in ["vmstate.snap", "mem.bin", "rootfs.ext4"] { + if !present.contains(OsStr::new(required)) { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source is missing regular artifact {required}" + ))); + } + } + + let mut metadata = match metadata_file { + Some(mut file) => { + let observed = file.metadata()?; + let metadata = read_json_bounded(&mut file, limits.max_metadata_bytes)?; + let current = file.metadata()?; + if !same_file_identity(&observed, ¤t) { + return Err(BlazeDaemonError::BadRequest( + "runtime template source metadata changed while it was imported".to_string(), + )); + } + metadata + } + None => json!({"name": name}), + }; + if !metadata.is_object() { + return Err(BlazeDaemonError::BadRequest( + "template.json must contain a JSON object".to_string(), + )); + } + metadata["name"] = json!(name); + if !description.is_empty() { + metadata["description"] = json!(description); + } + if metadata + .get("rootfs_size") + .and_then(serde_json::Value::as_u64) + .is_none() + { + metadata["rootfs_size"] = json!(8_u64 * 1024 * 1024 * 1024); + } + if metadata + .get("memory_size") + .and_then(serde_json::Value::as_u64) + .is_none() + { + metadata["memory_size"] = json!(4_u64 * 1024 * 1024 * 1024); + } + let metadata_bytes = serde_json::to_vec_pretty(&metadata)?; + let metadata_len = u64::try_from(metadata_bytes.len()).unwrap_or(u64::MAX); + if metadata_len > limits.max_metadata_bytes { + return Err(payload_too_large(metadata_len, limits.max_metadata_bytes)); + } + let reserved_bytes = artifact_bytes + .checked_add(metadata_len) + .ok_or_else(|| payload_too_large(u64::MAX, limits.max_bytes))?; + if reserved_bytes > limits.max_bytes { + return Err(payload_too_large(reserved_bytes, limits.max_bytes)); + } + + Ok(PreparedImport { + files, + metadata, + metadata_bytes, + reserved_bytes, + }) +} + +fn publish_prepared( + root: &Path, + name: &str, + prepared: PreparedImport, + cancellation: &CancellationToken, + claim: &mut ImportClaim, +) -> Result { + let destination = root.join(name); + if destination.exists() { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} already exists" + ))); + } + + let staging = root.join(format!(".import-{name}-{}.tmp", Uuid::new_v4())); + create_private_directory(&staging)?; + #[cfg(test)] + wait_for_copy_gate(&claim.inner, cancellation); + let result = populate_and_publish( + root, + &destination, + &staging, + name, + prepared, + cancellation, + claim, + ); + if result.is_err() && staging.exists() { + if let Err(cleanup_error) = std::fs::remove_dir_all(&staging) { + claim.block_catalog(format!( + "runtime template staging cleanup failed; restart after repairing the catalog: \ + {cleanup_error}" + )); + tracing::error!( + path = %staging.display(), + error = %cleanup_error, + "runtime template staging cleanup failed" + ); + } else if let Err(sync_error) = sync_directory(root) { + claim.block_catalog(format!( + "runtime template cleanup durability is unknown; restart after repairing the \ + catalog: {sync_error}" + )); + tracing::error!( + path = %root.display(), + error = %sync_error, + "runtime template cleanup durability is unknown" + ); + } + } + result +} + +fn populate_and_publish( + root: &Path, + destination: &Path, + staging: &Path, + name: &str, + mut prepared: PreparedImport, + cancellation: &CancellationToken, + claim: &mut ImportClaim, +) -> Result { + let mut actual_bytes = 0_u64; + for source in &mut prepared.files { + check_cancelled(cancellation)?; + let remaining = prepared + .reserved_bytes + .checked_sub(actual_bytes) + .and_then(|value| { + value.checked_sub(u64::try_from(prepared.metadata_bytes.len()).unwrap_or(u64::MAX)) + }) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + let destination_file = staging.join(&source.name); + let copied = + copy_regular_file(&mut source.file, &destination_file, remaining, cancellation)?; + let current = source.file.metadata()?; + if copied != source.observed_bytes + || current.len() != source.observed_bytes + || current.dev() != source.observed_dev + || current.ino() != source.observed_ino + || current.mtime() != source.observed_mtime + || current.mtime_nsec() != source.observed_mtime_nsec + || current.ctime() != source.observed_ctime + || current.ctime_nsec() != source.observed_ctime_nsec + { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source file {} changed while it was imported", + source.name.to_string_lossy() + ))); + } + actual_bytes = actual_bytes + .checked_add(copied) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + } + + let metadata_len = u64::try_from(prepared.metadata_bytes.len()).unwrap_or(u64::MAX); + actual_bytes = actual_bytes + .checked_add(metadata_len) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + if actual_bytes > prepared.reserved_bytes { + return Err(payload_too_large(actual_bytes, prepared.reserved_bytes)); + } + write_file_durable(&staging.join("template.json"), &prepared.metadata_bytes)?; + sync_directory(staging)?; + check_cancelled(cancellation)?; + + if destination.exists() { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} already exists" + ))); + } + rename_no_replace(staging, destination).map_err(|error| { + if destination.exists() { + BlazeDaemonError::Conflict(format!("runtime template {name} already exists")) + } else { + error.into() + } + })?; + + // The directory is now publicly owned even if the parent fsync fails. + // Account for it before reporting an uncertain durability result. + claim.publish(actual_bytes)?; + if let Err(error) = sync_directory(root) { + let message = format!( + "runtime template {name} was published but catalog durability is unknown: {error}" + ); + claim.block_catalog(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + Ok(prepared.metadata) +} + +fn open_import_source(import_root: &Path, relative: &Path) -> Result { + let mut directory = open_directory_no_follow(import_root).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open configured runtime template import root {}: {error}", + import_root.display() + )) + })?; + validate_source_directory(&directory.metadata()?, import_root)?; + for component in relative.components() { + let Component::Normal(name) = component else { + return Err(BlazeDaemonError::BadRequest( + "runtime template source must be a non-empty relative path below the configured \ + import root" + .to_string(), + )); + }; + directory = openat_directory(&directory, name).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open runtime template source {}: {error}", + relative.display() + )) + })?; + validate_source_directory(&directory.metadata()?, relative)?; + } + Ok(directory) +} + +fn source_entry_names(directory: &File) -> Result> { + let duplicated = unsafe { libc::dup(directory.as_raw_fd()) }; + if duplicated < 0 { + return Err(io::Error::last_os_error().into()); + } + let stream = unsafe { libc::fdopendir(duplicated) }; + if stream.is_null() { + let error = io::Error::last_os_error(); + unsafe { + libc::close(duplicated); + } + return Err(error.into()); + } + let mut names = Vec::new(); + loop { + clear_errno(); + let entry = unsafe { libc::readdir(stream) }; + if entry.is_null() { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(0) { + unsafe { + libc::closedir(stream); + } + return Err(error.into()); + } + break; + } + let name = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes(); + if name != b"." && name != b".." { + names.push(OsString::from_vec(name.to_vec())); + } + } + let close_result = unsafe { libc::closedir(stream) }; + if close_result != 0 { + return Err(io::Error::last_os_error().into()); + } + names.sort(); + Ok(names) +} + +#[cfg(target_os = "linux")] +fn clear_errno() { + unsafe { + *libc::__errno_location() = 0; + } +} + +#[cfg(not(target_os = "linux"))] +fn clear_errno() { + unsafe { + *libc::__error() = 0; + } +} + +fn open_directory_no_follow(path: &Path) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains NUL"))?; + let fd = unsafe { + libc::open( + path.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW, + ) + }; + file_from_fd(fd) +} + +fn openat_directory(parent: &File, name: &OsStr) -> io::Result { + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "name contains NUL"))?; + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW, + ) + }; + file_from_fd(fd) +} + +fn openat_regular(parent: &File, name: &OsStr) -> Result { + let name_c = CString::new(name.as_bytes()) + .map_err(|_| BlazeDaemonError::BadRequest("artifact name contains NUL".to_string()))?; + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name_c.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, + ) + }; + file_from_fd(fd).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open runtime template source entry {} without following links: {error}", + name.to_string_lossy() + )) + }) +} + +fn file_from_fd(fd: libc::c_int) -> io::Result { + if fd < 0 { + return Err(io::Error::last_os_error()); + } + Ok(unsafe { File::from_raw_fd(fd) }) +} + +fn validate_source_directory(metadata: &std::fs::Metadata, path: &Path) -> Result<()> { + if !metadata.is_dir() { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source {} is not a directory", + path.display() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid || metadata.mode() & 0o022 != 0 { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source directory {} must be owned by the daemon user and not \ + writable by group or other users", + path.display() + ))); + } + Ok(()) +} + +fn validate_source_file(metadata: &std::fs::Metadata, name: &OsStr) -> Result<()> { + if !metadata.file_type().is_file() { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source entry {} is not a regular file", + name.to_string_lossy() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid || metadata.mode() & 0o022 != 0 { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source file {} must be owned by the daemon user and not writable \ + by group or other users", + name.to_string_lossy() + ))); + } + Ok(()) +} + +fn same_file_identity(observed: &std::fs::Metadata, current: &std::fs::Metadata) -> bool { + observed.len() == current.len() + && observed.dev() == current.dev() + && observed.ino() == current.ino() + && observed.mtime() == current.mtime() + && observed.mtime_nsec() == current.mtime_nsec() + && observed.ctime() == current.ctime() + && observed.ctime_nsec() == current.ctime_nsec() +} + +fn create_catalog_root(root: &Path) -> Result<()> { + if !root.exists() { + DirBuilder::new() + .recursive(true) + .mode(CATALOG_DIR_MODE) + .create(root)?; + if let Some(parent) = root.parent() { + sync_directory(parent)?; + } + } + enforce_owned_mode(root, true, CATALOG_DIR_MODE) +} + +fn create_private_directory(path: &Path) -> Result<()> { + DirBuilder::new().mode(CATALOG_DIR_MODE).create(path)?; + enforce_owned_mode(path, true, CATALOG_DIR_MODE) +} + +fn enforce_owned_mode(path: &Path, directory: bool, mode: u32) -> Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() + || (directory && !metadata.is_dir()) + || (!directory && !metadata.file_type().is_file()) + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog path {} has an unexpected file type", + path.display() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog path {} is not owned by the daemon user", + path.display() + ))); + } + if metadata.mode() & 0o777 != mode { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?; + } + Ok(()) +} + +fn catalog_usage(root: &Path, limits: ImportLimits) -> Result { + let mut total = 0_u64; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if name.to_string_lossy().starts_with('.') { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains unresolved hidden entry {}", + entry.path().display() + ))); + } + enforce_owned_mode(&entry.path(), true, CATALOG_DIR_MODE)?; + let mut file_count = 0_usize; + let mut template_bytes = 0_u64; + for artifact in std::fs::read_dir(entry.path())? { + let artifact = artifact?; + enforce_owned_mode(&artifact.path(), false, CATALOG_FILE_MODE)?; + file_count = file_count.checked_add(1).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog file count overflow".to_string(), + ) + })?; + if file_count > limits.max_files { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} exceeds the configured file limit", + name.to_string_lossy() + ))); + } + let artifact_bytes = artifact.metadata()?.len(); + if artifact.file_name() == OsStr::new("template.json") + && artifact_bytes > limits.max_metadata_bytes + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} metadata exceeds the configured limit", + name.to_string_lossy() + ))); + } + template_bytes = template_bytes.checked_add(artifact_bytes).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template byte accounting overflow".to_string(), + ) + })?; + total = total.checked_add(artifact_bytes).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog byte accounting overflow".to_string(), + ) + })?; + } + if template_bytes > limits.max_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} exceeds the configured per-import byte limit", + name.to_string_lossy() + ))); + } + let name = name.into_string().map_err(|_| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog contains a non-UTF-8 published name".to_string(), + ) + })?; + validate_name(&name, "runtime template").map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains invalid published name {name}: {error}" + )) + })?; + read_published(root, &name, limits)?; + } + Ok(total) +} + +fn list_published(root: &Path, limits: ImportLimits) -> Result> { + let mut templates = Vec::new(); + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if name.to_string_lossy().starts_with('.') { + continue; + } + let name = name.into_string().map_err(|_| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains a non-UTF-8 name at {}", + entry.path().display() + )) + })?; + templates.push(read_published(root, &name, limits)?); + } + templates.sort_by(|left, right| { + left.get("name") + .and_then(serde_json::Value::as_str) + .cmp(&right.get("name").and_then(serde_json::Value::as_str)) + }); + Ok(templates) +} + +fn get_published(root: &Path, name: &str, limits: ImportLimits) -> Result { + let root = open_directory_no_follow(root).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot open runtime template catalog {}: {error}", + root.display() + )) + })?; + let directory = match openat_directory(&root, OsStr::new(name)) { + Ok(directory) => directory, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(BlazeDaemonError::NotFound(format!( + "runtime template {name}" + ))); + } + Err(error) => return Err(error.into()), + }; + read_published_directory(&directory, name, limits) +} + +fn read_published(root: &Path, name: &str, limits: ImportLimits) -> Result { + let root = open_directory_no_follow(root)?; + let directory = openat_directory(&root, OsStr::new(name)).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!("cannot open runtime template {name}: {error}")) + })?; + read_published_directory(&directory, name, limits) +} + +fn read_published_directory( + directory: &File, + expected_name: &str, + limits: ImportLimits, +) -> Result { + let names = source_entry_names(directory).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot inspect runtime template {expected_name}: {error}" + )) + })?; + if names.len() > limits.max_files { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} exceeds the configured file limit" + ))); + } + let mut total_bytes = 0_u64; + for name in names { + validate_artifact_name(&name).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} contains an invalid artifact: {error}" + )) + })?; + let file = openat_regular(directory, &name).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} contains an invalid artifact {}: {error}", + name.to_string_lossy() + )) + })?; + let metadata = file.metadata()?; + validate_published_file(&metadata, expected_name, &name)?; + if name == OsStr::new("template.json") && metadata.len() > limits.max_metadata_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} metadata exceeds the configured limit" + ))); + } + total_bytes = total_bytes.checked_add(metadata.len()).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} byte accounting overflow" + )) + })?; + } + if total_bytes > limits.max_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} exceeds the configured byte limit" + ))); + } + + let mut metadata = openat_regular(directory, OsStr::new("template.json")).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot open runtime template {expected_name} metadata: {error}" + )) + })?; + let value = read_json_bounded(&mut metadata, limits.max_metadata_bytes).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot read runtime template {expected_name} metadata: {error}" + )) + })?; + if !value.is_object() + || value.get("name").and_then(serde_json::Value::as_str) != Some(expected_name) + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} metadata does not match its catalog name" + ))); + } + for required in ["vmstate.snap", "mem.bin", "rootfs.ext4"] { + openat_regular(directory, OsStr::new(required)).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} is missing regular artifact {required}: {error}" + )) + })?; + } + Ok(value) +} + +fn validate_published_file( + metadata: &std::fs::Metadata, + template: &str, + name: &OsStr, +) -> Result<()> { + let expected_uid = unsafe { libc::geteuid() }; + if !metadata.file_type().is_file() + || metadata.uid() != expected_uid + || metadata.mode() & 0o777 != CATALOG_FILE_MODE + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {template} artifact {} has unexpected type, ownership, or mode", + name.to_string_lossy() + ))); + } + Ok(()) +} + +fn copy_regular_file( + source: &mut File, + destination: &Path, + max_bytes: u64, + cancellation: &CancellationToken, +) -> Result { + let mut destination = OpenOptions::new() + .write(true) + .create_new(true) + .mode(CATALOG_FILE_MODE) + .custom_flags(libc::O_NOFOLLOW) + .open(destination)?; + let mut buffer = vec![0_u8; COPY_BUFFER_BYTES]; + let mut copied = 0_u64; + loop { + check_cancelled(cancellation)?; + let read = source.read(&mut buffer)?; + if read == 0 { + break; + } + let next = copied + .checked_add(u64::try_from(read).unwrap_or(u64::MAX)) + .ok_or_else(|| payload_too_large(u64::MAX, max_bytes))?; + if next > max_bytes { + return Err(payload_too_large(next, max_bytes)); + } + destination.write_all(&buffer[..read])?; + copied = next; + } + destination.sync_all()?; + destination.set_permissions(std::fs::Permissions::from_mode(CATALOG_FILE_MODE))?; + Ok(copied) +} + +fn write_file_durable(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(CATALOG_FILE_MODE) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + file.set_permissions(std::fs::Permissions::from_mode(CATALOG_FILE_MODE))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn rename_no_replace(source: &Path, destination: &Path) -> io::Result<()> { + rename_no_replace_linux(source, destination) +} + +#[cfg(not(target_os = "linux"))] +fn rename_no_replace(source: &Path, destination: &Path) -> io::Result<()> { + if destination.exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "destination already exists", + )); + } + std::fs::rename(source, destination) +} + +#[cfg(target_os = "linux")] +fn rename_no_replace_linux(source: &Path, destination: &Path) -> io::Result<()> { + let source = CString::new(source.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "source path contains NUL"))?; + let destination = CString::new(destination.as_os_str().as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "destination path contains NUL") + })?; + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + source.as_ptr(), + libc::AT_FDCWD, + destination.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +fn read_json_bounded(file: &mut File, limit: u64) -> Result { + let mut bytes = Vec::new(); + file.take(limit.saturating_add(1)).read_to_end(&mut bytes)?; + let actual = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if actual > limit { + return Err(payload_too_large(actual, limit)); + } + Ok(serde_json::from_slice(&bytes)?) +} + +fn sync_directory(path: &Path) -> Result<()> { + open_directory_no_follow(path)?.sync_all()?; + Ok(()) +} + +fn cleanup_staging(root: &Path) -> Result { + let mut removed = 0; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if !is_staging_name(&name) { + continue; + } + let metadata = std::fs::symlink_metadata(entry.path())?; + let expected_uid = unsafe { libc::geteuid() }; + if !metadata.is_dir() || metadata.uid() != expected_uid { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template staging entry {} has unexpected ownership or type", + entry.path().display() + ))); + } + std::fs::remove_dir_all(entry.path())?; + removed += 1; + } + if removed > 0 { + sync_directory(root)?; + tracing::info!( + removed, + "removed stale runtime template staging directories" + ); + } + Ok(removed) +} + +fn is_staging_name(name: &OsStr) -> bool { + let name = name.to_string_lossy(); + name.starts_with(".import-") && name.ends_with(".tmp") +} + +fn validate_name(value: &str, label: &str) -> Result<()> { + let mut chars = value.chars(); + let first = chars.next(); + if value.len() > 128 + || !first.is_some_and(|ch| ch.is_ascii_alphanumeric()) + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + { + return Err(BlazeDaemonError::BadRequest(format!( + "{label} must start with an ASCII letter or digit and contain at most 128 \ + letters, digits, dots, dashes, or underscores" + ))); + } + Ok(()) +} + +fn validate_artifact_name(value: &OsStr) -> Result<()> { + let value = value.to_str().ok_or_else(|| { + BlazeDaemonError::BadRequest( + "runtime template artifact names must be valid UTF-8".to_string(), + ) + })?; + validate_name(value, "runtime template artifact") +} + +fn validate_relative_source(source: &Path) -> Result<()> { + if source.as_os_str().is_empty() + || source + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(BlazeDaemonError::BadRequest( + "runtime template source must be a non-empty relative path below the configured \ + import root" + .to_string(), + )); + } + Ok(()) +} + +fn check_cancelled(cancellation: &CancellationToken) -> Result<()> { + if cancellation.is_cancelled() { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template import cancelled during daemon shutdown".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +fn wait_for_copy_gate(inner: &CatalogInner, cancellation: &CancellationToken) { + let gate = inner.copy_gate.lock().expect("copy gate lock").clone(); + if let Some(gate) = gate { + let _ = gate.entered.send(()); + while !gate.release.load(Ordering::Acquire) && !cancellation.is_cancelled() { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } +} + +fn payload_too_large(actual: u64, limit: u64) -> BlazeDaemonError { + BlazeDaemonError::PayloadTooLarge { + actual, + limit: usize::try_from(limit).unwrap_or(usize::MAX), + } +} + +fn join_error(context: &'static str) -> impl FnOnce(tokio::task::JoinError) -> BlazeDaemonError { + move |error| BlazeDaemonError::Internal(format!("{context} task: {error}")) +} + +fn lock_catalog_state(inner: &CatalogInner) -> std::sync::MutexGuard<'_, CatalogState> { + match inner.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config(root: &Path, import_root: &Path) -> RuntimeTemplateSection { + RuntimeTemplateSection { + dir: root.to_path_buf(), + import_root: Some(import_root.to_path_buf()), + max_files: 8, + max_bytes: 1024, + max_metadata_bytes: 512, + max_total_bytes: 2048, + } + } + + #[tokio::test] + async fn import_publishes_artifacts_with_private_permissions() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + let metadata = catalog + .import( + "runtime-base".to_string(), + PathBuf::from("source"), + "base runtime template".to_string(), + ) + .await + .expect("import"); + let destination = root.join("runtime-base"); + + assert_eq!(metadata["name"], "runtime-base"); + assert_eq!(metadata["description"], "base runtime template"); + assert_eq!( + std::fs::symlink_metadata(&destination) + .expect("directory") + .mode() + & 0o777, + CATALOG_DIR_MODE + ); + for file in ["vmstate.snap", "mem.bin", "rootfs.ext4", "template.json"] { + assert_eq!( + std::fs::symlink_metadata(destination.join(file)) + .expect("artifact") + .mode() + & 0o777, + CATALOG_FILE_MODE + ); + } + } + + #[tokio::test] + async fn import_rejects_special_entries_and_cleans_staging() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let fifo = CString::new(source.join("fifo").as_os_str().as_bytes()).expect("fifo path"); + assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + catalog + .import("special".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("special file"); + + assert!(!root.join("special").exists()); + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[tokio::test] + async fn import_does_not_follow_source_links() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + symlink("mem.bin", source.join("linked-memory")).expect("source link"); + symlink("source", import_root.join("source-link")).expect("directory link"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + catalog + .import("file-link".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("file link"); + catalog + .import( + "directory-link".into(), + PathBuf::from("source-link"), + String::new(), + ) + .await + .expect_err("directory link"); + + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[tokio::test] + async fn metadata_and_catalog_capacity_are_enforced() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let mut config = test_config(&root, &import_root); + config.max_metadata_bytes = 64; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + + let error = catalog + .import("metadata".into(), PathBuf::from("source"), "x".repeat(128)) + .await + .expect_err("metadata limit"); + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + + let mut config = test_config(&root, &import_root); + config.max_total_bytes = 8; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + let error = catalog + .import("capacity".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("catalog capacity"); + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + } + + #[test] + fn concurrent_reservations_share_one_catalog_limit() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let mut config = test_config(&root, &import_root); + config.max_total_bytes = 100; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + let mut first = + ImportClaim::begin(Arc::clone(&catalog.inner), "first".into()).expect("first claim"); + let mut second = + ImportClaim::begin(Arc::clone(&catalog.inner), "second".into()).expect("second claim"); + + first.reserve(60).expect("first reservation"); + assert!(matches!( + second.reserve(60), + Err(BlazeDaemonError::PayloadTooLarge { .. }) + )); + } + + #[test] + fn accounting_failure_blocks_later_imports() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let mut claim = + ImportClaim::begin(Arc::clone(&catalog.inner), "first".into()).expect("claim"); + claim.reserve(10).expect("reservation"); + + let error = claim.publish(11).expect_err("reservation mismatch"); + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(matches!( + ImportClaim::begin(Arc::clone(&catalog.inner), "later".into()), + Err(BlazeDaemonError::RecoveryRequired(_)) + )); + } + + #[test] + fn copy_counts_bytes_read_after_preflight() { + let temp = tempfile::tempdir().expect("tempdir"); + let source_path = temp.path().join("source"); + std::fs::write(&source_path, b"one").expect("source"); + let mut source = OpenOptions::new() + .read(true) + .open(&source_path) + .expect("open source"); + std::fs::write(&source_path, b"longer").expect("grow source"); + let destination = temp.path().join("destination"); + + let error = copy_regular_file(&mut source, &destination, 3, &CancellationToken::new()) + .expect_err("actual bytes exceed reservation"); + + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + } + + #[tokio::test] + async fn shutdown_waits_for_registered_import_claims() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let claim = ImportClaim::begin(Arc::clone(&catalog.inner), "active".into()).expect("claim"); + assert_eq!(catalog.active_imports(), 1); + + catalog.cancel_imports(); + let waiting = tokio::spawn({ + let catalog = catalog.clone(); + async move { catalog.wait_for_imports().await } + }); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + + drop(claim); + waiting.await.expect("join").expect("imports stopped"); + assert_eq!(catalog.active_imports(), 0); + assert!(matches!( + ImportClaim::begin(Arc::clone(&catalog.inner), "late".into()), + Err(BlazeDaemonError::ServiceUnavailable(_)) + )); + } + + #[tokio::test] + async fn shutdown_cancels_copy_and_removes_staging() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let mut entered = catalog.install_copy_gate(); + let import = tokio::spawn({ + let catalog = catalog.clone(); + async move { + catalog + .import("cancelled".into(), PathBuf::from("source"), String::new()) + .await + } + }); + entered.recv().await.expect("copy entered"); + + catalog.cancel_imports(); + catalog.wait_for_imports().await.expect("imports quiescent"); + let error = import + .await + .expect("import task") + .expect_err("cancelled import"); + + assert!(matches!(error, BlazeDaemonError::ServiceUnavailable(_))); + assert!(!root.join("cancelled").exists()); + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[test] + fn list_reports_corrupt_published_metadata() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let published = root.join("published"); + create_private_directory(&published).expect("published"); + write_file_durable(&published.join("template.json"), b"{broken").expect("metadata"); + + let error = list_published(&catalog.inner.root, catalog.inner.limits) + .expect_err("corrupt metadata"); + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + } + + #[test] + fn startup_removes_owned_staging_directories() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + create_catalog_root(&root).expect("root"); + let staging = root.join(".import-pending-uuid.tmp"); + create_private_directory(&staging).expect("staging"); + + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + assert!(!staging.exists()); + } + + fn write_artifacts(source: &Path) { + std::fs::create_dir_all(source).expect("source directory"); + std::fs::write(source.join("vmstate.snap"), b"snapshot").expect("snapshot"); + std::fs::write(source.join("mem.bin"), b"memory").expect("memory"); + std::fs::write(source.join("rootfs.ext4"), b"rootfs").expect("rootfs"); + } +} diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index 0e33a3456c..0cd0a06879 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -2,6 +2,7 @@ //! Backend process ownership and runtime lifecycle abstraction. pub mod firecracker; +mod netns; use std::collections::HashMap; use std::fmt; @@ -14,8 +15,13 @@ use std::time::Duration; use std::time::Instant; use async_trait::async_trait; -use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::backend::{ + BackendKind, RestoreCapability, RestoreRequest, SnapshotRequest, SnapshotResult, SpawnRequest, +}; +use blaze_core::guest_protocol::DEFAULT_MAX_RESPONSE_BYTES; use blaze_core::{BlazeError, Result}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixListener; use tokio::process::{Child, Command}; use tokio::sync::Mutex; use tokio::task::JoinHandle; @@ -25,6 +31,8 @@ use uuid::Uuid; pub use firecracker::FirecrackerSpawner; const TERMINATION_GRACE: Duration = Duration::from_secs(5); +#[cfg(target_os = "linux")] +const PID_HANDOFF_GRACE: Duration = Duration::from_secs(1); const STOPPED_MARKER: &str = "backend.stopped"; /// Result reported when a backend process exits. @@ -41,14 +49,55 @@ pub struct SpawnResult { /// Owned runtime instance returned by a backend spawner. #[async_trait] pub trait BackendInstance: Send + Sync { + /// Stable sandbox identifier. + /// + /// The nil default prevents legacy or test-only owners from claiming a + /// real sandbox identity until they explicitly implement this contract. + fn instance_id(&self) -> Uuid { + Uuid::nil() + } /// Concrete backend implementation. fn backend(&self) -> BackendKind; + /// Backend version frozen into checkpoint metadata when available. + fn version(&self) -> Option<&str> { + None + } + /// Whether pause, resume, and full snapshot capture are implemented. + fn supports_checkpoint_capture(&self) -> bool { + false + } + /// Guest transport endpoint, or an empty path for guestless backends. + fn guest_socket_path(&self) -> &Path { + Path::new("") + } + /// Stable host-network slot required to reproduce snapshot device names. + fn network_slot(&self) -> Option { + None + } /// Report an observed backend exit without waiting. /// /// `None` means the owned process or task was running when checked. /// Once an exit is observed, later calls continue to report a completed /// result even though the underlying handle has already been consumed. async fn try_wait(&self) -> Result>; + /// Pause guest execution for a consistent snapshot. + async fn pause(&self) -> Result<()> { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint pause", self.backend()), + }) + } + /// Resume guest execution after snapshot capture. + async fn resume(&self) -> Result<()> { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint resume", self.backend()), + }) + } + /// Write a self-contained snapshot. + async fn snapshot(&self, _request: SnapshotRequest) -> Result { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint capture", self.backend()), + }) + } /// Terminate the process and release all backend-owned resources. async fn kill(&self) -> Result<()>; } @@ -56,6 +105,9 @@ pub trait BackendInstance: Send + Sync { /// Shared backend instance handle stored in the daemon runtime map. pub type DynBackendInstance = Arc; +/// Restore outcome that preserves ownership when cleanup cannot be confirmed. +pub type RestoreResult = std::result::Result; + /// Backend start failure that may retain ownership of a started process. pub struct SpawnFailure { source: BlazeError, @@ -135,17 +187,46 @@ impl From for SpawnFailure { /// Factory for owned backend runtime instances. #[async_trait] pub trait BackendSpawner: Send + Sync { + /// Persist backend-specific ownership metadata before spawn or restore. + async fn prepare_spawn(&self, _run_dir: &Path) -> Result<()> { + Ok(()) + } + /// Start a new sandbox. async fn spawn( &self, request: SpawnRequest, ) -> std::result::Result; + /// Report the restore identity of the requested backend executable. + /// + /// `None` means restore is unsupported. Implementations that return a + /// version must inspect `binary_path` for every call rather than reusing + /// mutable process-wide state. + async fn restore_capability(&self, _binary_path: &Path) -> Result> { + Ok(None) + } + + /// Start an owned backend from committed checkpoint artifacts. + /// + /// Callers prepare the PID handoff through [`Self::prepare_spawn`] first. + /// Failures transfer any owner whose cleanup could not be confirmed. + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let _ = request; + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "checkpoint restore is not supported by this backend".to_string(), + })) + } + /// Probe whether the configured backend executable is usable. async fn probe(&self, binary_path: &Path) -> Result; /// Clean up a backend process and resources whose in-memory handle was /// lost across daemon restart. + /// + /// Implementations must be safe to retry after a prior successful call. + /// They may update backend-specific files, but must not remove `run_dir` + /// because the caller retains its durable ownership journal there. async fn cleanup_orphan(&self, instance_id: Uuid, run_dir: &Path) -> Result<()>; } @@ -180,13 +261,20 @@ pub struct BubblewrapSpawner; #[async_trait] impl BackendSpawner for BubblewrapSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("backend.pid")) + } + async fn spawn( &self, request: SpawnRequest, ) -> std::result::Result { tokio::fs::create_dir_all(&request.run_dir).await?; remove_file_if_exists(&request.run_dir.join(STOPPED_MARKER)).await?; - let child = Command::new(&request.binary_path) + let pid_file = request.run_dir.join("backend.pid"); + let mut command = Command::new(&request.binary_path); + command .args([ "--ro-bind", "/", @@ -206,22 +294,12 @@ impl BackendSpawner for BubblewrapSpawner { ]) .env("BLAZE_INSTANCE_ID", request.instance_id.to_string()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - let pid_file = request.run_dir.join("backend.pid"); + .stderr(Stdio::null()); + let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + let child = command.spawn(); + drop(pid_handoff); + let child = child?; let stopped_marker = request.run_dir.join(STOPPED_MARKER); - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { - let owner: DynBackendInstance = Arc::new(ProcessInstance::new( - request.instance_id, - BackendKind::Bubblewrap, - child, - pid_file, - stopped_marker, - )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); - } let instance = ProcessInstance::new( request.instance_id, BackendKind::Bubblewrap, @@ -325,6 +403,76 @@ impl BackendSpawner for MockSpawner { .map_err(SpawnFailure::from) } + async fn restore_capability(&self, _binary_path: &Path) -> Result> { + Ok(Some(RestoreCapability { + backend: BackendKind::Mock, + version: Some("mock-v1".to_string()), + snapshot_kind: blaze_core::backend::SnapshotKind::Full, + })) + } + + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let RestoreRequest { + instance_id, + run_dir, + snapshot_path, + mem_path, + checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + network_slot: _, + .. + } = request; + if checkpoint_backend != BackendKind::Mock + || expected_version.as_deref() != Some("mock-v1") + || snapshot_kind != blaze_core::backend::SnapshotKind::Full + { + return Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "mock checkpoint identity is incompatible with the restore adapter" + .to_string(), + })); + } + let vmstate: serde_json::Value = match tokio::fs::read(&snapshot_path) + .await + .map_err(BlazeError::from) + .and_then(|bytes| { + serde_json::from_slice(&bytes).map_err(|error| BlazeError::BackendError { + msg: format!("decode mock VM state: {error}"), + }) + }) { + Ok(vmstate) => vmstate, + Err(error) => return Err(SpawnFailure::clean(error)), + }; + if vmstate.get("format").and_then(serde_json::Value::as_str) != Some("blaze-mock-v1") + || vmstate + .get("instance_id") + .and_then(serde_json::Value::as_str) + != Some(instance_id.to_string().as_str()) + || vmstate.get("kind").and_then(serde_json::Value::as_str) != Some("full") + { + return Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "mock VM state does not match the requested sandbox".to_string(), + })); + } + let files = match tokio::fs::read(&mem_path) + .await + .map_err(BlazeError::from) + .and_then(|bytes| { + serde_json::from_slice::>>(&bytes).map_err(|error| { + BlazeError::BackendError { + msg: format!("decode mock guest memory: {error}"), + } + }) + }) { + Ok(files) => files, + Err(error) => return Err(SpawnFailure::clean(error)), + }; + spawn_mock_instance_with_files(instance_id, run_dir, files, expose_guest_socket) + .await + .map_err(SpawnFailure::from) + } + async fn probe(&self, _binary_path: &Path) -> Result { Ok(true) } @@ -337,30 +485,91 @@ impl BackendSpawner for MockSpawner { struct MockInstance { instance_id: Uuid, + guest_socket_path: PathBuf, cancellation: CancellationToken, task: Mutex>>, + files: Arc>>>, killed: AtomicBool, } async fn spawn_mock_instance(instance_id: Uuid, run_dir: PathBuf) -> Result { + spawn_mock_instance_with_files(instance_id, run_dir, HashMap::new(), true).await +} + +async fn spawn_mock_instance_with_files( + instance_id: Uuid, + run_dir: PathBuf, + restored_files: HashMap>, + expose_guest_socket: bool, +) -> Result { tokio::fs::create_dir_all(&run_dir).await?; + let socket = run_dir.join("vsock.uds"); + if socket.exists() { + tokio::fs::remove_file(&socket).await?; + } let cancellation = CancellationToken::new(); let task_token = cancellation.clone(); - let task = tokio::spawn(async move { task_token.cancelled().await }); + let files = Arc::new(Mutex::new(restored_files)); + let task_files = files.clone(); + let (guest_socket_path, task) = if expose_guest_socket { + let listener = UnixListener::bind(&socket)?; + let task = tokio::spawn(async move { + loop { + tokio::select! { + _ = task_token.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + break; + }; + let files = task_files.clone(); + tokio::spawn(async move { + if let Err(error) = serve_mock_guest(stream, files).await { + tracing::debug!(%error, "mock guest connection ended"); + } + }); + } + } + } + }); + (socket, task) + } else { + let task = tokio::spawn(async move { + task_token.cancelled().await; + }); + (PathBuf::new(), task) + }; Ok(Arc::new(MockInstance { instance_id, + guest_socket_path, cancellation, task: Mutex::new(Some(task)), + files, killed: AtomicBool::new(false), })) } #[async_trait] impl BackendInstance for MockInstance { + fn instance_id(&self) -> Uuid { + self.instance_id + } + fn backend(&self) -> BackendKind { BackendKind::Mock } + fn version(&self) -> Option<&str> { + Some("mock-v1") + } + + fn supports_checkpoint_capture(&self) -> bool { + true + } + + fn guest_socket_path(&self) -> &Path { + &self.guest_socket_path + } + async fn try_wait(&self) -> Result> { let task = { let mut task = self.task.lock().await; @@ -386,6 +595,44 @@ impl BackendInstance for MockInstance { })) } + async fn pause(&self) -> Result<()> { + Ok(()) + } + + async fn resume(&self) -> Result<()> { + Ok(()) + } + + async fn snapshot(&self, request: SnapshotRequest) -> Result { + for path in [&request.snapshot_path, &request.mem_path] { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + tokio::fs::create_dir_all(parent).await?; + } + } + let vmstate = serde_json::to_vec(&serde_json::json!({ + "format": "blaze-mock-v1", + "instance_id": self.instance_id, + "kind": request.kind, + })) + .map_err(|error| BlazeError::BackendError { + msg: format!("serialize mock VM state: {error}"), + })?; + tokio::fs::write(&request.snapshot_path, vmstate).await?; + let memory = serde_json::to_vec(&*self.files.lock().await).map_err(|error| { + BlazeError::BackendError { + msg: format!("serialize mock guest state: {error}"), + } + })?; + tokio::fs::write(&request.mem_path, memory).await?; + Ok(SnapshotResult { + snapshot_path: request.snapshot_path, + mem_path: request.mem_path, + }) + } + async fn kill(&self) -> Result<()> { if self.killed.load(Ordering::Acquire) { return Ok(()); @@ -398,11 +645,97 @@ impl BackendInstance for MockInstance { if let Some(task) = task.take() { let _ = task.await; } + if self.guest_socket_path.exists() { + tokio::fs::remove_file(&self.guest_socket_path).await?; + } self.killed.store(true, Ordering::Release); Ok(()) } } +async fn serve_mock_guest( + mut stream: tokio::net::UnixStream, + files: Arc>>>, +) -> std::io::Result<()> { + use base64::Engine; + use base64::engine::general_purpose::STANDARD as BASE64; + + let connect = read_mock_line(&mut stream, 128).await?; + if !connect.starts_with(b"CONNECT ") { + return Ok(()); + } + stream.write_all(b"OK 5000\n").await?; + let request = read_mock_line(&mut stream, DEFAULT_MAX_RESPONSE_BYTES).await?; + let request: serde_json::Value = match serde_json::from_slice(&request) { + Ok(request) => request, + Err(_) => return Ok(()), + }; + let id = request.get("id").cloned().unwrap_or_default(); + let response = match request.get("op").and_then(serde_json::Value::as_str) { + Some("exec") => { + let command = request + .get("cmd") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + serde_json::json!({ + "id": id, + "ok": true, + "rc": 0, + "stdout_b64": BASE64.encode(command.as_bytes()), + "stderr_b64": "" + }) + } + Some("read") => { + let path = request + .get("path") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let data = files.lock().await.get(path).cloned().unwrap_or_default(); + serde_json::json!({"id": id, "ok": true, "data_b64": BASE64.encode(data)}) + } + Some("write") => { + let path = request + .get("path") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(); + let data = request + .get("data_b64") + .and_then(serde_json::Value::as_str) + .and_then(|encoded| BASE64.decode(encoded).ok()) + .unwrap_or_default(); + files.lock().await.insert(path, data); + serde_json::json!({"id": id, "ok": true}) + } + _ => serde_json::json!({"id": id, "ok": true}), + }; + let mut encoded = serde_json::to_vec(&response).unwrap_or_else(|_| b"{}".to_vec()); + encoded.push(b'\n'); + stream.write_all(&encoded).await +} + +async fn read_mock_line(stream: &mut R, limit: usize) -> std::io::Result> +where + R: AsyncRead + Unpin, +{ + let mut reader = BufReader::new(stream).take(limit.saturating_add(1) as u64); + let mut output = Vec::with_capacity(limit.min(8192)); + reader.read_until(b'\n', &mut output).await?; + if output.last() == Some(&b'\n') { + output.pop(); + if output.len() <= limit { + return Ok(output); + } + } + if output.len() > limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "mock guest line too long", + )); + } + Ok(output) +} + pub(super) async fn terminate_child(child: &mut Child, backend: &str) -> Result<()> { if child.try_wait()?.is_some() { return Ok(()); @@ -446,6 +779,147 @@ pub(super) fn stopped_marker(run_dir: &Path) -> PathBuf { run_dir.join(STOPPED_MARKER) } +#[cfg(unix)] +pub(super) struct PidHandoff { + _file: std::fs::File, +} + +#[cfg(not(unix))] +pub(super) struct PidHandoff; + +#[cfg(unix)] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + use std::ffi::CString; + use std::os::fd::FromRawFd; + use std::os::unix::ffi::OsStrExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC | libc::O_NOFOLLOW, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + file.sync_all()?; + if let Some(parent) = pid_file.parent() { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(not(unix))] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + let file = std::fs::File::create(pid_file)?; + file.sync_all()?; + Ok(()) +} + +#[cfg(unix)] +pub(super) fn configure_pid_handoff(command: &mut Command, pid_file: &Path) -> Result { + use std::ffi::CString; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::process::CommandExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CLOEXEC | libc::O_NOFOLLOW, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + let child_fd = file.as_raw_fd(); + // SAFETY: the closure calls only async-signal-safe libc functions and does + // not allocate after fork. The returned guard keeps `child_fd` open and + // locked until `Command::spawn` completes. + unsafe { + command + .as_std_mut() + .pre_exec(move || write_current_pid(child_fd)); + } + Ok(PidHandoff { _file: file }) +} + +#[cfg(not(unix))] +pub(super) fn configure_pid_handoff( + _command: &mut Command, + _pid_file: &Path, +) -> Result { + Ok(PidHandoff) +} + +#[cfg(unix)] +fn write_current_pid(fd: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::lseek(fd, 0, libc::SEEK_SET) } < 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::ftruncate(fd, 0) } != 0 { + return Err(std::io::Error::last_os_error()); + } + write_pid_and_sync(fd) +} + +#[cfg(unix)] +fn write_pid_and_sync(fd: libc::c_int) -> std::io::Result<()> { + let mut buffer = [0_u8; 16]; + let mut cursor = buffer.len(); + cursor -= 1; + buffer[cursor] = b'\n'; + let mut pid = unsafe { libc::getpid() } as u32; + loop { + cursor -= 1; + buffer[cursor] = b'0' + (pid % 10) as u8; + pid /= 10; + if pid == 0 { + break; + } + } + + let mut remaining = &buffer[cursor..]; + while !remaining.is_empty() { + let written = unsafe { + libc::write( + fd, + remaining.as_ptr().cast::(), + remaining.len(), + ) + }; + if written < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if written == 0 { + return Err(std::io::ErrorKind::WriteZero.into()); + } + remaining = &remaining[written as usize..]; + } + if unsafe { libc::fsync(fd) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + async fn cleanup_process_run_dir(instance_id: Uuid, run_dir: &Path, backend: &str) -> Result<()> { let stopped_marker = stopped_marker(run_dir); if stopped_marker.is_file() { @@ -477,17 +951,9 @@ pub(super) async fn terminate_recorded_process( pid_file: &Path, backend: &str, ) -> Result<()> { - let raw = match tokio::fs::read_to_string(pid_file).await { - Ok(raw) => raw, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Err(BlazeError::BackendError { - msg: format!( - "cannot confirm {backend} instance {instance_id} stopped: missing PID metadata {}", - pid_file.display() - ), - }); - } - Err(error) => return Err(error.into()), + let raw = match wait_for_pid_handoff(pid_file).await? { + Some(raw) => raw, + None => return Ok(()), }; let pid: u32 = raw .trim() @@ -537,6 +1003,74 @@ pub(super) async fn terminate_recorded_process( Ok(()) } +#[cfg(target_os = "linux")] +async fn wait_for_pid_handoff(pid_file: &Path) -> Result> { + let deadline = Instant::now() + PID_HANDOFF_GRACE; + loop { + match read_pid_handoff(pid_file)? { + PidHandoffState::NotStarted => return Ok(None), + PidHandoffState::Missing => { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: missing PID handoff {}", + pid_file.display() + ), + }); + } + PidHandoffState::Ready(raw) => return Ok(Some(raw)), + PidHandoffState::InProgress => {} + } + if Instant::now() >= deadline { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: PID handoff is still in progress at {}", + pid_file.display() + ), + }); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[cfg(target_os = "linux")] +enum PidHandoffState { + Missing, + NotStarted, + InProgress, + Ready(String), +} + +#[cfg(target_os = "linux")] +fn read_pid_handoff(pid_file: &Path) -> Result { + use std::io::{Read, Seek, SeekFrom}; + use std::os::fd::AsRawFd; + + let mut file = match std::fs::OpenOptions::new().read(true).open(pid_file) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(PidHandoffState::Missing); + } + Err(error) => return Err(error.into()), + }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EAGAIN) + || error.raw_os_error() == Some(libc::EWOULDBLOCK) + { + return Ok(PidHandoffState::InProgress); + } + return Err(error.into()); + } + file.seek(SeekFrom::Start(0))?; + let mut raw = String::new(); + file.read_to_string(&mut raw)?; + if raw.trim().is_empty() { + Ok(PidHandoffState::NotStarted) + } else { + Ok(PidHandoffState::Ready(raw)) + } +} + #[cfg(target_os = "linux")] async fn wait_for_process_exit(process_dir: &Path, timeout: Duration) -> Result { let deadline = Instant::now() + timeout; @@ -611,12 +1145,31 @@ mod tests { #[cfg(target_os = "linux")] use std::time::Duration; - use blaze_core::backend::SpawnRequest; + use blaze_core::backend::{RestoreRequest, SnapshotKind, SnapshotRequest, SpawnRequest}; use blaze_core::policy::BackendConfigs; use blaze_core::storage::StorageSlot; + use crate::guest::GuestClient; + use super::*; + struct UnsupportedInstance; + + #[async_trait] + impl BackendInstance for UnsupportedInstance { + fn backend(&self) -> BackendKind { + BackendKind::Bubblewrap + } + + async fn try_wait(&self) -> Result> { + Ok(None) + } + + async fn kill(&self) -> Result<()> { + Ok(()) + } + } + fn request(root: &Path) -> SpawnRequest { let id = Uuid::new_v4(); let slot_dir = root.join("slot"); @@ -660,18 +1213,129 @@ mod tests { } #[tokio::test] - async fn mock_instance_reports_liveness_and_supports_idempotent_kill() { + async fn mock_instance_supports_guest_io_and_idempotent_kill() { let temp = tempfile::tempdir().expect("temp"); let instance = MockSpawner .spawn(request(temp.path())) .await .expect("spawn"); + let client = GuestClient::new( + temp.path().join("run/vsock.uds"), + Duration::from_secs(1), + 1024, + ); + client + .write_file("/tmp/value".into(), b"hello") + .await + .expect("write"); + assert_eq!( + client.read_file("/tmp/value".into()).await.expect("read"), + b"hello" + ); assert_eq!(instance.try_wait().await.expect("try wait"), None); instance.kill().await.expect("kill"); assert!(instance.try_wait().await.expect("try wait").is_some()); instance.kill().await.expect("idempotent kill"); } + #[tokio::test] + async fn checkpoint_capture_defaults_fail_closed() { + let temp = tempfile::tempdir().expect("temp"); + let instance = UnsupportedInstance; + let request = SnapshotRequest { + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + kind: SnapshotKind::Full, + }; + + assert_eq!(instance.instance_id(), Uuid::nil()); + assert_eq!(instance.version(), None); + assert!(!instance.supports_checkpoint_capture()); + assert!(instance.pause().await.is_err()); + assert!(instance.resume().await.is_err()); + assert!(instance.snapshot(request).await.is_err()); + } + + #[tokio::test] + async fn checkpoint_restore_defaults_fail_closed_without_an_owner() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = request(temp.path()); + let restore = RestoreRequest { + instance_id: spawn.instance_id, + run_dir: spawn.run_dir, + binary_path: spawn.binary_path, + storage: spawn.storage, + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + checkpoint_backend: BackendKind::Bubblewrap, + expected_version: None, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, + }; + + assert!( + BubblewrapSpawner + .restore_capability(Path::new("")) + .await + .expect("capability") + .is_none() + ); + let failure = match BubblewrapSpawner.restore(restore).await { + Ok(_) => panic!("restore must remain unsupported"), + Err(failure) => failure, + }; + let (source, owner) = failure.into_parts(); + assert!(source.to_string().contains("restore is not supported")); + assert!(owner.is_none()); + } + + #[tokio::test] + async fn mock_instance_captures_self_contained_state() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = request(temp.path()); + let instance_id = spawn.instance_id; + let instance = MockSpawner.spawn(spawn).await.expect("spawn"); + let client = GuestClient::new( + temp.path().join("run/vsock.uds"), + Duration::from_secs(1), + 1024, + ); + client + .write_file("/tmp/value".into(), b"captured") + .await + .expect("write guest state"); + let snapshot_path = temp.path().join("checkpoint/vmstate.snap"); + let mem_path = temp.path().join("checkpoint/memory.snap"); + + assert_eq!(instance.instance_id(), instance_id); + assert_eq!(instance.version(), Some("mock-v1")); + assert!(instance.supports_checkpoint_capture()); + instance.pause().await.expect("pause"); + let result = instance + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + kind: SnapshotKind::Full, + }) + .await + .expect("snapshot"); + instance.resume().await.expect("resume"); + + assert_eq!(result.snapshot_path, snapshot_path); + assert_eq!(result.mem_path, mem_path); + let vmstate: serde_json::Value = + serde_json::from_slice(&std::fs::read(&result.snapshot_path).expect("VM state")) + .expect("VM state JSON"); + assert_eq!(vmstate["instance_id"], instance_id.to_string()); + assert_eq!(vmstate["kind"], "full"); + let memory: HashMap> = + serde_json::from_slice(&std::fs::read(&result.mem_path).expect("memory")) + .expect("memory JSON"); + assert_eq!(memory.get("/tmp/value"), Some(&b"captured".to_vec())); + instance.kill().await.expect("kill"); + } + #[cfg(target_os = "linux")] #[tokio::test] async fn child_termination_requests_graceful_exit_first() { @@ -720,19 +1384,90 @@ mod tests { #[cfg(target_os = "linux")] #[tokio::test] - async fn orphan_cleanup_rejects_missing_pid_without_stop_record() { + async fn orphan_cleanup_accepts_pre_spawn_handoff_without_pid() { + let temp = tempfile::tempdir().expect("temp"); + prepare_pid_handoff(&temp.path().join("backend.pid")).expect("prepare handoff"); + cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .await + .expect("an unlocked empty handoff proves the backend was not started"); + assert!(stopped_marker(temp.path()).is_file()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_rejects_missing_pid_handoff() { let temp = tempfile::tempdir().expect("temp"); let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect_err("missing metadata cannot prove termination"); - assert!(error.to_string().contains("missing PID metadata")); + .expect_err("missing handoff cannot prove backend ownership"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(!stopped_marker(temp.path()).exists()); + } - record_backend_stopped(&stopped_marker(temp.path())) + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_an_active_pid_handoff() { + let temp = tempfile::tempdir().expect("temp"); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); + + let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect("record stopped"); - cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .expect_err("an active handoff cannot prove the backend absent"); + + assert!( + error + .to_string() + .contains("PID handoff is still in progress") + ); + assert!(!stopped_marker(temp.path()).exists()); + drop(handoff); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn pid_handoff_is_visible_when_spawn_returns() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); + let mut child = command.spawn().expect("spawn child"); + drop(handoff); + wait_for_instance_marker(&child, instance_id).await; + + assert_eq!( + std::fs::read_to_string(&pid_file) + .expect("pid handoff") + .trim(), + child.id().expect("child pid").to_string() + ); + terminate_recorded_process(instance_id, &pid_file, "test") .await - .expect("durable stop record proves termination"); + .expect("terminate handed-off process"); + child.wait().await.expect("reap child"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_pid_handoff_preparation_does_not_start_backend() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("missing").join("backend.pid"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + + assert!(configure_pid_handoff(&mut command, &pid_file).is_err()); + assert!(!pid_file.exists()); } #[cfg(target_os = "linux")] diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index a037ee5c29..aaf8138486 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Firecracker process ownership and HTTP API over Unix domain sockets. +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::Arc; @@ -8,124 +9,506 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use async_trait::async_trait; -use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::backend::{ + BackendKind, RestoreCapability, RestoreRequest, SnapshotKind, SnapshotRequest, SnapshotResult, + SpawnRequest, +}; use blaze_core::policy::{FirecrackerConfig, VmConfig, parse_memory_value, to_mib_ceil}; use blaze_core::{BlazeError, Result}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Bytes; +use hyper::client::conn::http1; +use hyper::{Method, Request}; +use hyper_util::rt::TokioIo; use tokio::net::UnixStream; use tokio::process::{Child, Command}; use tokio::sync::Mutex; use uuid::Uuid; +use super::netns::{NetworkManager, NetworkSlot}; #[cfg(target_os = "linux")] use super::terminate_recorded_process; use super::{ - BackendInstance, BackendSpawner, DynBackendInstance, SpawnFailure, SpawnResult, - record_backend_stopped, remove_file_if_exists, spawn_result, stopped_marker, terminate_child, + BackendInstance, BackendSpawner, DynBackendInstance, RestoreResult, SpawnFailure, SpawnResult, + configure_pid_handoff, prepare_pid_handoff, record_backend_stopped, remove_file_if_exists, + spawn_result, stopped_marker, terminate_child, }; +const NETWORK_BOOT_IP: &str = "ip=169.254.0.2::169.254.0.1:255.255.255.252::eth0:off"; +const MAX_API_RESPONSE_BYTES: usize = 64 * 1024; + /// Firecracker backend factory. pub struct FirecrackerSpawner { images_dir: PathBuf, + api_timeout: Duration, socket_timeout: Duration, - version: Mutex>, + network: Arc, + network_required: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum NetworkProcessState { + PreSpawn, + #[default] + Launching, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct NetworkRecord { + slot: usize, + owner: Uuid, + #[serde(default)] + process_state: NetworkProcessState, } impl FirecrackerSpawner { - /// Create a spawner resolving the guest kernel from `images_dir`. + /// Create a spawner without requiring host networking during startup + /// probing. Individual network-enabled requests still run the full probe. pub fn new(images_dir: PathBuf) -> Self { Self { images_dir, + api_timeout: Duration::from_secs(30), socket_timeout: Duration::from_secs(5), - version: Mutex::new(None), + network: Arc::new(NetworkManager::default()), + network_required: false, + } + } + + /// Create a spawner whose startup probe includes network prerequisites + /// when at least one loaded policy enables Firecracker networking. + pub fn with_network_requirement(images_dir: PathBuf, network_required: bool) -> Self { + Self { + network_required, + ..Self::new(images_dir) + } + } + + async fn network_probe_ready(&self) -> Result { + if !self.network_required { + return Ok(true); } + self.network.probe().await + } + + async fn capture_for( + &self, + binary_path: &Path, + api_socket: PathBuf, + ) -> Result { + let backend_version = read_backend_version(binary_path).await?; + Ok(FirecrackerCapture::new( + api_socket, + self.api_timeout, + backend_version, + )) } async fn start( &self, request: SpawnRequest, + restore: Option, ) -> std::result::Result { validate_regular_file(&request.binary_path, "firecracker binary")?; validate_regular_file(&request.storage.rootfs_path, "rootfs")?; - validate_regular_file(&self.images_dir.join("vmlinux"), "vmlinux")?; - tokio::fs::create_dir_all(&request.run_dir).await?; + match &restore { + Some(restore) => { + validate_regular_file(&restore.snapshot_path, "VM-state snapshot")?; + validate_regular_file(&restore.mem_path, "memory snapshot")?; + } + None => validate_regular_file(&self.images_dir.join("vmlinux"), "vmlinux")?, + } let api_socket = request.run_dir.join("api.sock"); + let capture = self + .capture_for(&request.binary_path, api_socket.clone()) + .await?; + if let Some(restore) = &restore { + validate_restore_compatibility(restore, &capture.backend_version)?; + } + tokio::fs::create_dir_all(&request.run_dir).await?; let guest_socket = request.run_dir.join("vsock.uds"); let pid_file = request.run_dir.join("firecracker.pid"); let stopped_marker = stopped_marker(&request.run_dir); + let network_file = request.run_dir.join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); remove_if_exists(&api_socket).await?; remove_if_exists(&guest_socket).await?; - remove_if_exists(&pid_file).await?; remove_file_if_exists(&stopped_marker).await?; + remove_if_exists(&network_file).await?; + remove_if_exists(&network_temp_file).await?; let fc_config = request .backend .firecracker .as_ref() .cloned() .unwrap_or_default(); - let mut command = build_launch_command(&request.binary_path, &api_socket); - let config_path = write_vm_config(&self.images_dir, &request, &fc_config, &guest_socket)?; - command.arg("--config-file").arg(config_path); - configure_logs(&mut command, &request.run_dir, fc_config.serial_log)?; + let expose_guest_socket = restore.as_ref().map_or(fc_config.enable_vsock, |restore| { + restore.expose_guest_socket + }); + let network = if fc_config.enable_network { + if !self.network.probe().await? { + return Err(BlazeError::BackendError { + msg: "Firecracker networking is unavailable; it requires Linux root and executable ip, sysctl, and iptables commands".to_string(), + } + .into()); + } + let created = match restore.as_ref().and_then(|restore| restore.network_slot) { + Some(slot) => { + self.network + .create_at(request.instance_id, slot, |slot| { + write_network_metadata(&network_file, slot) + }) + .await + } + None => { + self.network + .create(request.instance_id, |slot| { + write_network_metadata(&network_file, slot) + }) + .await + } + }; + match created { + Ok(network) => Some(network), + Err(error) => { + let (source, residual) = error.into_parts(); + if let Some(network) = residual { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + Some(network), + self.network.clone(), + expose_guest_socket, + )); + return Err(SpawnFailure::compensate_started(source, owner).await); + } + if let Err(cleanup) = remove_if_exists(&network_file).await { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + None, + self.network.clone(), + expose_guest_socket, + )); + return Err(SpawnFailure::compensate_started( + BlazeError::BackendError { + msg: format!( + "{source}; network metadata cleanup failed: {cleanup}" + ), + }, + owner, + ) + .await); + } + if let Err(cleanup) = remove_if_exists(&network_temp_file).await { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + None, + self.network.clone(), + expose_guest_socket, + )); + return Err(SpawnFailure::compensate_started( + BlazeError::BackendError { + msg: format!( + "{source}; temporary network metadata cleanup failed: {cleanup}" + ), + }, + owner, + ) + .await); + } + return Err(source.into()); + } + } + } else { + None + }; + + let mut command = build_launch_command(&request.binary_path, network.as_ref(), &api_socket); + if restore.is_none() { + let config_path = match write_vm_config( + &self.images_dir, + &request, + &fc_config, + &guest_socket, + network.as_ref(), + ) { + Ok(path) => path, + Err(error) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + expose_guest_socket, + error, + ) + .await); + } + }; + command.arg("--config-file").arg(config_path); + } + if let Err(error) = configure_logs(&mut command, &request.run_dir, fc_config.serial_log) { + return Err(self + .compensate_before_spawn( + request.instance_id, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + expose_guest_socket, + error, + ) + .await); + } command.env("BLAZE_INSTANCE_ID", request.instance_id.to_string()); - let mut child = match command.spawn() { + if let Some(slot) = network.as_ref() + && let Err(error) = + write_network_record(&network_file, slot, NetworkProcessState::Launching) + { + return Err(self + .compensate_before_spawn( + request.instance_id, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + expose_guest_socket, + error, + ) + .await); + } + let pid_handoff = match configure_pid_handoff(&mut command, &pid_file) { + Ok(pid_handoff) => pid_handoff, + Err(error) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + expose_guest_socket, + error, + ) + .await); + } + }; + let child = command.spawn(); + drop(pid_handoff); + let mut child = match child { Ok(child) => child, - Err(source) => return Err(source.into()), + Err(source) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + capture.clone(), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + expose_guest_socket, + source.into(), + ) + .await); + } }; - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { + if let Err(error) = wait_for_socket(&api_socket, &mut child, self.socket_timeout).await { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, + Some(child), + capture, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + self.network.clone(), + expose_guest_socket, )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); + return Err(SpawnFailure::compensate_started(error, owner).await); } - if let Err(error) = wait_for_socket(&api_socket, &mut child, self.socket_timeout).await { - let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( - request.instance_id, - child, + + let instance = Arc::new(FirecrackerInstance::new( + request.instance_id, + Some(child), + capture, + runtime_files( api_socket, guest_socket, pid_file, stopped_marker, - )); + network_file, + ), + network, + self.network.clone(), + expose_guest_socket, + )); + if let Some(restore) = restore + && let Err(error) = instance.load_snapshot(&restore).await + { + let owner: DynBackendInstance = instance; return Err(SpawnFailure::compensate_started(error, owner).await); } + Ok(instance) + } - let instance = FirecrackerInstance::new( - request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, - ); - Ok(Arc::new(instance)) + async fn compensate_before_spawn( + &self, + instance_id: Uuid, + capture: FirecrackerCapture, + files: FirecrackerRuntimeFiles, + network: Option, + enable_vsock: bool, + source: BlazeError, + ) -> SpawnFailure { + if network.is_none() { + return SpawnFailure::clean(source); + } + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + instance_id, + None, + capture, + files, + network, + self.network.clone(), + enable_vsock, + )); + SpawnFailure::compensate_started(source, owner).await } } #[async_trait] impl BackendSpawner for FirecrackerSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("firecracker.pid")) + } + async fn spawn( &self, request: SpawnRequest, ) -> std::result::Result { - self.start(request).await + self.start(request, None).await + } + + async fn restore_capability(&self, binary_path: &Path) -> Result> { + validate_regular_file(binary_path, "firecracker binary")?; + Ok(Some(RestoreCapability { + backend: BackendKind::Firecracker, + version: Some(read_backend_version(binary_path).await?), + snapshot_kind: SnapshotKind::Full, + })) + } + + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let RestoreRequest { + instance_id, + run_dir, + binary_path, + storage, + snapshot_path, + mem_path, + checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + network_slot, + } = request; + let backend = blaze_core::policy::BackendConfigs { + firecracker: Some(blaze_core::policy::FirecrackerConfig { + enable_vsock: expose_guest_socket, + enable_network: network_slot.is_some(), + ..blaze_core::policy::FirecrackerConfig::default() + }), + }; + self.start( + SpawnRequest { + instance_id, + run_dir, + binary_path, + storage, + // Snapshot restore only reconstructs host-side resources that + // are required to make the captured runtime reachable. + backend, + vm: None, + }, + Some(FirecrackerRestore { + snapshot_path, + mem_path, + backend: checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + network_slot, + }), + ) + .await } async fn probe(&self, binary_path: &Path) -> Result { if !binary_path.is_file() || !executable_in_path("unshare") { return Ok(false); } + if !self.network_probe_ready().await? { + return Ok(false); + } match read_backend_version(binary_path).await { - Ok(version) => { - *self.version.lock().await = Some(version); - Ok(true) - } + Ok(_) => Ok(true), Err(error) => { tracing::debug!(%error, binary = %binary_path.display(), "firecracker version probe failed"); Ok(false) @@ -134,66 +517,186 @@ impl BackendSpawner for FirecrackerSpawner { } async fn cleanup_orphan(&self, instance_id: Uuid, run_dir: &Path) -> Result<()> { - cleanup_orphan_run_dir(instance_id, run_dir).await + cleanup_orphan_run_dir_with(instance_id, run_dir, &self.network).await } } struct FirecrackerInstance { instance_id: Uuid, child: Mutex>, + exit_result: Mutex>, + capture: FirecrackerCapture, + files: FirecrackerRuntimeFiles, + guest_socket: PathBuf, + network: Mutex>, + network_slot: Option, + network_manager: Arc, + cleanup_complete: AtomicBool, + killed: AtomicBool, +} + +struct FirecrackerRuntimeFiles { api_socket: PathBuf, guest_socket: PathBuf, pid_file: PathBuf, stopped_marker: PathBuf, - killed: AtomicBool, + network_file: PathBuf, +} + +fn runtime_files( + api_socket: PathBuf, + guest_socket: PathBuf, + pid_file: PathBuf, + stopped_marker: PathBuf, + network_file: PathBuf, +) -> FirecrackerRuntimeFiles { + FirecrackerRuntimeFiles { + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + } } impl FirecrackerInstance { fn new( instance_id: Uuid, - child: Child, - api_socket: PathBuf, - guest_socket: PathBuf, - pid_file: PathBuf, - stopped_marker: PathBuf, + child: Option, + capture: FirecrackerCapture, + files: FirecrackerRuntimeFiles, + network: Option, + network_manager: Arc, + enable_vsock: bool, ) -> Self { + let guest_socket = configured_guest_socket(enable_vsock, files.guest_socket.clone()); + let network_slot = network.as_ref().map(NetworkSlot::slot); Self { instance_id, - child: Mutex::new(Some(child)), - api_socket, + child: Mutex::new(child), + exit_result: Mutex::new(None), + capture, + files, guest_socket, - pid_file, - stopped_marker, + network: Mutex::new(network), + network_slot, + network_manager, + cleanup_complete: AtomicBool::new(false), killed: AtomicBool::new(false), } } + + async fn load_snapshot(&self, restore: &FirecrackerRestore) -> Result<()> { + self.capture.api.load_snapshot(restore).await + } } #[async_trait] impl BackendInstance for FirecrackerInstance { + fn instance_id(&self) -> Uuid { + self.instance_id + } + fn backend(&self) -> BackendKind { BackendKind::Firecracker } + fn version(&self) -> Option<&str> { + Some(&self.capture.backend_version) + } + + fn supports_checkpoint_capture(&self) -> bool { + true + } + + fn guest_socket_path(&self) -> &Path { + &self.guest_socket + } + + fn network_slot(&self) -> Option { + self.network_slot + } + async fn try_wait(&self) -> Result> { - let status = { + let result = { let mut guard = self.child.lock().await; let Some(child) = guard.as_mut() else { - return Ok(Some(SpawnResult { + let result = self.exit_result.lock().await.unwrap_or(SpawnResult { instance_id: self.instance_id, exit_code: None, signal: None, - })); + }); + drop(guard); + self.cleanup().await?; + return Ok(Some(result)); }; let Some(status) = child.try_wait()? else { return Ok(None); }; - record_backend_stopped(&self.stopped_marker).await?; + record_backend_stopped(&self.files.stopped_marker).await?; + let result = spawn_result(self.instance_id, status); + *self.exit_result.lock().await = Some(result); *guard = None; - status + result }; self.cleanup().await?; - Ok(Some(spawn_result(self.instance_id, status))) + Ok(Some(result)) + } + + async fn pause(&self) -> Result<()> { + self.capture + .api + .call_json( + Method::PATCH, + "/vm", + Some(serde_json::json!({"state": "Paused"})), + ) + .await?; + Ok(()) + } + + async fn resume(&self) -> Result<()> { + self.capture + .api + .call_json( + Method::PATCH, + "/vm", + Some(serde_json::json!({"state": "Resumed"})), + ) + .await?; + Ok(()) + } + + async fn snapshot(&self, request: SnapshotRequest) -> Result { + let SnapshotRequest { + snapshot_path, + mem_path, + kind: SnapshotKind::Full, + } = request; + for path in [&snapshot_path, &mem_path] { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + tokio::fs::create_dir_all(parent).await?; + } + } + self.capture + .api + .call_json( + Method::PUT, + "/snapshot/create", + Some(serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_file_path": mem_path, + "snapshot_type": "Full" + })), + ) + .await?; + Ok(SnapshotResult { + snapshot_path, + mem_path, + }) } async fn kill(&self) -> Result<()> { @@ -207,7 +710,7 @@ impl BackendInstance for FirecrackerInstance { if let Some(child) = guard.as_mut() { terminate_child(child, "firecracker").await?; } - record_backend_stopped(&self.stopped_marker).await?; + record_backend_stopped(&self.files.stopped_marker).await?; *guard = None; drop(guard); self.cleanup().await?; @@ -216,11 +719,189 @@ impl BackendInstance for FirecrackerInstance { } } +#[derive(Clone)] +struct FirecrackerCapture { + api: FirecrackerApiClient, + backend_version: String, +} + +#[derive(Debug)] +struct FirecrackerRestore { + snapshot_path: PathBuf, + mem_path: PathBuf, + backend: BackendKind, + expected_version: Option, + snapshot_kind: SnapshotKind, + expose_guest_socket: bool, + network_slot: Option, +} + +impl FirecrackerCapture { + fn new(api_socket: PathBuf, api_timeout: Duration, backend_version: String) -> Self { + Self { + api: FirecrackerApiClient::new(api_socket, api_timeout), + backend_version, + } + } +} + +#[derive(Debug, Clone)] +struct FirecrackerApiClient { + socket: PathBuf, + timeout: Duration, +} + +impl FirecrackerApiClient { + fn new(socket: PathBuf, timeout: Duration) -> Self { + Self { socket, timeout } + } + + async fn load_snapshot(&self, restore: &FirecrackerRestore) -> Result<()> { + self.call_json( + Method::PUT, + "/snapshot/load", + Some(serde_json::json!({ + "snapshot_path": path_string(&restore.snapshot_path, "VM-state snapshot")?, + "mem_backend": { + "backend_type": "File", + "backend_path": path_string(&restore.mem_path, "memory snapshot")? + }, + "resume_vm": true + })), + ) + .await?; + Ok(()) + } + + async fn call_json( + &self, + method: Method, + path: &str, + body: Option, + ) -> Result> { + let operation = async { + let stream = UnixStream::connect(&self.socket).await?; + let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) + .await + .map_err(backend_protocol_error)?; + tokio::spawn(async move { + if let Err(error) = connection.await { + tracing::debug!(%error, "firecracker API connection ended"); + } + }); + let bytes = match body { + Some(body) => { + serde_json::to_vec(&body).map_err(|error| BlazeError::BackendError { + msg: format!("serialize Firecracker API request: {error}"), + })? + } + None => Vec::new(), + }; + let mut builder = Request::builder() + .method(method.clone()) + .uri(format!("http://localhost{path}")); + if !bytes.is_empty() { + builder = builder.header("content-type", "application/json"); + } + let request = builder + .body(Full::new(Bytes::from(bytes))) + .map_err(|error| BlazeError::BackendError { + msg: format!("build Firecracker API request: {error}"), + })?; + let response = sender + .send_request(request) + .await + .map_err(backend_protocol_error)?; + let status = response.status(); + let mut response_body = response.into_body(); + let mut collected = Vec::new(); + while let Some(frame) = response_body.frame().await { + let frame = frame.map_err(backend_protocol_error)?; + if let Ok(data) = frame.into_data() { + let remaining = MAX_API_RESPONSE_BYTES.saturating_sub(collected.len()); + collected.extend_from_slice(&data[..data.len().min(remaining)]); + if data.len() > remaining { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} response exceeded \ + {MAX_API_RESPONSE_BYTES} bytes" + ), + }); + } + } + } + if !status.is_success() { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} returned {status}: {}", + String::from_utf8_lossy(&collected) + ), + }); + } + Ok(collected) + }; + tokio::time::timeout(self.timeout, operation) + .await + .map_err(|_| BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} timed out after {:?}", + self.timeout + ), + })? + } +} + +fn validate_restore_compatibility( + restore: &FirecrackerRestore, + actual_version: &str, +) -> Result<()> { + if restore.backend != BackendKind::Firecracker { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker cannot restore a {} checkpoint", + restore.backend + ), + }); + } + if restore.snapshot_kind != SnapshotKind::Full { + return Err(BlazeError::BackendError { + msg: "Firecracker restore accepts only full checkpoints".to_string(), + }); + } + let expected_version = + restore + .expected_version + .as_deref() + .ok_or_else(|| BlazeError::BackendError { + msg: "Firecracker restore requires a checkpoint backend version".to_string(), + })?; + if expected_version != actual_version { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker checkpoint version {expected_version:?} does not match \ + executable version {actual_version:?}" + ), + }); + } + Ok(()) +} + impl FirecrackerInstance { async fn cleanup(&self) -> Result<()> { - remove_if_exists(&self.api_socket).await?; - remove_if_exists(&self.guest_socket).await?; - remove_if_exists(&self.pid_file).await?; + if self.cleanup_complete.load(Ordering::Acquire) { + return Ok(()); + } + remove_if_exists(&self.files.api_socket).await?; + remove_if_exists(&self.files.guest_socket).await?; + remove_if_exists(&self.files.pid_file).await?; + let mut network = self.network.lock().await; + if let Some(slot) = network.as_ref().cloned() { + self.network_manager.destroy(&slot).await?; + *network = None; + } + remove_if_exists(&self.files.network_file).await?; + remove_if_exists(&network_metadata_temp(&self.files.network_file)).await?; + self.cleanup_complete.store(true, Ordering::Release); Ok(()) } } @@ -230,16 +911,43 @@ fn write_vm_config( request: &SpawnRequest, config: &FirecrackerConfig, guest_socket: &Path, + network: Option<&NetworkSlot>, ) -> Result { let vcpus = config .vcpus .or(request.vm.as_ref().map(|vm| vm.vcpus)) .unwrap_or(1); let memory_mib = resolve_memory(config, request.vm.as_ref())?; + let mut boot_args = config.boot_args.clone(); + if network.is_some() { + let network_arguments = boot_args + .split_whitespace() + .filter(|argument| argument.starts_with("ip=")) + .collect::>(); + match network_arguments.as_slice() { + [] => { + boot_args.push(' '); + boot_args.push_str(NETWORK_BOOT_IP); + } + [argument] if *argument == NETWORK_BOOT_IP => {} + arguments => { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker networking requires exactly {NETWORK_BOOT_IP:?}, found {}", + arguments + .iter() + .map(|argument| format!("{argument:?}")) + .collect::>() + .join(", ") + ), + }); + } + } + } let mut value = serde_json::json!({ "boot-source": { "kernel_image_path": path_string(&images_dir.join("vmlinux"), "vmlinux")?, - "boot_args": config.boot_args + "boot_args": boot_args }, "drives": [{ "drive_id": "rootfs", @@ -258,6 +966,13 @@ fn write_vm_config( "uds_path": path_string(guest_socket, "guest socket")? }); } + if let Some(network) = network { + value["network-interfaces"] = serde_json::json!([{ + "iface_id": "eth0", + "guest_mac": "02:FC:00:00:00:02", + "host_dev_name": network.tap_name() + }]); + } let path = request.run_dir.join("vmconfig.json"); std::fs::write( &path, @@ -281,9 +996,26 @@ fn resolve_memory(config: &FirecrackerConfig, vm: Option<&VmConfig>) -> Result Command { +fn build_launch_command( + binary: &Path, + network: Option<&NetworkSlot>, + api_socket: &Path, +) -> Command { #[cfg(target_os = "linux")] - let mut command = { + let mut command = if let Some(network) = network { + let mut command = Command::new("ip"); + command + .arg("netns") + .arg("exec") + .arg(network.netns()) + .arg("unshare") + .arg("--mount") + .arg("--propagation") + .arg("private") + .arg("--") + .arg(binary); + command + } else { let mut command = Command::new("unshare"); command .arg("--mount") @@ -294,7 +1026,10 @@ fn build_launch_command(binary: &Path, api_socket: &Path) -> Command { command }; #[cfg(not(target_os = "linux"))] - let mut command = Command::new(binary); + let mut command = { + let _ = network; + Command::new(binary) + }; command.arg("--api-sock").arg(api_socket); command.arg("--id").arg(format!( "fc-{}", @@ -323,14 +1058,29 @@ fn configure_logs(command: &mut Command, run_dir: &Path, serial_log: bool) -> Re } async fn read_backend_version(binary_path: &Path) -> Result { - let output = tokio::time::timeout( - Duration::from_secs(5), - Command::new(binary_path).arg("--version").output(), - ) - .await - .map_err(|_| BlazeError::BackendError { - msg: format!("firecracker probe timed out: {}", binary_path.display()), - })??; + let mut busy_retries = 0; + let output = loop { + match tokio::time::timeout( + Duration::from_secs(5), + Command::new(binary_path).arg("--version").output(), + ) + .await + { + Ok(Ok(output)) => break output, + Ok(Err(error)) + if error.kind() == std::io::ErrorKind::ExecutableFileBusy && busy_retries < 3 => + { + busy_retries += 1; + tokio::time::sleep(Duration::from_millis(5)).await; + } + Ok(Err(error)) => return Err(error.into()), + Err(_) => { + return Err(BlazeError::BackendError { + msg: format!("firecracker probe timed out: {}", binary_path.display()), + }); + } + } + }; if !output.status.success() { return Err(BlazeError::BackendError { msg: format!( @@ -398,6 +1148,9 @@ async fn wait_for_socket(socket: &Path, child: &mut Child, timeout: Duration) -> } async fn remove_if_exists(path: &Path) -> Result<()> { + if path.as_os_str().is_empty() { + return Ok(()); + } match tokio::fs::remove_file(path).await { Ok(()) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), @@ -405,6 +1158,61 @@ async fn remove_if_exists(path: &Path) -> Result<()> { } } +fn write_network_metadata(path: &Path, network: &NetworkSlot) -> Result<()> { + write_network_record(path, network, NetworkProcessState::PreSpawn) +} + +fn write_network_record( + path: &Path, + network: &NetworkSlot, + process_state: NetworkProcessState, +) -> Result<()> { + let parent = path.parent().ok_or_else(|| BlazeError::BackendError { + msg: format!("network metadata has no parent: {}", path.display()), + })?; + let temporary = network_metadata_temp(path); + (|| -> Result<()> { + let bytes = serde_json::to_vec_pretty(&NetworkRecord { + slot: network.slot(), + owner: network.owner(), + process_state, + }) + .map_err(|error| BlazeError::BackendError { + msg: format!("serialize network metadata: {error}"), + })?; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })() +} + +fn read_network_metadata(path: &Path) -> Result<(NetworkSlot, NetworkProcessState)> { + let record: NetworkRecord = serde_json::from_slice(&std::fs::read(path)?).map_err(|error| { + BlazeError::BackendError { + msg: format!("parse network metadata {}: {error}", path.display()), + } + })?; + Ok(( + NetworkSlot::from_record(record.slot, record.owner)?, + record.process_state, + )) +} + +fn network_metadata_temp(path: &Path) -> PathBuf { + path.with_extension("json.tmp") +} + +fn configured_guest_socket(enable_vsock: bool, socket: PathBuf) -> PathBuf { + if enable_vsock { socket } else { PathBuf::new() } +} + fn validate_regular_file(path: &Path, label: &str) -> Result<()> { if !path.is_file() { return Err(BlazeError::BackendError { @@ -462,30 +1270,87 @@ fn path_string<'a>(path: &'a Path, label: &str) -> Result<&'a str> { }) } -pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> Result<()> { - let stopped_marker = stopped_marker(run_dir); - if stopped_marker.is_file() { - return Ok(()); +fn backend_protocol_error(error: hyper::Error) -> BlazeError { + BlazeError::BackendError { + msg: format!("Firecracker API protocol error: {error}"), } +} + +async fn cleanup_orphan_run_dir_with( + instance_id: Uuid, + run_dir: &Path, + network_manager: &NetworkManager, +) -> Result<()> { + let stopped_marker = stopped_marker(run_dir); let pid_file = run_dir.join("firecracker.pid"); - #[cfg(target_os = "linux")] - { - terminate_recorded_process(instance_id, &pid_file, "firecracker").await?; - } - #[cfg(not(target_os = "linux"))] - { - let _ = instance_id; - if pid_file.exists() { - return Err(BlazeError::BackendError { - msg: format!( - "cannot validate Firecracker orphan {} outside Linux", - pid_file.display() - ), - }); + let network_file = run_dir.join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + let record_path = if network_file.is_file() { + Some(network_file.as_path()) + } else if network_temp_file.is_file() { + Some(network_temp_file.as_path()) + } else { + None + }; + let network_record = match record_path { + Some(path) => match read_network_metadata(path) { + Ok((network, state)) => { + if network.owner() != instance_id { + return Err(BlazeError::BackendError { + msg: format!( + "network record owner {} does not match instance {instance_id}", + network.owner() + ), + }); + } + Some((network, Some(state))) + } + Err(error) if path == network_temp_file.as_path() && !network_file.exists() => { + match network_manager.find_by_owner(instance_id).await? { + // The namespace name proves ownership, but it cannot prove + // whether the backend crossed the spawn boundary. + Some(network) => Some((network, None)), + None => return Err(error), + } + } + Err(error) => return Err(error), + }, + None => network_manager + .find_by_owner(instance_id) + .await? + .map(|network| (network, None)), + }; + let process_may_exist = pid_file.exists() + || network_record + .as_ref() + .is_none_or(|(_, state)| *state != Some(NetworkProcessState::PreSpawn)); + if !stopped_marker.is_file() { + #[cfg(target_os = "linux")] + { + if process_may_exist { + terminate_recorded_process(instance_id, &pid_file, "firecracker").await?; + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = instance_id; + if process_may_exist { + return Err(BlazeError::BackendError { + msg: format!( + "cannot validate Firecracker orphan {} outside Linux", + pid_file.display() + ), + }); + } } + record_backend_stopped(&stopped_marker).await?; } - record_backend_stopped(&stopped_marker).await?; + if let Some((network, _)) = network_record { + network_manager.destroy(&network).await?; + remove_if_exists(&network_file).await?; + } + remove_if_exists(&network_temp_file).await?; remove_if_exists(&run_dir.join("api.sock")).await?; remove_if_exists(&run_dir.join("vsock.uds")).await?; remove_if_exists(&pid_file).await?; @@ -494,7 +1359,28 @@ pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> #[cfg(test)] mod tests { + use std::collections::VecDeque; + #[cfg(target_os = "linux")] + use std::convert::Infallible; + use blaze_core::storage::StorageSlot; + #[cfg(target_os = "linux")] + use http_body_util::BodyExt; + #[cfg(target_os = "linux")] + use hyper::Response; + #[cfg(target_os = "linux")] + use hyper::server::conn::http1 as server_http1; + #[cfg(target_os = "linux")] + use hyper::service::service_fn; + #[cfg(target_os = "linux")] + use tokio::net::UnixListener; + #[cfg(target_os = "linux")] + use tokio::sync::oneshot; + + #[cfg(target_os = "linux")] + use crate::spawner::SpawnerRegistry; + + use crate::spawner::netns::{IpCommandRunner, IpOutput, NetworkManager, test_network_slot}; use super::*; @@ -516,16 +1402,654 @@ mod tests { } #[test] - fn vm_config_omits_network_until_the_network_capability_is_enabled() { - let temp = tempfile::tempdir().expect("temp"); - let request = spawn_request(temp.path()); + fn restore_compatibility_requires_the_matching_firecracker_version() { + let mut restore = FirecrackerRestore { + snapshot_path: PathBuf::from("vmstate.snap"), + mem_path: PathBuf::from("memory.snap"), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + network_slot: None, + }; - let path = write_vm_config( - &temp.path().join("images"), - &request, - &FirecrackerConfig::default(), - &temp.path().join("guest.sock"), - ) + validate_restore_compatibility(&restore, "Firecracker v1.16.0").expect("matching version"); + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.17.0") + .expect_err("mismatched version") + .to_string() + .contains("does not match executable version") + ); + restore.expected_version = None; + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.16.0") + .expect_err("missing version") + .to_string() + .contains("requires a checkpoint backend version") + ); + restore.expected_version = Some("Firecracker v1.16.0".to_string()); + restore.backend = BackendKind::Mock; + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.16.0") + .expect_err("wrong backend") + .to_string() + .contains("cannot restore a mock checkpoint") + ); + } + + #[test] + fn instance_preserves_its_network_slot_for_restore() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + None, + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + temp.path().join("vsock.uds"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + Some(test_network_slot(7)), + Arc::new(NetworkManager::default()), + false, + ); + + assert_eq!(instance.network_slot(), Some(7)); + } + + #[cfg(target_os = "linux")] + async fn spawn_api( + socket: &Path, + call_count: usize, + ) -> oneshot::Receiver> { + let listener = UnixListener::bind(socket).expect("bind"); + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + let observed = Arc::new(Mutex::new(Vec::with_capacity(call_count))); + for _ in 0..call_count { + let (stream, _) = listener.accept().await.expect("accept"); + let observed = observed.clone(); + let service = service_fn(move |request: Request| { + let observed = observed.clone(); + async move { + let method = request.method().clone(); + let path = request.uri().path().to_string(); + let body = request + .into_body() + .collect() + .await + .expect("request body") + .to_bytes(); + let body = if body.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_slice(&body).expect("request JSON") + }; + observed.lock().await.push((method, path, body)); + Ok::<_, Infallible>( + Response::builder() + .status(hyper::StatusCode::NO_CONTENT) + .body(Full::new(Bytes::new())) + .expect("response"), + ) + } + }); + server_http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("serve"); + } + let calls = observed.lock().await.clone(); + let _ = tx.send(calls); + }); + rx + } + + #[cfg(target_os = "linux")] + fn spawn_api_response( + socket: &Path, + status: hyper::StatusCode, + body: Vec, + delay: Duration, + ) -> tokio::task::JoinHandle<()> { + let listener = UnixListener::bind(socket).expect("bind"); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let body = Bytes::from(body); + let service = service_fn(move |_request: Request| { + let body = body.clone(); + async move { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + Ok::<_, Infallible>( + Response::builder() + .status(status) + .body(Full::new(body)) + .expect("response"), + ) + } + }); + let _ = server_http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }) + } + + #[cfg(target_os = "linux")] + fn write_version_binary(path: &Path, output: &str) { + use std::os::unix::fs::PermissionsExt; + + let staged = path.with_extension("new"); + std::fs::write(&staged, format!("#!/bin/sh\nprintf '%s\\n' '{output}'\n")) + .expect("write version binary"); + std::fs::set_permissions(&staged, std::fs::Permissions::from_mode(0o755)) + .expect("make version binary executable"); + std::fs::rename(staged, path).expect("replace version binary"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn launch_capture_reads_the_requested_binary_each_time() { + let temp = tempfile::tempdir().expect("temp"); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + write_version_binary(&first, "Firecracker v1.15.0"); + write_version_binary(&second, "Firecracker v1.16.0"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + let first_capture = spawner + .capture_for(&first, temp.path().join("first.sock")) + .await + .expect("first capture"); + let second_capture = spawner + .capture_for(&second, temp.path().join("second.sock")) + .await + .expect("second capture"); + assert_eq!(first_capture.backend_version, "Firecracker v1.15.0"); + assert_eq!(second_capture.backend_version, "Firecracker v1.16.0"); + + write_version_binary(&first, "Firecracker v1.17.0"); + let replaced_capture = spawner + .capture_for(&first, temp.path().join("replaced.sock")) + .await + .expect("replaced capture"); + assert_eq!(replaced_capture.backend_version, "Firecracker v1.17.0"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn probe_checks_each_requested_binary() { + let temp = tempfile::tempdir().expect("temp"); + let valid = temp.path().join("valid"); + let invalid = temp.path().join("invalid"); + write_version_binary(&valid, "Firecracker v1.16.0"); + write_version_binary(&invalid, "not a Firecracker version"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + assert!(spawner.probe(&valid).await.expect("valid probe")); + assert!(!spawner.probe(&invalid).await.expect("invalid probe")); + + write_version_binary(&invalid, "Firecracker v1.17.0"); + assert!(spawner.probe(&invalid).await.expect("replaced probe")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn registry_restore_capability_reads_each_requested_binary() { + let temp = tempfile::tempdir().expect("temp"); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + write_version_binary(&first, "Firecracker v1.15.0"); + write_version_binary(&second, "Firecracker v1.16.0"); + let mut registry = SpawnerRegistry::new(); + registry.insert( + BackendKind::Firecracker, + Arc::new(FirecrackerSpawner::new(temp.path().join("images"))), + ); + let adapter = registry + .get(BackendKind::Firecracker) + .expect("registered Firecracker adapter"); + + let first_capability = adapter + .restore_capability(&first) + .await + .expect("first capability") + .expect("restore supported"); + let second_capability = adapter + .restore_capability(&second) + .await + .expect("second capability") + .expect("restore supported"); + assert_eq!(first_capability.backend, BackendKind::Firecracker); + assert_eq!( + first_capability.version.as_deref(), + Some("Firecracker v1.15.0") + ); + assert_eq!(first_capability.snapshot_kind, SnapshotKind::Full); + assert_eq!( + second_capability.version.as_deref(), + Some("Firecracker v1.16.0") + ); + + write_version_binary(&first, "Firecracker v1.17.0"); + let replaced_capability = adapter + .restore_capability(&first) + .await + .expect("replaced capability") + .expect("restore supported"); + assert_eq!( + replaced_capability.version.as_deref(), + Some("Firecracker v1.17.0") + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn restore_rejects_a_binary_version_change_before_start() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = spawn_request(temp.path()); + let run_dir = spawn.run_dir.clone(); + std::fs::remove_dir_all(&run_dir).expect("remove fixture run dir"); + write_version_binary(&spawn.binary_path, "Firecracker v1.17.0"); + std::fs::create_dir_all(spawn.storage.rootfs_path.parent().expect("rootfs parent")) + .expect("slot"); + std::fs::write(&spawn.storage.rootfs_path, b"rootfs").expect("rootfs"); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + std::fs::write(&snapshot_path, b"vmstate").expect("VM state"); + std::fs::write(&mem_path, b"memory").expect("memory"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + let failure = match spawner + .restore(RestoreRequest { + instance_id: spawn.instance_id, + run_dir: spawn.run_dir, + binary_path: spawn.binary_path, + storage: spawn.storage, + snapshot_path, + mem_path, + checkpoint_backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + network_slot: None, + }) + .await + { + Ok(_) => panic!("version change must fail before process start"), + Err(failure) => failure, + }; + let (source, owner) = failure.into_parts(); + + assert!( + source + .to_string() + .contains("does not match executable version") + ); + assert!(owner.is_none()); + assert!(!run_dir.exists()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn instance_loads_a_full_snapshot_with_a_minimal_payload() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 1).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + Some(child), + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + PathBuf::new(), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), + false, + ); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + + instance + .load_snapshot(&FirecrackerRestore { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + network_slot: None, + }) + .await + .expect("load snapshot"); + + let calls = observed.await.expect("observed call"); + assert_eq!( + calls, + vec![( + Method::PUT, + "/snapshot/load".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_backend": { + "backend_type": "File", + "backend_path": mem_path, + }, + "resume_vm": true, + }), + )] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn restored_owner_exposes_the_guest_socket_when_requested() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 1).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let guest_socket = temp.path().join("vsock.uds"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + Some(child), + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + guest_socket.clone(), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), + true, + ); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + + assert_eq!(instance.guest_socket_path(), guest_socket); + instance + .load_snapshot(&FirecrackerRestore { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, + }) + .await + .expect("load snapshot"); + + let calls = observed.await.expect("observed call"); + assert_eq!( + calls, + vec![( + Method::PUT, + "/snapshot/load".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_backend": { + "backend_type": "File", + "backend_path": mem_path, + }, + "resume_vm": true, + }), + )] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_snapshot_load_retains_an_owner_when_cleanup_is_incomplete() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &api_socket, + hyper::StatusCode::BAD_REQUEST, + b"incompatible snapshot".to_vec(), + Duration::ZERO, + ); + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let guest_socket = temp.path().join("guest.sock"); + std::fs::create_dir(&guest_socket).expect("cleanup blocker"); + let instance_id = Uuid::new_v4(); + let instance = Arc::new(FirecrackerInstance::new( + instance_id, + Some(child), + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + guest_socket.clone(), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), + true, + )); + let restore = FirecrackerRestore { + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, + }; + + let load_error = instance + .load_snapshot(&restore) + .await + .expect_err("snapshot load must fail"); + server.await.expect("server"); + let owner: DynBackendInstance = instance; + let failure = SpawnFailure::compensate_started(load_error, owner).await; + let (source, retained) = failure.into_parts(); + + assert!(source.to_string().contains("cleanup failed")); + let retained = retained.expect("incomplete cleanup must retain ownership"); + assert_eq!(retained.instance_id(), instance_id); + assert_eq!(retained.version(), Some("Firecracker v1.16.0")); + + std::fs::remove_dir(&guest_socket).expect("remove cleanup blocker"); + retained.kill().await.expect("retry cleanup"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn instance_reports_version_and_captures_full_snapshot_over_uds() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 3).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let instance_id = Uuid::new_v4(); + let instance = FirecrackerInstance::new( + instance_id, + Some(child), + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + PathBuf::new(), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), + false, + ); + let snapshot_path = temp.path().join("checkpoint/vmstate.snap"); + let mem_path = temp.path().join("checkpoint/memory.snap"); + + assert_eq!(instance.instance_id(), instance_id); + assert_eq!(instance.version(), Some("Firecracker v1.16.0")); + assert!(instance.supports_checkpoint_capture()); + instance.pause().await.expect("pause"); + let result = instance + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + kind: SnapshotKind::Full, + }) + .await + .expect("snapshot"); + instance.resume().await.expect("resume"); + + assert_eq!(result.snapshot_path, snapshot_path); + assert_eq!(result.mem_path, mem_path); + let calls = observed.await.expect("observed calls"); + assert_eq!( + calls, + vec![ + ( + Method::PATCH, + "/vm".to_string(), + serde_json::json!({"state": "Paused"}), + ), + ( + Method::PUT, + "/snapshot/create".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_file_path": mem_path, + "snapshot_type": "Full", + }), + ), + ( + Method::PATCH, + "/vm".to_string(), + serde_json::json!({"state": "Resumed"}), + ), + ] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_reports_non_success_response_body() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::BAD_REQUEST, + b"invalid VM state".to_vec(), + Duration::ZERO, + ); + let client = FirecrackerApiClient::new(socket, Duration::from_secs(1)); + + let error = client + .call_json(Method::PATCH, "/vm", None) + .await + .expect_err("non-success response"); + server.await.expect("server"); + + let message = error.to_string(); + assert!(message.contains("400 Bad Request")); + assert!(message.contains("invalid VM state")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_rejects_an_oversized_response() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::OK, + vec![b'x'; MAX_API_RESPONSE_BYTES + 1], + Duration::ZERO, + ); + let client = FirecrackerApiClient::new(socket, Duration::from_secs(1)); + + let error = client + .call_json(Method::GET, "/vm", None) + .await + .expect_err("oversized response"); + server.await.expect("server"); + + assert!(error.to_string().contains("response exceeded 65536 bytes")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_times_out_a_stalled_response() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::OK, + Vec::new(), + Duration::from_millis(200), + ); + let client = FirecrackerApiClient::new(socket, Duration::from_millis(20)); + + let error = client + .call_json(Method::GET, "/vm", None) + .await + .expect_err("stalled response"); + server.await.expect("server"); + + assert!(error.to_string().contains("timed out after 20ms")); + } + + #[test] + fn vm_config_omits_network_until_the_network_capability_is_enabled() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + + let path = write_vm_config( + &temp.path().join("images"), + &request, + &FirecrackerConfig::default(), + &temp.path().join("guest.sock"), + None, + ) .expect("write config"); let value: serde_json::Value = serde_json::from_slice(&std::fs::read(path).expect("read config")) @@ -533,6 +2057,592 @@ mod tests { assert!(value.get("network-interfaces").is_none()); } + #[test] + fn vm_config_wires_an_allocated_network_slot() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + + let path = write_vm_config( + &temp.path().join("images"), + &request, + &FirecrackerConfig::default(), + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect("write config"); + let value: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read config")) + .expect("parse config"); + assert_eq!(value["network-interfaces"][0]["iface_id"], "eth0"); + assert_eq!(value["network-interfaces"][0]["host_dev_name"], "tap0"); + assert!( + value["boot-source"]["boot_args"] + .as_str() + .expect("boot args") + .contains("::eth0:off") + ); + } + + #[test] + fn vm_config_accepts_the_matching_network_boot_argument() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: format!("console=ttyS0 {NETWORK_BOOT_IP}"), + ..FirecrackerConfig::default() + }; + + write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect("matching network boot argument"); + } + + #[test] + fn vm_config_rejects_an_incompatible_network_boot_argument() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: "console=ttyS0 ip=dhcp".to_string(), + ..FirecrackerConfig::default() + }; + + let error = write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect_err("incompatible network boot argument"); + + assert!(error.to_string().contains("requires")); + assert!(error.to_string().contains("ip=dhcp")); + } + + #[test] + fn vm_config_rejects_conflicting_network_boot_arguments() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: format!("console=ttyS0 {NETWORK_BOOT_IP} ip=dhcp"), + ..FirecrackerConfig::default() + }; + + let error = write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect_err("conflicting network boot arguments"); + + assert!(error.to_string().contains("exactly")); + assert!(error.to_string().contains("ip=dhcp")); + } + + #[test] + fn network_metadata_is_published_atomically() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + let slot = test_network_slot(7); + + write_network_metadata(&path, &slot).expect("write metadata"); + + let (stored, state) = read_network_metadata(&path).expect("parse metadata"); + assert_eq!(stored, slot); + assert_eq!(state, NetworkProcessState::PreSpawn); + assert!(!network_metadata_temp(&path).exists()); + } + + #[test] + fn network_metadata_records_launch_intent_before_spawn() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + let slot = test_network_slot(7); + write_network_metadata(&path, &slot).expect("write pre-spawn metadata"); + + write_network_record(&path, &slot, NetworkProcessState::Launching) + .expect("record launch intent"); + + let (stored, state) = read_network_metadata(&path).expect("parse metadata"); + assert_eq!(stored, slot); + assert_eq!(state, NetworkProcessState::Launching); + assert!(!network_metadata_temp(&path).exists()); + } + + #[test] + fn network_metadata_rejects_out_of_range_slots() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + std::fs::write( + &path, + br#"{"slot":16383,"owner":"00000000-0000-0000-0000-000000000001"}"#, + ) + .expect("metadata"); + + let error = read_network_metadata(&path).expect_err("invalid slot"); + + assert!(error.to_string().contains("outside")); + } + + #[tokio::test] + async fn network_cleanup_failure_retains_a_retryable_backend_owner() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let slot = test_network_slot(0); + write_network_metadata(&network_file, &slot).expect("network metadata"); + let namespace = format!("{}\n", slot.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_failure("delete peer failed"), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = Arc::new(NetworkManager::with_runner(runner.clone())); + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + slot.owner(), + None, + FirecrackerCapture::new( + temp.path().join("api.sock"), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + network_file.clone(), + ), + Some(slot.clone()), + network_manager, + false, + )); + + owner.kill().await.expect_err("first cleanup must fail"); + assert!(network_file.exists()); + owner.kill().await.expect("retry cleanup"); + assert!(!network_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", slot.netns()]) + ); + assert!( + !runner + .calls() + .iter() + .any(|args| args == &["link", "del", "blz-veth-0"]) + ); + } + + #[tokio::test] + async fn try_wait_retries_cleanup_after_observing_process_exit() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let slot = test_network_slot(0); + write_network_metadata(&network_file, &slot).expect("network metadata"); + let namespace = format!("{}\n", slot.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_failure("delete peer failed"), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let child = Command::new("sh") + .arg("-c") + .arg("exit 7") + .spawn() + .expect("spawn child"); + let instance = FirecrackerInstance::new( + slot.owner(), + Some(child), + FirecrackerCapture::new( + temp.path().join("api.sock"), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + network_file.clone(), + ), + Some(slot.clone()), + Arc::new(NetworkManager::with_runner(runner.clone())), + false, + ); + + let first_error = loop { + match instance.try_wait().await { + Ok(None) => tokio::time::sleep(Duration::from_millis(5)).await, + Ok(Some(result)) => { + panic!("cleanup failure must not report completion: {result:?}") + } + Err(error) => break error, + } + }; + assert!(first_error.to_string().contains("delete peer failed")); + assert!(network_file.exists()); + + let result = instance + .try_wait() + .await + .expect("retry cleanup") + .expect("completed process"); + assert_eq!(result.exit_code, Some(7)); + assert!(!network_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", slot.netns()]) + ); + } + + #[tokio::test] + async fn stopped_orphan_still_releases_recorded_network() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("orphan cleanup"); + + assert!(!network_file.exists()); + let calls = runner.calls(); + assert!(calls.iter().any(|args| { + args == &[ + "netns", + "exec", + network.netns(), + "ip", + "link", + "del", + "blz-vpeer-0", + ] + })); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[tokio::test] + async fn orphan_cleanup_recovers_a_complete_temporary_network_record() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + let network = test_network_slot(0); + let bytes = serde_json::to_vec(&NetworkRecord { + slot: network.slot(), + owner: network.owner(), + process_state: NetworkProcessState::PreSpawn, + }) + .expect("serialize metadata"); + std::fs::write(&network_temp_file, bytes).expect("temporary metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("orphan cleanup"); + + assert!(!network_file.exists()); + assert!(!network_temp_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_a_truncated_network_record_without_pid_proof() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + std::fs::write(&network_temp_file, b"{").expect("truncated metadata"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("unknown launch state must fail closed"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(network_temp_file.exists()); + assert!(!stopped_marker(temp.path()).exists()); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_an_unrecorded_namespace_without_pid_proof() { + let temp = tempfile::tempdir().expect("temp"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("unknown launch state must fail closed"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(!stopped_marker(temp.path()).exists()); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + + #[tokio::test] + async fn stopped_orphan_releases_an_unrecorded_owner_namespace() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("stopped process permits network recovery"); + + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[tokio::test] + async fn network_record_owner_mismatch_issues_no_host_commands() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let runner = Arc::new(TestIpRunner::default()); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(Uuid::from_u128(2), temp.path(), &network_manager) + .await + .expect_err("mismatched owner must fail"); + + assert!(error.to_string().contains("does not match instance")); + assert!(network_file.exists()); + assert!(runner.calls().is_empty()); + } + + #[tokio::test] + async fn stale_network_record_does_not_delete_a_reused_slot() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let old_network = test_network_slot(0); + write_network_metadata(&network_file, &old_network).expect("network metadata"); + let new_network = + NetworkSlot::from_record(0, Uuid::from_u128(2)).expect("new network owner"); + let namespace = format!("{}\n", new_network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(old_network.owner(), temp.path(), &network_manager) + .await + .expect("retire stale record"); + + assert!(!network_file.exists()); + let calls = runner.calls(); + assert_eq!(calls, vec![vec!["netns".to_string(), "list".to_string()]]); + } + + #[tokio::test] + async fn pre_spawn_orphan_releases_network_without_pid_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("pre-spawn cleanup"); + + assert!(!network_file.exists()); + assert!(stopped_marker(temp.path()).exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn unconfirmed_process_ownership_retains_network_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_record(&network_file, &network, NetworkProcessState::Launching) + .expect("network metadata"); + let runner = Arc::new(TestIpRunner::default()); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("missing process metadata must block cleanup"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(network_file.exists()); + assert!(runner.calls().is_empty()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn launch_intent_with_empty_handoff_releases_network() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_record(&network_file, &network, NetworkProcessState::Launching) + .expect("network metadata"); + prepare_pid_handoff(&temp.path().join("firecracker.pid")).expect("prepare PID handoff"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("empty handoff proves launch did not start"); + + assert!(!network_file.exists()); + assert!(stopped_marker(temp.path()).exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[test] + fn vm_config_and_reported_guest_transport_agree() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let socket = temp.path().join("vsock.uds"); + let disabled = FirecrackerConfig::default(); + let disabled_path = write_vm_config( + &temp.path().join("images"), + &request, + &disabled, + &socket, + None, + ) + .expect("disabled config"); + let disabled_value: serde_json::Value = + serde_json::from_slice(&std::fs::read(disabled_path).expect("read disabled config")) + .expect("parse disabled config"); + assert!(disabled_value.get("vsock").is_none()); + assert!( + configured_guest_socket(disabled.enable_vsock, socket.clone()) + .as_os_str() + .is_empty() + ); + + let enabled = FirecrackerConfig { + enable_vsock: true, + ..FirecrackerConfig::default() + }; + let enabled_path = write_vm_config( + &temp.path().join("images"), + &request, + &enabled, + &socket, + None, + ) + .expect("enabled config"); + let enabled_value: serde_json::Value = + serde_json::from_slice(&std::fs::read(enabled_path).expect("read enabled config")) + .expect("parse enabled config"); + assert_eq!( + enabled_value["vsock"]["uds_path"], + path_string(&socket, "socket").unwrap() + ); + assert_eq!( + configured_guest_socket(enabled.enable_vsock, socket.clone()), + socket + ); + } + #[test] fn serial_log_rotates_before_reuse() { let temp = tempfile::tempdir().expect("temp"); @@ -567,6 +2677,27 @@ mod tests { assert!(is_executable_file(&tool)); } + #[tokio::test] + async fn backend_probe_skips_network_checks_when_no_policy_enables_them() { + let temp = tempfile::tempdir().expect("temp"); + let called = Arc::new(AtomicBool::new(false)); + let network = Arc::new(NetworkManager::with_runner(Arc::new( + UnavailableNetworkRunner { + called: called.clone(), + }, + ))); + let spawner = FirecrackerSpawner { + images_dir: temp.path().join("images"), + api_timeout: Duration::from_secs(1), + socket_timeout: Duration::from_secs(1), + network, + network_required: false, + }; + + assert!(spawner.network_probe_ready().await.expect("probe")); + assert!(!called.load(Ordering::Acquire)); + } + #[tokio::test] async fn start_failure_terminates_child_and_removes_process_metadata() { let temp = tempfile::tempdir().expect("temp"); @@ -583,11 +2714,22 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( Uuid::new_v4(), - child, - temp.path().join("api.sock"), - temp.path().join("guest.sock"), - pid_file.clone(), - stopped_marker(temp.path()), + Some(child), + FirecrackerCapture::new( + temp.path().join("api.sock"), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + pid_file.clone(), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), + true, )); let failure = SpawnFailure::compensate_started( BlazeError::BackendError { @@ -631,4 +2773,76 @@ mod tests { vm: None, } } + + #[derive(Default)] + struct TestIpRunner { + responses: std::sync::Mutex>, + calls: std::sync::Mutex>>, + } + + struct UnavailableNetworkRunner { + called: Arc, + } + + #[async_trait] + impl IpCommandRunner for UnavailableNetworkRunner { + async fn output(&self, _args: &[String], _timeout: Duration) -> Result { + self.called.store(true, Ordering::Release); + Ok(ip_failure("network commands unavailable")) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, _name: &str) -> bool { + false + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + false + } + } + + impl TestIpRunner { + fn with_responses(responses: [IpOutput; N]) -> Self { + Self { + responses: std::sync::Mutex::new(responses.into()), + calls: std::sync::Mutex::new(Vec::new()), + } + } + + fn calls(&self) -> Vec> { + self.calls.lock().expect("calls lock").clone() + } + } + + #[async_trait] + impl IpCommandRunner for TestIpRunner { + async fn output(&self, args: &[String], _timeout: Duration) -> Result { + self.calls.lock().expect("calls lock").push(args.to_vec()); + Ok(self + .responses + .lock() + .expect("responses lock") + .pop_front() + .unwrap_or_else(|| ip_success(b""))) + } + } + + fn ip_success(stdout: &[u8]) -> IpOutput { + IpOutput { + success: true, + status: "exit status: 0".to_string(), + stdout: stdout.to_vec(), + stderr: Vec::new(), + } + } + + fn ip_failure(stderr: &str) -> IpOutput { + IpOutput { + success: false, + status: "exit status: 1".to_string(), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } + } } diff --git a/src/blaze/crates/blazed/src/spawner/netns.rs b/src/blaze/crates/blazed/src/spawner/netns.rs new file mode 100644 index 0000000000..02a4d9a625 --- /dev/null +++ b/src/blaze/crates/blazed/src/spawner/netns.rs @@ -0,0 +1,1178 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Per-VM network namespace allocation and compensated setup. + +use std::collections::HashSet; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use blaze_core::{BlazeError, Result}; +use thiserror::Error; +use tokio::process::Command; +use uuid::Uuid; + +const NET_VETH_BASE: usize = 4; +const NET_VETH_TOP: usize = 0x1_0000; +const NET_MAX_SLOT: usize = (NET_VETH_TOP - NET_VETH_BASE) / 4; +const HOST_NETWORK_LOCK: &str = "/run/lock/blaze-network.lock"; +const HOST_LOCK_RETRY: Duration = Duration::from_millis(10); + +#[derive(Debug, Default)] +struct SlotState { + used: HashSet, + next: usize, +} + +#[derive(Debug, Clone)] +pub(super) struct IpOutput { + pub(super) success: bool, + pub(super) status: String, + pub(super) stdout: Vec, + pub(super) stderr: Vec, +} + +#[async_trait] +pub(super) trait IpCommandRunner: Send + Sync { + async fn output(&self, args: &[String], timeout: Duration) -> Result; + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, _name: &str) -> bool { + true + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + true + } +} + +struct SystemIpCommandRunner; + +#[async_trait] +impl IpCommandRunner for SystemIpCommandRunner { + async fn output(&self, args: &[String], timeout: Duration) -> Result { + let mut command = Command::new("ip"); + command.kill_on_drop(true).env("LC_ALL", "C").args(args); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| BlazeError::BackendError { + msg: format!("ip {} timed out", args.join(" ")), + })??; + Ok(IpOutput { + success: output.status.success(), + status: output.status.to_string(), + stdout: output.stdout, + stderr: output.stderr, + }) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, name: &str) -> bool { + executable_in_path(name) + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + // The current implementation relies on root-owned netns mounts, + // tap creation, forwarding changes, and NAT rules. + unsafe { libc::geteuid() == 0 } + } +} + +/// Process-local allocator and lifecycle owner for Blaze network namespaces. +pub(super) struct NetworkManager { + state: Mutex, + command_timeout: Duration, + runner: Arc, + coordination_file: Option, +} + +impl fmt::Debug for NetworkManager { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NetworkManager") + .field("command_timeout", &self.command_timeout) + .finish_non_exhaustive() + } +} + +impl Default for NetworkManager { + fn default() -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(5), + runner: Arc::new(SystemIpCommandRunner), + coordination_file: Some(PathBuf::from(HOST_NETWORK_LOCK)), + } + } +} + +struct HostNetworkGuard { + #[cfg(unix)] + _file: Option, +} + +/// One fully configured per-VM network namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct NetworkSlot { + slot: usize, + owner: Uuid, + netns: String, + tap_name: String, + veth_host: String, + veth_peer: String, +} + +/// Network setup failure with an optional residual slot owner. +#[derive(Debug, Error)] +#[error("{source}")] +pub(super) struct NetworkCreateError { + #[source] + source: BlazeError, + residual: Option, +} + +impl NetworkCreateError { + fn clean(source: BlazeError) -> Self { + Self { + source, + residual: None, + } + } + + fn with_residual(source: BlazeError, residual: NetworkSlot) -> Self { + Self { + source, + residual: Some(residual), + } + } + + /// Split the setup error from any network slot that still needs cleanup. + pub(super) fn into_parts(self) -> (BlazeError, Option) { + (self.source, self.residual) + } +} + +impl From for NetworkCreateError { + fn from(source: BlazeError) -> Self { + Self::clean(source) + } +} + +impl NetworkSlot { + pub(super) fn from_record(slot: usize, owner: Uuid) -> Result { + if slot >= NET_MAX_SLOT { + return Err(BlazeError::BackendError { + msg: format!("network slot {slot} is outside 0..{NET_MAX_SLOT}"), + }); + } + Ok(network_slot(slot, owner)) + } + + pub(super) fn slot(&self) -> usize { + self.slot + } + + pub(super) fn owner(&self) -> Uuid { + self.owner + } + + /// Network namespace name. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub(super) fn netns(&self) -> &str { + &self.netns + } + + /// Tap device visible inside the namespace. + pub(super) fn tap_name(&self) -> &str { + &self.tap_name + } +} + +impl NetworkManager { + #[cfg(test)] + pub(super) fn with_runner(runner: Arc) -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(1), + runner, + coordination_file: None, + } + } + + #[cfg(test)] + pub(super) fn with_runner_and_lock( + runner: Arc, + coordination_file: PathBuf, + ) -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(1), + runner, + coordination_file: Some(coordination_file), + } + } + + /// Check the commands and host conditions required by the network path. + pub(super) async fn probe(&self) -> Result { + #[cfg(not(target_os = "linux"))] + { + Ok(false) + } + #[cfg(target_os = "linux")] + { + if !self.runner.has_network_admin() + || ["ip", "sysctl", "iptables"] + .iter() + .any(|command| !self.runner.executable_in_path(command)) + { + return Ok(false); + } + let args = vec!["netns".to_string(), "list".to_string()]; + Ok(self.run_ip_output(&args).await?.success) + } + } + + /// Create an isolated namespace, veth uplink, tap, route, and NAT rule. + /// + /// `record` runs immediately after this manager creates the namespace and + /// before any dependent resources are added. Callers use it to publish + /// ownership without ever recording a namespace created by another process. + pub(super) async fn create( + &self, + owner: Uuid, + record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + self.create_with_slot(owner, None, record).await + } + + /// Recreate the exact slot whose device names are embedded in a snapshot. + pub(super) async fn create_at( + &self, + owner: Uuid, + slot: usize, + record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + NetworkSlot::from_record(slot, owner)?; + self.create_with_slot(owner, Some(slot), record).await + } + + async fn create_with_slot( + &self, + owner: Uuid, + requested_slot: Option, + mut record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + let _host_guard = self.acquire_host_guard().await?; + let mut blocked = self.existing_slots().await?; + let (slot, network) = loop { + let slot = match requested_slot { + Some(slot) => self.reserve(slot, &blocked)?, + None => self.allocate(&blocked)?, + }; + let network = network_slot(slot, owner); + let add_namespace = vec!["netns".into(), "add".into(), network.netns.clone()]; + match self.run_ip(&add_namespace).await { + Ok(()) => { + if let Err(error) = record(&network) { + return self.fail_setup(&network, error).await; + } + break (slot, network); + } + Err(error) => { + self.release(slot); + let refreshed = self.existing_slots().await?; + if refreshed.contains(&slot) { + if requested_slot.is_some() { + return Err(NetworkCreateError::clean(BlazeError::BackendError { + msg: format!( + "required network slot {slot} is unavailable during restore" + ), + })); + } + blocked.extend(refreshed); + continue; + } + return Err(NetworkCreateError::clean(error)); + } + } + }; + let (host_ip, peer_ip) = veth_ips(slot); + let add_veth = vec![ + "link".into(), + "add".into(), + network.veth_host.clone(), + "type".into(), + "veth".into(), + "peer".into(), + "name".into(), + network.veth_peer.clone(), + "netns".into(), + network.netns.clone(), + ]; + if let Err(error) = self.run_ip(&add_veth).await { + return self.fail_setup(&network, error).await; + } + let host_steps = vec![ + vec![ + "addr".into(), + "add".into(), + format!("{host_ip}/30"), + "dev".into(), + network.veth_host.clone(), + ], + vec![ + "link".into(), + "set".into(), + network.veth_host.clone(), + "up".into(), + ], + ]; + for args in host_steps { + if let Err(error) = self.run_ip(&args).await { + return self.fail_setup(&network, error).await; + } + } + + let ns_steps = vec![ + vec![ + "ip".into(), + "addr".into(), + "add".into(), + format!("{peer_ip}/30"), + "dev".into(), + network.veth_peer.clone(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + network.veth_peer.clone(), + "up".into(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + "lo".into(), + "up".into(), + ], + vec![ + "ip".into(), + "tuntap".into(), + "add".into(), + network.tap_name.clone(), + "mode".into(), + "tap".into(), + ], + vec![ + "ip".into(), + "addr".into(), + "add".into(), + "169.254.0.1/30".into(), + "dev".into(), + network.tap_name.clone(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + network.tap_name.clone(), + "up".into(), + ], + vec![ + "ip".into(), + "route".into(), + "add".into(), + "default".into(), + "via".into(), + host_ip, + ], + vec!["sysctl".into(), "-w".into(), "net.ipv4.ip_forward=1".into()], + vec![ + "iptables".into(), + "-t".into(), + "nat".into(), + "-A".into(), + "POSTROUTING".into(), + "-s".into(), + "169.254.0.2".into(), + "-o".into(), + network.veth_peer.clone(), + "-j".into(), + "SNAT".into(), + "--to".into(), + peer_ip, + ], + ]; + for command in ns_steps { + if let Err(error) = self.run_in_namespace(&network.netns, &command).await { + return self.fail_setup(&network, error).await; + } + } + Ok(network) + } + + /// Remove all resources for a slot and return it to the allocator. + pub(super) async fn destroy(&self, network: &NetworkSlot) -> Result<()> { + let _host_guard = self.acquire_host_guard().await?; + self.cleanup_commands(network).await?; + self.release(network.slot); + Ok(()) + } + + async fn acquire_host_guard(&self) -> Result { + let Some(path) = self.coordination_file.as_deref() else { + return Ok(HostNetworkGuard { + #[cfg(unix)] + _file: None, + }); + }; + acquire_host_guard(path, self.command_timeout).await + } + + fn allocate(&self, blocked: &HashSet) -> Result { + let mut state = self.state.lock().map_err(|_| BlazeError::BackendError { + msg: "network slot allocator lock poisoned".to_string(), + })?; + for offset in 0..NET_MAX_SLOT { + let slot = (state.next + offset) % NET_MAX_SLOT; + if !blocked.contains(&slot) && state.used.insert(slot) { + state.next = (slot + 1) % NET_MAX_SLOT; + return Ok(slot); + } + } + Err(BlazeError::BackendError { + msg: format!("network slots exhausted (max {NET_MAX_SLOT})"), + }) + } + + fn reserve(&self, slot: usize, blocked: &HashSet) -> Result { + if slot >= NET_MAX_SLOT { + return Err(BlazeError::BackendError { + msg: format!("network slot {slot} is outside 0..{NET_MAX_SLOT}"), + }); + } + if blocked.contains(&slot) { + return Err(BlazeError::BackendError { + msg: format!("required network slot {slot} is unavailable during restore"), + }); + } + let mut state = self.state.lock().map_err(|_| BlazeError::BackendError { + msg: "network slot allocator lock poisoned".to_string(), + })?; + if !state.used.insert(slot) { + return Err(BlazeError::BackendError { + msg: format!("required network slot {slot} is already reserved"), + }); + } + state.next = (slot + 1) % NET_MAX_SLOT; + Ok(slot) + } + + async fn existing_slots(&self) -> Result> { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(parse_existing_slots(&output.stdout)) + } + + fn release(&self, slot: usize) { + if let Ok(mut state) = self.state.lock() { + state.used.remove(&slot); + } + } + + async fn cleanup_commands(&self, network: &NetworkSlot) -> Result<()> { + if !self.namespace_exists(&network.netns).await? { + return Ok(()); + } + self.run_in_namespace_cleanup( + &network.netns, + &[ + "ip".into(), + "link".into(), + "del".into(), + network.veth_peer.clone(), + ], + ) + .await?; + self.delete_namespace(&network.netns).await?; + Ok(()) + } + + pub(super) async fn find_by_owner(&self, owner: Uuid) -> Result> { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(parse_existing_networks(&output.stdout) + .into_iter() + .find(|network| network.owner == owner)) + } + + async fn namespace_exists(&self, name: &str) -> Result { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .any(|candidate| candidate == name)) + } + + async fn delete_namespace(&self, name: &str) -> Result<()> { + let args = vec!["netns".to_string(), "del".to_string(), name.to_string()]; + let output = self.run_ip_output(&args).await?; + if output.success { + return Ok(()); + } + let deletion = command_error(&args, &output, "cleaning up"); + match self.namespace_exists(name).await { + Ok(false) => Ok(()), + Ok(true) => Err(deletion), + Err(confirmation) => Err(BlazeError::BackendError { + msg: format!( + "{deletion}; cannot confirm namespace {name} was removed: {confirmation}" + ), + }), + } + } + + async fn fail_setup( + &self, + network: &NetworkSlot, + original: BlazeError, + ) -> std::result::Result { + match self.cleanup_commands(network).await { + Ok(()) => { + self.release(network.slot); + Err(NetworkCreateError::clean(original)) + } + Err(cleanup) => Err(NetworkCreateError::with_residual( + BlazeError::BackendError { + msg: format!( + "network setup failed ({original}); cleanup failed ({cleanup}); slot {} retained", + network.slot + ), + }, + network.clone(), + )), + } + } + + async fn run_in_namespace(&self, netns: &str, command: &[String]) -> Result<()> { + let mut args = vec!["netns".to_string(), "exec".to_string(), netns.to_string()]; + args.extend_from_slice(command); + self.run_ip(&args).await + } + + async fn run_in_namespace_cleanup(&self, netns: &str, command: &[String]) -> Result<()> { + let mut args = vec!["netns".to_string(), "exec".to_string(), netns.to_string()]; + args.extend_from_slice(command); + self.run_ip_cleanup(&args).await + } + + async fn run_ip(&self, args: &[String]) -> Result<()> { + let output = self.run_ip_output(args).await?; + if output.success { + return Ok(()); + } + Err(command_error(args, &output, "running command")) + } + + async fn run_ip_cleanup(&self, args: &[String]) -> Result<()> { + let output = self.run_ip_output(args).await?; + if output.success { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if [ + "Cannot find device", + "No such file", + "Invalid \"netns\" value", + ] + .iter() + .any(|marker| stderr.contains(marker)) + { + return Ok(()); + } + Err(command_error(args, &output, "cleaning up")) + } + + async fn run_ip_output(&self, args: &[String]) -> Result { + self.runner.output(args, self.command_timeout).await + } +} + +fn command_error(args: &[String], output: &IpOutput, context: &str) -> BlazeError { + let stderr = String::from_utf8_lossy(&output.stderr); + let stderr = stderr.chars().take(4096).collect::(); + BlazeError::BackendError { + msg: format!( + "ip {} failed while {context} ({}): {}", + args.join(" "), + output.status, + stderr.trim() + ), + } +} + +#[cfg(unix)] +async fn acquire_host_guard(path: &Path, timeout: Duration) -> Result { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let deadline = Instant::now() + timeout; + loop { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path)?; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(HostNetworkGuard { _file: Some(file) }); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::EAGAIN) + && error.raw_os_error() != Some(libc::EWOULDBLOCK) + { + return Err(error.into()); + } + if Instant::now() >= deadline { + return Err(BlazeError::BackendError { + msg: format!( + "timed out acquiring host network allocation lock {}", + path.display() + ), + }); + } + tokio::time::sleep(HOST_LOCK_RETRY).await; + } +} + +#[cfg(not(unix))] +async fn acquire_host_guard(_path: &Path, _timeout: Duration) -> Result { + Ok(HostNetworkGuard {}) +} + +#[cfg(target_os = "linux")] +fn executable_in_path(name: &str) -> bool { + let Some(path) = std::env::var_os("PATH") else { + return false; + }; + std::env::split_paths(&path).any(|directory| { + let candidate = directory.join(name); + if !candidate.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(candidate) + .map(|metadata| metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + } + #[cfg(not(unix))] + { + true + } + }) +} + +fn parse_existing_slots(stdout: &[u8]) -> HashSet { + String::from_utf8_lossy(stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|name| name.strip_prefix("blz-ns-")) + .filter_map(|suffix| suffix.split('-').next()) + .filter_map(|slot| slot.parse::().ok()) + .filter(|slot| *slot < NET_MAX_SLOT) + .collect() +} + +fn parse_existing_networks(stdout: &[u8]) -> Vec { + String::from_utf8_lossy(stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|name| name.strip_prefix("blz-ns-")) + .filter_map(|suffix| suffix.split_once('-')) + .filter_map(|(slot, owner)| { + Some(network_slot( + slot.parse::().ok()?, + owner.parse::().ok()?, + )) + }) + .filter(|network| network.slot < NET_MAX_SLOT) + .collect() +} + +fn network_slot(slot: usize, owner: Uuid) -> NetworkSlot { + NetworkSlot { + slot, + owner, + netns: format!("blz-ns-{slot}-{owner}"), + tap_name: "tap0".to_string(), + veth_host: format!("blz-veth-{slot}"), + veth_peer: format!("blz-vpeer-{slot}"), + } +} + +#[cfg(test)] +pub(super) fn test_network_slot(slot: usize) -> NetworkSlot { + network_slot(slot, Uuid::from_u128(1)) +} + +fn veth_ips(slot: usize) -> (String, String) { + let base = NET_VETH_BASE + slot * 4; + let third = (base >> 8) & 0xff; + ( + format!("169.254.{third}.{}", (base & 0xff) + 1), + format!("169.254.{third}.{}", (base & 0xff) + 2), + ) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use super::*; + + #[test] + fn allocator_is_unique_and_recycles() { + let manager = NetworkManager::default(); + let first = manager.allocate(&HashSet::new()).expect("first"); + let second = manager.allocate(&HashSet::new()).expect("second"); + assert_ne!(first, second); + manager.release(first); + for _ in 0..NET_MAX_SLOT { + if manager.allocate(&HashSet::new()).expect("slot") == first { + return; + } + } + panic!("released slot was not recycled"); + } + + #[test] + fn addresses_follow_the_slot_layout() { + assert_eq!( + veth_ips(0), + ("169.254.0.5".to_string(), "169.254.0.6".to_string()) + ); + assert_eq!( + veth_ips(63), + ("169.254.1.1".to_string(), "169.254.1.2".to_string()) + ); + } + + #[test] + fn existing_slot_parser_ignores_unrelated_and_invalid_names() { + let slots = parse_existing_slots( + b"blz-ns-0-00000000-0000-0000-0000-000000000001\n\ + blz-ns-17 (id: 2)\nunrelated\nblz-ns-nope\nblz-ns-16383\n", + ); + assert_eq!(slots, HashSet::from([0, 17])); + } + + #[cfg(unix)] + #[tokio::test] + async fn host_lock_serializes_independent_network_managers() { + let temp = tempfile::tempdir().expect("temp"); + let lock = temp.path().join("network.lock"); + let first = + NetworkManager::with_runner_and_lock(Arc::new(FakeIpRunner::default()), lock.clone()); + let second = NetworkManager::with_runner_and_lock(Arc::new(FakeIpRunner::default()), lock); + + let first_guard = first.acquire_host_guard().await.expect("first lock"); + let blocked = + tokio::time::timeout(Duration::from_millis(50), second.acquire_host_guard()).await; + assert!( + blocked.is_err(), + "a second daemon must not allocate while the host lock is held" + ); + + drop(first_guard); + second + .acquire_host_guard() + .await + .expect("lock becomes available"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn network_probe_checks_tools_privileges_and_ip_access() { + let missing_tool = Arc::new(FakeIpRunner::without_tool("iptables")); + let manager = NetworkManager::with_runner(missing_tool.clone()); + assert!(!manager.probe().await.expect("missing tool probe")); + assert!(missing_tool.calls().is_empty()); + + let unprivileged = Arc::new(FakeIpRunner::without_admin()); + let manager = NetworkManager::with_runner(unprivileged.clone()); + assert!(!manager.probe().await.expect("privilege probe")); + assert!(unprivileged.calls().is_empty()); + + let available = Arc::new(FakeIpRunner::with_responses([success(b"")])); + let manager = NetworkManager::with_runner(available.clone()); + assert!(manager.probe().await.expect("ready probe")); + assert_eq!( + available.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + + let inaccessible = Arc::new(FakeIpRunner::with_responses([failure( + "network namespace access denied", + )])); + let manager = NetworkManager::with_runner(inaccessible); + assert!(!manager.probe().await.expect("inaccessible probe")); + } + + #[tokio::test] + async fn create_skips_namespaces_owned_by_another_process() { + let runner = Arc::new(FakeIpRunner::with_responses([success( + b"blz-ns-0 (id: 4)\n", + )])); + let manager = NetworkManager::with_runner(runner.clone()); + + let owner = Uuid::from_u128(1); + let slot = manager + .create(owner, |_| Ok(())) + .await + .expect("create slot"); + + assert_eq!(slot.slot, 1); + let calls = runner.calls(); + assert!( + calls + .iter() + .any(|args| args == &["netns", "add", test_network_slot(1).netns()]) + ); + assert!( + !calls + .iter() + .any(|args| args == &["netns", "del", "blz-ns-0"]) + ); + } + + #[tokio::test] + async fn restore_recreates_the_requested_network_slot() { + let runner = Arc::new(FakeIpRunner::default()); + let manager = NetworkManager::with_runner(runner.clone()); + manager.state.lock().expect("state lock").next = 11; + + let slot = manager + .create_at(Uuid::from_u128(1), 7, |_| Ok(())) + .await + .expect("restore slot"); + + assert_eq!(slot.slot(), 7); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "add", test_network_slot(7).netns()]) + ); + } + + #[tokio::test] + async fn restore_rejects_an_occupied_network_slot() { + let namespace = format!("{}\n", test_network_slot(7).netns()); + let runner = Arc::new(FakeIpRunner::with_responses([success( + namespace.as_bytes(), + )])); + let manager = NetworkManager::with_runner(runner.clone()); + + let error = manager + .create_at(Uuid::from_u128(1), 7, |_| Ok(())) + .await + .expect_err("occupied restore slot"); + + assert!(error.to_string().contains("slot 7 is unavailable")); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + + #[tokio::test] + async fn create_retries_when_namespace_appears_during_allocation() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + failure("namespace already exists"), + success(b"blz-ns-0\n"), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + let owner = Uuid::from_u128(1); + let slot = manager + .create(owner, |_| Ok(())) + .await + .expect("create slot"); + + assert_eq!(slot.slot, 1); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "add", test_network_slot(1).netns()]) + ); + } + + #[tokio::test] + async fn failed_veth_creation_only_removes_new_namespace() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + success(b""), + failure("veth already exists"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + manager + .create(Uuid::from_u128(1), |_| Ok(())) + .await + .expect_err("veth creation must fail"); + + let calls = runner.calls(); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", test_network_slot(0).netns()]) + ); + assert!( + !calls + .iter() + .any(|args| args == &["link", "del", "blz-veth-0"]) + ); + } + + #[tokio::test] + async fn failed_cleanup_returns_the_residual_slot_owner() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + success(b""), + success(b""), + failure("host address failed"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + failure("delete peer failed"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + success(b""), + success(b""), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + let failure = manager + .create(Uuid::from_u128(1), |_| Ok(())) + .await + .expect_err("setup must fail"); + let (source, residual) = failure.into_parts(); + let residual = residual.expect("cleanup failure must retain the slot"); + + assert!(source.to_string().contains("slot 0 retained")); + assert_eq!(residual, test_network_slot(0)); + manager + .destroy(&residual) + .await + .expect("a later cleanup can release the retained slot"); + let calls = runner.calls(); + assert!(calls.iter().any(|args| { + args == &[ + "netns", + "exec", + test_network_slot(0).netns(), + "ip", + "link", + "del", + "blz-vpeer-0", + ] + })); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", test_network_slot(0).netns()]) + ); + } + + #[tokio::test] + async fn namespace_delete_failure_retains_a_present_slot() { + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(FakeIpRunner::with_responses([ + success(namespace.as_bytes()), + success(b""), + failure("Cannot remove namespace file: Permission denied"), + success(namespace.as_bytes()), + ])); + let manager = NetworkManager::with_runner(runner); + manager + .state + .lock() + .expect("state lock") + .used + .insert(network.slot); + + let error = manager + .destroy(&network) + .await + .expect_err("present namespace must retain its slot"); + + assert!(error.to_string().contains("Permission denied")); + assert!( + manager + .state + .lock() + .expect("state lock") + .used + .contains(&network.slot) + ); + } + + #[tokio::test] + async fn namespace_delete_failure_accepts_confirmed_absence() { + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(FakeIpRunner::with_responses([ + success(namespace.as_bytes()), + success(b""), + failure("Cannot remove namespace file: already removed"), + success(b""), + ])); + let manager = NetworkManager::with_runner(runner); + manager + .state + .lock() + .expect("state lock") + .used + .insert(network.slot); + + manager + .destroy(&network) + .await + .expect("confirmed absence completes cleanup"); + + assert!( + !manager + .state + .lock() + .expect("state lock") + .used + .contains(&network.slot) + ); + } + + struct FakeIpRunner { + responses: Mutex>, + calls: Mutex>>, + #[cfg(target_os = "linux")] + unavailable_tool: Option, + #[cfg(target_os = "linux")] + network_admin: bool, + } + + impl Default for FakeIpRunner { + fn default() -> Self { + Self { + responses: Mutex::new(VecDeque::new()), + calls: Mutex::new(Vec::new()), + #[cfg(target_os = "linux")] + unavailable_tool: None, + #[cfg(target_os = "linux")] + network_admin: true, + } + } + } + + impl FakeIpRunner { + fn with_responses(responses: [IpOutput; N]) -> Self { + Self { + responses: Mutex::new(responses.into()), + calls: Mutex::new(Vec::new()), + #[cfg(target_os = "linux")] + unavailable_tool: None, + #[cfg(target_os = "linux")] + network_admin: true, + } + } + + #[cfg(target_os = "linux")] + fn without_tool(tool: &str) -> Self { + Self { + unavailable_tool: Some(tool.to_string()), + ..Self::default() + } + } + + #[cfg(target_os = "linux")] + fn without_admin() -> Self { + Self { + network_admin: false, + ..Self::default() + } + } + + fn calls(&self) -> Vec> { + self.calls.lock().expect("calls lock").clone() + } + } + + #[async_trait] + impl IpCommandRunner for FakeIpRunner { + async fn output(&self, args: &[String], _timeout: Duration) -> Result { + self.calls.lock().expect("calls lock").push(args.to_vec()); + Ok(self + .responses + .lock() + .expect("responses lock") + .pop_front() + .unwrap_or_else(|| success(b""))) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, name: &str) -> bool { + self.unavailable_tool.as_deref() != Some(name) + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + self.network_admin + } + } + + fn success(stdout: &[u8]) -> IpOutput { + IpOutput { + success: true, + status: "exit status: 0".to_string(), + stdout: stdout.to_vec(), + stderr: Vec::new(), + } + } + + fn failure(stderr: &str) -> IpOutput { + IpOutput { + success: false, + status: "exit status: 1".to_string(), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } + } +} diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index 0dcece9a63..6e656db86f 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -1,14 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 -//! Daemon-wide shared state: configuration, policy engine, pool, template -//! and hook registries, and the in-memory instance map. All API handlers -//! receive an [`Arc`] and acquire the relevant `Mutex<...>` -//! lock just long enough to read or mutate the piece they need — locks -//! are never held across `.await` boundaries. +//! Daemon-wide shared state: configuration, policy engine, pool, template and +//! hook registries, plus the sandbox manager. API paths that change runtime +//! ownership enter through the manager so its per-instance lock spans every +//! asynchronous resource mutation. use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::{Arc, Mutex}; +#[cfg(test)] +use std::path::PathBuf; + use blaze_core::backend::BackendKind; use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; @@ -17,38 +19,38 @@ use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use blaze_core::template::TemplateRegistry; -use tokio::sync::Mutex as AsyncMutex; use uuid::Uuid; -use crate::error::Result; +use crate::error::{BlazeDaemonError, Result}; use crate::metrics::Metrics; -use crate::spawner::{DynBackendInstance, DynSpawner, SpawnerRegistry}; +use crate::sandbox::template::RuntimeTemplateCatalog; +use crate::sandbox::{SandboxManager, SandboxManagerInit}; +use crate::spawner::SpawnerRegistry; /// All daemon mutable state. Cloning is via `Arc` (see the `state.clone()` /// idiom in `daemon.rs`); the struct itself is never `Clone`. pub struct ServerState { pub config: Mutex, pub policy: Mutex, - pub pool: Mutex, + pub pool: Arc>, pub template: Mutex, pub hook: Mutex, - pub instances: Mutex>, - pub backend_instances: Mutex>, - operation_locks: Mutex>>>, - pub spawners: SpawnerRegistry, + #[cfg(test)] + pub instances: Arc>>, + pub manager: Arc, /// The backend kind that `build_spawner` actually probed and selected. /// API handlers use this to constrain availability to the single active /// backend rather than reporting all configured binaries. pub active_backend: BackendKind, pub storage: Arc, + #[cfg(test)] pub state_dir: PathBuf, - pub metrics: Metrics, + pub metrics: Arc, } impl ServerState { /// Build a server state, scanning `state_dir` to repopulate the - /// `instances` map from previous runs (best-effort; corrupt entries - /// are skipped with a warning). + /// `instances` map from previous runs. #[allow(clippy::too_many_arguments)] pub fn build( config: DaemonConfig, @@ -59,58 +61,54 @@ impl ServerState { spawners: SpawnerRegistry, active_backend: BackendKind, storage: Arc, - ) -> Self { + ) -> Result { let state_dir = config.daemon.state_dir.clone(); - let instances = scan_state_dir(&state_dir).unwrap_or_else(|err| { - tracing::warn!(error = %err, "failed to scan state_dir, starting empty"); - HashMap::new() - }); - let operation_locks = instances - .keys() - .copied() - .map(|id| (id, Arc::new(AsyncMutex::new(())))) - .collect(); - - Self { + let instances = scan_state_dir(&state_dir)?; + let runtime_templates = RuntimeTemplateCatalog::open(&config.runtime_templates)?; + let (manager, resources) = SandboxManager::new(SandboxManagerInit { + instances, + pool, + spawners, + active_backend, + storage: storage.clone(), + state_dir: state_dir.clone(), + rootfs_size: config.storage.rootfs_size, + mem_size: config.storage.mem_size, + pool_size: config.storage.pool_size, + prefork: config.storage.prefork, + default_warm_ttl: config.pool.default_warm_ttl.clone(), + gc_interval: config.pool.gc_interval.clone(), + runtime_templates, + })?; + + Ok(Self { config: Mutex::new(config), policy: Mutex::new(policy), - pool: Mutex::new(pool), + pool: resources.pool, template: Mutex::new(template), hook: Mutex::new(hook), - instances: Mutex::new(instances), - backend_instances: Mutex::new(HashMap::new()), - operation_locks: Mutex::new(operation_locks), - spawners, + #[cfg(test)] + instances: resources.instances, + manager: Arc::new(manager), active_backend, storage, + #[cfg(test)] state_dir, - metrics: Metrics::new(), - } + metrics: resources.metrics, + }) } /// Return the async operation lock that serializes one sandbox mutation. - pub fn operation_lock(&self, id: Uuid) -> Arc> { - match self.operation_locks.lock() { - Ok(mut locks) => locks - .entry(id) - .or_insert_with(|| Arc::new(AsyncMutex::new(()))) - .clone(), - Err(poisoned) => poisoned - .into_inner() - .entry(id) - .or_insert_with(|| Arc::new(AsyncMutex::new(()))) - .clone(), - } - } - - /// Return the implementation responsible for a persisted backend kind. - pub fn spawner_for(&self, kind: BackendKind) -> Option { - self.spawners.get(kind) + #[cfg(test)] + pub fn operation_lock(&self, id: Uuid) -> Arc> { + self.manager.operation_lock(id) } } -/// Best-effort: walk `{state_dir}//state.json` and rebuild the -/// instance map. Used both at boot and (in the future) by `daemon doctor`. +/// Walk `{state_dir}//state.json` and rebuild the instance map. +/// +/// A valid UUID directory is owned lifecycle state. If its record cannot be +/// loaded, startup must stop rather than hide resources from later cleanup. fn scan_state_dir(state_dir: &Path) -> Result> { let mut out = HashMap::new(); if !state_dir.exists() { @@ -128,15 +126,85 @@ fn scan_state_dir(state_dir: &Path) -> Result> { let Ok(id) = Uuid::parse_str(name_str) else { continue; }; - match SandboxInstance::load(state_dir, id) { - Ok(inst) => { - out.insert(id, inst); - } - Err(err) => { - tracing::warn!(instance = %id, error = %err, "skipping corrupt instance state"); - } + let instance = SandboxInstance::load(state_dir, id).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot load persisted instance {id}: {error}" + )) + })?; + if instance.id != id { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "persisted instance id {} does not match owned directory {id}", + instance.id + ))); } + out.insert(id, instance); } tracing::info!(instances = out.len(), "rehydrated instances from state_dir"); Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scan_ignores_runtime_pool_root() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_slot = temp + .path() + .join("runtime-pool") + .join(Uuid::new_v4().to_string()); + std::fs::create_dir_all(&runtime_slot).expect("runtime slot"); + std::fs::write(runtime_slot.join("ownership.json"), b"{not-lifecycle-state") + .expect("ownership marker"); + + let instances = scan_state_dir(temp.path()).expect("scan state"); + + assert!(instances.is_empty()); + } + + #[test] + fn scan_rejects_corrupt_owned_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let id = Uuid::new_v4(); + let instance_dir = temp.path().join(id.to_string()); + std::fs::create_dir_all(&instance_dir).expect("instance dir"); + std::fs::write(instance_dir.join("state.json"), b"{not-json").expect("state"); + + let error = scan_state_dir(temp.path()).expect_err("corrupt state must stop startup"); + + assert!(matches!( + error, + BlazeDaemonError::RecoveryRequired(message) + if message.contains(&id.to_string()) + && message.contains("cannot load persisted instance") + )); + } + #[test] + fn scan_rejects_state_owned_by_a_different_directory() { + let temp = tempfile::tempdir().expect("tempdir"); + let instance = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:mismatched-id".into(), + blaze_core::lifecycle::StartPath::Cold, + "test".into(), + ); + instance.persist(temp.path()).expect("persist state"); + let directory_id = Uuid::new_v4(); + std::fs::rename( + temp.path().join(instance.id.to_string()), + temp.path().join(directory_id.to_string()), + ) + .expect("rename directory"); + + let error = scan_state_dir(temp.path()).expect_err("mismatched state must stop startup"); + + assert!(matches!( + error, + BlazeDaemonError::RecoveryRequired(message) + if message.contains(&instance.id.to_string()) + && message.contains(&directory_id.to_string()) + )); + } +} diff --git a/src/blaze/dist/blazed.service b/src/blaze/dist/blazed.service index aad769fe72..984f25e4e5 100644 --- a/src/blaze/dist/blazed.service +++ b/src/blaze/dist/blazed.service @@ -15,7 +15,7 @@ LimitNPROC=infinity LimitCORE=infinity KillMode=process KillSignal=SIGTERM -TimeoutStopSec=30 +TimeoutStopSec=80s [Install] WantedBy=multi-user.target diff --git a/src/blaze/docs/design/runtime-slot-ownership.md b/src/blaze/docs/design/runtime-slot-ownership.md new file mode 100644 index 0000000000..7cfeb48b25 --- /dev/null +++ b/src/blaze/docs/design/runtime-slot-ownership.md @@ -0,0 +1,283 @@ +# Runtime Slot Ownership + +## Purpose + +Background runtime capacity reduces the work on a compatible sandbox create +without allowing partially built storage or backend processes to lose an +owner. Each prepared slot has independent storage and may also own a running +backend. The daemon records who owns those resources before and during every +handoff, cleanup, restart, and shutdown transition. + +This design covers background construction, claim, ownership transfer, and +recovery. It does not add a public runtime-capacity management API, restore +ready slots after restart, or change the existing lifecycle recycling pool +served by `/v1/pools`. + +## Inputs and Activation + +The daemon reads these runtime-capacity settings: + +| Setting | Effect | +| --- | --- | +| `storage.pool_size` | Maximum pool-owned and in-flight slot count; zero disables construction | +| `storage.prefork` | Whether a slot starts its selected backend before becoming ready | +| `pool.default_warm_ttl` | Ready-slot lifetime when an eligible policy has no override | +| `pool.gc_interval` | Maintenance interval for expiry, cleanup, and refill work | +| policy `pool.enabled` | Whether creates evaluated by that policy may configure or claim a slot | +| policy `pool.warm_ttl` | Optional lifetime override for that policy's accepted build shape | + +The first eligible create fixes one prototype for the current daemon run. +Later creates must produce the same prototype fingerprint to claim its slots. +An incompatible request continues through the existing create flow and does +not replace the active prototype. + +## Components + +```mermaid +flowchart TB + API["Sandbox create API"] --> Manager["SandboxManager"] + Manager --> Pool["RuntimeWarmPool"] + Pool --> Storage["StorageProvider"] + Pool --> Spawners["SpawnerRegistry"] + Pool --> Journal["Runtime ownership journal"] + Manager --> State["Sandbox lifecycle state"] + Manager --> Spawners + Manager --> Storage + Journal <--> Recovery["Startup reconciler"] + State <--> Recovery + Recovery --> Storage + Recovery --> Spawners +``` + +- `RuntimeWarmPool` owns the bounded worker, ready queue, leases, and pending + cleanup. +- `StorageProvider` creates independent slots. Non-zero capacity requires a + provider that can enumerate every ID it owns and release an ID even when + creation stopped partway through. +- `SpawnerRegistry` starts the configured backend. During recovery, the + persisted backend kind selects its cleanup implementation; recovery does not + restart or adopt the old process. +- `SandboxManager` is the transfer boundary between pool ownership and normal + sandbox lifecycle ownership. +- The startup reconciler compares runtime journals, provider inventory, and + durable lifecycle state before the API starts listening. + +Provider- and backend-specific behavior remains behind the existing traits. +The pool records stable IDs and ownership facts; it does not persist a concrete +provider implementation or choose a replacement backend after failure. + +## Slot Construction + +```mermaid +sequenceDiagram + participant C as Create request + participant M as SandboxManager + participant P as RuntimeWarmPool + participant S as StorageProvider + participant B as Backend spawner + + C->>M: create(eligible decision) + M->>P: configure(prototype) + alt prototype is incompatible or capacity is disabled + P-->>M: rejected + M-->>C: existing create flow + else prototype is accepted + par Background work + P->>P: wake worker toward pool_size + P->>S: acquire independent storage + opt prefork enabled + P->>B: prepare and start backend + P->>B: check guest readiness when applicable + end + P->>P: persist Ready + and Request path + M->>P: acquire() + alt no ready slot + P-->>M: miss + M-->>C: existing create flow + else ready slot + P-->>M: lease + M->>P: persist Handoff(token) + M->>M: persist lifecycle owner(token) + M->>P: persist LifecycleOwned(token) + opt storage-only slot + M->>B: start backend + end + M-->>C: Running, start_path=warm + end + end + end +``` + +Construction is asynchronous. The request that first configures the prototype +does not wait for a slot to finish. The worker fills only while the number of +physical pool owners is below `pool_size`. + +Before calling provider allocation, the journal conservatively records that +storage may be owned. This permits release by ID if cancellation occurs after +the provider created an artifact but before it returned a complete slot. When +`prefork` is enabled, backend ownership is recorded as it moves from not +started, through starting, to running. A slot becomes `Ready` only after all +configured preparation succeeds. + +## Ownership States + +The persisted runtime journal uses these phases: + +```mermaid +stateDiagram-v2 + [*] --> Building + Building --> Ready: storage and optional backend prepared + Building --> PoolCleanup: build cannot complete + Ready --> Handoff: lease records token + Ready --> PoolCleanup: expiry or failed liveness check + Handoff --> LifecycleOwned: matching lifecycle owner is durable + Handoff --> PoolCleanup: lifecycle publication is absent + Handoff --> LifecycleCleanup: visible lifecycle owner begins cleanup + LifecycleOwned --> LifecycleCleanup: destroy or failed create cleanup + PoolCleanup --> [*]: backend, storage, and directory released + LifecycleCleanup --> [*]: runtime released; lifecycle state retained as Destroyed +``` + +The handoff token links one runtime journal to one lifecycle record. A phase +change is accepted only with the matching instance ID, backend, backend +ownership, and token. This prevents a stale cleanup path from reclaiming a +slot that a sandbox already owns. + +An in-memory lease has one synchronous drop owner: + +- before lifecycle publication, drop returns the slot to pool cleanup; +- after lifecycle state is retained, drop removes it from pool accounting and + leaves cleanup to the lifecycle owner; +- when publication cannot be classified, the pool retains an unresolved + handoff instead of choosing either owner. + +## Capacity Accounting + +The target counts physical resources, not only ready entries: + +```text +physical count = + ready + + building + + leased + + quarantined + + unresolved handoffs + + cleanup in progress +``` + +The worker starts another build only when this sum is below +`storage.pool_size`. A failed cleanup therefore cannot silently create excess +capacity. Once a handoff completes and the lease leaves pool accounting, the +active sandbox is governed by lifecycle ownership and no longer consumes the +runtime-slot target. + +## Failure Decisions + +| Failure point | Durable decision | Result | +| --- | --- | --- | +| Before or during storage allocation | `Building`, with conservative storage ownership before the call | Release by stable ID; retry cleanup if release fails | +| Backend prepare, start, or readiness | `Building` plus the latest backend ownership | Keep residual owners, move to pool cleanup, and retry | +| Ready slot expired or backend exited | Pool still owns the lease | Quarantine and clean it; try another ready slot | +| Lifecycle publication is definitely absent | `Handoff(token)` remains pool-authorized | Lease drop schedules pool cleanup | +| Lifecycle publication is visible but reports failure | Matching lifecycle record is retained | Finish lifecycle ownership, then run failed-create cleanup | +| Lifecycle publication is ambiguous | `Handoff(token)` is retained as unresolved | Keep it counted and require startup reconciliation | +| Pool or lifecycle cleanup stops partway through removal | The cleanup journal remains; a deletion proof also remains if directory removal began | Resume the same cleanup after restart | +| Pool cleanup attempt fails | Pool owner remains in its cleanup phase | Retry with backoff without returning the slot to ready | +| Lifecycle cleanup attempt fails | Lifecycle owner remains retryable | Report it; retry only through destroy, startup, or shutdown | + +Failure handling preserves the original error together with cleanup errors. +The caller never receives a successful warm create until lifecycle state is +`Running` and no create operation remains open. + +## Startup Reconciliation + +Runtime reconciliation runs before socket binding: + +```mermaid +flowchart TD + A["Load durable sandbox state"] --> B["Enumerate provider-owned slot IDs"] + B --> C["Read runtime journals and cleanup proofs"] + C --> D{"Ownership classification"} + D -- "Pool-owned or unclaimed" --> E["Clean backend, then storage and directory"] + D -- "Matching lifecycle owner" --> F["Preserve for lifecycle reconciliation"] + D -- "Incomplete cleanup" --> G["Resume recorded cleanup"] + D -- "Ambiguous or inconsistent" --> H["Stop daemon startup with an error"] + B -- "Inventory error" --> H + C -- "Read error" --> H + E -- "Cleanup error" --> H + G -- "Cleanup error" --> H + E -- "Success" --> J["Run sandbox lifecycle reconciliation"] + F --> J + G -- "Success" --> J + J --> K{"Any sandbox cleanup failure?"} + K -- "Yes" --> L["Retain and report each failure"] + K -- "No" --> I["Open API listeners"] + L --> I +``` + +The ready queue is process-local and is not restored. Valid unclaimed slots +are cleaned, while slots with a matching durable lifecycle owner are protected +for normal sandbox reconciliation. A runtime inventory, journal read, or +runtime cleanup error stops startup. Once runtime ownership is consistent, +ordinary sandbox reconciliation runs; one sandbox cleanup failure is retained +and reported without preventing listener startup. Unknown entries, mismatched +tokens, unexpected aliases, or conflicting ownership evidence also stop +startup rather than selecting an owner by guesswork. + +Recovery uses the backend recorded in the journal and the provider configured +for the same owned directories. Operators must keep those roots and provider +selection stable across restart. + +## Shutdown + +Shutdown first stops accepting work, cancels readiness waits, and drains +accepted connections. One shared deadline then covers concurrent lifecycle +cleanup and runtime-pool shutdown. The pool joins its single worker and cleans +pool-owned resources, while sandbox cleanup runs through per-sandbox operation +locks and lifecycle ownership. + +If the shutdown future itself is cancelled, the worker handle is retained so a +later shutdown attempt can still join it. A timeout stops new construction but +does not rewrite unresolved ownership as success. + +## Concurrency Rules + +- One maintenance task serializes build and cleanup actions. +- The in-memory state lock protects queues and counters but is not held across + provider or backend calls. +- One operation lock serializes create, destroy, and guest work for a sandbox. +- Runtime handoff is published before lifecycle mutation continues. +- Shutdown uses a fixed lock order and a shared deadline so worker join and + resource cleanup cannot wait indefinitely on one another. + +## Public Surface + +The feature changes `POST /v1/sandboxes` and its `/v1/instances` alias: +compatible requests may return a top-level `start_path` value of `"warm"` and +a nested `instance.runtime_location` value of `"warm-pool"`. `start_path` +remains a generic classification; `runtime_location` identifies this ownership +path. Configuration is the only public control surface for runtime-slot +capacity in this design. + +The following existing surfaces are separate: + +- `/v1/pools` exposes the separate lifecycle recycling-pool management + contract, but public reset still returns `501` and no production path returns + a used sandbox to that pool; +- the health response's `storage_pool` object reports provider storage-pool + status; +- neither surface reports, drains, or refills background runtime slots. + +## Test Mapping + +| Contract | Representative coverage | +| --- | --- | +| Storage-only and prefork claims complete create and destroy | `non_prefork_runtime_claim_completes_create_and_destroy`, `prefork_runtime_claim_completes_create_and_destroy` | +| Effective daemon TTL is visible when policy TTL is omitted | `omitted_policy_ttl_is_resolved_in_create_response` | +| All owned and in-flight states count toward the target | `every_owned_state_consumes_physical_capacity` | +| Failed allocation or prefork work retains cleanup ownership | `residual_acquire_failure_remains_owned_until_cleanup`, `residual_prefork_failure_remains_owned_until_cleanup` | +| Handoff ambiguity remains counted until reconciliation | `ambiguous_lifecycle_publish_remains_counted_until_restart`, `unresolved_handoff_counts_capacity_and_blocks_shutdown` | +| Startup protects lifecycle owners and cleans pool owners | `reconcile_protects_durable_lifecycle_owner`, `reconcile_releases_provider_only_slot_by_id` | +| Interrupted cleanup resumes from durable evidence | `reconcile_finishes_deletion_when_only_the_proof_remains`, `reconcile_resumes_lifecycle_tombstone_after_journal_unlink` | +| Shutdown joins the worker and preserves cancellation recovery | `shutdown_joins_worker_and_shares_one_deadline`, `cancelled_shutdown_retains_worker_for_a_joining_retry` | diff --git a/src/blaze/docs/design/runtime-template-catalog.md b/src/blaze/docs/design/runtime-template-catalog.md new file mode 100644 index 0000000000..6f13ddb095 --- /dev/null +++ b/src/blaze/docs/design/runtime-template-catalog.md @@ -0,0 +1,88 @@ +# Runtime Template Catalog + +The daemon can publish a reusable runtime artifact set into the directory +configured by `runtime_templates.dir`. Imports are disabled unless an operator +also configures `runtime_templates.import_root`. + +The catalog is separate from the existing in-memory `/v1/templates` registry. +It provides durable publication and lookup, but sandbox creation does not +select catalog entries. + +## Import request + +```http +POST /v1/runtime-templates/import +Content-Type: application/json + +{ + "name": "runtime-base", + "source": "runtime-base", + "description": "base runtime" +} +``` + +`source` is a relative path below the configured import root. Absolute paths, +parent traversal, and symbolic links in the path are rejected. Every source +directory and file must be owned by the daemon user and must not be writable +by group or other users. + +The source must contain top-level regular files named `vmstate.snap`, +`mem.bin`, and `rootfs.ext4`. An optional `template.json` must contain a JSON +object. Nested directories, links, and special files are rejected. The daemon +sets `name` from the request, applies a non-empty request description, and +fills numeric `rootfs_size` and `memory_size` defaults when they are absent. +It returns `409 Conflict` when the destination exists or another import of the +same name is active. + +## Limits and owned paths + +The following settings bound work before data is published: + +| Setting | Meaning | +|---------|---------| +| `max_files` | Maximum files in one published entry, including `template.json` | +| `max_bytes` | Maximum artifact and generated metadata bytes in one entry | +| `max_metadata_bytes` | Maximum input and generated metadata size | +| `max_total_bytes` | Maximum committed bytes plus concurrent reservations | + +`runtime_templates.dir` and `runtime_templates.import_root` must be absolute, +must not contain parent components, and must not overlap each other. They also +must not overlap the storage image, storage instance, or configured template +directories. + +The catalog, staging directories, and published directories use mode `0700`. +Published files use mode `0600`. + +## Publication and recovery + +The importer opens source entries without following links, reserves catalog +capacity, and copies them into a private, uniquely named staging directory. +It checks the source identity and size again after copying. The complete +directory is synchronized and renamed into place without replacing an +existing entry, so readers see either no entry or the complete entry. + +A failed import removes its staging directory. If cleanup cannot be completed, +or publication has occurred but catalog durability cannot be confirmed, the +daemon rejects later imports until the catalog is repaired and the daemon +restarts. Startup removes owned staging directories left by an interrupted +run and validates the type, ownership, permissions, contents, and capacity of +published entries. + +During graceful shutdown, the daemon rejects new imports, requests +cancellation of active imports, waits for their file handles and staging data +to be released, and then continues normal runtime cleanup. + +## Lookup and current limits + +Published metadata is available through: + +- `GET /v1/runtime-templates` +- `GET /v1/runtime-templates/{name}` + +Catalog listing is sorted by template name and reports corrupt published +metadata instead of silently hiding it. These routes manage stored artifacts +only. Validation is structural; it does not prove that a snapshot is bootable +or compatible with a particular backend. This capability does not make +sandbox creation select an imported template, reference-count imported +entries, or remove their directories through the existing `/v1/templates/gc` +registry route. diff --git a/src/blaze/docs/design/runtime-template-catalog_zh.md b/src/blaze/docs/design/runtime-template-catalog_zh.md new file mode 100644 index 0000000000..1bd351b7b1 --- /dev/null +++ b/src/blaze/docs/design/runtime-template-catalog_zh.md @@ -0,0 +1,76 @@ +# Runtime 模板目录 + +daemon 可以将一组可复用的 runtime artifact 发布到 +`runtime_templates.dir` 配置的目录。只有同时配置 +`runtime_templates.import_root` 后，导入功能才会启用。 + +该目录与已有的内存 `/v1/templates` registry 相互独立。它提供持久化发布和 +查询，但 sandbox create 不会选择其中的条目。 + +## 导入请求 + +```http +POST /v1/runtime-templates/import +Content-Type: application/json + +{ + "name": "runtime-base", + "source": "runtime-base", + "description": "base runtime" +} +``` + +`source` 是配置的导入根目录下的相对路径。绝对路径、父目录跳转和路径中的 +符号链接都会被拒绝。每一级源目录和源文件都必须属于 daemon 用户，并且不能 +允许 group 或其他用户写入。 + +源目录必须包含顶层普通文件 `vmstate.snap`、`mem.bin` 和 `rootfs.ext4`。 +可选的 `template.json` 必须是 JSON object。嵌套目录、链接和特殊文件都会 +被拒绝。daemon 会使用请求中的 `name`，采用请求中非空的 `description`， +并在缺少数值类型的 `rootfs_size` 或 `memory_size` 时填入默认值。目标名称 +已存在或同名导入正在执行时返回 `409 Conflict`。 + +## 上限和目录边界 + +以下配置会在发布前限制一次导入所做的工作： + +| 配置 | 含义 | +|------|------| +| `max_files` | 单个发布条目的文件数上限，包括 `template.json` | +| `max_bytes` | 单个条目的 artifact 与生成后元数据的总字节数上限 | +| `max_metadata_bytes` | 输入与生成后元数据的大小上限 | +| `max_total_bytes` | 已发布字节与并发预留字节之和的上限 | + +`runtime_templates.dir` 与 `runtime_templates.import_root` 必须是绝对路径， +不能包含父目录组件，也不能互相重叠。它们还不能与存储镜像、存储实例或已有 +模板配置目录重叠。 + +catalog、staging 目录和已发布目录的权限为 `0700`，已发布文件的权限为 +`0600`。 + +## 发布与恢复 + +导入器打开源条目时不会跟随链接；它会先预留 catalog 容量，再把文件复制到 +私有且名称唯一的 staging 目录。复制完成后会再次检查源文件的身份和大小。 +完整目录同步后以不覆盖已有条目的方式改名发布，因此读取方只会看到“没有条目” +或完整条目。 + +导入失败会移除对应的 staging 目录。如果清理无法完成，或者条目已经发布但 +catalog 持久性无法确认，daemon 会拒绝后续导入，直到修复 catalog 并重启。 +启动时，daemon 会移除上次中断后遗留且归自己所有的 staging 目录，并校验 +已发布条目的类型、所有者、权限、内容和容量。 + +正常关闭时，daemon 会拒绝新导入，请求取消正在执行的导入，等待相关文件句柄 +和 staging 数据释放，然后继续既有的 runtime 清理。 + +## 查询与当前限制 + +已发布的元数据可通过以下接口查询： + +- `GET /v1/runtime-templates` +- `GET /v1/runtime-templates/{name}` + +列表按模板名称排序；已发布元数据损坏时会返回错误，而不是静默隐藏条目。这些 +接口只管理已经保存的 artifact，校验范围仅限结构，不证明快照能够启动或与某个 +backend 兼容。它不会让 sandbox create 自动选择导入模板，不会为导入条目维护 +引用计数，也不会通过既有 `/v1/templates/gc` registry 路由删除其目录。 diff --git a/src/blaze/docs/design/storage-synchronization.md b/src/blaze/docs/design/storage-synchronization.md new file mode 100644 index 0000000000..a3fd753b5a --- /dev/null +++ b/src/blaze/docs/design/storage-synchronization.md @@ -0,0 +1,65 @@ +# Storage Synchronization + +[中文版](storage-synchronization_zh.md) + +Blaze can periodically ask the configured `StorageProvider` to synchronize +data owned by running sandboxes. This closes the gap between a provider that +can synchronize one slot and a daemon that schedules the operation safely +across all eligible sandboxes. + +Periodic synchronization is disabled by default. Set +`storage.flush_interval` to a positive duration to enable it. +`storage.flush_timeout` is a positive duration that bounds each provider call +and defaults to 30 seconds. + +## Which sandboxes are synchronized + +At the beginning of a sweep, the manager selects records whose lifecycle state +is `Running`. Before it calls the provider for one sandbox, it enters the same +operation lock used by create and destroy and checks the record again. + +The provider call runs only when all of these conditions still hold: + +- the lifecycle state is `Running`; +- there is no unfinished lifecycle operation; +- metadata says the backend is running and the daemon still owns that backend; +- the provider can reconstruct a complete slot from the sandbox ID. + +A sandbox that changed state while waiting for the operation lock is skipped. +An inconsistent Running record is reported as a failed item instead of being +silently omitted. The remaining sandboxes in the sweep still run. + +The first sweep starts after one complete interval. Missed ticks are skipped +instead of queued, so a slow sweep cannot create an unbounded backlog. + +## Failure and retry behavior + +Each provider call has its own deadline. A failure or timeout leaves the slot +owned by the sandbox and does not change lifecycle state. A later sweep or +destroy can therefore retry the provider operation. + +`StorageProvider::flush_dirty` is the provider-specific persistence boundary. +Implementations must leave a cancelled call safe to retry or release. The file +provider synchronizes the canonical files in its independent sandbox slot. +Other providers can use a different mechanism while preserving the same +ownership and cancellation contract. + +Storage synchronization does not save VM memory or device state. It is not a +substitute for saving and restoring a complete runtime. + +## Daemon shutdown + +The daemon supervises the periodic worker while serving requests. If the +worker exits unexpectedly, the daemon stops accepting work and follows the +normal coordinated shutdown path. + +During normal shutdown, the daemon performs these steps in order: + +1. stop accepting new connections; +2. cancel and join the synchronization worker; +3. drain accepted connections; +4. release owned runtime and storage resources. + +The worker is gone before destroy starts, so a periodic provider call cannot +race with teardown of the same sandbox. If both the worker and a later +shutdown stage fail, the daemon reports both failures. diff --git a/src/blaze/docs/design/storage-synchronization_zh.md b/src/blaze/docs/design/storage-synchronization_zh.md new file mode 100644 index 0000000000..366de3e221 --- /dev/null +++ b/src/blaze/docs/design/storage-synchronization_zh.md @@ -0,0 +1,58 @@ +# 存储同步 + +[English](storage-synchronization.md) + +Blaze 可以定期要求已配置的 `StorageProvider` 同步 running sandbox 持有的 +数据。这样，provider 不仅具备同步单个 slot 的能力，daemon 也能安全调度 +所有符合条件的 sandbox。 + +周期同步默认关闭。将 `storage.flush_interval` 设置为正数 duration 后启用。 +`storage.flush_timeout` 是单次 provider 调用的最长时间，默认值为 30 秒。 + +## 哪些 sandbox 会被同步 + +每轮开始时，manager 先选择 lifecycle 状态为 `Running` 的记录。调用某个 +sandbox 的 provider 前，会取得 create 和 destroy 共用的 operation lock， +然后重新检查记录。 + +只有同时满足以下条件时才会调用 provider： + +- lifecycle 状态仍为 `Running`； +- 没有未结束的 lifecycle operation； +- metadata 记录 backend 正在运行，而且 daemon 仍持有该 backend； +- provider 可以根据 sandbox ID 重建完整 slot。 + +等待 operation lock 期间改变状态的 sandbox 会被跳过。状态为 Running 但 +ownership 不完整的记录会计为失败，而不是被静默遗漏；本轮的其他 sandbox +仍会继续处理。 + +第一次 sweep 在一个完整 interval 后开始。错过的 tick 会被跳过，不会排队， +因此耗时较长的一轮不会形成无界积压。 + +## 失败与重试 + +每次 provider 调用都有独立 deadline。失败或超时后，slot 仍归 sandbox +持有，lifecycle 状态也不会改变；后续 sweep 或 destroy 可以再次尝试。 + +`StorageProvider::flush_dirty` 是 provider 特定的持久化边界。实现必须保证 +调用被取消后仍可安全重试或释放。file provider 会同步独立 sandbox slot +中的规范文件；其他 provider 可以采用不同机制，但必须保持相同的 ownership +和取消合同。 + +存储同步不会保存 VM 内存或设备状态，不能代替完整 runtime 的保存和恢复。 + +## Daemon 关闭 + +daemon 提供请求服务时会同时监控周期 worker。如果 worker 意外退出，daemon +会停止接收工作，并进入正常的协调关闭流程。 + +正常关闭按以下顺序进行： + +1. 停止接收新连接； +2. 取消并等待同步 worker 退出； +3. 排空已经接收的连接； +4. 释放仍持有的 runtime 和 storage 资源。 + +destroy 开始前 worker 已经退出，因此周期 provider 调用不会和同一 +sandbox 的清理并发。如果 worker 与后续关闭阶段都失败，daemon 会同时报告 +两项失败。 diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 774f1e25fc..4b0b6c9a06 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -34,6 +34,14 @@ socket = "/run/blaze/api.sock" # Default: "" (disabled). http_addr = "" +# --------------------------------------------------------------------------- +# HTTP API request limits. +# --------------------------------------------------------------------------- +[api] +# Maximum request body collected by the daemon. Requests above this limit +# receive HTTP 413 before the full body is buffered. Default: 1 MiB. +max_body_bytes = 1048576 + # --------------------------------------------------------------------------- # Backend binary paths. # Maps backend name → absolute path to the executable. @@ -88,25 +96,31 @@ provider = "file" rootfs_size = 8589934592 mem_size = 4294967296 -# Number of pre-warmed storage slots to keep ready in the pool. -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) +# Number of recoverable runtime slots to keep ready for compatible requests. +# Zero disables background construction. Default: 0. +pool_size = 0 -# Whether to prefork (pre-allocate) storage slots at daemon startup. -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) +# Start each slot's backend before it becomes ready. When false, the slot owns +# storage only and starts its backend after a request claims it. Default: false. +prefork = false -# Interval between flush-dirty sweeps. -# flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) +# Periodic provider synchronization is opt-in. Set a positive duration to +# synchronize running sandbox slots after each interval. +flush_interval = "disabled" + +# Maximum duration of one provider synchronization attempt. +flush_timeout = "30s" # --------------------------------------------------------------------------- -# Instance pool settings (global defaults for warm-pool recycling). -# Per-workload pool settings are configured in each policy file. +# Runtime slot maintenance defaults. +# A policy-specific [pool].warm_ttl overrides the daemon-wide TTL. # --------------------------------------------------------------------------- [pool] -# Default time-to-live for warm (idle) instances before the GC reclaims them. +# Default time-to-live for ready runtime slots before the worker reclaims them. # Format: duration string (e.g. "30m", "1h", "90s"). Default: "30m". default_warm_ttl = "30m" -# How often the garbage collector runs to reap expired warm instances. +# How often the runtime worker checks for expired slots and missing capacity. # Format: duration string. Default: "5m". gc_interval = "5m" @@ -126,6 +140,23 @@ gc_interval = "10m" # Format: duration string. Default: "1h". idle_ttl = "1h" +# --------------------------------------------------------------------------- +# Published runtime artifact catalog. +# --------------------------------------------------------------------------- +[runtime_templates] +# Destination for atomically published runtime artifact sets. +dir = "/var/lib/blaze/runtime-templates" + +# Optional operator-controlled source root. Imports are disabled when omitted. +# API requests name a relative directory below this root. +# import_root = "/srv/blaze/runtime-template-imports" + +# Per-import resource bounds and aggregate catalog capacity. +max_files = 32 +max_bytes = 274877906944 +max_metadata_bytes = 1048576 +max_total_bytes = 1099511627776 + # --------------------------------------------------------------------------- # Metrics / observability. # --------------------------------------------------------------------------- diff --git a/src/blaze/examples/policies/agent-rl.toml b/src/blaze/examples/policies/agent-rl.toml index 8fe5e79695..151df04dc6 100644 --- a/src/blaze/examples/policies/agent-rl.toml +++ b/src/blaze/examples/policies/agent-rl.toml @@ -54,27 +54,32 @@ templates = [] fallback_on_missing_hook = "degrade" # --------------------------------------------------------------------------- -# Warm-pool configuration. Enables pre-created instances for fast cold-start. +# Warm-path policy metadata. Background slots also require a non-zero +# [storage].pool_size in config.toml. # --------------------------------------------------------------------------- [pool] -# Whether the warm pool is enabled for this policy. Default: false. +# Whether this policy is eligible for warm paths. Background runtime slots +# require this setting. Default: false. enabled = true -# Minimum number of warm instances to maintain. Default: 0. +# Reserved lifecycle recycling-pool schema metadata. The runtime-slot worker +# does not consume min/target/max, and /v1/pools does not apply them from policy. +# Reserved minimum. Default: 0. min = 4 -# Target pool size the autoscaler aims for. Default: 0. +# Reserved target. Default: 0. target = 16 -# Maximum pool size (hard cap). Default: 0. +# Reserved maximum. Default: 0. max = 64 -# Time-to-live for idle warm instances before the GC reclaims them. -# Overrides [pool].default_warm_ttl from config.toml for this policy. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". +# Ready-slot time-to-live override for this policy. If omitted, the daemon uses +# [pool].default_warm_ttl from config.toml (whose default is "30m"). +# Format: duration string (e.g. "30m", "1h"). warm_ttl = "30m" -# How the pool resets an instance after use so it can be returned to warm. +# Reserved lifecycle reset metadata. The public reset/return workflow is not +# connected; background slots are not returned to ready after destroy. # Options: "mm-template" (memory-template rollback, requires kernel hook), # "overlayfs-rollback" (filesystem-level rollback), # "full-recreate" (destroy and recreate from scratch). @@ -117,7 +122,8 @@ memory = "4G" # --------------------------------------------------------------------------- [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" -enable_vsock = true # parsed but not yet wired (Phase 2+) +enable_vsock = true # requires the compatible guest agent on vsock port 5000 +enable_network = false # opt in to a per-sandbox netns, tap, veth, and NAT serial_log = false # --------------------------------------------------------------------------- diff --git a/src/blaze/examples/policies/agent-tool.toml b/src/blaze/examples/policies/agent-tool.toml index 1d77a9b410..68c6ae7608 100644 --- a/src/blaze/examples/policies/agent-tool.toml +++ b/src/blaze/examples/policies/agent-tool.toml @@ -47,26 +47,32 @@ templates = [] fallback_on_missing_hook = "fail" # --------------------------------------------------------------------------- -# Warm-pool configuration. +# Warm-path policy metadata. Background slots also require a non-zero +# [storage].pool_size in config.toml. # --------------------------------------------------------------------------- [pool] -# Whether the warm pool is enabled for this policy. Default: false. +# Whether this policy is eligible for warm paths. Background runtime slots +# require this setting. Default: false. enabled = true -# Minimum number of warm instances to maintain. Default: 0. +# Reserved lifecycle recycling-pool schema metadata. The runtime-slot worker +# does not consume min/target/max, and /v1/pools does not apply them from policy. +# Reserved minimum. Default: 0. min = 2 -# Target pool size the autoscaler aims for. Default: 0. +# Reserved target. Default: 0. target = 8 -# Maximum pool size (hard cap). Default: 0. +# Reserved maximum. Default: 0. max = 32 -# Time-to-live for idle warm instances before the GC reclaims them. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". +# Ready-slot time-to-live override for this policy. If omitted, the daemon uses +# [pool].default_warm_ttl from config.toml (whose default is "30m"). +# Format: duration string (e.g. "30m", "1h"). warm_ttl = "30m" -# How the pool resets an instance after use. +# Reserved lifecycle reset metadata. The public reset/return workflow is not +# connected; background slots are not returned to ready after destroy. # Options: "mm-template", "overlayfs-rollback", "full-recreate". # Default: "mm-template". reset_mode = "full-recreate" @@ -105,7 +111,8 @@ memory = "1G" # --------------------------------------------------------------------------- [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" -enable_vsock = false # parsed but not yet wired (Phase 2+) +enable_vsock = false # enable only when the image runs the compatible guest agent +enable_network = false # opt in to a per-sandbox netns, tap, veth, and NAT memory = "2G" vcpus = 2 serial_log = false