From adbe1c1220b426e3340877c0599f31d9c2f4237c Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 7 Aug 2026 04:00:21 +0800 Subject: [PATCH] fix(blaze)!: reject incomplete reset and pool use Reset changed only lifecycle metadata before returning the same runtime to an instance pool. It did not reset runtime or storage, so callers could receive a false success and later reuse state they expected to discard. Return 501 for running reset requests while preserving 400, 404, and 422 for malformed, missing, and invalid-state requests. Remove the inactive pool implementation because reset was its only source of reusable instances. Keep the four pool routes as 501 compatibility endpoints. Reject unsupported pool settings, but accept and ignore only the exact defaults shipped in older RPM configuration and policy files. This narrow exception prevents an administrator-modified %config(noreplace) file from blocking the upgraded binary before the operator can merge its .rpmnew replacement. Retain decoding for legacy Reset and Warm states, including persisted start_path = warm records, so startup cleanup can release their resources. Remove the three metrics that no longer describe a supported operation. This is a breaking correction for clients using the incomplete pool API, unsupported pool configuration, or removed metrics. Complete reusable-instance support remains out of scope and must arrive with sandbox creation that uses it. Fixes: 1f0cfac498ec ("feat(anvil): scaffold local orchestrator crate skeleton") Signed-off-by: Weisson Han --- docs/user-guide/en/runtime/blaze.md | 65 +- docs/user-guide/zh/runtime/blaze.md | 54 +- src/blaze/AGENTS.md | 8 +- src/blaze/README.md | 92 +- src/blaze/README_zh.md | 80 +- src/blaze/crates/blaze-core/src/config.rs | 141 ++- src/blaze/crates/blaze-core/src/lib.rs | 2 - src/blaze/crates/blaze-core/src/lifecycle.rs | 82 +- src/blaze/crates/blaze-core/src/policy.rs | 257 +++-- src/blaze/crates/blaze-core/src/pool.rs | 288 ----- src/blaze/crates/blaze-core/src/storage.rs | 13 +- src/blaze/crates/blazed/src/api.rs | 995 ++++++------------ src/blaze/crates/blazed/src/daemon.rs | 26 +- src/blaze/crates/blazed/src/error.rs | 6 +- src/blaze/crates/blazed/src/file_provider.rs | 12 +- src/blaze/crates/blazed/src/metrics.rs | 44 +- .../crates/blazed/src/sandbox/manager.rs | 297 +----- .../crates/blazed/src/sandbox/storage_sync.rs | 11 +- src/blaze/crates/blazed/src/state.rs | 18 +- src/blaze/crates/blazed/src/state_store.rs | 2 - src/blaze/dist/blaze.spec | 4 +- .../design/lifecycle-state-consistency.md | 67 +- .../design/lifecycle-state-consistency_zh.md | 50 +- src/blaze/examples/config.toml | 19 - src/blaze/examples/policies/agent-rl.toml | 32 +- src/blaze/examples/policies/agent-tool.toml | 25 - src/blaze/manifests/blaze.toml | 5 - 27 files changed, 1061 insertions(+), 1634 deletions(-) delete mode 100644 src/blaze/crates/blaze-core/src/pool.rs diff --git a/docs/user-guide/en/runtime/blaze.md b/docs/user-guide/en/runtime/blaze.md index 0da29b9155..fc28a94591 100644 --- a/docs/user-guide/en/runtime/blaze.md +++ b/docs/user-guide/en/runtime/blaze.md @@ -104,12 +104,7 @@ Guest operations are available only while a sandbox is `Running` and its backend reports a compatible guest endpoint. A cold create that reports such an endpoint waits for the guest agent before publishing `Running`. Backends without an endpoint, including production mock fallback, skip that wait and -return HTTP 409 for guest operations. Warm-pool activation validates the -retained backend owner and storage before publishing `Running`, but it does -not repeat the guest readiness probe. `Running` on this path therefore does -not guarantee that the guest endpoint is still responsive: the first guest -request performs the normal bounded connection and can return a guest error. -Callers should apply the retry and outcome rules below to that first request. +return HTTP 409 for guest operations. Guest operations and lifecycle changes use the same per-sandbox operation lock. After obtaining the lock, the manager checks `Running` again so a request @@ -158,6 +153,64 @@ Leave `listen.http_addr` disabled in production until Daemon shutdown also does not yet wait for every active HTTP handler or release all runtime owners, so an in-flight request may observe a closed connection. +## Reset and Reusable-Instance Management + +`POST /v1/instances/{id}/reset` does not report success until Blaze can reset +both runtime and storage. A malformed identifier returns HTTP 400, an unknown +instance returns HTTP 404, an instance that is not running returns HTTP 422, +and a running instance returns HTTP 501 without changing its state or owned +resources. + +The four `/v1/pools` management routes also return HTTP 501. Blaze rejects +`storage.pool_size`, `storage.prefork`, and every `[pool]` section except the +exact historical package defaults. During an upgrade, it temporarily accepts +and ignores only those defaults from the older daemon configuration and two +default policy files, and logs a warning. This exception prevents an +administrator-modified file retained by RPM `%config(noreplace)` from blocking +the new daemon. It does not enable reusable instances. Merge each `.rpmnew` +file or remove the legacy section; later releases may remove this exception. +Any other policy `[pool]` section fails policy loading. At startup, +`policy.on_load_error = "fail"` stops the daemon, while `"warn"` starts with an +empty policy set. A failed administrative or signal-driven reload keeps the +currently active policies unchanged. + +The accepted daemon section is exactly: + +```toml +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +An accepted policy section must contain exactly these six fields and belong to +one of the two packaged policy identities: + +| Policy name | Workload class | `min` | `target` | `max` | +|---|---|---:|---:|---:| +| `agent-rl-default` | `agent-rl` | 4 | 16 | 64 | +| `agent-tool-default` | `agent-tool` | 2 | 8 | 32 | + +Both rows require `enabled = true`, `warm_ttl = "30m"`, and +`reset_mode = "full-recreate"`. A missing or additional field, a changed value +or type, a different policy name or workload class, or any other `[pool]` +section is rejected. Accepted compatibility values are ignored and omitted +when configuration is serialized. + +Blaze continues to decode persisted `Reset`, `Warm`, and +`start_path = "warm"` values written by earlier releases. Startup +reconciliation treats non-terminal records containing those values as cleanup +candidates and never reuses them. A failed cleanup retains the in-memory record +as `RecoveryRequired` and attempts to persist that state. If persistence also +fails, the startup warning includes the additional error and the durable record +may still contain its previous state. Reconciliation continues with other +accepted records. +The metrics endpoint no longer publishes `blaze_instances_resets_total`, +`blaze_pool_hits_total`, or `blaze_pool_misses_total`. + +The lifecycle invariants behind these compatibility responses are recorded in +the +[lifecycle state consistency and compatibility design](../../../../src/blaze/docs/design/lifecycle-state-consistency.md). + ## Storage Artifact Synchronization Blaze can periodically persist the already-written host artifacts and directory diff --git a/docs/user-guide/zh/runtime/blaze.md b/docs/user-guide/zh/runtime/blaze.md index dae16f2f55..cb55515d8b 100644 --- a/docs/user-guide/zh/runtime/blaze.md +++ b/docs/user-guide/zh/runtime/blaze.md @@ -88,11 +88,7 @@ Blaze 负责配置 sandbox 本地的网络路径。主机以外的路由和 DNS 只有 sandbox 处于 `Running` 且 backend 报告兼容的 guest endpoint 时， 才能执行 guest 操作。冷启动 backend 如果报告了该 endpoint，创建流程会在 发布 `Running` 前等待 guest agent。没有 endpoint 的 backend（包括生产环境 -mock fallback）会跳过等待，guest 操作返回 HTTP 409。当前从 warm pool 激活 -实例时，manager 会先验证保留的 backend owner 和 storage，再发布 `Running`， -但不会再次执行 guest readiness 探测。因此 warm 路径的 `Running` 不保证 guest -endpoint 仍然可响应；第一次 guest 请求仍会执行有界连接，并可能返回 guest -错误。调用方应对第一次请求采用下文说明的重试和结果判定规则。 +mock fallback）会跳过等待，guest 操作返回 HTTP 409。 Guest 操作和 lifecycle 变更使用同一个 sandbox operation lock。取得锁后， manager 会再次检查 `Running`，避免并发 lifecycle 变更后请求仍访问旧 runtime。 @@ -134,6 +130,54 @@ read 响应过大时返回 HTTP 502 和 保持 `listen.http_addr` 关闭。Daemon 停止时也不会等待全部 HTTP handler 或 释放所有 runtime owner，因此正在执行的请求可能看到连接关闭。 +## 重置与可复用实例管理 + +在 Blaze 能够同时重置运行环境和存储之前， +`POST /v1/instances/{id}/reset` 不会返回成功。实例编号格式错误时返回 +HTTP 400，实例不存在时返回 HTTP 404，实例不处于运行状态时返回 HTTP 422， +运行中的实例返回 HTTP 501，且不会改变其状态或已占用资源。 + +四个 `/v1/pools` 管理接口同样返回 HTTP 501。`storage.pool_size` 和 +`storage.prefork` 始终会被拒绝；除历史软件包的精确默认值外，任何 `[pool]` +配置段也会失败。软件包升级时，只会临时接受并忽略旧版守护进程配置和两份默认策略 +原样附带的 `[pool]` 默认值，同时记录警告。这项例外用于避免 RPM 通过 +`%config(noreplace)` 保留的管理员自定义文件阻止新版服务启动，并不会启用 +可复用实例。管理员应合并每个 `.rpmnew` 文件，或删除旧配置段；后续版本可能 +取消这项兼容。其他策略 `[pool]` 配置会导致策略加载失败。启动时， +`policy.on_load_error = "fail"` 会让守护进程停止，`"warn"` 则会使用空策略集 +继续启动。通过管理接口或信号重新加载策略失败时，当前生效的策略保持不变。 + +可以接受的 daemon `[pool]` 配置段必须恰好包含以下两个键值： + +```toml +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +可以接受的策略配置必须恰好包含六个字段，并且属于以下两个软件包内置策略之一： + +| 策略名称 | 工作负载类型 | `min` | `target` | `max` | +|---|---|---:|---:|---:| +| `agent-rl-default` | `agent-rl` | 4 | 16 | 64 | +| `agent-tool-default` | `agent-tool` | 2 | 8 | 32 | + +两行都要求 `enabled = true`、`warm_ttl = "30m"` 和 +`reset_mode = "full-recreate"`。缺少或增加字段、改变值或类型、策略名称或工作 +负载类型不同，或者出现任何其他 `[pool]` 配置，都会被拒绝。接受的兼容值会被 +忽略，序列化配置时也会省略。 + +Blaze 仍可读取旧版本写入的 `Reset`、`Warm` 和 `start_path = "warm"` 持久化 +值。启动恢复会把包含这些值的未终止记录作为清理对象，且不会复用这些记录。 +清理失败时，内存记录会保留为 `RecoveryRequired`，并尝试持久化该状态。如果 +持久化也失败，启动警告会记录附加错误，磁盘上的记录可能仍是先前状态。其他已通过 +校验的记录仍会继续恢复。监控接口不再输出 `blaze_instances_resets_total`、 +`blaze_pool_hits_total` 和 `blaze_pool_misses_total`。 + +这些兼容响应背后的生命周期约束记录在 +[生命周期状态一致性与兼容性设计](../../../../src/blaze/docs/design/lifecycle-state-consistency_zh.md) +中。 + ## 存储制品同步 Blaze 可以定期持久化 running sandbox 中已经写入的宿主机制品和目录元数据。 diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index 8ac0cc52bc..d2c9b1f92d 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -8,7 +8,7 @@ blaze is a **daemon-only** per-host sandbox orchestrator. All sandbox management Two-crate workspace: -- **blaze-core** (library): policy engine, lifecycle state machine, backend selector, pool manager, kernel hook registry, config schema. Zero I/O beyond local TOML/JSON parsing. +- **blaze-core** (library): policy engine, lifecycle state machine, backend selector, kernel hook registry, config schema. Zero I/O beyond local TOML/JSON parsing. - **blazed** (binary): daemon HTTP server (UDS + TCP), spawner implementations, metrics endpoint, CLI for daemon lifecycle commands. Dependency direction: `blazed` → `blaze-core`. No reverse dependency. @@ -26,12 +26,12 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw ## Key Design Constraints -- **Daemon-only API model**: No CLI client for sandbox operations. All instance/pool/template management is done via HTTP endpoints on UDS (`/run/blaze/api.sock`) or TCP (`:14159`). The CLI subcommands (`daemon start`, `daemon reload`, `daemon doctor`) only manage daemon lifecycle. +- **Daemon-only API model**: No CLI client for sandbox operations. All instance and template management is done via HTTP endpoints on UDS (`/run/blaze/api.sock`) or TCP (`:14159`). The CLI subcommands (`daemon start`, `daemon reload`, `daemon doctor`) only manage daemon lifecycle. - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. - **Lifecycle state machine**: 9 states. The main branches are Pending → - Creating → Running, Running ↔ Paused → Checkpointed, and Running → Reset → - Warm → Creating. Any non-terminal state can enter Destroyed; incomplete + Creating → Running and Running ↔ Paused → Checkpointed. Any non-terminal + state can enter Destroyed; incomplete cleanup enters RecoveryRequired. State transitions are enforced by `blaze_core::lifecycle`. Do not bypass via direct field mutation. - **MockSpawner fallback**: When the configured backend binary is missing or fails `probe()`, the daemon auto-downgrades to `MockSpawner` with a warning. This keeps API/integration tests functional without a real backend. diff --git a/src/blaze/README.md b/src/blaze/README.md index 7c1c03aa3b..2eafb972be 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -5,22 +5,20 @@ Per-host sandbox orchestrator daemon for AI Agent workloads. Blaze manages sandbox instance lifecycles via HTTP API with policy-driven -backend selection. It supports warm-pool pre-allocation, multi-backend -fallback (Firecracker → Bubblewrap → Mock), and Prometheus metrics export. +backend selection. It supports multi-backend fallback +(Firecracker → Bubblewrap → Mock) and Prometheus metrics export. Designed as the per-host agent for E2B-style orchestrator platforms. ## Features - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **Policy-driven backend selection** — workload class → backend priority list -- **Lifecycle state machine** — 9 states: Pending, Creating, Running, Paused, - Checkpointed, RecoveryRequired, Reset, Warm, and Destroyed +- **Lifecycle state machine** — durable state with restart recovery - **Guest operations** — bounded command execution and file transfer for running backends that expose a guest endpoint -- **Warm pool management** — pre-warmed instances with TTL-based GC - **Template catalog** — bounded import and atomic publication of reusable artifacts - **Kernel hook registry** — state tracking for pre/post hooks -- **Prometheus metrics** — request counts, instance gauges, pool sizes +- **Prometheus metrics** — request and instance counters - **Spawners** — FirecrackerSpawner, BubblewrapSpawner, MockSpawner - **Optional VM networking** — isolated namespace, tap, veth, and NAT per Firecracker VM @@ -108,12 +106,23 @@ provider = "file" # Storage provider selection. Currently supported: "file # "auto" probes available providers in priority order (currently equivalent to "file"). # Other values will log a warning and fall back to file. images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) sync_interval = "disabled" # Set a positive duration to persist already-written slot artifacts. sync_timeout = "30s" # Maximum scheduler wait for reconstruction plus artifact sync. ``` +Reusable-instance settings are not supported. Blaze rejects +`storage.pool_size`, `storage.prefork`, and every `[pool]` section except the +exact historical package defaults. Blaze temporarily accepts and ignores those +defaults from older `config.toml`, +`agent-rl.toml`, and `agent-tool.toml` files, and logs a warning. This lets an +administrator-modified file retained by RPM `%config(noreplace)` reach the new +daemon without enabling an incomplete feature. Merge the corresponding +`.rpmnew` file or remove the old `[pool]` section; later releases may remove +this exception. Any other policy `[pool]` section fails policy loading. At +startup, `policy.on_load_error = "fail"` stops the daemon, while `"warn"` starts +with an empty policy set. A failed administrative or signal-driven reload +keeps the currently active policies unchanged. + The `file` provider uses standard filesystem operations for sandbox storage. The `auto` provider probes available backends in priority order (currently equivalent to `file`). Unrecognized values will log a warning and fall back to `file`. When periodic synchronization is enabled, a completed provider failure is isolated from later sandboxes. If a provider cannot stop its filesystem work at @@ -146,11 +155,11 @@ for configuration, selection, retry, and worker shutdown behavior. | POST | `/v1/instances/{id}/read` | Compatible guest file read action | | POST | `/v1/instances/{id}/write` | Compatible guest file write action | | POST | `/v1/instances/{id}/checkpoint` | Record checkpoint state | -| POST | `/v1/instances/{id}/reset` | Record reset and return to the warm pool | -| GET | `/v1/pools` | List warm pools | -| GET | `/v1/pools/{backend}/{class}` | Get pool status | -| POST | `/v1/pools/{backend}/{class}/drain` | Drain a pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a pool | +| POST | `/v1/instances/{id}/reset` | Reserved for running instances; returns `501` until runtime and storage reset are implemented | +| GET | `/v1/pools` | Reserved; returns `501` | +| GET | `/v1/pools/{backend}/{class}` | Reserved; returns `501` | +| POST | `/v1/pools/{backend}/{class}/drain` | Reserved; returns `501` | +| PUT | `/v1/pools/{backend}/{class}/sizing` | Reserved; returns `501` | | GET | `/v1/templates` | List published template names | | GET | `/v1/templates/{name}` | Inspect published template metadata | | POST | `/v1/templates/import` | Publish a template from the configured import root | @@ -159,6 +168,45 @@ for configuration, selection, retry, and worker shutdown behavior. | GET | `/v1/metrics` | Prometheus metrics | | POST | `/v1/admin/reload` | Hot-reload policies | +For reset requests, a malformed instance identifier returns `400`, an unknown +instance returns `404`, and an instance that is not running returns `422`. A +running instance returns `501` without changing its lifecycle state or its +runtime and storage resources. Clients that require a fresh sandbox must +successfully destroy the old sandbox and create a new one; `501` does not mean +that reset completed. + +Upgrade compatibility accepts and ignores only this exact daemon section: + +```toml +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +An accepted policy section must contain exactly these six fields and belong to +one of the two packaged policy identities: + +| Policy name | Workload class | `min` | `target` | `max` | +|---|---|---:|---:|---:| +| `agent-rl-default` | `agent-rl` | 4 | 16 | 64 | +| `agent-tool-default` | `agent-tool` | 2 | 8 | 32 | + +Both rows require `enabled = true`, `warm_ttl = "30m"`, and +`reset_mode = "full-recreate"`. A missing or additional field, a changed value +or type, a different policy name or workload class, any other `[pool]` section, +and every `storage.pool_size` or `storage.prefork` setting are rejected. The +accepted values do not enable reusable instances and are omitted when the +configuration is serialized. + +Blaze continues to decode persisted `Reset`, `Warm`, and +`start_path = "warm"` values written by earlier releases. Startup +reconciliation treats non-terminal records containing those values as cleanup +candidates and never reuses them. A failed cleanup retains the in-memory record +as `RecoveryRequired` and attempts to persist that state. If persistence also +fails, the startup warning includes the additional error and the durable record +may still contain its previous state. Reconciliation continues with other +accepted records. + The `/v1/templates` routes are the single operator-facing template catalog. Importing an entry does not yet make sandbox creation select it; future create support will resolve optional names from this same catalog. See the @@ -196,15 +244,17 @@ map lock until publication. Direct file changes by a process that bypasses the state-root lock are unsupported. See the -[lifecycle state consistency design](docs/design/lifecycle-state-consistency.md) -for the writer-coordination, inventory-publication, and failure boundaries. +[lifecycle state consistency and compatibility design](docs/design/lifecycle-state-consistency.md) +for writer coordination, inventory publication, reset rejection, legacy-state +cleanup, and failure boundaries. The operation journal records the operation and start time, not completion of each resource step. An interrupted create is cleaned up rather than resumed, and an existing backend process is not adopted after restart. Failed recovery -does not run in a background retry loop. The checkpoint and reset endpoints -retain their existing metadata transitions; this recovery flow does not add -backend snapshot or restore operations. +does not run in a background retry loop. The checkpoint endpoint retains its +existing metadata transition. Reset remains unavailable until runtime and +storage can be reset together; this recovery flow does not add backend snapshot +or restore operations. ### Guest operations @@ -217,13 +267,13 @@ boundaries. #### Health Check -`GET /v1/health` returns daemon status including storage pool readiness: +`GET /v1/health` returns daemon status including storage capacity: ```json { "status": "ok", "version": "0.3.0", - "storage_pool": { "ready": 0, "capacity": 0, "pending": 0 } + "storage_pool": { "ready": 0, "capacity": 0, "pending": 0, "quarantined": 0 } } ``` @@ -232,7 +282,7 @@ boundaries. ``` src/blaze/ ├── crates/ -│ ├── blaze-core/ # Library: policy, lifecycle, pool, template, kernel, config +│ ├── blaze-core/ # Library: policy, lifecycle, template, kernel, config │ └── blazed/ # Binary: daemon, API server, spawners, metrics ├── examples/ # config.toml, policies/ ├── dist/ # blazed.service, blaze.spec, tmpfiles diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index dad6c0e123..e834093778 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -5,20 +5,18 @@ 面向 AI Agent 工作负载的单机 sandbox 编排 daemon。 Blaze 通过 HTTP API 管理 sandbox 实例的完整生命周期，支持策略驱动的后端选择。 -它提供 warm pool 预分配、多后端回退（Firecracker → Bubblewrap → Mock）以及 -Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 +它提供多后端回退（Firecracker → Bubblewrap → Mock）和 Prometheus 指标导出， +设计为 E2B 类编排平台的单机执行代理。 ## 特性 - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **策略驱动后端选择** — workload class → 后端优先级列表 -- **生命周期状态机** — 9 种状态：Pending、Creating、Running、Paused、 - Checkpointed、RecoveryRequired、Reset、Warm 和 Destroyed +- **生命周期状态机** — 持久化状态，并支持重启恢复 - **Guest 操作** — 对提供 guest endpoint 的运行中后端执行有界命令和文件传输 -- **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC - **Template catalog** — 有界导入并原子发布可复用 artifact - **内核 hook 注册** — 前/后置 hook 状态追踪 -- **Prometheus 指标** — 请求计数、实例 gauge、池大小 +- **Prometheus 指标** — 请求和实例计数 - **Spawner 后端** — FirecrackerSpawner、BubblewrapSpawner、MockSpawner - **可选 VM 网络** — 每台 Firecracker VM 独立使用 netns、tap、veth 和 NAT @@ -103,12 +101,21 @@ provider = "file" # 存储 provider 选择。当前支持："file"、"auto # "auto" 按优先级探测可用 provider（当前等同于 "file"）。 # 其他值将记录告警并回退到 file。 images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] 预热存储槽位数（尚未启用） -# prefork = false # [Reserved] 是否在槽位中预启动 VM（尚未启用） sync_interval = "disabled" # 设置正数 duration 后持久化 slot 中已经写入的制品。 sync_timeout = "30s" # scheduler 等待 slot 重建与制品同步的最长时间。 ``` +Blaze 当前不支持可复用实例设置。`storage.pool_size` 和 `storage.prefork` +始终会导致配置校验失败；除历史软件包的精确默认值外，任何 `[pool]` 配置段 +也会失败。软件包升级时有一项临时例外：旧版 `config.toml`、`agent-rl.toml` 和 +`agent-tool.toml` 原样附带的 `[pool]` 默认值会被接受并忽略，同时记录警告。 +这样，RPM 通过 `%config(noreplace)` 保留的管理员自定义文件不会阻止新版服务 +启动，但也不会启用尚未完整实现的功能。管理员应合并对应的 `.rpmnew` 文件， +或删除旧 `[pool]` 配置段；后续版本可能取消这项兼容。其他策略 `[pool]` 配置 +会导致策略加载失败。启动时，`policy.on_load_error = "fail"` 会让守护进程停止， +`"warn"` 则会使用空策略集继续启动。通过管理接口或信号重新加载策略失败时， +当前生效的策略保持不变。 + `file` provider 使用标准文件系统操作管理 sandbox 存储。`auto` 按优先级探测可用 provider（当前等同于 `file`）。无法识别的值将记录告警并回退到 `file`。 启用周期同步后，已经返回的 provider 失败不会中断后续 sandbox。如果 provider 在 deadline 到达时仍无法停止文件系统操作，该操作会继续持有 sandbox operation @@ -139,11 +146,11 @@ lock 和唯一的同步许可直至完成；后续同步会被推迟而不会不 | POST | `/v1/instances/{id}/read` | Guest 文件读取兼容入口 | | POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | | POST | `/v1/instances/{id}/checkpoint` | 记录 checkpoint 状态 | -| POST | `/v1/instances/{id}/reset` | 记录 reset 并返回 warm pool | -| GET | `/v1/pools` | 列出 warm pool | -| GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | -| POST | `/v1/pools/{backend}/{class}/drain` | 排空 pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | 调整 pool 大小 | +| POST | `/v1/instances/{id}/reset` | 运行中实例的预留接口；运行时和存储重置实现前返回 `501` | +| GET | `/v1/pools` | 预留接口；返回 `501` | +| GET | `/v1/pools/{backend}/{class}` | 预留接口；返回 `501` | +| POST | `/v1/pools/{backend}/{class}/drain` | 预留接口；返回 `501` | +| PUT | `/v1/pools/{backend}/{class}/sizing` | 预留接口；返回 `501` | | GET | `/v1/templates` | 列出已发布 template 的名称 | | GET | `/v1/templates/{name}` | 查看已发布 template 的 metadata | | POST | `/v1/templates/import` | 从配置的导入根目录发布 template | @@ -152,6 +159,38 @@ lock 和唯一的同步许可直至完成；后续同步会被推迟而不会不 | GET | `/v1/metrics` | Prometheus 指标 | | POST | `/v1/admin/reload` | 热加载策略 | +重置请求中的实例编号格式错误时返回 `400`，实例不存在时返回 `404`，实例 +不处于运行状态时返回 `422`。运行中的实例返回 `501`，且不会改变其生命周期 +状态，也不会改变其运行环境和存储资源。需要全新沙箱的客户端必须先成功销毁 +旧沙箱，再创建新沙箱；`501` 不表示重置已经完成。 + +升级兼容仅接受并忽略以下内容完全一致的 daemon `[pool]` 配置段： + +```toml +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +可以接受的策略配置必须恰好包含六个字段，并且属于以下两个软件包内置策略之一： + +| 策略名称 | 工作负载类型 | `min` | `target` | `max` | +|---|---|---:|---:|---:| +| `agent-rl-default` | `agent-rl` | 4 | 16 | 64 | +| `agent-tool-default` | `agent-tool` | 2 | 8 | 32 | + +两行都要求 `enabled = true`、`warm_ttl = "30m"` 和 +`reset_mode = "full-recreate"`。缺少或增加字段、改变值或类型、策略名称或工作 +负载类型不同、任何其他 `[pool]` 配置，以及所有 `storage.pool_size` 或 +`storage.prefork` 设置都会被拒绝。接受这些值不会启用实例复用；序列化配置时也会 +省略这些值。 + +Blaze 仍可读取旧版本写入的 `Reset`、`Warm` 和 `start_path = "warm"` 持久化 +值。启动恢复会把包含这些值的未终止记录作为清理对象，且不会复用这些记录。 +清理失败时，内存记录会保留为 `RecoveryRequired`，并尝试持久化该状态。如果 +持久化也失败，启动警告会记录附加错误，磁盘上的记录可能仍是先前状态。其他已通过 +校验的记录仍会继续恢复。 + `/v1/templates` 是唯一面向运维人员的 template catalog。导入条目目前不会让 sandbox create 自动选择它；后续 create 支持会从同一个 catalog 解析可选名称。 配置方法、接受的 artifact、上限和发布规则参见 @@ -180,13 +219,14 @@ daemon 才会逐个处理未结束的 sandbox。后续逐项恢复期间，如 也会持有进程内 ownership map 锁直至发布。绕过 state root 锁直接修改文件的外部 进程不在支持范围内。 -写入协调、清单发布和失败边界参见 -[生命周期状态一致性设计](docs/design/lifecycle-state-consistency_zh.md)。 +写入协调、清单发布、重置拒绝、旧状态清理和失败边界参见 +[生命周期状态一致性与兼容性设计](docs/design/lifecycle-state-consistency_zh.md)。 操作记录只保存操作类型和开始时间，不记录每个资源步骤是否已经完成。中断的 创建会被清理而不是从原位置继续，重启后也不会接管先前的后端进程。恢复失败 -后目前没有后台循环自动重试。checkpoint 和 reset 接口保持原有的元数据状态 -变化；这里的恢复流程没有增加后端 snapshot 或 restore 操作。 +后目前没有后台循环自动重试。检查点接口保持原有的元数据状态变化。重置接口 +在运行环境和存储能够一起重置前不可用；这里的恢复流程没有增加后端快照或 +恢复操作。 ### Guest 操作 @@ -197,13 +237,13 @@ daemon 才会逐个处理未结束的 sandbox。后续逐项恢复期间，如 #### 健康检查 -`GET /v1/health` 返回 daemon 状态，包含存储池就绪信息： +`GET /v1/health` 返回 daemon 状态，包含存储容量信息： ```json { "status": "ok", "version": "0.3.0", - "storage_pool": { "ready": 0, "capacity": 0, "pending": 0 } + "storage_pool": { "ready": 0, "capacity": 0, "pending": 0, "quarantined": 0 } } ``` @@ -212,7 +252,7 @@ daemon 才会逐个处理未结束的 sandbox。后续逐项恢复期间，如 ``` src/blaze/ ├── crates/ -│ ├── blaze-core/ # 库：策略、生命周期、池、模板、内核、配置 +│ ├── blaze-core/ # 库：策略、生命周期、模板、内核、配置 │ └── blazed/ # 二进制：daemon、API server、spawner、指标 ├── examples/ # config.toml、policies/ ├── dist/ # blazed.service、blaze.spec、tmpfiles diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index 376061cf91..fc3c26ea70 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -26,8 +26,12 @@ pub struct DaemonConfig { pub policy: PolicySection, #[serde(default)] pub storage: StorageSection, - #[serde(default)] - pub pool: PoolSection, + /// Legacy `[pool]` input retained for package-upgrade compatibility. + /// + /// The exact defaults shipped by older packages are accepted but ignored. + /// Serialization omits the section, and any other value fails validation. + #[serde(default, skip_serializing)] + pub pool: Option, #[serde(default)] pub template: TemplateSection, #[serde(default)] @@ -87,23 +91,6 @@ pub enum PolicyLoadErrorMode { Warn, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PoolSection { - #[serde(default = "default_pool_warm_ttl")] - pub default_warm_ttl: String, - #[serde(default = "default_pool_gc_interval")] - pub gc_interval: String, -} - -impl Default for PoolSection { - fn default() -> Self { - Self { - default_warm_ttl: default_pool_warm_ttl(), - gc_interval: default_pool_gc_interval(), - } - } -} - /// Published template catalog and its local import boundary. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TemplateSection { @@ -176,15 +163,17 @@ pub struct StorageSection { #[serde(default = "default_storage_provider")] pub provider: String, - /// Warm pool target size (0 = no pool). - /// NOTE: Reserved for future use. Not yet wired into runtime. - #[serde(default)] - pub pool_size: usize, + /// Legacy `storage.pool_size` input retained for an explicit validation error. + /// + /// Serialization omits the value, and validation fails whenever it is present. + #[serde(default, skip_serializing)] + pub pool_size: Option, - /// Whether to pre-start VMs in pool slots. - /// NOTE: Reserved for future use. Not yet wired into runtime. - #[serde(default)] - pub prefork: bool, + /// Legacy `storage.prefork` input retained for an explicit validation error. + /// + /// Serialization omits the value, and validation fails whenever it is present. + #[serde(default, skip_serializing)] + pub prefork: Option, /// Interval for persisting already-written provider-owned artifacts. /// @@ -211,8 +200,8 @@ impl Default for StorageSection { images_dir: default_images_dir(), instances_dir: default_instances_dir(), provider: default_storage_provider(), - pool_size: 0, - prefork: false, + pool_size: None, + prefork: None, sync_interval: default_sync_interval(), sync_timeout: default_sync_timeout(), rootfs_size: default_rootfs_size(), @@ -257,12 +246,29 @@ impl DaemonConfig { let raw = fs::read_to_string(path)?; let cfg: DaemonConfig = toml::from_str(&raw)?; cfg.validate()?; + if cfg.pool.is_some() { + tracing::warn!( + path = %path.display(), + "ignoring legacy packaged [pool] defaults; remove this section because reusable-instance management is unavailable" + ); + } tracing::info!(path = %path.display(), "loaded blaze daemon config"); Ok(cfg) } /// Validate cross-field invariants that serde cannot express. pub fn validate(&self) -> Result<()> { + if let Some(pool) = &self.pool { + if !is_legacy_packaged_pool_defaults(pool) { + return Err(unsupported_pool_config("[pool]")); + } + } + if self.storage.pool_size.is_some() { + return Err(unsupported_pool_config("storage.pool_size")); + } + if self.storage.prefork.is_some() { + return Err(unsupported_pool_config("storage.prefork")); + } validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir)?; self.storage.sync_schedule()?; self.storage.sync_timeout_duration()?; @@ -326,6 +332,23 @@ impl DaemonConfig { } } +fn is_legacy_packaged_pool_defaults(value: &toml::Value) -> bool { + let Some(table) = value.as_table() else { + return false; + }; + table.len() == 2 + && table.get("default_warm_ttl").and_then(toml::Value::as_str) == Some("30m") + && table.get("gc_interval").and_then(toml::Value::as_str) == Some("5m") +} + +fn unsupported_pool_config(field: &str) -> BlazeError { + BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "{field} is not supported because warm pool management is not implemented" + )), + } +} + fn invalid_storage_duration(name: &str, value: &str, allow_disabled: bool) -> BlazeError { let expected = if allow_disabled { "a positive duration or \"disabled\"" @@ -499,12 +522,6 @@ fn default_policy_dir() -> PathBuf { fn default_on_load_error() -> PolicyLoadErrorMode { PolicyLoadErrorMode::Fail } -fn default_pool_warm_ttl() -> String { - "30m".to_string() -} -fn default_pool_gc_interval() -> String { - "5m".to_string() -} fn default_template_dir() -> PathBuf { PathBuf::from("/var/lib/blaze/templates") } @@ -589,6 +606,60 @@ mod tests { assert_eq!(cfg.backends.len(), 2); } + #[test] + fn rejects_unsupported_pool_configuration() { + for input in [ + "[pool]\n", + "[pool]\ndefault_warm_ttl = \"30m\"\n", + "[pool]\ngc_interval = \"5m\"\n", + "[pool]\ndefault_warm_ttl = \"31m\"\ngc_interval = \"5m\"\n", + "[pool]\ndefault_warm_ttl = \"30m\"\ngc_interval = \"6m\"\n", + "[pool]\ndefault_warm_ttl = 30\ngc_interval = \"5m\"\n", + "[pool]\ndefault_warm_ttl = \"30m\"\ngc_interval = \"5m\"\nextra = true\n", + "[storage]\npool_size = 0\n", + "[storage]\nprefork = false\n", + ] { + let cfg: DaemonConfig = toml::from_str(input).expect("compatibility parse"); + let error = cfg.validate().expect_err("unsupported pool setting"); + assert!( + error + .to_string() + .contains("warm pool management is not implemented"), + "{error}" + ); + } + } + + #[test] + fn accepts_only_the_legacy_packaged_pool_defaults_without_serializing_them() { + let cfg: DaemonConfig = + toml::from_str("[pool]\ndefault_warm_ttl = \"30m\"\ngc_interval = \"5m\"\n") + .expect("legacy packaged configuration parses"); + + cfg.validate() + .expect("legacy packaged defaults remain upgrade-compatible"); + let serialized = toml::to_string(&cfg).expect("serialize configuration"); + assert!(!serialized.contains("[pool]")); + assert!(!serialized.contains("default_warm_ttl")); + assert!(!serialized.contains("gc_interval")); + } + + #[test] + fn load_accepts_a_preserved_packaged_pool_section() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("config.toml"); + std::fs::write( + &path, + "[daemon]\nlog_level = \"debug\"\n\n[pool]\ndefault_warm_ttl = \"30m\"\ngc_interval = \"5m\"\n", + ) + .expect("write legacy packaged configuration"); + + let cfg = DaemonConfig::load(&path).expect("load preserved packaged configuration"); + + assert!(cfg.pool.is_some()); + assert_eq!(cfg.daemon.log_level, "debug"); + } + #[test] fn rejects_equal_or_nested_storage_roots() { for (images, instances) in [ diff --git a/src/blaze/crates/blaze-core/src/lib.rs b/src/blaze/crates/blaze-core/src/lib.rs index 7ecc680d9a..cc6707d7d2 100644 --- a/src/blaze/crates/blaze-core/src/lib.rs +++ b/src/blaze/crates/blaze-core/src/lib.rs @@ -11,7 +11,6 @@ //! - [`backend`]: backend kinds + selection / fallback //! - [`guest_protocol`]: guest-agent wire DTOs //! - [`lifecycle`]: sandbox state machine + JSON persistence -//! - [`pool`]: warm-pool key/stat/manager //! - [`kernel`]: kernel hook registry, per-hook mutex //! - [`error`]: unified [`BlazeError`] error enum @@ -22,7 +21,6 @@ pub mod guest_protocol; pub mod kernel; pub mod lifecycle; pub mod policy; -pub mod pool; pub mod storage; pub use error::{BlazeError, Result}; diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index d2d9fab9d4..ec0b81c058 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -69,12 +69,16 @@ impl std::fmt::Display for SandboxState { } } -/// Whether a request entered `creating` from cold boot or via a warm -/// pool reuse — used as the primary latency / capacity SLO dimension. +/// Persisted record of whether sandbox startup used a reusable instance. +/// +/// New instances always use [`StartPath::Cold`]. [`StartPath::Warm`] remains +/// readable so startup reconciliation can clean records written by older releases. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum StartPath { + /// Sandbox creation started without a reusable instance. Cold, + /// Legacy reusable-instance start retained only for persisted-state compatibility. Warm, } @@ -113,14 +117,11 @@ pub struct SandboxInstance { } impl SandboxInstance { - /// Create a new instance in [`SandboxState::Pending`] with `start_path` - /// pre-classified by the caller (cold for fresh boots, warm for - /// pool reuses). + /// Create a new instance in [`SandboxState::Pending`]. pub fn new( backend: BackendKind, workload_class: WorkloadClass, image_digest: String, - start_path: StartPath, policy_name: String, ) -> Self { let now = Utc::now(); @@ -130,7 +131,7 @@ impl SandboxInstance { backend, workload_class, image_digest, - start_path, + start_path: StartPath::Cold, created_at: now, updated_at: now, policy_name, @@ -167,15 +168,6 @@ impl SandboxInstance { let prev = self.state; self.state = target; self.updated_at = Utc::now(); - // entering `creating` re-classifies the start path: warm-pool - // reuse goes warm → creating, fresh boots go pending → creating. - if target == SandboxState::Creating { - self.start_path = if prev == SandboxState::Warm { - StartPath::Warm - } else { - StartPath::Cold - }; - } tracing::info!( instance = %self.id, from = %prev, @@ -217,7 +209,7 @@ impl SandboxInstance { fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { use SandboxState::{ - Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Reset, Running, Warm, + Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Running, }; if to == Destroyed { // `* → destroyed` is always valid (terminal sink). @@ -230,11 +222,8 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { (Pending, Creating) => true, (Creating, Running) => true, (Running, Paused) => true, - (Running, Reset) => true, (Paused, Checkpointed) => true, (Paused, Running) => true, // resume - (Reset, Warm) => true, - (Warm, Creating) => true, // pool reuse / warm path _ => false, } } @@ -248,7 +237,6 @@ mod tests { BackendKind::KataFc, WorkloadClass::AgentRl, "sha256:deadbeef".into(), - StartPath::Cold, "agent-rl-default".into(), ) } @@ -268,18 +256,6 @@ mod tests { } } - #[test] - fn happy_path_warm_reuse() { - let mut inst = fresh(); - inst.transition(SandboxState::Creating).expect("ok"); - inst.transition(SandboxState::Running).expect("ok"); - inst.transition(SandboxState::Reset).expect("ok"); - inst.transition(SandboxState::Warm).expect("ok"); - // warm → creating must flip start_path to Warm. - inst.transition(SandboxState::Creating).expect("ok"); - assert_eq!(inst.start_path, StartPath::Warm); - } - #[test] fn destroy_is_always_legal_except_from_destroyed() { let mut inst = fresh(); @@ -322,23 +298,14 @@ mod tests { } #[test] - fn illegal_running_to_warm() { + fn reset_and_warm_are_not_runtime_transition_targets() { let mut inst = fresh(); inst.transition(SandboxState::Creating).expect("ok"); inst.transition(SandboxState::Running).expect("ok"); - let err = inst.transition(SandboxState::Warm).expect_err("illegal"); - assert!(matches!(err, BlazeError::InvalidStateTransition { .. })); - } - - #[test] - fn illegal_warm_to_running() { - let mut inst = fresh(); - inst.transition(SandboxState::Creating).expect("ok"); - inst.transition(SandboxState::Running).expect("ok"); - inst.transition(SandboxState::Reset).expect("ok"); - inst.transition(SandboxState::Warm).expect("ok"); - let err = inst.transition(SandboxState::Running).expect_err("illegal"); - assert!(matches!(err, BlazeError::InvalidStateTransition { .. })); + for target in [SandboxState::Reset, SandboxState::Warm] { + let error = inst.transition(target).expect_err("legacy-only state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + } } #[test] @@ -373,6 +340,27 @@ mod tests { assert_eq!(loaded.backend_ownership, BackendOwnership::Unknown); } + #[test] + fn legacy_reset_and_warm_states_deserialize() { + let inst = fresh(); + for state in ["reset", "warm"] { + let value = serde_json::json!({ + "id": inst.id, + "state": state, + "backend": "mock", + "workload_class": "agent-rl", + "image_digest": "sha256:old", + "start_path": "warm", + "created_at": inst.created_at, + "updated_at": inst.updated_at, + "policy_name": "legacy" + }); + let loaded: SandboxInstance = serde_json::from_value(value).expect("legacy state"); + assert_eq!(loaded.state.as_str(), state); + assert_eq!(loaded.start_path, StartPath::Warm); + } + } + #[test] fn create_journal_round_trips() { let tmp = tempfile::tempdir().expect("tmp"); diff --git a/src/blaze/crates/blaze-core/src/policy.rs b/src/blaze/crates/blaze-core/src/policy.rs index 627fb7d65e..4307945082 100644 --- a/src/blaze/crates/blaze-core/src/policy.rs +++ b/src/blaze/crates/blaze-core/src/policy.rs @@ -84,15 +84,6 @@ pub enum FallbackOnMissingHook { Continue, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -pub enum ResetMode { - #[default] - MmTemplate, - OverlayfsRollback, - FullRecreate, -} - /// Checkpoint strategy selection. /// NOTE(Phase 3): v0.1 stores strategy in policy config but does NOT invoke /// kernel syscalls. Real checkpoint/restore via UFFD-WP deferred to Phase 3. @@ -117,8 +108,12 @@ pub struct PolicyFile { #[serde(rename = "match")] pub match_: PolicyMatch, pub select: PolicySelect, - #[serde(default)] - pub pool: Option, + /// Legacy policy `[pool]` input retained for package-upgrade compatibility. + /// + /// Exact defaults from the two older packaged policies are accepted but ignored. + /// Serialization omits the section, and any other value fails validation. + #[serde(default, skip_serializing)] + pub pool: Option, #[serde(default)] pub checkpoint: Option, #[serde(default)] @@ -152,6 +147,16 @@ pub struct PolicySelect { impl PolicyFile { /// Validate internal consistency constraints of a policy file. pub fn validate(&self) -> Result<()> { + if let Some(pool) = &self.pool { + if !is_legacy_packaged_policy_pool(self, pool) { + return Err(BlazeError::PolicyEvalError { + reason: format!( + "policy \"{}\": [pool] is not supported because warm pool management is not implemented", + self.policy_name + ), + }); + } + } // Validate [vm] vcpus/memory if present. if let Some(vm) = &self.vm { validate_vm_resource(&self.policy_name, "[vm]", Some(&vm.memory), Some(vm.vcpus))?; @@ -184,23 +189,28 @@ impl PolicyFile { }); } - // Validate [pool].warm_ttl format (e.g. "30s", "30m", "1h", "1d"; pure numbers are illegal). - if let Some(pool) = self.pool.as_ref() - && parse_duration(&pool.warm_ttl).is_none() - { - return Err(BlazeError::PolicyEvalError { - reason: format!( - "policy \"{policy_name}\": [pool].warm_ttl must be a duration like \"30s\", \"30m\", \"1h\", \"1d\", got \"{warm_ttl}\"", - policy_name = self.policy_name, - warm_ttl = pool.warm_ttl - ), - }); - } - Ok(()) } } +fn is_legacy_packaged_policy_pool(policy: &PolicyFile, value: &toml::Value) -> bool { + let expected_sizes = match (policy.policy_name.as_str(), policy.match_.workload_class) { + ("agent-rl-default", WorkloadClass::AgentRl) => (4, 16, 64), + ("agent-tool-default", WorkloadClass::AgentTool) => (2, 8, 32), + _ => return false, + }; + let Some(table) = value.as_table() else { + return false; + }; + table.len() == 6 + && table.get("enabled").and_then(toml::Value::as_bool) == Some(true) + && table.get("min").and_then(toml::Value::as_integer) == Some(expected_sizes.0) + && table.get("target").and_then(toml::Value::as_integer) == Some(expected_sizes.1) + && table.get("max").and_then(toml::Value::as_integer) == Some(expected_sizes.2) + && table.get("warm_ttl").and_then(toml::Value::as_str) == Some("30m") + && table.get("reset_mode").and_then(toml::Value::as_str) == Some("full-recreate") +} + fn validate_vm_resource( policy_name: &str, field: &str, @@ -287,26 +297,6 @@ pub fn parse_duration(s: &str) -> Option { Some(Duration::from_secs(secs)) } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PolicyPool { - #[serde(default)] - pub enabled: bool, - #[serde(default)] - pub min: u32, - #[serde(default)] - pub target: u32, - #[serde(default)] - pub max: u32, - #[serde(default = "default_warm_ttl")] - pub warm_ttl: String, - #[serde(default)] - pub reset_mode: ResetMode, -} - -fn default_warm_ttl() -> String { - "30m".to_string() -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PolicyCheckpoint { #[serde(default)] @@ -557,9 +547,7 @@ pub struct ImageMetadata { pub kernel_version: Option, } -/// Result of evaluating a request against the policy library. Drives -/// backend selection, hook activation, and pool eligibility -/// for a single sandbox instance. +/// Result of evaluating a request against the policy library for one sandbox. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RuntimeDecision { pub policy_name: String, @@ -568,13 +556,11 @@ pub struct RuntimeDecision { pub kernel_hooks: Vec, pub templates: Vec, pub fallback_on_missing_hook: FallbackOnMissingHook, - pub pool: Option, pub checkpoint: Option, pub quota: Option, pub hooks: PolicyHooks, pub backend: BackendConfigs, pub vm: Option, - pub pool_eligible: bool, } // --------------------------------------------------------------------------- @@ -619,6 +605,14 @@ impl PolicyEngine { } let policy = load_one(&path)?; policy.validate()?; + if policy.pool.is_some() { + tracing::warn!( + path = %path.display(), + policy_name = %policy.policy_name, + workload_class = %policy.match_.workload_class, + "ignoring legacy packaged policy [pool] defaults; remove this section because reusable-instance management is unavailable" + ); + } warn_vm_config(&policy); tracing::info!( policy_name = %policy.policy_name, @@ -760,7 +754,6 @@ fn warn_vm_config(policy: &PolicyFile) { } fn build_decision(policy: &PolicyFile) -> RuntimeDecision { - let pool_eligible = policy.pool.as_ref().map(|p| p.enabled).unwrap_or(false); RuntimeDecision { policy_name: policy.policy_name.clone(), workload_class: policy.match_.workload_class, @@ -768,13 +761,11 @@ fn build_decision(policy: &PolicyFile) -> RuntimeDecision { kernel_hooks: policy.select.kernel_hooks.clone(), templates: policy.select.templates.clone(), fallback_on_missing_hook: policy.select.fallback_on_missing_hook, - pool: policy.pool.clone(), checkpoint: policy.checkpoint.clone(), quota: policy.quota.clone(), hooks: policy.hooks.clone(), backend: policy.backend.clone(), vm: policy.vm.clone(), - pool_eligible, } } @@ -799,14 +790,6 @@ kernel_hooks = ["mm-template", "uffd-wp"] templates = ["mm-template"] fallback_on_missing_hook = "fail" -[pool] -enabled = true -min = 4 -target = 16 -max = 64 -warm_ttl = "30m" -reset_mode = "mm-template" - [checkpoint] enabled = true strategy = "uffd-wp-async" @@ -837,7 +820,6 @@ sequence = ["template-reg:bind-mm-template"] assert_eq!(pf.policy_name, "agent-rl-default"); assert_eq!(pf.match_.workload_class, WorkloadClass::AgentRl); assert_eq!(pf.select.backend_priority[0], BackendKind::KataFc); - assert!(pf.pool.as_ref().expect("pool").enabled); assert!( pf.backend .firecracker @@ -866,7 +848,6 @@ sequence = ["template-reg:bind-mm-template"] let decision = engine.evaluate(&labels, &img).expect("matches"); assert_eq!(decision.policy_name, "agent-rl-override"); assert_eq!(decision.backend_priority, vec![BackendKind::Rund]); - assert!(decision.pool_eligible); } #[test] @@ -1085,7 +1066,7 @@ cpu_shares = 0 } #[test] - fn validate_rejects_bare_number_warm_ttl() { + fn validate_rejects_unsupported_pool_section() { let raw = r#" manifest_version = 1 policy_name = "test" @@ -1097,14 +1078,152 @@ workload_class = "agent-rl" backend_priority = ["firecracker"] [pool] -enabled = true -min = 0 -target = 0 -max = 0 -warm_ttl = "300" +enabled = false "#; let pf: PolicyFile = toml::from_str(raw).expect("parse"); - assert!(pf.validate().is_err()); + let error = pf.validate().expect_err("unsupported pool section"); + assert!( + error + .to_string() + .contains("warm pool management is not implemented"), + "{error}" + ); + } + + fn packaged_pool_policy_toml( + policy_name: &str, + workload_class: &str, + min: i64, + target: i64, + max: i64, + ) -> String { + format!( + r#" +manifest_version = 1 +policy_name = "{policy_name}" +priority = 777 + +[match] +workload_class = "{workload_class}" + +[select] +backend_priority = ["firecracker"] + +[pool] +enabled = true +min = {min} +target = {target} +max = {max} +warm_ttl = "30m" +reset_mode = "full-recreate" +"# + ) + } + + #[test] + fn validate_accepts_only_packaged_policy_pool_defaults_without_serializing_them() { + for (policy_name, workload_class, min, target, max) in [ + ("agent-rl-default", "agent-rl", 4, 16, 64), + ("agent-tool-default", "agent-tool", 2, 8, 32), + ] { + let raw = packaged_pool_policy_toml(policy_name, workload_class, min, target, max); + let policy: PolicyFile = toml::from_str(&raw).expect("packaged policy parses"); + + policy + .validate() + .expect("packaged policy remains upgrade-compatible"); + let serialized = toml::to_string(&policy).expect("serialize policy"); + assert!(!serialized.contains("[pool]")); + assert!(!serialized.contains("warm_ttl")); + assert!(!serialized.contains("reset_mode")); + } + } + + #[test] + fn validate_rejects_modified_packaged_policy_pool_defaults() { + let raw = packaged_pool_policy_toml("agent-tool-default", "agent-tool", 2, 8, 32); + let base: PolicyFile = toml::from_str(&raw).expect("packaged policy parses"); + let mut modified = Vec::new(); + + let mut wrong_name = base.clone(); + wrong_name.policy_name = "renamed-agent-tool".to_string(); + modified.push(wrong_name); + + let mut wrong_class = base.clone(); + wrong_class.match_.workload_class = WorkloadClass::AgentRl; + modified.push(wrong_class); + + let rl_raw = packaged_pool_policy_toml("agent-rl-default", "agent-rl", 4, 16, 64); + let mut wrong_rl_class: PolicyFile = + toml::from_str(&rl_raw).expect("packaged policy parses"); + wrong_rl_class.match_.workload_class = WorkloadClass::AgentTool; + modified.push(wrong_rl_class); + + let mut missing = base.clone(); + missing + .pool + .as_mut() + .and_then(toml::Value::as_table_mut) + .expect("pool table") + .remove("max"); + modified.push(missing); + + let mut extra = base.clone(); + extra + .pool + .as_mut() + .and_then(toml::Value::as_table_mut) + .expect("pool table") + .insert("extra".to_string(), toml::Value::Boolean(true)); + modified.push(extra); + + for (key, value) in [ + ("enabled", toml::Value::Boolean(false)), + ("min", toml::Value::Integer(3)), + ("target", toml::Value::Integer(9)), + ("max", toml::Value::Integer(33)), + ("warm_ttl", toml::Value::String("31m".to_string())), + ( + "reset_mode", + toml::Value::String("reuse-runtime".to_string()), + ), + ("min", toml::Value::String("2".to_string())), + ] { + let mut policy = base.clone(); + policy + .pool + .as_mut() + .and_then(toml::Value::as_table_mut) + .expect("pool table") + .insert(key.to_string(), value); + modified.push(policy); + } + + for policy in modified { + policy + .validate() + .expect_err("modified pool settings remain unsupported"); + } + } + + #[test] + fn load_dir_accepts_both_preserved_packaged_policy_files() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write( + tmp.path().join("agent-rl.toml"), + packaged_pool_policy_toml("agent-rl-default", "agent-rl", 4, 16, 64), + ) + .expect("write agent-rl policy"); + fs::write( + tmp.path().join("agent-tool.toml"), + packaged_pool_policy_toml("agent-tool-default", "agent-tool", 2, 8, 32), + ) + .expect("write agent-tool policy"); + + let engine = PolicyEngine::load_dir(tmp.path()).expect("load preserved policies"); + + assert_eq!(engine.policies().len(), 2); + assert!(engine.policies().iter().all(|policy| policy.pool.is_some())); } #[test] diff --git a/src/blaze/crates/blaze-core/src/pool.rs b/src/blaze/crates/blaze-core/src/pool.rs deleted file mode 100644 index e1703dfc70..0000000000 --- a/src/blaze/crates/blaze-core/src/pool.rs +++ /dev/null @@ -1,288 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -//! Warm pool key/config/manager. -//! -//! v0.1: in-memory only. The daemon owns pool persistence indirectly -//! via [`crate::lifecycle::SandboxInstance::persist`] for each warm -//! instance — pool itself is rebuilt by scanning the state dir on -//! restart. - -use std::collections::{HashMap, VecDeque}; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use crate::backend::BackendKind; -use crate::policy::{ResetMode, WorkloadClass}; - -/// One warm pool exists per `(backend, workload_class, image_digest)` -/// triple. Cross-backend mixing is forbidden. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct PoolKey { - pub backend: BackendKind, - pub workload_class: WorkloadClass, - pub image_digest: String, -} - -impl PoolKey { - pub fn new(backend: BackendKind, workload_class: WorkloadClass, image_digest: String) -> Self { - Self { - backend, - workload_class, - image_digest, - } - } -} - -/// Per-pool sizing & reset config; usually derived from -/// [`crate::policy::PolicyPool`] at evaluate time and pinned for the -/// pool's lifetime. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PoolConfig { - pub enabled: bool, - pub min: u32, - pub target: u32, - pub max: u32, - /// Serialized as humantime-style string; manager treats it opaquely. - #[serde(with = "duration_secs")] - pub warm_ttl: Duration, - pub reset_mode: ResetMode, -} - -impl Default for PoolConfig { - fn default() -> Self { - Self { - enabled: false, - min: 0, - target: 0, - max: 0, - warm_ttl: Duration::from_secs(30 * 60), - reset_mode: ResetMode::default(), - } - } -} - -mod duration_secs { - use serde::{Deserialize, Deserializer, Serializer}; - use std::time::Duration; - - pub fn serialize(d: &Duration, ser: S) -> Result { - ser.serialize_u64(d.as_secs()) - } - - pub fn deserialize<'de, D: Deserializer<'de>>(de: D) -> Result { - let secs = u64::deserialize(de)?; - Ok(Duration::from_secs(secs)) - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PoolStats { - pub warm_count: u32, - pub total_hits: u64, - pub total_misses: u64, - pub evict_count: u64, - /// Candidates removed from service because their runtime resources failed validation. - pub quarantine_count: u64, -} - -#[derive(Debug, Default)] -struct PoolBucket { - config: PoolConfig, - /// FIFO of warm instances ready for reuse. - warm: VecDeque, - stats: PoolStats, -} - -/// Coordinator for all warm pools on a single host. -#[derive(Debug, Default)] -pub struct PoolManager { - pools: HashMap, -} - -impl PoolManager { - pub fn new() -> Self { - Self::default() - } - - /// Try to reuse a warm instance. Returns the instance id and bumps - /// `total_hits`. On miss returns `None` and bumps `total_misses`. - pub fn lookup(&mut self, key: &PoolKey) -> Option { - let bucket = self.pools.entry(key.clone()).or_default(); - if let Some(id) = bucket.warm.pop_front() { - bucket.stats.warm_count = bucket.stats.warm_count.saturating_sub(1); - bucket.stats.total_hits += 1; - tracing::info!(?key, %id, "pool hit"); - Some(id) - } else { - bucket.stats.total_misses += 1; - tracing::info!(?key, "pool miss"); - None - } - } - - /// Record that a candidate returned by [`Self::lookup`] failed validation. - /// - /// Invalid candidates are not returned to the ready queue. Reclassifying - /// the optimistic lookup as a miss keeps hit metrics aligned with usable - /// warm activations. - pub fn quarantine(&mut self, key: &PoolKey, instance_id: Uuid) { - let bucket = self.pools.entry(key.clone()).or_default(); - bucket.stats.total_hits = bucket.stats.total_hits.saturating_sub(1); - bucket.stats.total_misses += 1; - bucket.stats.quarantine_count += 1; - tracing::warn!(?key, %instance_id, "quarantined invalid warm instance"); - } - - /// Restore a claimed warm instance after a retryable activation failure. - /// - /// Capacity is not re-evaluated because the instance occupied this slot - /// immediately before [`Self::lookup`] removed it. - pub fn restore_lookup(&mut self, key: PoolKey, instance_id: Uuid) { - let bucket = self.pools.entry(key.clone()).or_default(); - if !bucket.warm.contains(&instance_id) { - bucket.warm.push_front(instance_id); - } - bucket.stats.warm_count = bucket.warm.len() as u32; - tracing::warn!(?key, %instance_id, "restored failed pool activation"); - } - - /// Push an instance back into its pool after reset. - pub fn return_to_pool(&mut self, key: PoolKey, instance_id: Uuid) { - let bucket = self.pools.entry(key.clone()).or_default(); - // enforce max if configured - if bucket.config.max > 0 && bucket.warm.len() as u32 >= bucket.config.max { - bucket.stats.evict_count += 1; - tracing::warn!(?key, %instance_id, "pool full, evicting on return"); - return; - } - bucket.warm.push_back(instance_id); - bucket.stats.warm_count = bucket.warm.len() as u32; - tracing::info!(?key, %instance_id, "returned instance to pool"); - } - - /// Drain every pool whose `(backend, workload_class)` matches; used - /// by `POST /v1/pools/{backend}/{class}/drain`. Returns the evicted - /// instance ids so the caller can destroy them. - pub fn drain(&mut self, backend: BackendKind, class: WorkloadClass) -> Vec { - let mut drained = Vec::new(); - for (key, bucket) in self.pools.iter_mut() { - if key.backend == backend && key.workload_class == class { - let count = bucket.warm.len(); - drained.extend(bucket.warm.drain(..)); - bucket.stats.warm_count = 0; - bucket.stats.evict_count += count as u64; - tracing::info!(?key, drained = count, "drained pool"); - } - } - drained - } - - /// Update sizing/reset config for a pool (creating it if missing). - pub fn resize(&mut self, key: &PoolKey, config: PoolConfig) { - let bucket = self.pools.entry(key.clone()).or_default(); - bucket.config = config; - tracing::info!(?key, "pool sizing updated"); - } - - pub fn config(&self, key: &PoolKey) -> Option<&PoolConfig> { - self.pools.get(key).map(|b| &b.config) - } - - /// Snapshot of stats for a pool. Returns the zero value when the - /// pool has not been registered yet. - pub fn stats(&self, key: &PoolKey) -> PoolStats { - self.pools - .get(key) - .map(|b| b.stats.clone()) - .unwrap_or_default() - } - - /// Snapshot of every known pool — drives `GET /v1/pools`. - pub fn list_pools(&self) -> Vec<(PoolKey, PoolStats)> { - self.pools - .iter() - .map(|(k, b)| (k.clone(), b.stats.clone())) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn key() -> PoolKey { - PoolKey::new( - BackendKind::KataFc, - WorkloadClass::AgentTool, - "sha256:abc".into(), - ) - } - - #[test] - fn lookup_miss_then_return_then_hit() { - let mut mgr = PoolManager::new(); - assert!(mgr.lookup(&key()).is_none()); - - let id = Uuid::new_v4(); - mgr.return_to_pool(key(), id); - assert_eq!(mgr.lookup(&key()), Some(id)); - assert!(mgr.lookup(&key()).is_none()); - - let stats = mgr.stats(&key()); - assert_eq!(stats.total_hits, 1); - assert_eq!(stats.total_misses, 2); - } - - #[test] - fn drain_clears_matching_pools() { - let mut mgr = PoolManager::new(); - mgr.return_to_pool(key(), Uuid::new_v4()); - mgr.return_to_pool(key(), Uuid::new_v4()); - - let drained = mgr.drain(BackendKind::KataFc, WorkloadClass::AgentTool); - assert_eq!(drained.len(), 2); - assert_eq!(mgr.stats(&key()).warm_count, 0); - } - - #[test] - fn quarantine_reclassifies_an_invalid_candidate_as_a_miss() { - let mut mgr = PoolManager::new(); - let id = Uuid::new_v4(); - mgr.return_to_pool(key(), id); - - assert_eq!(mgr.lookup(&key()), Some(id)); - mgr.quarantine(&key(), id); - - let stats = mgr.stats(&key()); - assert_eq!(stats.warm_count, 0); - assert_eq!(stats.total_hits, 0); - assert_eq!(stats.total_misses, 1); - assert_eq!(stats.quarantine_count, 1); - } - - #[test] - fn resize_caps_growth() { - let mut mgr = PoolManager::new(); - let cfg = PoolConfig { - enabled: true, - max: 1, - ..PoolConfig::default() - }; - mgr.resize(&key(), cfg); - - mgr.return_to_pool(key(), Uuid::new_v4()); - mgr.return_to_pool(key(), Uuid::new_v4()); // evicted by max - let stats = mgr.stats(&key()); - assert_eq!(stats.warm_count, 1); - assert_eq!(stats.evict_count, 1); - } - - #[test] - fn list_pools_returns_all() { - let mut mgr = PoolManager::new(); - mgr.return_to_pool(key(), Uuid::new_v4()); - let listed = mgr.list_pools(); - assert_eq!(listed.len(), 1); - } -} diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index 051a45a9f4..1e2d96f44e 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -2,7 +2,7 @@ //! Generic storage provider abstraction. //! //! Different providers may offer different performance characteristics -//! (warm pools, copy-on-write, content-addressable dedup) but present +//! (copy-on-write, content-addressable dedup) but present //! a uniform interface to the daemon layer. use std::path::PathBuf; @@ -32,13 +32,13 @@ pub struct StorageSlot { pub instance_dir: PathBuf, } -/// Pool readiness status. +/// Storage provider capacity reported by the health endpoint. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct PoolStatus { pub ready: usize, pub capacity: usize, pub pending: usize, - /// Slots retained because backend or storage cleanup must be retried. + /// Slots retained because cleanup must be retried. pub quarantined: usize, } @@ -101,7 +101,7 @@ pub trait StorageProvider: Send + Sync { /// Probe whether this provider is available in the current environment. async fn probe(&self) -> Result; - /// Acquire a ready storage slot (may come from a warm pool). + /// Acquire a storage slot for one sandbox. async fn acquire( &self, opts: &AcquireOpts, @@ -136,9 +136,6 @@ pub trait StorageProvider: Send + Sync { /// synchronization or cleanup must remain safe after completion. async fn sync_artifacts(&self, slot: &StorageSlot) -> Result<()>; - /// Query warm pool status. + /// Return the provider's current storage capacity. fn pool_status(&self) -> PoolStatus; - - /// Drain all ready slots from the warm pool. - async fn drain_pool(&self) -> Result; } diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 32c478f2ec..a45e76b989 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -8,7 +8,6 @@ use std::collections::HashMap; use std::convert::Infallible; use std::path::PathBuf; -use std::str::FromStr; use std::sync::Arc; use base64::Engine; @@ -17,7 +16,6 @@ use blaze_core::backend::{BackendKind, BackendStatus, select_backend}; use blaze_core::kernel::HookKind; use blaze_core::lifecycle::{SandboxInstance, SandboxState, StartPath}; use blaze_core::policy::{ImageMetadata, RuntimeDecision, WorkloadClass}; -use blaze_core::pool::{PoolConfig, PoolKey}; use http_body_util::{BodyExt, Full}; use hyper::body::{Body, Bytes, Incoming}; use hyper::header::CONTENT_TYPE; @@ -161,12 +159,10 @@ async fn dispatch( ("DELETE", ["v1", "instances", id]) | ("DELETE", ["v1", "sandboxes", id]) | ("POST", ["v1", "instances", id, "destroy"]) => destroy_instance(state, id).await, - ("GET", ["v1", "pools"]) => list_pools(state), - ("GET", ["v1", "pools", backend, class]) => pool_status(state, backend, class), - ("POST", ["v1", "pools", backend, class, "drain"]) => drain_pool(state, backend, class), - ("PUT", ["v1", "pools", backend, class, "sizing"]) => { - resize_pool(state, backend, class, &body) - } + ("GET", ["v1", "pools"]) + | ("GET", ["v1", "pools", _, _]) + | ("POST", ["v1", "pools", _, _, "drain"]) + | ("PUT", ["v1", "pools", _, _, "sizing"]) => pool_operation_unavailable(), ("GET", ["v1", "templates"]) => list_templates(state).await, ("GET", ["v1", "templates", name]) => get_template(state, name).await, ("POST", ["v1", "templates", "import"]) => import_template(state, &body).await, @@ -367,34 +363,24 @@ async fn reset_instance(state: &Arc, id: &str) -> Result, id: &str) -> Result>> { @@ -513,139 +499,10 @@ fn decode_guest_file(encoded: &str, limit: usize) -> Result> { Ok(data) } -// --------------------------------------------------------------------------- -// Pools -// --------------------------------------------------------------------------- - -fn list_pools(state: &Arc) -> Result>> { - let pool = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))?; - let listed: Vec<_> = pool - .list_pools() - .into_iter() - .map(|(k, s)| { - json!({ - "key": { - "backend": k.backend.as_str(), - "workload_class": k.workload_class.as_str(), - "image_digest": k.image_digest, - }, - "stats": s, - }) - }) - .collect(); - json_ok(&listed) -} - -fn pool_status( - state: &Arc, - backend: &str, - class: &str, -) -> Result>> { - let pool = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))?; - let backend_kind = BackendKind::from_str(backend) - .map_err(|e| BlazeDaemonError::BadRequest(format!("backend: {e}")))?; - let class_kind = WorkloadClass::from_str(class) - .map_err(|e| BlazeDaemonError::BadRequest(format!("class: {e}")))?; - - let listed: Vec<_> = pool - .list_pools() - .into_iter() - .filter(|(k, _)| k.backend == backend_kind && k.workload_class == class_kind) - .map(|(k, s)| { - json!({ - "key": { - "backend": k.backend.as_str(), - "workload_class": k.workload_class.as_str(), - "image_digest": k.image_digest, - }, - "stats": s, - }) - }) - .collect(); - json_ok(&listed) -} - -fn drain_pool( - state: &Arc, - backend: &str, - class: &str, -) -> Result>> { - let backend_kind = BackendKind::from_str(backend) - .map_err(|e| BlazeDaemonError::BadRequest(format!("backend: {e}")))?; - let class_kind = WorkloadClass::from_str(class) - .map_err(|e| BlazeDaemonError::BadRequest(format!("class: {e}")))?; - // TODO(v0.2): after removing instance IDs from the pool, walk - // spawn_handles and kill the underlying processes so that drain - // actually frees host resources. - let drained = { - let mut pool = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))?; - pool.drain(backend_kind, class_kind) - }; - json_ok(&json!({ - "drained": drained, - "count": drained.len(), - })) -} - -#[derive(Debug, Deserialize)] -struct ResizeReq { - #[serde(default)] - enabled: Option, - min: u32, - target: u32, - max: u32, - #[serde(default)] - image_digest: Option, - #[serde(default)] - warm_ttl_secs: Option, -} - -fn resize_pool( - state: &Arc, - backend: &str, - class: &str, - body: &[u8], -) -> Result>> { - let req: ResizeReq = serde_json::from_slice(body) - .map_err(|e| BlazeDaemonError::BadRequest(format!("invalid resize body: {e}")))?; - let backend_kind = BackendKind::from_str(backend) - .map_err(|e| BlazeDaemonError::BadRequest(format!("backend: {e}")))?; - let class_kind = WorkloadClass::from_str(class) - .map_err(|e| BlazeDaemonError::BadRequest(format!("class: {e}")))?; - let key = PoolKey::new( - backend_kind, - class_kind, - req.image_digest.clone().unwrap_or_default(), - ); - let cfg = PoolConfig { - enabled: req.enabled.unwrap_or(true), - min: req.min, - target: req.target, - max: req.max, - warm_ttl: std::time::Duration::from_secs(req.warm_ttl_secs.unwrap_or(30 * 60)), - reset_mode: blaze_core::policy::ResetMode::default(), - }; - { - let mut pool = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))?; - pool.resize(&key, cfg); - } - json_ok(&json!({ - "resized": true, - "backend": backend, - "class": class, - })) +fn pool_operation_unavailable() -> Result>> { + Err(BlazeDaemonError::UnsupportedOperation( + "warm pool management is not implemented".to_string(), + )) } // --------------------------------------------------------------------------- @@ -784,9 +641,8 @@ mod tests { use blaze_core::lifecycle::{BackendOwnership, OperationKind}; use blaze_core::policy::{ BackendConfigs, FallbackOnMissingHook, PolicyEngine, PolicyFile, PolicyHooks, PolicyMatch, - PolicyPool, PolicySelect, ResetMode, WorkloadClass, + PolicySelect, WorkloadClass, }; - use blaze_core::pool::PoolManager; use blaze_core::storage::{ AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, }; @@ -823,7 +679,7 @@ mod tests { config } - fn test_policy(kind: BackendKind, pooled: bool) -> PolicyFile { + fn test_policy(kind: BackendKind) -> PolicyFile { PolicyFile { manifest_version: 1, policy_name: "ownership-test".into(), @@ -838,14 +694,7 @@ mod tests { templates: vec![], fallback_on_missing_hook: FallbackOnMissingHook::default(), }, - pool: pooled.then_some(PolicyPool { - enabled: true, - min: 0, - target: 0, - max: 1, - warm_ttl: "30m".into(), - reset_mode: ResetMode::FullRecreate, - }), + pool: None, checkpoint: None, quota: None, hooks: PolicyHooks::default(), @@ -873,7 +722,6 @@ mod tests { ServerState::build( config, PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), HookRegistry::new(), registry, active_backend, @@ -884,7 +732,7 @@ mod tests { } #[cfg(feature = "test-failpoints")] - fn mock_state(temp: &tempfile::TempDir, pooled: bool) -> Arc { + fn mock_state(temp: &tempfile::TempDir) -> Arc { let config = test_config(temp); let storage: Arc = Arc::new(FileStorageProvider::with_images( config.storage.images_dir.clone(), @@ -892,7 +740,7 @@ mod tests { )); build_test_state( config, - test_policy(BackendKind::Mock, pooled), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -900,7 +748,7 @@ mod tests { } #[cfg(feature = "test-failpoints")] - fn guest_mock_state(temp: &tempfile::TempDir, pooled: bool) -> Arc { + fn guest_mock_state(temp: &tempfile::TempDir) -> Arc { let config = test_config(temp); let storage: Arc = Arc::new(FileStorageProvider::with_images( config.storage.images_dir.clone(), @@ -908,7 +756,7 @@ mod tests { )); build_test_state( config, - test_policy(BackendKind::Mock, pooled), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(GuestMockSpawner)), BackendKind::Mock, storage, @@ -974,63 +822,6 @@ mod tests { (status, value) } - struct TransientReconstructStorage { - inner: FileStorageProvider, - fail_reconstruct: AtomicBool, - } - - impl TransientReconstructStorage { - fn new(images_dir: std::path::PathBuf, instances_dir: std::path::PathBuf) -> Self { - Self { - inner: FileStorageProvider::with_images(images_dir, instances_dir), - fail_reconstruct: AtomicBool::new(false), - } - } - } - - #[async_trait] - impl StorageProvider for TransientReconstructStorage { - async fn probe(&self) -> blaze_core::Result { - self.inner.probe().await - } - - async fn acquire( - &self, - opts: &AcquireOpts, - ) -> std::result::Result { - self.inner.acquire(opts).await - } - - async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { - self.inner.release(slot).await - } - - async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { - self.inner.release_by_id(instance_id).await - } - - async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { - if self.fail_reconstruct.load(Ordering::Acquire) { - return Err(BlazeError::StorageError { - msg: "transient reconstruct failure".into(), - }); - } - self.inner.reconstruct(instance_id).await - } - - async fn sync_artifacts(&self, slot: &StorageSlot) -> blaze_core::Result<()> { - self.inner.sync_artifacts(slot).await - } - - fn pool_status(&self) -> PoolStatus { - self.inner.pool_status() - } - - async fn drain_pool(&self) -> blaze_core::Result { - self.inner.drain_pool().await - } - } - struct OwnershipObservingStorage { inner: FileStorageProvider, state_dir: PathBuf, @@ -1078,10 +869,6 @@ mod tests { fn pool_status(&self) -> PoolStatus { self.inner.pool_status() } - - async fn drain_pool(&self) -> blaze_core::Result { - self.inner.drain_pool().await - } } struct FailOnceOwner { @@ -1371,10 +1158,6 @@ mod tests { fn pool_status(&self) -> PoolStatus { self.inner.pool_status() } - - async fn drain_pool(&self) -> blaze_core::Result { - self.inner.drain_pool().await - } } #[cfg(feature = "test-failpoints")] @@ -1399,7 +1182,7 @@ mod tests { }); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners( BackendKind::Mock, Arc::new(CountingSpawner { @@ -1423,7 +1206,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -1458,6 +1241,71 @@ mod tests { assert_eq!(sandbox, instance); } + #[tokio::test] + async fn reserved_pool_routes_return_not_implemented() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + for (method, path) in [ + (Method::GET, "/v1/pools"), + (Method::GET, "/v1/pools/mock/agent-tool"), + (Method::POST, "/v1/pools/mock/agent-tool/drain"), + (Method::PUT, "/v1/pools/mock/agent-tool/sizing"), + ] { + let (status, body) = handled_json(&state, method, path, Vec::new()).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{path}"); + assert_eq!(body["status"], 501, "{path}"); + assert!( + body["error"] + .as_str() + .expect("error") + .contains("warm pool management is not implemented"), + "{path}" + ); + } + + let (status, body) = handled_json(&state, Method::GET, "/v1/pools/mock", Vec::new()).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(body["status"], 404); + } + + #[tokio::test] + async fn health_keeps_storage_pool_status() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let (status, body) = handled_json(&state, Method::GET, "/v1/health", Vec::new()).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(body["status"], "ok"); + assert_eq!(body["storage_pool"]["ready"], 0); + assert_eq!(body["storage_pool"]["capacity"], 0); + assert_eq!(body["storage_pool"]["pending"], 0); + assert_eq!(body["storage_pool"]["quarantined"], 0); + } + #[tokio::test] async fn destroy_route_forms_share_managed_cleanup() { let temp = tempfile::tempdir().expect("temp"); @@ -1468,7 +1316,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -1566,7 +1414,6 @@ mod tests { ServerState::build( config, engine, - PoolManager::new(), HookRegistry::new(), spawners(BackendKind::Firecracker, spawner), BackendKind::Firecracker, @@ -1601,123 +1448,125 @@ mod tests { } #[tokio::test] - async fn warm_claim_validates_runtime_and_quarantines_dead_owner() { + async fn reset_never_reports_success_without_runtime_and_storage_reset() { let temp = tempfile::tempdir().expect("temp"); - let mut config = DaemonConfig::default(); - config.daemon.state_dir = temp.path().join("state"); - config.storage.images_dir = temp.path().join("images"); - config.storage.instances_dir = temp.path().join("instances"); - config.template.dir = temp.path().join("templates"); - std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); - std::fs::create_dir_all(&config.storage.images_dir).expect("images"); - std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); - - let policy = PolicyFile { - manifest_version: 1, - policy_name: "warm-validation".into(), - priority: 100, - match_: PolicyMatch { - workload_class: WorkloadClass::AgentTool, - image_labels: HashMap::new(), - }, - select: PolicySelect { - backend_priority: vec![BackendKind::Mock], - kernel_hooks: vec![], - templates: vec![], - fallback_on_missing_hook: FallbackOnMissingHook::default(), - }, - pool: Some(PolicyPool { - enabled: true, - min: 0, - target: 0, - max: 1, - warm_ttl: "30m".into(), - reset_mode: ResetMode::FullRecreate, - }), - checkpoint: None, - quota: None, - hooks: PolicyHooks::default(), - backend: BackendConfigs::default(), - vm: None, - }; - let storage: Arc = - Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - config.storage.instances_dir.clone(), - )); - let state = Arc::new( - ServerState::build( - config, - PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), - HookRegistry::new(), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage, - ) - .expect("state"), + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, ); - let request = serde_json::to_vec(&json!({ - "workload_class": "agent-tool", - "image_digest": "sha256:warm-validation" - })) - .expect("request"); - let cold = create_instance(&state, &request) - .await - .expect("cold create"); - let cold: serde_json::Value = - serde_json::from_slice(&cold.into_body().collect().await.expect("body").to_bytes()) - .expect("cold json"); - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("return to pool"); - - let warm = create_instance(&state, &request) - .await - .expect("warm create"); - let warm: serde_json::Value = - serde_json::from_slice(&warm.into_body().collect().await.expect("body").to_bytes()) - .expect("warm json"); - assert_eq!(warm["instance"]["id"], id); - assert_eq!(warm["start_path"], "warm"); - - reset_instance(&state, &id) - .await - .expect("return live owner"); - let owner = state - .manager - .backend_owner(Uuid::parse_str(&id).expect("uuid")) - .expect("owner"); - owner.kill().await.expect("simulate backend exit"); + let (status, body) = handled_json( + &state, + Method::POST, + "/v1/instances/not-a-uuid/reset", + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["status"], 400); - let replacement = create_instance(&state, &request) - .await - .expect("cold fallback"); - let replacement: serde_json::Value = serde_json::from_slice( - &replacement - .into_body() - .collect() - .await - .expect("body") - .to_bytes(), + let missing_id = Uuid::nil(); + let (status, body) = handled_json( + &state, + Method::POST, + &format!("/v1/instances/{missing_id}/reset"), + Vec::new(), ) - .expect("replacement json"); - assert_ne!(replacement["instance"]["id"], id); - assert_eq!(replacement["start_path"], "cold"); - let key = PoolKey::new( + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(body["status"], 404); + + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + + let (status, body) = handled_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/reset"), + Vec::new(), + ) + .await; + + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + assert_eq!(body["status"], 501); + assert!( + body["error"] + .as_str() + .expect("error") + .contains("runtime and storage") + ); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Running + ); + let persisted = state + .state_store + .load(uuid) + .expect("persisted running state"); + assert_eq!(persisted.state, SandboxState::Running); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn reset_preserves_invalid_state_error_and_persisted_state() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, - WorkloadClass::AgentTool, - "sha256:warm-validation".into(), + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + { + let mut instances = state.instances.lock().expect("instances"); + let instance = instances.get_mut(&uuid).expect("instance"); + instance.transition(SandboxState::Paused).expect("pause"); + state.state_store.persist(instance).expect("persist pause"); + } + + let (status, body) = handled_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/reset"), + Vec::new(), + ) + .await; + + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(body["status"], 422); + assert!( + body["error"] + .as_str() + .expect("error") + .contains("paused -> reset") ); assert_eq!( - state - .pool - .lock() - .expect("pool") - .stats(&key) - .quarantine_count, - 1 + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Paused ); + let persisted = state + .state_store + .load(uuid) + .expect("persisted paused state"); + assert_eq!(persisted.state, SandboxState::Paused); + assert!(state.manager.backend_owner(uuid).is_some()); } #[tokio::test] @@ -1730,7 +1579,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(GuestMockSpawner)), BackendKind::Mock, storage, @@ -1829,7 +1678,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -1864,7 +1713,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(GuestMockSpawner)), BackendKind::Mock, storage, @@ -1928,7 +1777,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -2056,7 +1905,7 @@ mod tests { }); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, storage, @@ -2083,7 +1932,7 @@ mod tests { let reached = Arc::new(Notify::new()); let state = build_test_state( config.clone(), - test_policy(BackendKind::Bubblewrap, false), + test_policy(BackendKind::Bubblewrap), spawners( BackendKind::Bubblewrap, Arc::new(PreSpawnBoundarySpawner { @@ -2141,7 +1990,7 @@ mod tests { )); let recovered = build_test_state( config.clone(), - test_policy(BackendKind::Bubblewrap, false), + test_policy(BackendKind::Bubblewrap), spawners(BackendKind::Bubblewrap, Arc::new(BubblewrapSpawner)), BackendKind::Bubblewrap, recovered_storage, @@ -2199,7 +2048,6 @@ mod tests { BackendKind::Bubblewrap, WorkloadClass::AgentTool, "sha256:locked-handoff".into(), - StartPath::Cold, "pid-handoff-test".into(), ); instance @@ -2238,7 +2086,7 @@ mod tests { ); let state = build_test_state( config.clone(), - test_policy(BackendKind::Bubblewrap, false), + test_policy(BackendKind::Bubblewrap), spawners(BackendKind::Bubblewrap, Arc::new(BubblewrapSpawner)), BackendKind::Bubblewrap, storage, @@ -2297,38 +2145,6 @@ mod tests { )); } - #[tokio::test] - async fn mock_fallback_uses_runtime_backend_for_warm_reuse() { - let temp = tempfile::tempdir().expect("temp"); - let config = test_config(&temp); - let storage: Arc = Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - config.storage.instances_dir.clone(), - )); - let state = build_test_state( - config, - test_policy(BackendKind::Firecracker, true), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage, - ); - let request = test_request(); - - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - assert_eq!(cold["instance"]["backend"], "mock"); - assert_eq!(cold["selected_backend"], "mock"); - assert!(cold["instance"]["operation"].is_null()); - - reset_instance(&state, &id).await.expect("return to pool"); - let warm = created_json(&state, &request).await; - assert_eq!(warm["instance"]["id"], id); - assert_eq!(warm["instance"]["backend"], "mock"); - assert_eq!(warm["selected_backend"], "mock"); - assert_eq!(warm["start_path"], "warm"); - assert!(warm["instance"]["operation"].is_null()); - } - #[tokio::test] async fn partial_spawn_failure_retains_owner_and_storage_for_destroy() { let temp = tempfile::tempdir().expect("temp"); @@ -2340,7 +2156,7 @@ mod tests { )); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(PartialSpawnSpawner)), BackendKind::Mock, storage, @@ -2395,7 +2211,6 @@ mod tests { BackendKind::Bubblewrap, WorkloadClass::AgentTool, "sha256:recovery".into(), - StartPath::Cold, "recovery-test".into(), ); instance @@ -2430,7 +2245,7 @@ mod tests { ); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), registry, BackendKind::Mock, storage, @@ -2454,7 +2269,7 @@ mod tests { )); let initial_state = build_test_state( config.clone(), - test_policy(BackendKind::Firecracker, false), + test_policy(BackendKind::Firecracker), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, initial_storage, @@ -2486,7 +2301,7 @@ mod tests { )); let restarted = build_test_state( config, - test_policy(BackendKind::Firecracker, false), + test_policy(BackendKind::Firecracker), registry, BackendKind::Mock, restarted_storage, @@ -2509,7 +2324,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:write-ahead".into(), - StartPath::Cold, "write-ahead-test".into(), ); instance @@ -2528,7 +2342,7 @@ mod tests { )); let restarted = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners( BackendKind::Mock, Arc::new(RecordingSpawner { @@ -2550,220 +2364,12 @@ mod tests { assert!(!instances_dir.join(id.to_string()).exists()); } - #[tokio::test] - async fn warm_reconstruct_restores_transient_failure_for_retry() { - let temp = tempfile::tempdir().expect("temp"); - let config = test_config(&temp); - let storage = Arc::new(TransientReconstructStorage::new( - config.storage.images_dir.clone(), - config.storage.instances_dir.clone(), - )); - let state = build_test_state( - config, - test_policy(BackendKind::Mock, true), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage.clone(), - ); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); - - storage.fail_reconstruct.store(true, Ordering::Release); - let error = create_instance(&state, &request) - .await - .expect_err("transient error must preserve claim"); - assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); - let uuid = Uuid::parse_str(&id).expect("uuid"); - assert_eq!( - state.instances.lock().expect("instances")[&uuid].state, - SandboxState::Warm - ); - let key = PoolKey::new( - BackendKind::Mock, - WorkloadClass::AgentTool, - "sha256:ownership-test".into(), - ); - assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 1); - - storage.fail_reconstruct.store(false, Ordering::Release); - let retried = created_json(&state, &request).await; - assert_eq!(retried["instance"]["id"], id); - assert_eq!(retried["start_path"], "warm"); - } - - #[tokio::test] - async fn warm_reconstruct_quarantines_an_incomplete_slot() { - let temp = tempfile::tempdir().expect("temp"); - let config = test_config(&temp); - let instances_dir = config.storage.instances_dir.clone(); - let storage: Arc = Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - instances_dir.clone(), - )); - let state = build_test_state( - config, - test_policy(BackendKind::Mock, true), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage, - ); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); - std::fs::remove_file(instances_dir.join(&id).join("mem.bin")).expect("remove artifact"); - - let replacement = created_json(&state, &request).await; - assert_ne!(replacement["instance"]["id"], id); - assert_eq!(replacement["start_path"], "cold"); - let uuid = Uuid::parse_str(&id).expect("uuid"); - let replacement_id = Uuid::parse_str( - replacement["instance"]["id"] - .as_str() - .expect("replacement id"), - ) - .expect("replacement uuid"); - assert_eq!( - state.instances.lock().expect("instances")[&uuid].state, - SandboxState::Destroyed - ); - assert!(matches!( - state.state_store.run_dir(uuid), - Err(BlazeDaemonError::NotFound(_)) - )); - assert!(state.state_store.run_dir(replacement_id).is_ok()); - } - - #[tokio::test] - async fn warm_quarantine_cleanup_failure_retains_resources_for_destroy_retry() { - let temp = tempfile::tempdir().expect("temp"); - let config = test_config(&temp); - let instances_dir = config.storage.instances_dir.clone(); - let storage: Arc = Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - instances_dir.clone(), - )); - let state = build_test_state( - config, - test_policy(BackendKind::Mock, true), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage, - ); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - let uuid = Uuid::parse_str(&id).expect("uuid"); - reset_instance(&state, &id).await.expect("warm"); - state - .manager - .insert_backend_owner( - uuid, - Arc::new(FailOnceOwner { - instance_id: uuid, - attempts: AtomicUsize::new(0), - }), - ) - .expect("replace backend owner"); - std::fs::remove_file(instances_dir.join(&id).join("mem.bin")).expect("remove artifact"); - - let replacement = created_json(&state, &request).await; - - assert_ne!(replacement["instance"]["id"], id); - let retained = state.instances.lock().expect("instances")[&uuid].clone(); - assert_eq!(retained.state, SandboxState::RecoveryRequired); - assert_eq!(retained.backend_ownership, BackendOwnership::Running); - assert_eq!( - retained.operation.as_ref().map(|operation| operation.kind), - Some(OperationKind::Destroy) - ); - assert!(state.manager.backend_owner(uuid).is_some()); - assert!(state.state_store.run_dir(uuid).is_ok()); - assert!(instances_dir.join(&id).is_dir()); - - destroy_instance(&state, &id) - .await - .expect("retry quarantined destroy"); - - assert_eq!( - state.instances.lock().expect("instances")[&uuid].state, - SandboxState::Destroyed - ); - assert!(state.manager.backend_owner(uuid).is_none()); - assert!(matches!( - state.state_store.run_dir(uuid), - Err(BlazeDaemonError::NotFound(_)) - )); - assert!(!instances_dir.join(&id).exists()); - } - - #[cfg(feature = "test-failpoints")] - async fn assert_warm_state_commit_failure_restores_claim(failpoint: &'static str) { - let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, true); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - let uuid = Uuid::parse_str(&id).expect("uuid"); - reset_instance(&state, &id).await.expect("warm"); - let owner = state.manager.backend_owner(uuid).expect("backend owner"); - let key = PoolKey::new( - BackendKind::Mock, - WorkloadClass::AgentTool, - "sha256:ownership-test".into(), - ); - - let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); - let error = hook - .run(create_instance(&state, &request)) - .await - .expect_err("state commit failure"); - - assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); - let restored = state.instances.lock().expect("instances")[&uuid].clone(); - assert_eq!(restored.state, SandboxState::Warm); - assert!(restored.operation.is_none()); - let persisted = state.state_store.load(uuid).expect("persisted warm state"); - assert_eq!(persisted.state, SandboxState::Warm); - assert_eq!(persisted.backend_ownership, BackendOwnership::Running); - assert!(persisted.operation.is_none()); - let retained_owner = state.manager.backend_owner(uuid).expect("retained owner"); - assert!(Arc::ptr_eq(&owner, &retained_owner)); - assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 1); - assert!(retained_owner.try_wait().await.expect("liveness").is_none()); - - let retried = created_json(&state, &request).await; - assert_eq!(retried["instance"]["id"], id); - assert_eq!(retried["start_path"], "warm"); - let persisted = state - .state_store - .load(uuid) - .expect("persisted running state"); - assert_eq!(persisted.state, SandboxState::Running); - assert_eq!(persisted.backend_ownership, BackendOwnership::Running); - assert!(persisted.operation.is_none()); - } - - #[cfg(feature = "test-failpoints")] - #[tokio::test] - async fn warm_intent_commit_failure_restores_the_claim() { - assert_warm_state_commit_failure_restores_claim("warm-intent-state-commit").await; - } - - #[cfg(feature = "test-failpoints")] - #[tokio::test] - async fn warm_final_commit_failure_restores_the_claim() { - assert_warm_state_commit_failure_restores_claim("warm-final-state-commit").await; - } - #[cfg(feature = "test-failpoints")] #[tokio::test] async fn guest_readiness_failure_compensates_owned_resources() { let request = test_request(); let temp = tempfile::tempdir().expect("temp"); - let state = guest_mock_state(&temp, false); + let state = guest_mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(&["create-guest-ready"]); hook.run(create_instance(&state, &request)) @@ -2795,7 +2401,7 @@ mod tests { let request = test_request(); let spawn_temp = tempfile::tempdir().expect("temp"); - let spawn_state = mock_state(&spawn_temp, false); + let spawn_state = mock_state(&spawn_temp); let spawn_hook = crate::failpoint::TestFailpoint::new(&["create-spawn"]); spawn_hook .run(create_instance(&spawn_state, &request)) @@ -2812,7 +2418,7 @@ mod tests { assert_eq!(spawn_instance.state, SandboxState::Destroyed); let commit_temp = tempfile::tempdir().expect("temp"); - let commit_state = mock_state(&commit_temp, false); + let commit_state = mock_state(&commit_temp); let commit_hook = crate::failpoint::TestFailpoint::new(&["create-state-commit"]); commit_hook .run(create_instance(&commit_state, &request)) @@ -2835,7 +2441,7 @@ mod tests { ); let destroy_temp = tempfile::tempdir().expect("temp"); - let destroy_state = mock_state(&destroy_temp, false); + let destroy_state = mock_state(&destroy_temp); let created = created_json(&destroy_state, &request).await; let id = created["instance"]["id"].as_str().expect("id").to_string(); let kill_hook = crate::failpoint::TestFailpoint::new(&["destroy-kill"]); @@ -2859,7 +2465,7 @@ mod tests { .expect("destroy retry"); let release_temp = tempfile::tempdir().expect("temp"); - let release_state = mock_state(&release_temp, false); + let release_state = mock_state(&release_temp); let created = created_json(&release_state, &request).await; let id = created["instance"]["id"].as_str().expect("id").to_string(); let release_hook = crate::failpoint::TestFailpoint::new(&["storage-release"]); @@ -2889,7 +2495,7 @@ mod tests { failpoints: &'static [&'static str], ) { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(failpoints); let error = hook @@ -2947,7 +2553,7 @@ mod tests { #[tokio::test] async fn initial_publication_failure_before_publish_touches_no_resources() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(&["state-before-first-publication"]); hook.run(create_instance(&state, &test_request())) @@ -2977,7 +2583,7 @@ mod tests { #[tokio::test] async fn initial_publication_sync_failure_is_rolled_back_terminally() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(&["state-first-publication-root-sync"]); hook.run(create_instance(&state, &test_request())) @@ -3020,7 +2626,7 @@ mod tests { #[tokio::test] async fn unconfirmed_initial_publication_is_retained_for_recovery() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(&["state-post-publication-identity"]); let error = hook @@ -3092,7 +2698,7 @@ mod tests { #[tokio::test] async fn unconfirmed_publication_rejects_a_replaced_directory_on_retry() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let hook = crate::failpoint::TestFailpoint::new(&["state-post-publication-identity"]); hook.run(create_instance(&state, &test_request())) @@ -3332,7 +2938,7 @@ mod tests { #[tokio::test] async fn acquire_rollback_failure_retains_a_destroyable_record() { let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, false); + let state = mock_state(&temp); let acquire_hook = crate::failpoint::TestFailpoint::new(&[ "storage-acquire-artifacts", "storage-acquire-rollback", @@ -3380,7 +2986,7 @@ mod tests { )); let initial_state = build_test_state( config.clone(), - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, initial_storage, @@ -3437,7 +3043,7 @@ mod tests { )); let restarted = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners( BackendKind::Mock, Arc::new(RecordingSpawner { @@ -3466,54 +3072,6 @@ mod tests { assert!(!instances_dir.join(id.to_string()).exists()); } - #[cfg(feature = "test-failpoints")] - #[tokio::test] - async fn warm_activation_and_destroy_are_serialized_per_instance() { - let temp = tempfile::tempdir().expect("temp"); - let state = mock_state(&temp, true); - let request = test_request(); - let cold = created_json(&state, &request).await; - let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); - - let pause_hook = crate::failpoint::TestFailpoint::new(&["warm-before-state-commit"]); - let create_state = state.clone(); - let create_request = request.clone(); - let activation_hook = pause_hook.clone(); - let activation = tokio::spawn(async move { - activation_hook - .run(create_instance(&create_state, &create_request)) - .await - }); - pause_hook.wait_until_paused().await; - let uuid = Uuid::parse_str(&id).expect("uuid"); - assert_eq!( - state.instances.lock().expect("instances")[&uuid] - .operation - .as_ref() - .map(|operation| operation.kind), - Some(OperationKind::Create) - ); - - let destroy_state = state.clone(); - let destroy_id = id.clone(); - let destroy = - tokio::spawn(async move { destroy_instance(&destroy_state, &destroy_id).await }); - tokio::task::yield_now().await; - assert!(!destroy.is_finished(), "destroy must wait for activation"); - - pause_hook.release(); - activation - .await - .expect("activation task") - .expect("activation"); - destroy.await.expect("destroy task").expect("destroy"); - assert_eq!( - state.instances.lock().expect("instances")[&uuid].state, - SandboxState::Destroyed - ); - } - #[tokio::test] async fn startup_reconciliation_continues_after_one_cleanup_failure() { let temp = tempfile::tempdir().expect("temp"); @@ -3529,7 +3087,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:reconcile".into(), - StartPath::Cold, "reconcile-test".into(), ); instance.id = id; @@ -3551,7 +3108,7 @@ mod tests { let cleanup_count = Arc::new(AtomicUsize::new(0)); let state = build_test_state( config.clone(), - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners( BackendKind::Mock, Arc::new(SelectiveCleanupSpawner { @@ -3601,6 +3158,89 @@ mod tests { assert_eq!(created["instance"]["state"], "running"); } + #[tokio::test] + async fn startup_reconciliation_destroys_legacy_reset_and_warm_records() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage: Arc = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + let mut ids = Vec::new(); + for state_name in ["reset", "warm"] { + let id = Uuid::new_v4(); + ids.push(id); + let now = chrono::Utc::now(); + let record = json!({ + "id": id, + "state": state_name, + "backend": "mock", + "workload_class": "agent-tool", + "image_digest": "sha256:legacy", + "start_path": "warm", + "created_at": now, + "updated_at": now, + "policy_name": "legacy", + "backend_ownership": "running" + }); + let run_dir = config.daemon.state_dir.join(id.to_string()); + std::fs::create_dir(&run_dir).expect("legacy run directory"); + std::fs::write( + run_dir.join("state.json"), + serde_json::to_vec_pretty(&record).expect("legacy state JSON"), + ) + .expect("legacy state record"); + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("legacy storage"); + } + + let kill_count = Arc::new(AtomicUsize::new(0)); + let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: orphan_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 2); + assert!(report.failures.is_empty()); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 2); + assert_eq!(release_count.load(Ordering::Acquire), 2); + for id in ids { + assert_eq!( + state.instances.lock().expect("instances")[&id].state, + SandboxState::Destroyed + ); + assert!(!config.storage.instances_dir.join(id.to_string()).exists()); + assert!(matches!( + state.state_store.run_dir(id), + Err(BlazeDaemonError::NotFound(_)) + )); + } + } + #[tokio::test] async fn startup_reconciliation_skips_cleanup_for_known_stopped_states() { let temp = tempfile::tempdir().expect("temp"); @@ -3620,7 +3260,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:not-started".into(), - StartPath::Cold, "reconcile-test".into(), ); not_started.id = not_started_id; @@ -3635,7 +3274,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:stopped".into(), - StartPath::Cold, "reconcile-test".into(), ); stopped.id = stopped_id; @@ -3660,7 +3298,7 @@ mod tests { let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); let state = build_test_state( config, - test_policy(BackendKind::Mock, false), + test_policy(BackendKind::Mock), spawners( BackendKind::Mock, Arc::new(CountingSpawner { @@ -3724,7 +3362,6 @@ mod tests { ServerState::build( config, PolicyEngine::with_policies(Vec::new()), - PoolManager::new(), HookRegistry::new(), spawners(BackendKind::Mock, Arc::new(MockSpawner)), BackendKind::Mock, diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 6ccdc8040b..7a228d2939 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -9,7 +9,6 @@ use blaze_core::backend::BackendKind; use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode, StorageSyncSchedule}; use blaze_core::kernel::HookRegistry; use blaze_core::policy::PolicyEngine; -use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use http_body_util::Full; use hyper::body::Bytes; @@ -50,6 +49,12 @@ fn load_daemon_config(config_path: &Path) -> Result { let mut config: DaemonConfig = toml::from_str(&raw).map_err(blaze_core::BlazeError::from)?; absolutize_backend_paths(&mut config)?; config.validate()?; + if config.pool.is_some() { + tracing::warn!( + path = %config_path.display(), + "ignoring legacy packaged [pool] defaults; remove this section because reusable-instance management is unavailable" + ); + } tracing::info!(?config_path, "loaded daemon config"); Ok(LoadedDaemonConfig { config, source }) } @@ -87,7 +92,6 @@ async fn run_loaded_config(loaded: LoadedDaemonConfig) -> Result<()> { // storage initialization so later code cannot reopen a replacement path. let state_store = StateStore::open(config.daemon.state_dir.clone())?; let policy = load_policy_engine(&config, policy_load)?; - let pool = PoolManager::new(); let hook = HookRegistry::new(); let network_required = policy.policies().iter().any(|policy| { policy @@ -134,7 +138,6 @@ async fn run_loaded_config(loaded: LoadedDaemonConfig) -> Result<()> { let state = Arc::new(ServerState::build_with_store( config, policy, - pool, hook, spawners, active_backend, @@ -934,6 +937,23 @@ backend_priority = ["bubblewrap"] ); } + #[test] + fn config_load_accepts_a_preserved_packaged_pool_section() { + let current_dir = std::env::current_dir().expect("current directory"); + let temp = tempfile::tempdir_in(¤t_dir).expect("tempdir below current directory"); + let config_path = temp.path().join("config.toml"); + std::fs::write( + &config_path, + "[daemon]\nlog_level = \"debug\"\n\n[pool]\ndefault_warm_ttl = \"30m\"\ngc_interval = \"5m\"\n", + ) + .expect("write legacy packaged configuration"); + + let loaded = load_daemon_config(&config_path).expect("load preserved package config"); + + assert!(loaded.config.pool.is_some()); + assert_eq!(loaded.config.daemon.log_level, "debug"); + } + #[tokio::test] async fn loaded_config_inside_catalog_fails_before_startup_changes_catalog() { let temp = tempfile::tempdir().expect("tempdir"); diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index d60c98e483..40015abb33 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -12,7 +12,7 @@ use thiserror::Error; pub type Result = std::result::Result; #[derive(Debug, Error)] -pub enum BlazeDaemonError { +pub(crate) enum BlazeDaemonError { #[error("core error: {0}")] Core(#[from] blaze_core::BlazeError), @@ -54,6 +54,9 @@ pub enum BlazeDaemonError { #[error("not found: {0}")] NotFound(String), + #[error("unsupported operation: {0}")] + UnsupportedOperation(String), + #[error("conflict: {0}")] Conflict(String), @@ -107,6 +110,7 @@ impl BlazeDaemonError { match self { BlazeDaemonError::BadRequest(_) => 400, BlazeDaemonError::NotFound(_) => 404, + BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::Conflict(_) => 409, BlazeDaemonError::ServiceUnavailable(_) => 503, BlazeDaemonError::PayloadTooLarge { .. } => 413, diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index 6feabbc04c..51ba6fb755 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! File-based storage provider: creates per-instance directories with //! rootfs and memory files on a local filesystem. Base images and mutable -//! instance slots use separate roots; runtime pooling is owned by the daemon. +//! instance slots use separate roots. use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -349,10 +349,6 @@ impl StorageProvider for FileStorageProvider { fn pool_status(&self) -> PoolStatus { PoolStatus::default() } - - async fn drain_pool(&self) -> Result { - Ok(0) - } } async fn open_required_slot_path( @@ -503,15 +499,15 @@ mod tests { assert!(!dir.exists()); } - #[tokio::test] - async fn pool_status_returns_defaults() { + #[test] + fn pool_status_returns_current_capacity() { let tmp = tempfile::TempDir::new().unwrap(); let provider = FileStorageProvider::new(tmp.path().to_path_buf()); let status = provider.pool_status(); assert_eq!(status.ready, 0); assert_eq!(status.capacity, 0); assert_eq!(status.pending, 0); - assert_eq!(provider.drain_pool().await.unwrap(), 0); + assert_eq!(status.quarantined, 0); } #[tokio::test] diff --git a/src/blaze/crates/blazed/src/metrics.rs b/src/blaze/crates/blazed/src/metrics.rs index 3a5cdeea32..7665693d91 100644 --- a/src/blaze/crates/blazed/src/metrics.rs +++ b/src/blaze/crates/blazed/src/metrics.rs @@ -10,9 +10,6 @@ pub struct Metrics { pub requests_total: AtomicU64, pub instances_created: AtomicU64, pub instances_destroyed: AtomicU64, - pub instances_resets: AtomicU64, - pub pool_hits: AtomicU64, - pub pool_misses: AtomicU64, pub policy_eval_failures: AtomicU64, } @@ -44,21 +41,6 @@ impl Metrics { "Total sandbox instances destroyed", self.instances_destroyed.load(Ordering::Relaxed), ), - ( - "blaze_instances_resets_total", - "Total sandbox instances reset", - self.instances_resets.load(Ordering::Relaxed), - ), - ( - "blaze_pool_hits_total", - "Warm pool hits (instance reused)", - self.pool_hits.load(Ordering::Relaxed), - ), - ( - "blaze_pool_misses_total", - "Warm pool misses (cold boot)", - self.pool_misses.load(Ordering::Relaxed), - ), ( "blaze_policy_eval_failures_total", "Number of failed policy evaluations", @@ -74,3 +56,29 @@ impl Metrics { out } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_keeps_active_counters_and_omits_removed_pool_counters() { + let metrics = Metrics::new(); + metrics.inc(&metrics.requests_total); + metrics.inc(&metrics.instances_created); + + let rendered = metrics.render(); + + assert!(rendered.contains("blaze_requests_total 1")); + assert!(rendered.contains("blaze_instances_created_total 1")); + assert!(rendered.contains("blaze_instances_destroyed_total 0")); + assert!(rendered.contains("blaze_policy_eval_failures_total 0")); + for removed in [ + "blaze_instances_resets_total", + "blaze_pool_hits_total", + "blaze_pool_misses_total", + ] { + assert!(!rendered.contains(removed)); + } + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index d40d76e1ea..21c6b093d5 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//! Recoverable sandbox create, warm activation, destroy, and startup cleanup. +//! Recoverable sandbox create, destroy, and startup cleanup. use std::collections::{HashMap, HashSet}; use std::path::PathBuf; @@ -8,11 +8,8 @@ use std::time::Duration; use blaze_core::BlazeError; use blaze_core::backend::{BackendKind, SpawnRequest}; -use blaze_core::lifecycle::{ - BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, -}; +use blaze_core::lifecycle::{BackendOwnership, OperationKind, SandboxInstance, SandboxState}; use blaze_core::policy::RuntimeDecision; -use blaze_core::pool::{PoolKey, PoolManager}; use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard, Semaphore}; use tokio_util::sync::CancellationToken; @@ -34,7 +31,7 @@ const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); pub struct CreateSandbox { /// Policy decision for this request. pub decision: RuntimeDecision, - /// Image identity used by storage and warm-pool matching. + /// Image identity used by storage allocation. pub image_digest: String, /// Concrete backend selected from the policy and daemon availability. pub runtime_backend: BackendKind, @@ -74,7 +71,7 @@ pub struct ReconcileReport { /// Owns durable lifecycle metadata and non-serializable runtime handles. /// /// The maps are shared with read-only and non-lifecycle API paths. All -/// create, warm activation, destroy, and restart cleanup mutations enter +/// Create, destroy, and restart cleanup mutations enter /// through this type and are serialized by a per-sandbox async lock. pub struct SandboxManager { instances: Arc>>, @@ -82,7 +79,6 @@ pub struct SandboxManager { operation_locks: Mutex>>>, pub(super) storage_sync_inflight: Arc>>, pub(super) storage_sync_permits: Arc, - pool: Arc>, spawners: Arc, active_backend: BackendKind, storage: Arc, @@ -96,7 +92,6 @@ pub struct SandboxManager { /// Construction inputs grouped to keep daemon wiring explicit. pub struct SandboxManagerInit { pub instances: HashMap, - pub pool: PoolManager, pub spawners: SpawnerRegistry, pub active_backend: BackendKind, pub storage: Arc, @@ -110,7 +105,6 @@ pub struct SandboxManagerInit { /// ownership, such as checkpoint and reset. pub struct SandboxManagerResources { pub instances: Arc>>, - pub pool: Arc>, pub metrics: Arc, } @@ -119,7 +113,6 @@ impl SandboxManager { pub fn new(init: SandboxManagerInit) -> (Self, SandboxManagerResources) { let SandboxManagerInit { instances, - pool, spawners, active_backend, storage, @@ -135,11 +128,9 @@ impl SandboxManager { .collect(); let instances = Arc::new(Mutex::new(instances)); let backend_instances = Arc::new(Mutex::new(HashMap::new())); - let pool = Arc::new(Mutex::new(pool)); let metrics = Arc::new(Metrics::new()); let resources = SandboxManagerResources { instances: instances.clone(), - pool: pool.clone(), metrics: metrics.clone(), }; ( @@ -151,7 +142,6 @@ impl SandboxManager { // The periodic worker is sequential. Retain that bound when a // timed-out provider operation has to finish in the background. storage_sync_permits: Arc::new(Semaphore::new(1)), - pool, spawners: Arc::new(spawners), active_backend, storage, @@ -262,26 +252,12 @@ impl SandboxManager { .map_err(BlazeDaemonError::from) } - /// Create a cold sandbox or activate a compatible warm runtime. + /// Create a sandbox from a fresh runtime allocation. pub async fn create(&self, request: CreateSandbox) -> Result { - let pool_key = PoolKey::new( - request.runtime_backend, - request.decision.workload_class, - request.image_digest.clone(), - ); - if request.decision.pool_eligible { - if let Some(result) = self.activate_warm(&pool_key).await? { - self.metrics.inc(&self.metrics.pool_hits); - return Ok(result); - } - self.metrics.inc(&self.metrics.pool_misses); - } - let mut instance = SandboxInstance::new( request.runtime_backend, request.decision.workload_class, request.image_digest, - StartPath::Cold, request.decision.policy_name.clone(), ); let operation_lock = self.operation_lock(instance.id); @@ -499,269 +475,6 @@ impl SandboxManager { }) } - async fn activate_warm(&self, key: &PoolKey) -> Result> { - let candidate = self.pool.lock().map_err(|_| poisoned("pool"))?.lookup(key); - let Some(id) = candidate else { - return Ok(None); - }; - let operation_lock = self.operation_lock(id); - let _operation = operation_lock.lock().await; - - let instance = self - .instances - .lock() - .map_err(|_| poisoned("instances"))? - .get(&id) - .cloned(); - let backend = self - .backend_instances - .lock() - .map_err(|_| poisoned("backend_instances"))? - .get(&id) - .cloned(); - - let invalid_reason = match (&instance, &backend) { - (None, _) => Some("lifecycle metadata is missing".to_string()), - (_, None) => Some("backend owner is missing".to_string()), - (Some(instance), Some(_)) - if instance.state != SandboxState::Warm || instance.operation.is_some() => - { - Some(format!("lifecycle state is {}", instance.state)) - } - (Some(instance), Some(backend)) if backend.backend() != instance.backend => { - Some(format!( - "backend owner is {}, metadata is {}", - backend.backend(), - instance.backend - )) - } - (_, Some(backend)) => match backend.try_wait().await { - Ok(None) => None, - Ok(Some(status)) => Some(format!("backend exited: {status:?}")), - Err(error) => Some(format!("backend liveness check failed: {error}")), - }, - }; - if let Some(reason) = invalid_reason { - self.quarantine_warm(key, id, instance, backend, &reason) - .await; - return Ok(None); - } - - let original = instance.expect("validated warm metadata"); - match self.storage.reconstruct(&id.to_string()).await { - Ok(_) => {} - Err(error @ BlazeError::StorageIncomplete { .. }) => { - self.quarantine_warm( - key, - id, - Some(original), - backend, - &format!("storage validation failed: {error}"), - ) - .await; - return Ok(None); - } - Err(error) => { - return Err(self.restore_warm_claim(key, original, error.into())); - } - } - - let mut activating = original.clone(); - activating.begin_operation(OperationKind::Create); - if let Err(error) = crate::failpoint::state("warm-intent-state-commit") - .and_then(|_| self.state_store.persist(&activating)) - { - return Err(self.restore_warm_claim(key, original, error)); - } - if let Some(error) = self.retain_instance(activating.clone()) { - return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); - } - - crate::failpoint::pause("warm-before-state-commit").await; - let selected_backend = backend.expect("validated warm backend").backend(); - if let Err(error) = activating - .transition(SandboxState::Creating) - .and_then(|_| activating.transition(SandboxState::Running)) - { - return Err(self.restore_warm_claim(key, original, error.into())); - } - activating.finish_operation(); - if let Err(error) = crate::failpoint::state("warm-final-state-commit") - .and_then(|_| self.state_store.persist(&activating)) - { - return Err(self.restore_warm_claim(key, original, error)); - } - if let Some(error) = self.retain_instance(activating.clone()) { - return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); - } - Ok(Some(CreateSandboxResult { - instance: activating, - selected_backend, - })) - } - - fn restore_warm_claim( - &self, - key: &PoolKey, - instance: SandboxInstance, - cause: BlazeDaemonError, - ) -> BlazeDaemonError { - let id = instance.id; - let mut errors = Vec::new(); - if let Err(error) = self.state_store.persist(&instance) { - errors.push(format!("restore warm state persistence failed: {error}")); - } - if let Some(error) = self.retain_instance(instance) { - errors.push(error); - } - match self.pool.lock() { - Ok(mut pool) => pool.restore_lookup(key.clone(), id), - Err(poisoned) => { - poisoned.into_inner().restore_lookup(key.clone(), id); - errors.push("pool lock poisoned while restoring warm claim".to_string()); - } - } - let details = if errors.is_empty() { - "warm claim restored for retry".to_string() - } else { - format!("warm claim restored with errors: {}", errors.join("; ")) - }; - BlazeDaemonError::RecoveryRequired(format!("{cause}; instance {id}: {details}")) - } - - async fn quarantine_warm( - &self, - key: &PoolKey, - id: Uuid, - mut instance: Option, - backend: Option, - reason: &str, - ) { - match self.pool.lock() { - Ok(mut pool) => pool.quarantine(key, id), - Err(poisoned) => poisoned.into_inner().quarantine(key, id), - } - tracing::warn!(instance = %id, reason, "warm instance validation failed"); - - let Some(metadata) = instance.as_mut() else { - if let Some(backend) = backend.as_ref() - && let Err(error) = backend.kill().await - { - tracing::error!( - instance = %id, - %error, - "quarantined backend cleanup failed" - ); - } - tracing::error!( - instance = %id, - "quarantined lifecycle metadata missing; retaining storage" - ); - return; - }; - metadata.begin_operation(OperationKind::Destroy); - if let Err(error) = self.state_store.persist(metadata) { - tracing::error!(instance = %id, %error, "quarantine intent commit failed"); - return; - } - if let Some(error) = self.retain_instance(metadata.clone()) { - tracing::error!(instance = %id, %error, "quarantine intent retention failed"); - return; - } - - let backend_stopped = match backend.as_ref() { - Some(backend) => match backend.kill().await { - Ok(()) => true, - Err(error) => { - tracing::error!(instance = %id, %error, "quarantined backend cleanup failed"); - false - } - }, - None if matches!( - metadata.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) => - { - true - } - None => match self.spawners.get(metadata.backend) { - Some(spawner) => match self.state_store.run_dir(id) { - Ok(run_dir) => match spawner.cleanup_orphan(id, &run_dir).await { - Ok(()) => true, - Err(error) => { - tracing::error!( - instance = %id, - %error, - "quarantined orphan cleanup failed" - ); - false - } - }, - Err(error) => { - tracing::error!( - instance = %id, - %error, - "quarantined run-directory ownership failed" - ); - false - } - }, - None => { - tracing::error!( - instance = %id, - backend = %metadata.backend, - "quarantined backend has no recovery spawner" - ); - false - } - }, - }; - if !backend_stopped { - let _ = self.mark_recovery(id); - return; - } - - metadata.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = self.state_store.persist(metadata) { - tracing::error!(instance = %id, %error, "quarantined stop state commit failed"); - let _ = self.mark_recovery(id); - return; - } - if let Some(error) = self.retain_instance(metadata.clone()) { - tracing::error!(instance = %id, %error, "quarantined stop state retention failed"); - let _ = self.mark_recovery(id); - return; - } - if let Err(error) = self.storage.release_by_id(&id.to_string()).await { - tracing::error!(instance = %id, %error, "quarantined storage cleanup failed"); - let _ = self.mark_recovery(id); - return; - } - - if metadata.state != SandboxState::Destroyed - && let Err(error) = metadata.transition(SandboxState::Destroyed) - { - tracing::error!(instance = %id, %error, "quarantined lifecycle cleanup failed"); - let _ = self.mark_recovery(id); - return; - } - metadata.finish_operation(); - if let Err(error) = self.state_store.persist(metadata) { - tracing::error!(instance = %id, %error, "quarantined state commit failed"); - let _ = self.mark_recovery(id); - return; - } - let _ = self.retain_instance(metadata.clone()); - match self.backend_instances.lock() { - Ok(mut instances) => { - instances.remove(&id); - } - Err(poisoned) => { - poisoned.into_inner().remove(&id); - } - } - } - /// Idempotently destroy one sandbox and its owned runtime resources. pub async fn destroy(&self, id: Uuid) -> Result { let operation_lock = self.operation_lock(id); diff --git a/src/blaze/crates/blazed/src/sandbox/storage_sync.rs b/src/blaze/crates/blazed/src/sandbox/storage_sync.rs index ad23c39eec..ced7f555b9 100644 --- a/src/blaze/crates/blazed/src/sandbox/storage_sync.rs +++ b/src/blaze/crates/blazed/src/sandbox/storage_sync.rs @@ -450,11 +450,8 @@ mod tests { use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::config::TemplateSection; use blaze_core::error::{BlazeError, Result as CoreResult}; - use blaze_core::lifecycle::{ - BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, - }; + use blaze_core::lifecycle::{BackendOwnership, OperationKind, SandboxInstance, SandboxState}; use blaze_core::policy::{BackendConfigs, WorkloadClass}; - use blaze_core::pool::PoolManager; use blaze_core::storage::{ AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, }; @@ -637,10 +634,6 @@ mod tests { fn pool_status(&self) -> PoolStatus { self.inner.pool_status() } - - async fn drain_pool(&self) -> CoreResult { - self.inner.drain_pool().await - } } fn manager( @@ -665,7 +658,6 @@ mod tests { spawners.insert(BackendKind::Mock, Arc::new(MockSpawner)); let (manager, resources) = SandboxManager::new(SandboxManagerInit { instances: HashMap::new(), - pool: PoolManager::new(), spawners, active_backend: BackendKind::Mock, storage, @@ -725,7 +717,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:sync-test".into(), - StartPath::Cold, "sync-test".into(), ); metadata.id = id; diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index efa8d485f3..2c5247cdd1 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//! Daemon-wide shared state: configuration, policy engine, pool, hook registry, +//! Daemon-wide shared state: configuration, policy engine, hook registry, //! and the in-memory instance map. All API handlers //! receive an [`Arc`] and acquire the relevant `Mutex<...>` //! lock just long enough to read or mutate the piece they need — locks @@ -13,7 +13,6 @@ use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; use blaze_core::lifecycle::SandboxInstance; use blaze_core::policy::PolicyEngine; -use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use uuid::Uuid; @@ -31,7 +30,6 @@ use crate::state_store::StateStore; pub struct ServerState { pub config: Mutex, pub policy: Mutex, - pub pool: Arc>, pub hook: Mutex, pub instances: Arc>>, pub manager: Arc, @@ -50,7 +48,6 @@ impl ServerState { pub fn build_with_store( config: DaemonConfig, policy: PolicyEngine, - pool: PoolManager, hook: HookRegistry, spawners: SpawnerRegistry, active_backend: BackendKind, @@ -61,7 +58,6 @@ impl ServerState { Self::assemble( config, policy, - pool, hook, spawners, active_backend, @@ -76,7 +72,6 @@ impl ServerState { pub fn build( config: DaemonConfig, policy: PolicyEngine, - pool: PoolManager, hook: HookRegistry, spawners: SpawnerRegistry, active_backend: BackendKind, @@ -97,7 +92,6 @@ impl ServerState { Self::assemble( config, policy, - pool, hook, spawners, active_backend, @@ -111,7 +105,6 @@ impl ServerState { fn assemble( config: DaemonConfig, policy: PolicyEngine, - pool: PoolManager, hook: HookRegistry, spawners: SpawnerRegistry, active_backend: BackendKind, @@ -122,7 +115,6 @@ impl ServerState { let instances = state_store.scan()?; let (manager, resources) = SandboxManager::new(SandboxManagerInit { instances, - pool, spawners, active_backend, storage: storage.clone(), @@ -135,7 +127,6 @@ impl ServerState { Ok(Self { config: Mutex::new(config), policy: Mutex::new(policy), - pool: resources.pool, hook: Mutex::new(hook), instances: resources.instances, manager: Arc::new(manager), @@ -154,7 +145,7 @@ impl ServerState { #[cfg(test)] mod tests { - use blaze_core::lifecycle::{BackendOwnership, SandboxState, StartPath}; + use blaze_core::lifecycle::{BackendOwnership, SandboxState}; use blaze_core::policy::WorkloadClass; use crate::file_provider::FileStorageProvider; @@ -195,7 +186,6 @@ mod tests { ServerState::build_with_store( config, PolicyEngine::new(), - PoolManager::new(), HookRegistry::new(), spawners, BackendKind::Mock, @@ -235,7 +225,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:existing".into(), - StartPath::Cold, "default".into(), ); existing @@ -253,7 +242,6 @@ mod tests { let state = ServerState::build_with_store( config, PolicyEngine::new(), - PoolManager::new(), HookRegistry::new(), spawners, BackendKind::Mock, @@ -274,7 +262,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:new".into(), - StartPath::Cold, "default".into(), ); state @@ -318,7 +305,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:terminal".into(), - StartPath::Cold, "default".into(), ); stored diff --git a/src/blaze/crates/blazed/src/state_store.rs b/src/blaze/crates/blazed/src/state_store.rs index 0655b48ca5..042d5f2da8 100644 --- a/src/blaze/crates/blazed/src/state_store.rs +++ b/src/blaze/crates/blazed/src/state_store.rs @@ -999,7 +999,6 @@ impl OwnedRunDir { #[cfg(test)] mod tests { use blaze_core::backend::BackendKind; - use blaze_core::lifecycle::StartPath; use blaze_core::policy::WorkloadClass; use super::*; @@ -1009,7 +1008,6 @@ mod tests { BackendKind::Mock, WorkloadClass::AgentTool, "sha256:test".into(), - StartPath::Cold, "default".into(), ) } diff --git a/src/blaze/dist/blaze.spec b/src/blaze/dist/blaze.spec index 3fe3ad17c2..746145098f 100644 --- a/src/blaze/dist/blaze.spec +++ b/src/blaze/dist/blaze.spec @@ -18,8 +18,8 @@ Provides: anolisa-component(blaze) %description Blaze is the ANOLISA per-host sandbox orchestrator daemon. It manages sandbox instance lifecycles via HTTP API with policy-driven backend selection, supporting -Firecracker microVM, Bubblewrap, and Mock backends. Features include warm-pool -pre-allocation, multi-backend fallback, and Prometheus metrics export. +Firecracker microVM, Bubblewrap, and Mock backends. Features include +multi-backend fallback and Prometheus metrics export. %prep %setup -q diff --git a/src/blaze/docs/design/lifecycle-state-consistency.md b/src/blaze/docs/design/lifecycle-state-consistency.md index 2429049310..fa7244c2a8 100644 --- a/src/blaze/docs/design/lifecycle-state-consistency.md +++ b/src/blaze/docs/design/lifecycle-state-consistency.md @@ -1,14 +1,18 @@ -# Lifecycle State Consistency +# Lifecycle State Consistency and Compatibility [中文版](lifecycle-state-consistency_zh.md) -Blaze must reconstruct a complete persisted sandbox inventory before it can -reconcile resources or serve API requests. This document defines how the daemon -coordinates lifecycle-state writers, validates the startup inventory, and -publishes that inventory without exposing a partial result. +Blaze has two related lifecycle boundaries. Before serving requests, it must +reconstruct a complete persisted sandbox inventory without exposing a partial +result. While serving requests, it must reject reset and reusable-instance +operations that cannot preserve runtime and storage ownership. Retired `Reset`, +`Warm`, and `start_path = "warm"` values remain decodable so startup can clean +non-terminal records that contain them. -This protocol does not change the HTTP API, configuration keys, or the -persisted JSON format. +This document defines both boundaries. The inventory-publication protocol does +not change the HTTP API, configuration keys, or persisted JSON format. The reset +and reusable-instance section defines the public compatibility protocol that +follows from the reachable lifecycle states. ## Terms and owned objects @@ -88,8 +92,45 @@ entries remains separate from rejected-record handling. After a complete inventory has been accepted, startup reconciliation processes each non-terminal sandbox independently. A cleanup failure for one sandbox can -leave that sandbox in `RecoveryRequired` without turning the already validated -inventory into a partial one. +retain that sandbox in memory as `RecoveryRequired` without turning the already +validated inventory into a partial one. Blaze attempts to persist the recovery +state; if that write also fails, reconciliation reports the additional error +and the durable record may still contain its previous state. + +## Reset and reusable-instance compatibility boundary + +`POST /v1/instances/{id}/reset` has no successful path until Blaze can reset +runtime and storage as one operation. A malformed identifier returns `400 Bad +Request`, an unknown sandbox returns `404 Not Found`, and an existing sandbox +that is not `Running` returns `422 Unprocessable Entity`. A running sandbox +returns `501 Not Implemented`. Every rejection occurs before any in-memory or +persisted lifecycle change and before any change to runtime or storage +ownership. + +The following reserved management routes also return `501 Not Implemented` and +do not manage reusable capacity: + +- `GET /v1/pools`; +- `GET /v1/pools/{backend}/{class}`; +- `POST /v1/pools/{backend}/{class}/drain`; and +- `PUT /v1/pools/{backend}/{class}/sizing`. + +`GET /v1/health` retains its `storage_pool` object for response compatibility; +the file provider reports zero ready, capacity, pending, and quarantined slots. +The metrics endpoint no longer publishes reset, pool-hit, or pool-miss counters +because those operations have no supported success path. + +New sandbox creation always records `start_path = "cold"`. Lifecycle +transitions cannot enter `Reset` or `Warm`, so no supported path can produce or +reactivate a reusable sandbox. Blaze retains decoding of legacy `Reset`, `Warm`, and +`start_path = "warm"` values only so startup can release resources owned by +records written by earlier releases. After the complete inventory passes +validation, reconciliation destroys each such non-terminal record. Successful +cleanup reaches `Destroyed`. Failed cleanup retains the in-memory record as +`RecoveryRequired` and attempts to persist that state; a persistence failure is +reported and may leave the prior durable state intact. Reconciliation continues +with other accepted records. Create requests never select or reactivate a +legacy record. ## Consistency boundary @@ -113,6 +154,10 @@ Future lifecycle-state changes must preserve these rules: - startup holds the run-directory map lock until the complete inventory is accepted or rejected; - the final UUID enumeration completes before retained objects are - revalidated; and + revalidated; - no request handler can observe either startup map before all inventory - checks have passed. + checks have passed; +- reset and pool-management rejections occur before lifecycle, runtime, or + storage ownership changes; and +- lifecycle operations cannot enter or reactivate `Reset` or `Warm`; legacy + values are cleanup inputs only. diff --git a/src/blaze/docs/design/lifecycle-state-consistency_zh.md b/src/blaze/docs/design/lifecycle-state-consistency_zh.md index ba0b8c78e2..358a5abd0e 100644 --- a/src/blaze/docs/design/lifecycle-state-consistency_zh.md +++ b/src/blaze/docs/design/lifecycle-state-consistency_zh.md @@ -1,12 +1,14 @@ -# 生命周期状态一致性 +# 生命周期状态一致性与兼容性 [English](lifecycle-state-consistency.md) -Blaze 必须完整重建已经持久化的 sandbox 清单，才能开始资源恢复或提供 API -请求。本设计说明 daemon 如何协调生命周期状态写入、校验启动清单，以及如何在 -不暴露部分结果的前提下发布该清单。 +Blaze 有两个相互关联的生命周期边界。提供请求服务前，它必须完整重建已经持久化 +的 sandbox 清单，且不能暴露部分结果。提供请求服务期间，它必须拒绝无法保持运行 +环境与存储所有权的重置和实例复用操作。已停用的 `Reset`、`Warm` 和 +`start_path = "warm"` 值继续可解析，以便启动恢复清理包含这些值的非终态记录。 -这一协议不改变 HTTP API、配置项或持久化 JSON 格式。 +本设计定义这两个边界。清单发布流程不改变 HTTP API、配置项或持久化 JSON 格式。 +重置与实例复用章节定义由可达生命周期状态决定的公开兼容协议。 ## 概念与持有对象 @@ -73,8 +75,38 @@ Blaze 会保留被拒绝的 UUID 目录及其 `state.json`，供运维人员检 已有的状态发布 staging 条目清理流程与拒绝记录的处理相互独立。 完整清单通过校验后，启动恢复会分别处理每个非终态 sandbox。单个 sandbox -清理失败时可以保留为 `RecoveryRequired`，但不会把已经通过校验的清单变成 -部分清单。 +清理失败时可以在内存中保留为 `RecoveryRequired`，但不会把已经通过校验的清单 +变成部分清单。Blaze 会尝试持久化恢复状态；如果这次写入也失败，启动恢复会报告 +附加错误，持久化记录仍可能保留先前的状态。 + +## 重置与可复用实例兼容性边界 + +在 Blaze 能够把运行环境和存储作为一个整体完成重置之前， +`POST /v1/instances/{id}/reset` 没有成功路径。格式错误的标识符返回 +`400 Bad Request`，不存在的 sandbox 返回 `404 Not Found`，处于非 `Running` +状态的已有 sandbox 返回 `422 Unprocessable Entity`，运行中的 sandbox 返回 +`501 Not Implemented`。所有拒绝都发生在修改内存或持久化生命周期状态之前， +也不会改变运行环境或存储所有权。 + +以下保留的管理路由同样返回 `501 Not Implemented`，并且不会管理复用容量： + +- `GET /v1/pools`； +- `GET /v1/pools/{backend}/{class}`； +- `POST /v1/pools/{backend}/{class}/drain`； +- `PUT /v1/pools/{backend}/{class}/sizing`。 + +为了保持响应兼容，`GET /v1/health` 会继续返回 `storage_pool` 对象；文件存储 +提供者报告的就绪、容量、待处理和隔离槽位数量均为零。由于重置、资源池命中和 +资源池未命中都没有受支持的成功路径，监控接口不再发布对应的计数指标。 + +新建 sandbox 始终记录 `start_path = "cold"`。生命周期状态转换不能进入 `Reset` +或 `Warm`，因此没有受支持的路径可以产生或重新启用可复用 sandbox。Blaze 继续 +解析旧版本写入的 `Reset`、`Warm` 和 `start_path = "warm"`，唯一目的是让启动 +恢复释放这些记录拥有的资源。完整清单通过校验后，启动恢复会销毁每一条这样的 +非终态记录。清理成功后记录进入 `Destroyed`。清理失败后，内存记录保留为 +`RecoveryRequired`，并尝试持久化该状态；持久化失败会被报告，磁盘上的记录可能 +仍是先前状态。启动恢复会继续处理其他已通过校验的记录。新建请求绝不会选择或 +重新启用旧版记录。 ## 一致性边界 @@ -94,4 +126,6 @@ lock 的 daemon 进程。advisory lock 不会阻止无关进程直接修改该 写入生命周期状态期间持续持有； - 启动过程必须持有 run-directory map lock，直到完整清单被接受或拒绝； - 必须先完成最终 UUID 枚举，再复验保留对象； -- 所有清单检查完成前，request handler 不能观察到任何一个启动 map。 +- 所有清单检查完成前，request handler 不能观察到任何一个启动 map； +- 重置和资源池管理请求必须在生命周期、运行环境或存储所有权发生变化前被拒绝； +- 生命周期操作不能进入或重新启用 `Reset` 或 `Warm`；旧值只能用于清理。 diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 9803ae5722..1017f6ea2c 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -88,12 +88,6 @@ provider = "file" rootfs_size = 8589934592 mem_size = 4294967296 -# Number of pre-warmed storage slots to keep ready in the pool. -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) - -# Whether to prefork (pre-allocate) storage slots at daemon startup. -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) - # Periodic artifact synchronization is opt-in. Set a positive duration to # persist already-written sandbox slot files and directory metadata. sync_interval = "disabled" @@ -101,19 +95,6 @@ sync_interval = "disabled" # Maximum time the scheduler waits for one artifact synchronization attempt. sync_timeout = "30s" -# --------------------------------------------------------------------------- -# Instance pool settings (global defaults for warm-pool recycling). -# Per-workload pool settings are configured in each policy file. -# --------------------------------------------------------------------------- -[pool] -# Default time-to-live for warm (idle) instances before the GC reclaims them. -# Format: duration string (e.g. "30m", "1h", "90s"). Default: "30m". -default_warm_ttl = "30m" - -# How often the garbage collector runs to reap expired warm instances. -# Format: duration string. Default: "5m". -gc_interval = "5m" - # --------------------------------------------------------------------------- # Published template catalog. # --------------------------------------------------------------------------- diff --git a/src/blaze/examples/policies/agent-rl.toml b/src/blaze/examples/policies/agent-rl.toml index a044b5a847..98b2b32488 100644 --- a/src/blaze/examples/policies/agent-rl.toml +++ b/src/blaze/examples/policies/agent-rl.toml @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Default policy for the agent-rl workload class. # -# This policy targets long-running RL rollout workloads that benefit from -# warm-pool recycling and high resource quotas. +# This policy targets long-running RL rollout workloads with high resource +# quotas. # Policy manifest version. Must be 1 (only supported version). manifest_version = 1 @@ -53,34 +53,6 @@ templates = [] # "continue" (silently skip). Default: "fail". fallback_on_missing_hook = "degrade" -# --------------------------------------------------------------------------- -# Warm-pool configuration. Enables pre-created instances for fast cold-start. -# --------------------------------------------------------------------------- -[pool] -# Whether the warm pool is enabled for this policy. Default: false. -enabled = true - -# Minimum number of warm instances to maintain. Default: 0. -min = 4 - -# Target pool size the autoscaler aims for. Default: 0. -target = 16 - -# Maximum pool size (hard cap). Default: 0. -max = 64 - -# Time-to-live for idle warm instances before the GC reclaims them. -# Overrides [pool].default_warm_ttl from config.toml for this policy. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". -warm_ttl = "30m" - -# How the pool resets an instance after use so it can be returned to warm. -# Options: "mm-template" (memory-template rollback, requires kernel hook), -# "overlayfs-rollback" (filesystem-level rollback), -# "full-recreate" (destroy and recreate from scratch). -# Default: "mm-template". -reset_mode = "full-recreate" - # --------------------------------------------------------------------------- # Resource quotas applied to each sandbox instance (cgroup v2 knobs). # All fields are optional; omitting a field means no limit for that resource. diff --git a/src/blaze/examples/policies/agent-tool.toml b/src/blaze/examples/policies/agent-tool.toml index edf3bfa2b6..f0fa698b61 100644 --- a/src/blaze/examples/policies/agent-tool.toml +++ b/src/blaze/examples/policies/agent-tool.toml @@ -46,31 +46,6 @@ templates = [] # "continue" (silently skip). Default: "fail". fallback_on_missing_hook = "fail" -# --------------------------------------------------------------------------- -# Warm-pool configuration. -# --------------------------------------------------------------------------- -[pool] -# Whether the warm pool is enabled for this policy. Default: false. -enabled = true - -# Minimum number of warm instances to maintain. Default: 0. -min = 2 - -# Target pool size the autoscaler aims for. Default: 0. -target = 8 - -# Maximum pool size (hard cap). Default: 0. -max = 32 - -# Time-to-live for idle warm instances before the GC reclaims them. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". -warm_ttl = "30m" - -# How the pool resets an instance after use. -# Options: "mm-template", "overlayfs-rollback", "full-recreate". -# Default: "mm-template". -reset_mode = "full-recreate" - # --------------------------------------------------------------------------- # Resource quotas (cgroup v2 knobs). Optional; omit to leave uncapped. # --------------------------------------------------------------------------- diff --git a/src/blaze/manifests/blaze.toml b/src/blaze/manifests/blaze.toml index 77434cdee9..3b34577838 100644 --- a/src/blaze/manifests/blaze.toml +++ b/src/blaze/manifests/blaze.toml @@ -37,8 +37,3 @@ runtime = ["firecracker", "bubblewrap"] name = "function_tier" label = "Linux-sandbox function tier backend" default = true - -[[features]] -name = "warm_pool" -label = "Warm sandbox pool management" -default = true