diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index cb47dd4e..38066460 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -30,7 +30,7 @@ const ( CacheBackendTypeExternal CacheBackendType = "External" ) -// +kubebuilder:validation:Enum=Redis;LMCacheServer;Mooncake +// +kubebuilder:validation:Enum=Redis;LMCacheServer;Mooncake;NFS // CacheBackendRemoteStorageProvider identifies the technology used for the // optional shared/remote cache tier. @@ -40,6 +40,7 @@ const ( CacheBackendRemoteStorageProviderRedis CacheBackendRemoteStorageProvider = "Redis" CacheBackendRemoteStorageProviderLMCacheServer CacheBackendRemoteStorageProvider = "LMCacheServer" CacheBackendRemoteStorageProviderMooncake CacheBackendRemoteStorageProvider = "Mooncake" + CacheBackendRemoteStorageProviderNFS CacheBackendRemoteStorageProvider = "NFS" ) // +kubebuilder:validation:Enum=Managed;External @@ -133,6 +134,18 @@ const ( SGLangHiCacheMemoryPageHead SGLangHiCacheMemoryLayout = "page_head" ) +// +kubebuilder:validation:Enum=best_effort;wait_complete;timeout + +// SGLangHiCacheStoragePrefetchPolicy controls when a storage-tier prefetch +// stops before request execution continues. +type SGLangHiCacheStoragePrefetchPolicy string + +const ( + SGLangHiCacheStoragePrefetchBestEffort SGLangHiCacheStoragePrefetchPolicy = "best_effort" + SGLangHiCacheStoragePrefetchWaitComplete SGLangHiCacheStoragePrefetchPolicy = "wait_complete" + SGLangHiCacheStoragePrefetchTimeout SGLangHiCacheStoragePrefetchPolicy = "timeout" +) + // SGLangHiCacheSpec configures SGLang's native, engine-local host-memory cache. // Exactly one of SizeGB and Ratio must be set. Optional tuning fields are // passed to SGLang only when explicitly configured, so the engine version owns @@ -162,6 +175,11 @@ type SGLangHiCacheSpec struct { // MemoryLayout maps to --hicache-mem-layout. // +optional MemoryLayout SGLangHiCacheMemoryLayout `json:"memoryLayout,omitempty"` + + // StoragePrefetchPolicy maps to --hicache-storage-prefetch-policy when + // remoteStorage.provider=NFS. It is rejected without that storage tier. + // +optional + StoragePrefetchPolicy SGLangHiCacheStoragePrefetchPolicy `json:"storagePrefetchPolicy,omitempty"` } // CacheBackendHostMemorySpec configures engine-side host memory. Capacity is @@ -248,6 +266,24 @@ type MooncakeRemoteStorageSpec struct { Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } +// NFSRemoteStorageSpec identifies an existing NFS export mounted into each +// selected engine Pod. NFS is externally owned; inference-cache creates no +// mount target, export, PV, PVC, or StorageClass. +type NFSRemoteStorageSpec struct { + // Server is the NFS mount-target hostname or IP address. + // +kubebuilder:validation:MinLength=1 + Server string `json:"server"` + + // Path is the absolute path exported by the NFS server. + // +kubebuilder:validation:MinLength=1 + Path string `json:"path"` + + // MountPath is the absolute path at which the export is mounted in the + // engine container. + // +kubebuilder:validation:MinLength=1 + MountPath string `json:"mountPath"` +} + // CacheBackendRemoteStorageSpec configures the optional shared/remote tier. // Omitting this object in the canonical API requests an engine-local, // host-only hierarchy and never implicitly provisions infrastructure. @@ -259,8 +295,9 @@ type CacheBackendRemoteStorageSpec struct { // workload or connects to operator-managed infrastructure. Ownership CacheBackendRemoteStorageOwnership `json:"ownership"` - // Endpoint is required for External ownership and rejected for Managed - // ownership, whose endpoint is controller-observed. + // Endpoint is required for network providers with External ownership and + // rejected for Managed ownership, whose endpoint is controller-observed. + // NFS is mounted from its typed server/path fields and has no endpoint. // +optional Endpoint string `json:"endpoint,omitempty"` @@ -275,6 +312,11 @@ type CacheBackendRemoteStorageSpec struct { // Mooncake contains Mooncake-owned configuration. // +optional Mooncake *MooncakeRemoteStorageSpec `json:"mooncake,omitempty"` + + // NFS identifies an externally owned NFS export and its engine-container + // mount path. It is valid only with provider=NFS and ownership=External. + // +optional + NFS *NFSRemoteStorageSpec `json:"nfs,omitempty"` } // CacheBackendObservationSpec configures KV-event observation independently @@ -658,21 +700,23 @@ type CacheBackendIntegrationSpec struct { FirstEventTimeout *metav1.Duration `json:"firstEventTimeout,omitempty"` // FailOpen controls whether the engine treats cache lookups as a soft - // dependency. When true (the default), an unreachable or degraded cache - // backend MUST fall back to local prefill and never fail a serving - // request — the cache is an optimization, not a serving dependency. When - // explicitly set to false the engine fails requests on cache - // unreachability ("fail-closed"); the cache becomes a serving - // dependency, which is loud and visible via a Warning Event on the - // owning CacheBackend. + // dependency. For integrations admitted with true (the default), an + // unreachable or degraded cache backend MUST fall back to local prefill + // and never fail a serving request — the cache is an optimization, not a + // serving dependency. When explicitly set to false the cache becomes a + // fail-closed serving dependency, which is loud and visible via a Warning + // Event on the owning CacheBackend. // // The flag is plumbed by the engine adapter as INFERENCECACHE_FAIL_OPEN // (both shipping LMCache adapters — vLLM+LMCache and SGLang+LMCache — // inject it). Per-request fail-open enforcement at the engine level is the // engine/connector's responsibility; the cache plane surfaces the bit so // the engine can honor it. - // SGLangHiCache accepts only the default true value and does not inject this - // env var because native HiCache exposes no equivalent fail-closed control. + // Host-only SGLangHiCache accepts the default true value and does not inject + // this env var because native HiCache exposes no equivalent control. + // NFS-backed SGLangHiCache instead requires an explicit false value: kubelet + // mounts the inline NFS volume before containers start, so export + // availability is inherently a serving dependency. // +optional // +kubebuilder:default=true FailOpen *bool `json:"failOpen,omitempty"` diff --git a/api/v1alpha1/cachebackend_types_test.go b/api/v1alpha1/cachebackend_types_test.go index 26c6d4ec..b262fe94 100644 --- a/api/v1alpha1/cachebackend_types_test.go +++ b/api/v1alpha1/cachebackend_types_test.go @@ -82,13 +82,17 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { remoteStorageSchema := mustProperty(t, specSchema, "remoteStorage") requireRequired(t, remoteStorageSchema, "provider") requireRequired(t, remoteStorageSchema, "ownership") - requireEnum(t, mustProperty(t, remoteStorageSchema, "provider"), []string{"Redis", "LMCacheServer", "Mooncake"}) + requireEnum(t, mustProperty(t, remoteStorageSchema, "provider"), []string{"Redis", "LMCacheServer", "Mooncake", "NFS"}) requireEnum(t, mustProperty(t, remoteStorageSchema, "ownership"), []string{"Managed", "External"}) - for _, field := range []string{"endpoint", "redis", "lmCacheServer", "mooncake"} { + for _, field := range []string{"endpoint", "redis", "lmCacheServer", "mooncake", "nfs"} { if !hasProperty(remoteStorageSchema, field) { t.Fatalf("spec.remoteStorage.%s is missing from CRD schema", field) } } + nfsSchema := mustProperty(t, remoteStorageSchema, "nfs") + for _, field := range []string{"server", "path", "mountPath"} { + requireRequired(t, nfsSchema, field) + } observationSchema := mustProperty(t, specSchema, "observation") for _, field := range []string{"modelID", "firstEventTimeout"} { if !hasProperty(observationSchema, field) { @@ -141,6 +145,11 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { "page_first_kv_split", "page_head", }) + requireEnum(t, mustProperty(t, hiCacheSchema, "storagePrefetchPolicy"), []string{ + "best_effort", + "wait_complete", + "timeout", + }) firstEventTimeoutSchema := mustPath[map[string]any](t, integrationSchema, "properties", "firstEventTimeout") if got, ok := firstEventTimeoutSchema["default"].(string); !ok || got != "5m" { t.Fatalf("integration.firstEventTimeout default = %v, want \"5m\"", firstEventTimeoutSchema["default"]) @@ -266,6 +275,11 @@ func TestCacheBackendDeepCopyCopiesNestedFields(t *testing.T) { Limits: corev1.ResourceList{corev1.ResourceMemory: providerMemory}, }, }, + NFS: &NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, }, Observation: &CacheBackendObservationSpec{ ModelID: "model-a", @@ -286,8 +300,9 @@ func TestCacheBackendDeepCopyCopiesNestedFields(t *testing.T) { MatchLabels: map[string]string{"inferencecache.io/cache-enabled": "true"}, }, HiCache: &SGLangHiCacheSpec{ - SizeGB: &hiCacheSize, - WritePolicy: SGLangHiCacheWriteThrough, + SizeGB: &hiCacheSize, + WritePolicy: SGLangHiCacheWriteThrough, + StoragePrefetchPolicy: SGLangHiCacheStoragePrefetchWaitComplete, }, BackendConfig: map[string]string{"evictionPolicy": "LRU"}, Template: &CacheBackendPodSpecOverride{ @@ -336,6 +351,7 @@ func TestCacheBackendDeepCopyCopiesNestedFields(t *testing.T) { *backend.Spec.LMCache.WorkerPort = 6666 backend.Spec.RemoteStorage.LMCacheServer.Command[0] = "changed" backend.Spec.RemoteStorage.LMCacheServer.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") + backend.Spec.RemoteStorage.NFS.Server = "changed" backend.Spec.Observation.ModelID = "changed" backend.Spec.Observation.FirstEventTimeout.Duration = time.Hour backend.Spec.Integration.FirstEventTimeout.Duration = time.Hour @@ -386,6 +402,9 @@ func TestCacheBackendDeepCopyCopiesNestedFields(t *testing.T) { copied.Spec.RemoteStorage.LMCacheServer.Resources == nil { t.Fatalf("remoteStorage.lmCacheServer nested fields were not deep-copied") } + if copied.Spec.RemoteStorage.NFS == nil || copied.Spec.RemoteStorage.NFS.Server != "10.0.0.25" { + t.Fatalf("copied NFS storage = %+v, want independent original values", copied.Spec.RemoteStorage.NFS) + } copiedProviderMemory := copied.Spec.RemoteStorage.LMCacheServer.Resources.Limits[corev1.ResourceMemory] if copiedProviderMemory.Cmp(resource.MustParse("2Gi")) != 0 { t.Fatalf("remoteStorage.lmCacheServer resources were not deep-copied") diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4397bc79..9fbec46a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -310,6 +310,11 @@ func (in *CacheBackendRemoteStorageSpec) DeepCopyInto(out *CacheBackendRemoteSto *out = new(MooncakeRemoteStorageSpec) (*in).DeepCopyInto(*out) } + if in.NFS != nil { + in, out := &in.NFS, &out.NFS + *out = new(NFSRemoteStorageSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendRemoteStorageSpec. @@ -968,6 +973,21 @@ func (in *MooncakeRemoteStorageSpec) DeepCopy() *MooncakeRemoteStorageSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NFSRemoteStorageSpec) DeepCopyInto(out *NFSRemoteStorageSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NFSRemoteStorageSpec. +func (in *NFSRemoteStorageSpec) DeepCopy() *NFSRemoteStorageSpec { + if in == nil { + return nil + } + out := new(NFSRemoteStorageSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PDAcceleratorTypeSpec) DeepCopyInto(out *PDAcceleratorTypeSpec) { *out = *in diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 01d7a811..412a2e61 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -21,6 +21,7 @@ import ( "github.com/cachebox-project/inference-cache/internal/controller" podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" cachewebhookv1alpha1 "github.com/cachebox-project/inference-cache/internal/webhook/v1alpha1" + backendprovider "github.com/cachebox-project/inference-cache/pkg/adapters/backend/provider" adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" externaladapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime/external" sglangadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime/sglang" @@ -160,6 +161,10 @@ func main() { adapterruntime.WithSubscriberImage(opts.subscriberImage), adapterruntime.WithPolicyServerGRPCAddress(opts.policyServerGRPCAddress), )) + // Keep the controller and pod webhook on the same remote-provider + // capability registry, just as they share the runtime-adapter registry. + // Today it contains External NFS but no Managed NFS implementation. + backendRegistry := backendprovider.DefaultRegistry() // /probe wrapper for the CacheBackend reconciler's functional-probe gate. // An empty ProbeURL disables the gate — useful for local-dev runs that @@ -169,13 +174,14 @@ func main() { probeClient := &controller.ProbeClient{ProbeURL: opts.serverProbeURL} if err := (&controller.CacheBackendReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: ctrl.Log.WithName("controllers").WithName("CacheBackend"), - Recorder: mgr.GetEventRecorder("cachebackend-controller"), - APIReader: mgr.GetAPIReader(), - Registry: adapterRegistry, - ProbeClient: probeClient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName("CacheBackend"), + Recorder: mgr.GetEventRecorder("cachebackend-controller"), + APIReader: mgr.GetAPIReader(), + Registry: adapterRegistry, + BackendRegistry: backendRegistry, + ProbeClient: probeClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CacheBackend") os.Exit(1) @@ -241,9 +247,10 @@ func main() { // avoid a cold-cache window on controller startup. mgr.GetWebhookServer().Register(podwebhook.WebhookPath, &webhook.Admission{ Handler: &podwebhook.EngineInjector{ - Reader: mgr.GetAPIReader(), - Registry: adapterRegistry, - Log: ctrl.Log.WithName("webhooks").WithName("pod-injector"), + Reader: mgr.GetAPIReader(), + Registry: adapterRegistry, + BackendRegistry: backendRegistry, + Log: ctrl.Log.WithName("webhooks").WithName("pod-injector"), }, }) diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 89ced3b0..33011b7b 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -249,6 +249,15 @@ spec: format: int32 minimum: 1 type: integer + storagePrefetchPolicy: + description: |- + StoragePrefetchPolicy maps to --hicache-storage-prefetch-policy when + remoteStorage.provider=NFS. It is rejected without that storage tier. + enum: + - best_effort + - wait_complete + - timeout + type: string writePolicy: description: WritePolicy maps to --hicache-write-policy. enum: @@ -523,21 +532,23 @@ spec: default: true description: |- FailOpen controls whether the engine treats cache lookups as a soft - dependency. When true (the default), an unreachable or degraded cache - backend MUST fall back to local prefill and never fail a serving - request — the cache is an optimization, not a serving dependency. When - explicitly set to false the engine fails requests on cache - unreachability ("fail-closed"); the cache becomes a serving - dependency, which is loud and visible via a Warning Event on the - owning CacheBackend. + dependency. For integrations admitted with true (the default), an + unreachable or degraded cache backend MUST fall back to local prefill + and never fail a serving request — the cache is an optimization, not a + serving dependency. When explicitly set to false the cache becomes a + fail-closed serving dependency, which is loud and visible via a Warning + Event on the owning CacheBackend. The flag is plumbed by the engine adapter as INFERENCECACHE_FAIL_OPEN (both shipping LMCache adapters — vLLM+LMCache and SGLang+LMCache — inject it). Per-request fail-open enforcement at the engine level is the engine/connector's responsibility; the cache plane surfaces the bit so the engine can honor it. - SGLangHiCache accepts only the default true value and does not inject this - env var because native HiCache exposes no equivalent fail-closed control. + Host-only SGLangHiCache accepts the default true value and does not inject + this env var because native HiCache exposes no equivalent control. + NFS-backed SGLangHiCache instead requires an explicit false value: kubelet + mounts the inline NFS volume before containers start, so export + availability is inherently a serving dependency. type: boolean firstEventTimeout: default: 5m @@ -693,8 +704,9 @@ spec: properties: endpoint: description: |- - Endpoint is required for External ownership and rejected for Managed - ownership, whose endpoint is controller-observed. + Endpoint is required for network providers with External ownership and + rejected for Managed ownership, whose endpoint is controller-observed. + NFS is mounted from its typed server/path fields and has no endpoint. type: string lmCacheServer: description: LMCacheServer contains standalone lmcache-server-owned @@ -847,6 +859,32 @@ spec: type: object type: object type: object + nfs: + description: |- + NFS identifies an externally owned NFS export and its engine-container + mount path. It is valid only with provider=NFS and ownership=External. + properties: + mountPath: + description: |- + MountPath is the absolute path at which the export is mounted in the + engine container. + minLength: 1 + type: string + path: + description: Path is the absolute path exported by the NFS + server. + minLength: 1 + type: string + server: + description: Server is the NFS mount-target hostname or IP + address. + minLength: 1 + type: string + required: + - mountPath + - path + - server + type: object ownership: description: |- Ownership identifies whether inference-cache manages the provider @@ -861,6 +899,7 @@ spec: - Redis - LMCacheServer - Mooncake + - NFS type: string redis: description: Redis contains Redis-owned configuration. diff --git a/config/samples/README.md b/config/samples/README.md index 350dae5a..33a638b6 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -15,8 +15,10 @@ multi-tenant, Namespaces): - **`cachebackend-*.yaml`** — focused hand-curated canonical CacheBackend examples, including the [`cachebackend-sglang-hicache.yaml`](cachebackend-sglang-hicache.yaml) - engine-local example. The `recipe-*.yaml` catalog is the maintained entry - point for LMCache scenarios. + engine-local example and the + [`cachebackend-sglang-hicache-l3-nfs.yaml`](cachebackend-sglang-hicache-l3-nfs.yaml) + externally owned NFS L3 example. The `recipe-*.yaml` catalog is the + maintained entry point for LMCache scenarios. ## Recipe catalog @@ -44,19 +46,28 @@ into two namespaces of its own. pods to the cache. For *managed* backends the wiring becomes available once the controller publishes `status.endpoint`, so a pod admitted before then races past injection and runs unwired until recreated (see each recipe's header); externally -owned backends wire straight from `spec.remoteStorage.endpoint` and have no such -race. KV reuse then works, but a *managed* backend only reaches `Ready=True` +owned network backends wire straight from `spec.remoteStorage.endpoint` and +have no such race. NFS is mounted directly into the engine Pod and therefore +has no endpoint. KV reuse then works, but a *managed* backend only reaches `Ready=True` and reports index entries once the `kvevent-subscriber` sidecar is auto-attached, which requires the controller to run with `--kvevent-subscriber-image` set (empty by default); otherwise it holds at `AwaitingFirstKVEvent` and then -degrades to `NoKVEventsObserved`. Externally owned backends are exempt from that gate — -they go `Ready` as soon as admission accepts the endpoint. See the +degrades to `NoKVEventsObserved`. Externally owned endpoint backends are exempt +from that gate — they go `Ready` as soon as admission accepts the endpoint. +SGLang HiCache with NFS is endpoint-free and follows the lifecycle described +below. +See the [quickstart](../../docs/quickstart.md). `SGLangHiCache` is endpoint-free and has no endpoint publication race. Its first implementation intentionally publishes no `Ready` condition; the matching Pod's injection annotations are the available wiring signal until the -separate HiCache readiness contract ships. +separate HiCache readiness contract ships. With `remoteStorage.provider: NFS`, +the manifest must explicitly set `integration.failOpen: false`: kubelet must +mount the export before starting the engine container, so an unavailable NFS +server blocks Pod startup. The admitted Pod shape also carries the `file` +storage arguments, storage-directory environment variable, NFS volume, and +engine-container mount. `recipe-multi-tenant.yaml` spans two namespaces, so it carries a `# verify-samples: skip` marker — server-side dry-run can't create the diff --git a/config/samples/cachebackend-sglang-hicache-l3-nfs.yaml b/config/samples/cachebackend-sglang-hicache-l3-nfs.yaml new file mode 100644 index 00000000..01048281 --- /dev/null +++ b/config/samples/cachebackend-sglang-hicache-l3-nfs.yaml @@ -0,0 +1,31 @@ +# SGLang HiCache with an externally owned NFS L3 tier. +# Replace the documentation-only values with your NFS server address and export +# path before applying this sample to a runtime cluster. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-hicache-l3-nfs +spec: + runtime: SGLang + type: SGLangHiCache + # Inline NFS volumes are kubelet startup dependencies, so this shape must + # explicitly opt into fail-closed serving instead of using the true default. + integration: + failOpen: false + engineSelector: + matchLabels: + app: sglang + hiCache: + ratio: "2.0" + storagePrefetchPolicy: wait_complete + remoteStorage: + provider: NFS + ownership: External + nfs: + server: 192.0.2.10 + path: /hicache + mountPath: /mnt/hicache + # Optional: enables the KV-event subscriber when the controller was started + # with --kvevent-subscriber-image. + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 2c735ffe..686e0113 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -61,13 +61,14 @@ Every finding carries a stable, greppable code. Codes are permanent identifiers | `PB001` | FAIL | `/probe` route not wired (connection refused, or HTTP 404 = route not mounted) | | `PB002` | OK | `/probe` route is wired (2xx / 401 / 403 / 405) | | `PB003` | WARN | `/probe` mounted but answered an unexpected status (e.g. 5xx) | -| `CB001` | WARN | CacheBackend `Ready` is not `True` | +| `CB001` | WARN | controller has not observed the current CacheBackend generation, or its current `Ready` condition is missing/not `True` | | `CB002` | WARN | managed backend with a selector matches 0 engine pods (LikelySelectorMismatch) | | `CB003` | WARN | no KV event ever observed for the backend (EngineNotReportingState) | | `CB004` | WARN | last KV event is stale (EngineStale) | | `CB005` | WARN | `status.endpoint` empty or unreachable | | `CB006` | OK | CacheBackend healthy on every applicable axis | | `CB007` | WARN | `FunctionalProbeOK` condition present but not `True` — the controller's functional self-test is failing for this backend (explains a Ready downgrade) | +| `CB008` | INFO | NFS-backed CacheBackend passed observable engine/index checks, but NFS mount and HiCache L3 store/read readiness were not verified | | `EP001` | WARN | matched engine pod missing an injection marker (no `inferencecache.io/injected-by` annotation and no Event) | | `EP002` | OK | matched engine pod is injected (annotation or Event) | | `OP001` | WARN | orphaned engine pod (NoMatchingCacheBackend; forward-looking — see note below) | @@ -79,6 +80,14 @@ Every finding carries a stable, greppable code. Codes are permanent identifiers Notes: +- **CacheBackend status must be current before health is interpreted.** Doctor + first requires `status.observedGeneration == metadata.generation`; when a + `Ready` condition is present, its own `observedGeneration` must also match. + Otherwise it emits `CB001` and does not interpret controller-owned endpoint, + matched-pod, index, or condition values that may describe the previous spec + generation. The selector check can still list live Pods against the current + spec, preserving useful pre-reconcile diagnostics without trusting stale + status. - **`CB003` keys off KV-event observation, not prefix count.** Zero warm prefixes is a valid state for an up-but-idle backend, so doctor flags "engine not reporting" only when *no* KV event has ever been observed — which means BOTH @@ -89,11 +98,19 @@ Notes: `lastEventAt` has been cleared by the poller. `CB004` fires only when `lastEventAt` IS present but has gone stale — an idle backend with a fresh event is healthy (`CB006`). -- **Externally owned remote storage** (`spec.remoteStorage.ownership=External`, +- **Externally owned network storage** (`spec.remoteStorage.ownership=External`, including the legacy `spec.type=External` shape) is checked for `Ready` and endpoint reachability only. Engine-pod matching (`CB002`) and index - participation (`CB003`/`CB004`) are managed-backend concerns and are skipped, - so a valid external config is not spuriously flagged. + participation (`CB003`/`CB004`) are skipped. External NFS is the exception: + it has no endpoint and is diagnosed through the same engine-pod and index + axes as host-only caching. Passing those observable axes emits `CB008` INFO, + not `CB006` OK, because inference-cache does not probe the NFS mount or the + HiCache L3 store/read data path. The missing-`Ready` exception applies only + after the controller has observed the current generation. An explicit + non-`True` `Ready` condition still wins and is reported as `CB001`; the + controller uses that condition to expose adapter/provider capability + rejection for stored or admission-bypassed resources, so doctor does not + maintain its own combination whitelist. - **Host-only backends** (no `spec.remoteStorage`) retain the managed engine and index checks but skip endpoint reachability (`CB005`), because they have no provider endpoint to publish or dial. diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 417366d0..cf95ad54 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -151,12 +151,13 @@ and [`config/samples/cachebackend-mooncake.yaml`](../../config/samples/cacheback | `type` | string | Engine-side cache implementation identifier. Defaults to `LMCache`. `Mooncake` and `External` remain accepted only as legacy compatibility values. | | `lmCache` | object | Typed LMCache engine configuration: chunk size, host-memory capacity, MP-worker image/port, and remote serde. | | `remoteStorage` | object | Optional remote tier. Omitting it means host-only and provisions no provider workload. | -| `remoteStorage.provider` | enum | `Redis`, `LMCacheServer`, or `Mooncake`. | +| `remoteStorage.provider` | enum | `Redis`, `LMCacheServer`, `Mooncake`, or `NFS`. | | `remoteStorage.ownership` | enum | `Managed` or `External`. | -| `remoteStorage.endpoint` | string | Required for `External`, rejected for `Managed`; managed endpoints are controller-observed in status. Bare `host:port` is portable across all providers. `LMCacheServer` also accepts `lm://host:port`, `Mooncake` also accepts `mooncakestore://host:port`, and `Redis` accepts only bare `host:port` with a numeric port in `1-65535`. Admission rejects schemes belonging to another provider. | +| `remoteStorage.endpoint` | string | Required for network providers with `External` ownership and rejected for `Managed`; managed endpoints are controller-observed in status. `NFS` is the exception: it uses the typed server/export fields below and forbids an endpoint. Bare `host:port` is portable across the network providers. `LMCacheServer` also accepts `lm://host:port`, `Mooncake` also accepts `mooncakestore://host:port`, and `Redis` accepts only bare `host:port` with a numeric port in `1-65535`. Admission rejects schemes belonging to another provider. | | `remoteStorage.redis` | object | Redis-owned image and resource configuration. | | `remoteStorage.lmCacheServer` | object | Standalone LMCache-server-owned image, command, and resource configuration. | | `remoteStorage.mooncake` | object | Mooncake-owned image, command, and resource configuration. | +| `remoteStorage.nfs` | object | Existing NFS export used by SGLang HiCache L3: mount-target `server`, exported absolute `path`, and absolute engine-container `mountPath`. Valid only with `provider: NFS`, `ownership: External`, and explicit `integration.failOpen: false`, because kubelet volume setup is a Pod startup dependency. | | `observation` | object | Observation-owned `modelID` and `firstEventTimeout`. | | `deploymentKind` | enum | Managed workload kind: `Deployment` or `StatefulSet`. Defaults to `Deployment`. | | `replicas` | integer | Desired managed backend replicas. Defaults to `1`. Minimum `0`. See [Defaulting](#defaulting-mutating) for the interaction with `spec.autoscaling.minReplicas` (first-apply-only). | @@ -166,7 +167,7 @@ and [`config/samples/cachebackend-mooncake.yaml`](../../config/samples/cacheback | `integration.engine` | string | Deprecated runtime identity retained for legacy manifests. Use `runtime`. | | `integration.mode` | enum | Which cache tiers the engine is wired for: `Offload` (default) or `EventsOnly`. `Offload` is full participation — cache-aware routing (tier-1) plus the KV-offload connector (tier-2). It may remain host-only, connect to externally owned remote storage, or provision a provider workload when `remoteStorage.ownership` is `Managed`. `EventsOnly` wires routing only: the kvevent-subscriber sidecar is injected (when the controller runs with `--kvevent-subscriber-image` set and an observation model ID is present via canonical `observation.modelID` or legacy `backendConfig.model` — otherwise the append is skipped fail-open), but no KV connector is loaded into the engine and no backend server is provisioned. See [Events-only mode](#events-only-mode-specintegrationmode--eventsonly). | | `integration.role` | enum | Engine participation mode: `ReadOnly`, `WriteOnly`, or `ReadWrite`. Defaults to `ReadWrite`. | -| `integration.failOpen` | boolean | Default `true`. When `true`, engine pods fall back to local prefill on cache unreachability — the cache is an optimization, never a serving dependency. Setting it to `false` is an advanced opt-in to fail-closed serving (the cache becomes a serving dependency); the controller surfaces this as a Warning Kubernetes Event on the owning `CacheBackend`. **Pair-specific exception — `(sglang, LMCache)`:** SGLang has no cacheless code path while `--enable-lmcache` is on, so its co-scheduled MP worker is a *serving prerequisite* (a worker that never starts wedges the engine), not a remote dependency that degrades to local prefill. `failOpen` is still honored at the tier that can actually be "unavailable" — the shared L2 (the worker comes up L1-only when Redis is unreachable). This is a documented, accepted boundary; see the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md) and [SGLang engine support](#sglang-engine-support). | +| `integration.failOpen` | boolean | Default `true`. When `true`, engine pods fall back to local prefill on cache unreachability — the cache is an optimization, never a serving dependency. Setting it to `false` is an advanced opt-in to fail-closed serving (the cache becomes a serving dependency); the controller surfaces this as a Warning Kubernetes Event on the owning `CacheBackend`. NFS-backed SGLang HiCache requires an explicit `false`, because kubelet must mount its inline volume before the engine container starts; admission rejects the default `true` rather than claiming that local prefill can bypass volume setup. **Pair-specific exception — `(sglang, LMCache)`:** SGLang has no cacheless code path while `--enable-lmcache` is on, so its co-scheduled MP worker is a *serving prerequisite* (a worker that never starts wedges the engine), not a remote dependency that degrades to local prefill. `failOpen` is still honored at the tier that can actually be "unavailable" — the shared L2 (the worker comes up L1-only when Redis is unreachable). This is a documented, accepted boundary; see the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md) and [SGLang engine support](#sglang-engine-support). | | `integration.firstEventTimeout` | duration | Deprecated observation timeout retained for legacy manifests. Use `observation.firstEventTimeout`. | | `integration.engineOverrides` | object | Optional engine-injection overrides applied to the args/env the pod-mutating webhook would otherwise inject into the engine container. See [Engine-injection overrides](#engine-injection-overrides-specintegrationengineoverrides). | | `engineSelector.matchLabels` | map | Equality-based label selector matched against engine **pod** labels (the pod template's `metadata.labels`, not Deployment, DaemonSet, or any other workload-level labels). Every key/value here must appear on the pod for it to match. `matchExpressions` is intentionally not exposed in v1alpha1 — the surface is `matchLabels` only. | @@ -316,7 +317,8 @@ SGLang supports two peer cache integrations: |---|---|---| | `(SGLang, LMCache)` without `remoteStorage` | Node-local LMCache MP worker, host-only | None | | `(SGLang, LMCache)` with Managed Redis | Node-local LMCache MP worker with a shared Redis remote tier | Redis Deployment and Service | -| `(sglang, SGLangHiCache)` | Native engine-local host cache | None | +| `(SGLang, SGLangHiCache)` without `remoteStorage` | Native engine-local host cache | None | +| `(SGLang, SGLangHiCache)` with External NFS | Native host cache plus file-backed L3 mounted into the engine Pod | None; the NFS export is externally owned | #### SGLang LMCache MP mode @@ -392,12 +394,14 @@ args/env only. #### SGLang native HiCache -The `(sglang, SGLangHiCache)` pair configures the selected SGLang engine Pods -directly. It does not create a cache-server Deployment, Service, HPA, or -endpoint. The first implementation intentionally publishes no `Ready` -condition: Kubernetes Pod readiness proves that SGLang is serving, but does not -prove a HiCache host-tier write/read round trip. A dedicated readiness contract -is a separate follow-up. +The `(SGLang, SGLangHiCache)` pair configures the selected SGLang engine Pods +directly. With no `remoteStorage` it is host-only; with an External NFS binding +it mounts an existing export as HiCache's file-backed L3 tier. In both shapes +inference-cache creates no cache-server Deployment, Service, HPA, endpoint, +PV, PVC, StorageClass, NFS export, or mount target. The first implementation +intentionally publishes no `Ready` condition: Kubernetes Pod readiness proves +that SGLang is serving, but does not prove a HiCache host- or storage-tier +write/read round trip. A dedicated readiness contract is a separate follow-up. The required integration shape is: @@ -428,13 +432,68 @@ the SGLang CLI: - `memoryLayout`: `layer_first`, `page_first`, `page_first_direct`, `page_first_kv_split`, `page_head` +An NFS-backed L3 tier uses the canonical provider hierarchy: + +```yaml +spec: + runtime: SGLang + type: SGLangHiCache + integration: + # Required acknowledgement: kubelet NFS mount failure blocks Pod startup. + failOpen: false + engineSelector: + matchLabels: + app: sglang + hiCache: + ratio: "2.0" + storagePrefetchPolicy: wait_complete + remoteStorage: + provider: NFS + ownership: External + nfs: + server: 10.0.0.25 + path: /hicache + mountPath: /mnt/hicache +``` + +`storagePrefetchPolicy` is required with NFS and rejected without it; accepted +values are `best_effort`, `wait_complete`, and `timeout`. The adapter derives +SGLang's file implementation from `provider: NFS`; the API does not repeat a +`storageBackend: file` selector. At Pod CREATE it adds: + +- `--hicache-storage-backend file` +- `--hicache-storage-prefetch-policy ` +- `SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR=` +- an inline Kubernetes NFS volume for `:` and a mount on the + `sglang` engine container at `` + +The cluster nodes must already have NFS client support and network access to +the mount target. Unlike a dialed cache endpoint, a kubelet NFS mount is a Pod +startup prerequisite: a mount failure leaves the engine Pod unready. Admission +therefore requires the operator to acknowledge this serving dependency with +explicit `integration.failOpen: false`; omitting the field selects its `true` +default and is rejected. This initial surface intentionally does not expose NFS +mount options, credentials, provisioning, or PVC indirection. + +Pod-level `NFSVolumeSource` exposes no mount-option control, so this inline +shape cannot select `hard`/`soft`, `timeo`, or `retrans`. Linux NFS clients +default to a `hard` mount when neither `hard` nor `soft` is specified: after a +successful mount, an outage can therefore leave SGLang's HiCache file I/O +retrying instead of returning an error promptly. `storagePrefetchPolicy` +controls how long request execution waits for a storage prefetch; it does not +change the kernel mount policy or interrupt an NFS syscall already blocked in a +HiCache storage thread. In particular, `wait_complete` can leave an affected +request queued until the NFS path recovers. Operators must treat NFS server and +network-path availability as part of the inference serving SLO; the current +controller and doctor checks do not verify this mounted L3 store/read path. + The webhook injects `--enable-hierarchical-cache` plus the corresponding -`--hicache-*` flags at Pod CREATE time. A one-container Pod may use any +`--hicache-*` flags and, when configured, the NFS wiring at Pod CREATE time. A one-container Pod may use any container name; a multi-container Pod must name its engine container `sglang`. Existing matching arguments are left byte-for-byte unchanged. A different -value, the opposite capacity mode, a malformed/duplicate argument, or existing -SGLang LMCache flags causes the whole injection to fail open without partial -HiCache wiring. +value, the opposite capacity mode, a malformed/duplicate argument, conflicting +storage environment/volume/mount, or existing SGLang LMCache flags causes the +whole injection to fail open without partial HiCache wiring. Changing or deleting the CacheBackend does not mutate live Pods. Roll the SGLang workload to apply a new configuration or switch between LMCache and @@ -633,7 +692,7 @@ Engine-side: the adapter injects the same `--kv-transfer-config '{"kv_connector" | Field | Type | Purpose | |---|---|---| -| `endpoint` | string | Observed endpoint clients should use. For External ownership this mirrors canonical `spec.remoteStorage.endpoint` (or legacy `spec.endpoint`); for Managed remote storage it is populated from the controller-rendered Service. It stays **empty** for host-only and events-only backends because neither has a remote provider address to publish. | +| `endpoint` | string | Observed endpoint clients should use. For External network-provider ownership this mirrors canonical `spec.remoteStorage.endpoint` (or legacy `spec.endpoint`); for Managed remote storage it is populated from the controller-rendered Service. It stays **empty** for host-only, events-only, and file/NFS-backed HiCache because none has a dialable provider address to publish. | | `matchedEnginePods` | integer | Snapshot count, at the last reconcile, of pods in the CacheBackend's namespace whose labels satisfy `spec.engineSelector`. Pointer in Go so nil ("not yet computed") is distinguishable from an observed `0` ("computed and zero pods match"). Refreshed at reconcile cadence — not a real-time per-pod counter. The steady cadence is 30s; during known churn the reconciler uses a conditional 5s cadence when the observed matching Pod count differs from the desired replica sum of Deployments whose pod-template labels match the selector. This keeps the no-Pod-watch design while reducing stale operator output during rolling restarts. The field stays nil when no claim-capable selector is configured — both when `spec.engineSelector` is absent AND when `spec.engineSelector.matchLabels` is present but empty (the webhook treats an empty match map as no-claim by design, so the count is no-claim too). A CR that previously had a non-empty selector and just lost it gets its prior value cleared back to nil so the printer column does not advertise a stale match. | | `engineSelectorMessage` | string | Operator-facing diagnosis for selector drift. Set when `spec.engineSelector.matchLabels` is configured and `matchedEnginePods` is observed as `0` while engine pods are expected; the message echoes the selector (`spec.engineSelector.matchLabels={...}`) and states that no Pods in the namespace match. If the selector matches a Deployment that is intentionally scaled to zero, `matchedEnginePods` still reports the observed `0`, but this message stays empty because no engine pods are expected. Cleared once at least one pod matches, the matching Deployment is scaled to zero, or the selector is removed. The controller also emits a Normal `EngineSelectorUnmatched` Event when the initial observation is zero, when a previously non-zero match count transitions to zero, or when upgrading an existing zero-count status that did not yet have the diagnostic message; steady-state zero with an unchanged message does not re-emit. | | `failOpen` | boolean | Observed echo of the effective `spec.integration.failOpen`. Represented as a pointer in Go so an explicit `false` is serialized and operators can read the current mode from status alone. | @@ -649,9 +708,10 @@ Engine-side: the adapter injects the same `--kv-transfer-config '{"kv_connector" The set of published condition types depends on the backend's integration mode and type: - **Offload-managed backends** (`spec.integration.mode=Offload` on a managed type, where the controller renders a Deployment + Service) publish up to seven: `Ready`, `Degraded`, `Progressing`, `FunctionalProbeOK`, `EngineKernelsHealthy` (when a matched engine pod runs the lmcache kernel-check), `T2Degraded` (once a tier-2/LMCache backend has been exercised), and `EngineCompatibility` (when an injected engine pod is observed crash-looping after connector injection). -- **Host-only backends** (canonical resources with no `spec.remoteStorage`) publish `Ready`, `Degraded`, and `Progressing`, plus the engine-side advisory conditions when applicable. Their endpoint stays empty. `HostOnlyActive` is the base `Ready=True` reason before the KV-event gate overlays `AwaitingFirstKVEvent`, `KVEventsObserved`, or `NoKVEventsObserved`. +- **Host-only backends** (canonical resources with no `spec.remoteStorage`) publish `Ready`, `Degraded`, and `Progressing`, plus the engine-side advisory conditions when applicable. Their endpoint stays empty. `HostOnlyActive` is the base `Ready=True` reason before the KV-event gate overlays `AwaitingFirstKVEvent`, `KVEventsObserved`, or `NoKVEventsObserved`. Native SGLang HiCache is the current exception described below. - **Events-only backends** (`spec.integration.mode=EventsOnly`) publish exactly three: `Ready`, `Degraded`, `Progressing`. `FunctionalProbeOK`, `T2Degraded`, `EngineKernelsHealthy`, and `EngineCompatibility` are **Offload-managed-only** and are **never** published on an events-only backend — there is no provisioned server to functionally probe, no tier-2 offload to mark degraded, no LMCache native kernels to check, and no injected KV connector that could be incompatible (events-only injects none); an Offload→EventsOnly flip clears all four (see [Events-only mode](#events-only-mode-specintegrationmode--eventsonly)). -- **Externally owned remote storage** publishes `Ready` + `Progressing` only (there is no rollout to degrade and no probe to drive; the operator manages the provider out-of-band and the controller only validates and mirrors the endpoint). +- **Externally owned network storage** publishes `Ready` + `Progressing` only (there is no rollout to degrade and no probe to drive; the operator manages the provider out-of-band and the controller only validates and mirrors the endpoint). +- **SGLang HiCache**, with either host-only memory or External NFS, currently publishes no conditions. Its endpoint stays empty, the controller acknowledges `observedGeneration`, and matching engine Pods carry the webhook injection receipt. A dedicated readiness contract is tracked separately. The `Ready` / `Degraded` / `Progressing` semantics below apply to both Offload-managed and events-only backends (an events-only backend has no workload to roll out, so it is "up" the moment it exists and the KV-event gate starts immediately — see [Events-only mode](#events-only-mode-specintegrationmode--eventsonly)); the `FunctionalProbeOK`, `T2Degraded`, `EngineKernelsHealthy`, and `EngineCompatibility` rows are Offload-managed-only. @@ -659,8 +719,8 @@ The `Ready` / `Degraded` / `Progressing` semantics below apply to both Offload-m | Type | Meaning | |---|---| -| `Ready` | True once the backend Deployment has rolled out its current generation, has enough updated + available replicas to serve traffic, **and** — when the [KV-event readiness gate](#kv-event-readiness-gate) applies — at least one KV event has been observed for the backend (reason `KVEventsObserved`), **and** — when the [functional-probe gate](#functional-probe-gate) applies — the most recent probe call succeeded across every stage the backend runs. Workload Available but no event yet is `Ready=False`, reason `AwaitingFirstKVEvent`. Workload Available and KV-event observed but the probe reported a stage failure is `Ready=False` with reason `ProbeIngestFailed` / `ProbeRoutingFailed` / `ProbeT2Failed`. Deployment-level reasons: `BackendReady` (both gates disabled and Available), `RolloutInProgress`, `ScaledToZero`, `ReplicasUnavailable`. **And** — when a matched engine pod admitted in **strict** kernel-check mode reports a kernel load failure — `Ready=False` with reason `EngineKernelDegraded` (see [`EngineKernelsHealthy`](#conditions)); report-only mode never downgrades `Ready`. The `BackendDegraded` / `BackendRecovered` Events narrate the `ReplicasUnavailable` → `BackendReady` / `KVEventsObserved` transitions. | -| `Degraded` | True when the backend is in a terminal unhealthy state: rolled out but replicas unavailable (reason `ReplicasUnavailable`), or the managed workload is Available but no KV event observed within `firstEventTimeout` (reason `NoKVEventsObserved`). False (`NotDegraded`) otherwise. The functional-probe gate does NOT participate in `Degraded` — a probe failure is reflected only in `Ready` and `FunctionalProbeOK`, leaving `Degraded` reserved for managed-Deployment health (so an operator can tell "the probe says the cache plane is broken" apart from "the workload itself is in a terminal state"). | +| `Ready` | True once the backend Deployment has rolled out its current generation, has enough updated + available replicas to serve traffic, **and** — when the [KV-event readiness gate](#kv-event-readiness-gate) applies — at least one KV event has been observed for the backend (reason `KVEventsObserved`), **and** — when the [functional-probe gate](#functional-probe-gate) applies — the most recent probe call succeeded across every stage the backend runs. Workload Available but no event yet is `Ready=False`, reason `AwaitingFirstKVEvent`. Workload Available and KV-event observed but the probe reported a stage failure is `Ready=False` with reason `ProbeIngestFailed` / `ProbeRoutingFailed` / `ProbeT2Failed`. Deployment-level reasons: `BackendReady` (both gates disabled and Available), `RolloutInProgress`, `ScaledToZero`, `ReplicasUnavailable`. **And** — when a matched engine pod admitted in **strict** kernel-check mode reports a kernel load failure — `Ready=False` with reason `EngineKernelDegraded` (see [`EngineKernelsHealthy`](#conditions)); report-only mode never downgrades `Ready`. A stored or admission-bypassed configuration rejected by the registered runtime/provider capabilities is also `Ready=False`, with reason `UnsupportedRuntimeBackend`, `UnsupportedRemoteStorage`, or `UnsupportedRemoteBinding`; the message carries the registry rejection. The `BackendDegraded` / `BackendRecovered` Events narrate readiness transitions. | +| `Degraded` | True when the backend is in a terminal unhealthy state: its stored configuration is unsupported, it rolled out but replicas remain unavailable (reason `ReplicasUnavailable`), or the managed workload is Available but no KV event was observed within `firstEventTimeout` (reason `NoKVEventsObserved`). False (`NotDegraded`) otherwise. The functional-probe gate does NOT participate in `Degraded` — a probe failure is reflected only in `Ready` and `FunctionalProbeOK`, so an operator can distinguish a probe failure from a terminal workload or configuration failure. | | `Progressing` | True while the controller is still driving the live state toward the desired state (rollout in flight, first apply, awaiting first KV event). False once converged (`Synced`), stuck (`Degraded`), or scaled to zero (`ScaledToZero`). The pair (`Ready=False`, `Progressing=True`) means "still converging"; (`Ready=False`, `Progressing=False`) means "stuck/degraded" (or scaled to zero). | | `T2Degraded` | **Advisory** tier-2 (external offload, e.g. LMCache) health, derived from `status.indexParticipation.t2HitRate` (written by the CacheIndex poller). Published only once the tier has been **exercised** (external lookups observed): `True`/`T2ZeroHitRate` when it was queried but served **zero** reloads (wired but useless — a store/connection failure, an under-sized remote server, or a scheduler/worker hash mismatch); `False`/`T2Serving` when it is serving reloads (hit-rate > 0). Absent entirely until the tier is exercised (distinct from `False`). It **never gates `Ready`** — tier-2 is an optimization, not a serving dependency (fail-open). For Prometheus alerting use the `inferencecache_backend_t2_hit_rate{backend}` gauge (CR `.status` is not scraped). The signal is **lifetime-cumulative** — it flags a tier that has *never* served a reload (the silent-from-start failures: scheduler/worker hash mismatch, server OOM, version skew); a mid-life regression (served reloads before, now zero) keeps hit-rate > 0 and so does **not** trip `T2Degraded`. That windowed case is caught instead by the per-pod `LMCacheT2NoHits` alert. | | `FunctionalProbeOK` | The most recent functional-probe outcome. `True/ProbeOK` when every enabled stage (ingest, routing, and — for LMCache — tier-2 put/get) round-tripped; `True/ProbeBypassed` when the operator opted this CR out via the `inferencecache.io/skip-functional-probe: "true"` annotation; `False/ProbeIngestFailed`, `False/ProbeRoutingFailed`, or `False/ProbeT2Failed` when the named stage failed, with the server's diagnostic in `.message`; `Unknown/ProbeError` when the controller could not reach the server's `/probe` endpoint at all (transport error, 5xx) AND no prior stage failure existed. **Sticky-False**: an HTTP error while a `False/Probe*Failed` is already published preserves the prior failure and keeps `Ready` downgraded, so a transient server outage cannot fade a known per-stage failure back to `Unknown` and then to `Ready=True`. See [functional-probe gate](#functional-probe-gate). | @@ -669,9 +729,9 @@ The `Ready` / `Degraded` / `Progressing` semantics below apply to both Offload-m When the desired replica count is owned by an HPA (`spec.autoscaling` set) the controller compares the Ready condition against the HPA-written Deployment `spec.replicas` rather than the user-set `spec.replicas`. -**Externally owned remote storage**: +**Externally owned network storage**: -Canonical resources express this shape with +Canonical Redis, LMCacheServer, and Mooncake resources express this shape with `spec.remoteStorage.ownership: External`, the selected provider, and `spec.remoteStorage.endpoint`. The legacy `spec.type: External` + `spec.endpoint` shape maps to an external LMCacheServer binding. There is no @@ -692,7 +752,16 @@ trusting the operator is part of External ownership. A future enhancement could degrade `Ready` on a probe failure, but that is deliberately out of scope today (fail-soft, never a serving dependency). -`kubectl get cachebackend` displays a `Ready` column sourced from `status.conditions[?(@.type=="Ready")].status` (the standard K8s pattern — operators read readiness through conditions, not through a custom enum field), the observed `status.endpoint`, a `Matched` column sourced from `status.matchedEnginePods`, plus `status.indexParticipation.prefixCount` (as `PREFIXES`) and `status.indexParticipation.lastEventAt` (as `LASTEVENT`). Managed provider backends therefore show readiness, the endpoint, the operator-actionable engine-fleet count, and live index participation once reconciliation has populated them and the poller has observed a `/snapshot` tick. An empty `Matched` cell means the count has not yet been computed (e.g. cold start before the first reconcile) or the CR has no `spec.engineSelector` configured. Externally owned bindings display the operator-supplied endpoint immediately. Their `indexParticipation` is typically unset — the operator-managed provider itself has no observation sidecar — but it is not special-cased by ownership: the poller attributes replicas by engine-pod selector/annotation. An externally owned binding whose engine pods run the subscriber therefore projects `indexParticipation` the same way as a managed binding. The readiness gate still never applies to External ownership (readiness comes from endpoint acceptance), so a populated `lastEventAt` affects the displayed columns but not readiness. +An adapter-accepted External NFS binding is deliberately excluded from this +table. It has no endpoint to accept or probe. The controller acknowledges the +current generation and publishes no endpoint or synthetic `Ready` condition; +the Pod webhook applies the [SGLang native HiCache](#sglang-native-hicache) +file/NFS wiring directly to matching engine Pods. If the registered runtime +adapter rejects the file binding, the defensive controller path instead +publishes `Ready=False/UnsupportedRemoteBinding`; it does not duplicate the +capability table in status logic. + +`kubectl get cachebackend` displays a `Ready` column sourced from `status.conditions[?(@.type=="Ready")].status` (the standard K8s pattern — operators read readiness through conditions, not through a custom enum field), the observed `status.endpoint`, a `Matched` column sourced from `status.matchedEnginePods`, plus `status.indexParticipation.prefixCount` (as `PREFIXES`) and `status.indexParticipation.lastEventAt` (as `LASTEVENT`). Managed provider backends therefore show readiness, the endpoint, the operator-actionable engine-fleet count, and live index participation once reconciliation has populated them and the poller has observed a `/snapshot` tick. An empty `Matched` cell means the count has not yet been computed (e.g. cold start before the first reconcile) or the CR has no `spec.engineSelector` configured. Externally owned network bindings display the operator-supplied endpoint immediately; NFS-backed HiCache keeps both `Ready` and endpoint empty while the controller still reports selector matching. `indexParticipation` is typically unset for an operator-managed provider itself, but it is not special-cased by ownership: the poller attributes replicas by engine-pod selector/annotation. An externally owned binding whose engine pods run the subscriber therefore projects `indexParticipation` the same way as a managed binding. The KV-event readiness gate does not apply to External ownership: network providers use endpoint acceptance, while endpoint-free NFS publishes no synthetic readiness in this version. ### Supported-model matrix @@ -716,7 +785,7 @@ The signal source is `status.indexParticipation.lastEventAt` (written by the Cac - **at least one event ever observed** → `Ready=True/KVEventsObserved`, `Degraded=False`. An event already present on the first reconcile counts — there is no required transition through `AwaitingFirstKVEvent`. "Ever observed" is durable: the first observation is latched into `status.firstKVEventObservedAt`, because the poller's `lastEventAt` is a current-view value it clears when the backend's replicas drain — without the latch a drained-but-healthy backend would wrongly fall back to `AwaitingFirstKVEvent`. The gate is a first-event startup probe, not an ongoing liveness check. - **no event by `firstEventTimeout`** → `Ready=False/NoKVEventsObserved`, `Degraded=True/NoKVEventsObserved`. Once Degraded it stays Degraded until an event arrives, then transitions to Ready. -The gate is **on by default** and opt-out per CR with the annotation `inferencecache.io/require-kv-events: "false"` (alpha soft-rollout knob; an annotation rather than a spec field so it can be retired once the gate is trusted). External ownership is **always exempt** — the control plane does not own a provider workload to gate, so readiness is determined by accepting the provider-specific `spec.remoteStorage.endpoint` (or legacy `spec.endpoint`) as described above. +The gate is **on by default** and opt-out per CR with the annotation `inferencecache.io/require-kv-events: "false"` (alpha soft-rollout knob; an annotation rather than a spec field so it can be retired once the gate is trusted). External ownership is **always exempt** — external network providers use endpoint acceptance, while endpoint-free External NFS publishes no synthetic `Ready` condition in this version. **Operator note.** If a backend is stuck at `Ready=False/AwaitingFirstKVEvent` (and then flips to `Degraded=True/NoKVEventsObserved` after `firstEventTimeout`), either no engine pods are attached to the backend or the engine's KV-event publisher is mis-configured — check that engine pods are wired to the backend, then the engine's `--kv-events-config` and that its ZMQ socket bound. In `kubectl get cachebackend` the `Ready` column shows `False` and the `LASTEVENT` column shows ``; the specific reason (`AwaitingFirstKVEvent` / `NoKVEventsObserved`) and the remediation hint live in the `Ready` / `Degraded` conditions, which `kubectl describe` surfaces along with the `NoKVEventsObserved` Warning Event. @@ -733,7 +802,7 @@ Composition order on a managed CacheBackend's Reconcile is `managed-readiness - If `FunctionalProbeOK` is already `False/Probe*Failed` (a prior stage failure), **preserve the existing condition AND keep `Ready=False` with the prior failure's reason** (sticky-False). Letting an HTTP error fade a known per-stage failure to `Unknown` (and therefore back to `Ready=True`) would mask a real regression every time the server happens to be transiently unreachable. The False condition is sticky until a successful probe explicitly resolves it. - **rate-limited reconcile** → no probe call. If the existing `FunctionalProbeOK` is already `False`, the Ready downgrade is re-applied so the status patch doesn't silently overwrite the prior failure with the upstream KV gate's `Ready=True`. The reconciler also schedules a requeue at the rate-limit window expiry so a quiet stuck backend re-probes even without external watch events. -The gate is **on by default** when the controller is wired with a `--server-probe-url`. The operator escape hatch is the **annotation `inferencecache.io/skip-functional-probe: "true"`** (alpha soft-rollout knob; annotation not spec field so it can be retired once the gate is trusted): when set, the probe call is skipped entirely and `FunctionalProbeOK=True/ProbeBypassed` is published — the Ready gate does not downgrade. Disabling the controller-side gate entirely is achieved by passing `--server-probe-url=""` to the controller binary; the CacheBackend reconciler then never calls the endpoint and clears any stale `FunctionalProbeOK` condition the next time it processes the CR. External ownership is **always exempt** — there is no managed Deployment for the gate to compose with, and the controller does not drive a cache-plane round trip for an operator-managed endpoint. +The gate is **on by default** when the controller is wired with a `--server-probe-url`. The operator escape hatch is the **annotation `inferencecache.io/skip-functional-probe: "true"`** (alpha soft-rollout knob; annotation not spec field so it can be retired once the gate is trusted): when set, the probe call is skipped entirely and `FunctionalProbeOK=True/ProbeBypassed` is published — the Ready gate does not downgrade. Disabling the controller-side gate entirely is achieved by passing `--server-probe-url=""` to the controller binary; the CacheBackend reconciler then never calls the endpoint and clears any stale `FunctionalProbeOK` condition the next time it processes the CR. External ownership is **always exempt** — there is no managed Deployment for the gate to compose with. Network providers are operator-managed endpoints, while endpoint-free NFS-backed HiCache publishes no synthetic readiness or functional-probe condition in this version. **Operator note.** If a backend is stuck at `Ready=False/Probe*Failed`, read the condition's `.message` — the server populates a stage-specific diagnostic (e.g. "synthesized event not in index — in-process index ingest path is broken"). For `ProbeIngestFailed` the cache-server's ingest path is broken; for `ProbeRoutingFailed` the index routing / key-derivation layer is broken; for `ProbeT2Failed` the configured tier-2 backend rejected the put or returned nothing on the get. `FunctionalProbeOK=Unknown/ProbeError` means the controller could not reach `/probe` at all — check that `--server-probe-url` is reachable from the controller pod (intra-cluster `inference-cache-server:8081` by default), that the projected SA token is mounted (`/var/run/secrets/inferencecache.io/controller-token/token`), and that the audience-bound TokenReview accepts the controller SA. @@ -756,7 +825,7 @@ Only ONE CacheBackend ever claims a given replica — overlapping selectors must ## Contract Notes -- Lookup paths fail open by default. `spec.integration.failOpen` defaults to `true` and the engine adapter MUST fall back to local prefill on unreachability of a **remote/shared** cache tier — that tier is an optimization, never a serving dependency. **One pair-specific exception applies** to the *co-scheduled* component of the SGLang MP wire: `(sglang, LMCache)` has no cacheless engine path while `--enable-lmcache` is on, so its in-pod MP worker is a serving prerequisite (fail-open is still honored at the tier that can be unavailable — the shared Redis L2, which degrades to L1-only). See the `integration.failOpen` row above and the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md). Operators may opt into fail-closed serving by setting `failOpen: false`, which is loud and visible: the controller emits a Warning `FailClosedEnabled` Event on the `CacheBackend` to make it explicit that the cache has been promoted to a serving dependency. +- Lookup paths fail open by default. `spec.integration.failOpen` defaults to `true` and the engine adapter MUST fall back to local prefill on unreachability of a **remote/shared** cache tier — that tier is an optimization, never a serving dependency. **One pair-specific exception applies** to the *co-scheduled* component of the SGLang MP wire: `(sglang, LMCache)` has no cacheless engine path while `--enable-lmcache` is on, so its in-pod MP worker is a serving prerequisite (fail-open is still honored at the tier that can be unavailable — the shared Redis L2, which degrades to L1-only). NFS-backed SGLang HiCache is not a second exception: admission requires the operator to opt into fail-closed serving explicitly with `failOpen: false`, accurately acknowledging that kubelet volume setup precedes engine startup. See the `integration.failOpen` row above and the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md). Fail-closed mode is loud and visible: the controller emits a Warning `FailClosedEnabled` Event on the `CacheBackend` to make it explicit that the cache has been promoted to a serving dependency. - The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); `EngineSelectorUnmatched` (Normal) when a configured selector first observes zero matching pods while engine pods are expected, transitions from matched to zero, or gains the diagnostic message during an upgrade from an older zero-count status. One advisory Event is recorded on the `CacheBackend` but triggered by engine-pod state rather than a CacheBackend condition transition: `InjectedEngineCrashLooping` (Warning) is emitted once when an injected engine pod's engine container is first observed in CrashLoopBackOff after connector injection — commonly a connector incompatibility (esp. a hybrid-attention model), surfaced as `EngineCompatibility=False/InjectedEngineCrashLooping`, but a crash-loop can also be a bad image/command/secret/OOM, so the cause is verified via the engine logs, not asserted by the Event. The controller does not watch engine pod status — it detects this on the next `CacheBackend` reconcile that lists the pods, so the Event reflects observation time, not the instant the container entered CrashLoopBackOff; a transient pod-list failure preserves the prior condition rather than re-firing it. - A `Normal InjectedByCacheBackend` Event is emitted on engine pods the mutating webhook stamps with both `inferencecache.io/injected-by` AND `inferencecache.io/injected-by-uid`, where the UID annotation matches the live CacheBackend's `metadata.uid` at reconcile time. The controller deliberately skips emission when (a) the named CR cannot be looked up (NotFound), (b) the UID annotation is absent (failurePolicy=Ignore forgery shape), or (c) the UID does not match the live CR (forgery or CR was recreated under the same name). Non-NotFound lookup errors surface as reconcile errors so controller-runtime retries with backoff. A pod explicitly opted out with a truthy `inferencecache.io/skip-inject` is instead stamped with `inferencecache.io/inject-skipped: skip-inject-annotation`; the same post-create controller emits a `Normal SkippedByOperator` Event only when both the truthy opt-out annotation and the webhook's skipped marker are present. The Events are recorded by a Pod-watching controller, not by the webhook itself: at mutating-admission time the apiserver hasn't assigned `metadata.uid` to the pod yet, so an event recorded from the webhook would carry `involvedObject.uid=""` and be invisible to describe (which filters events by UID). Routing the emission through a post-create controller is what guarantees the event reaches the user-visible surface. There is no `NoMatchingCacheBackend` Event; the no-match signals are `status.matchedEnginePods == 0`, `status.engineSelectorMessage`, and `EngineSelectorUnmatched` on the CacheBackend. - Optional nested specs are pointer fields in Go so omitted objects stay absent in JSON and server-side apply does not claim empty nested objects. **`spec.integration` is the deliberate exception** — the defaulting webhook materialises it on admission, derives `engine` from canonical `spec.runtime` (or uses legacy `vllm`), and gives the nested schema-level defaults a parent object to apply to. The apiserver then applies the `+kubebuilder:default=` markers on `mode` (`Offload`), `role` (`ReadWrite`), `failOpen` (`true`), and `firstEventTimeout` (`5m`) before persisting the CR. Operators reading the persisted CR therefore see the effective compatibility fields explicitly. The `IntegrationFailOpen` reader helper still exists (nil spec or nil field ⇒ `true`) as defence-in-depth for callers that bypass admission. Other optional nested specs (`spec.autoscaling`, `spec.template`, `spec.engineSelector`) are NOT materialised — omitted means absent. Webhook-stamped and apiserver-stamped fields are owned by their respective field managers, not the operator's SSA apply, so SSA semantics for operator-set fields are unaffected. @@ -787,10 +856,11 @@ Rejects structurally-broken specs that the reconciler cannot do anything useful | Rule | Rejects | |---|---| | Canonical hierarchy fields cannot conflict | `spec.runtime` conflicts with deprecated `integration.engine`; `type` uses legacy provider/ownership values `Mooncake` or `External`; deprecated top-level `backendConfig` or `resources` is supplied; or a provider-specific typed block does not match `remoteStorage.provider`/`ownership`. | -| External remote storage requires an endpoint | Canonical `remoteStorage.ownership=External` without `remoteStorage.endpoint`, or legacy `spec.type=External` without `spec.endpoint`; managed ownership rejects a user-supplied endpoint. | -| Engine wire must accept the provider binding | Every canonical `(runtime, type)` adapter must explicitly implement the remote-binding contract, and admission rejects a binding it does not accept (`lm`, `resp`, `mooncakestore`, or host-only). Native SGLang HiCache accepts only the nil host-only binding; attaching any `remoteStorage` is rejected. Deprecated legacy resources retain their endpoint-based compatibility fallback. | +| External remote storage identifies its connection | Canonical External network storage without `remoteStorage.endpoint`, legacy `spec.type=External` without `spec.endpoint`, or External NFS without `remoteStorage.nfs.{server,path,mountPath}`. NFS forbids an endpoint; managed ownership rejects a user-supplied endpoint. | +| Engine wire must accept the provider binding | Every canonical `(runtime, type)` adapter must explicitly implement the remote-binding contract, and admission rejects a binding it does not accept (`lm`, `resp`, `mooncakestore`, `file`, or host-only). Native SGLang HiCache accepts host-only and file/NFS bindings; other remote providers are rejected. Deprecated legacy resources retain their endpoint-based compatibility fallback. | +| NFS-backed HiCache is explicitly fail-closed | `(SGLang, SGLangHiCache)` with `remoteStorage.provider: NFS` unless `integration.failOpen` is explicitly `false`; host-only HiCache continues to require the default `true`. The NFS volume is mounted by kubelet before containers start, so admission does not promise fail-open behavior that the Pod shape cannot provide. | | Provider resources must be valid | Typed provider resource blocks are checked with the same request/limit, claims, quantity, resource-name, extended-resource, and hugepage rules as legacy `spec.resources`, with errors reported at the typed provider path. | -| Endpoint ownership is explicit | Canonical `spec.remoteStorage.endpoint` is required for External ownership and rejected for Managed ownership. Legacy `spec.endpoint` is valid only with legacy `spec.type=External`. A managed endpoint always comes from the live Service the controller provisions, so a user-supplied value would be misleading. Whitespace-only values are treated as empty. | +| Endpoint ownership is explicit | Canonical `spec.remoteStorage.endpoint` is required for External network providers, forbidden for External NFS, and rejected for Managed ownership. Legacy `spec.endpoint` is valid only with legacy `spec.type=External`. A managed endpoint always comes from the live Service the controller provisions, so a user-supplied value would be misleading. Whitespace-only values are treated as empty. | | Cross-namespace endpoint requires opt-in | Canonical `spec.remoteStorage.endpoint` or legacy `spec.endpoint` resolves to a Service in a namespace other than the CacheBackend's, while `spec.allowCrossNamespace` is `false`. Crossing the namespace is a tenancy boundary the operator must acknowledge. Bare hostnames, IPs, and unqualified names pass through because no namespace can be inferred. | | `spec.replicas=0` + autoscaling requires explicit `minReplicas` | `spec.replicas=0` with `spec.autoscaling != nil` and `spec.autoscaling.minReplicas == nil`. The defaulter declines to compute `minReplicas` from a 0 replicas value (it would violate the schema's `Minimum=1`), so without this rule the apiserver accepts the CR and the reconciler's HPA fallback silently picks `1` — overriding the operator's "scale to zero" intent with no notification. The rejection tells the operator to either set `minReplicas` explicitly or remove `spec.autoscaling` to scale to zero unconditionally. | | `spec.integration.engineOverrides` cannot touch reserved args/env | An entry in `engineOverrides.args` / `engineOverrides.suppressArgs` matches a leading flag token the adapter declares as `ReservedArgs()`, or an entry in `engineOverrides.env` / `engineOverrides.suppressEnv` matches a name in `ReservedEnv()`. The rejection names both the offending flag/env and the adapter so the operator can fix the spec rather than wait for the engine to crash. The reserved set is per-adapter (the vLLM+LMCache adapter reserves `--kv-transfer-config`, `VLLM_USE_V1`, `LMCACHE_REMOTE_URL`, `INFERENCECACHE_FAIL_OPEN`, `PYTHONHASHSEED`). | @@ -909,9 +979,9 @@ A separate mutating admission webhook on `corev1/v1.Pod` (`name: mpod.inferencec | Aspect | Behavior | |---|---| | Selection | Lists `CacheBackend`s in the pod's namespace via the manager's **APIReader** (uncached live client; an informer-cache miss on a freshly-Ready backend would leave the pod permanently unwired since pod CREATE is a one-shot), then matches `pod.Labels` against each `Spec.EngineSelector.MatchLabels`. The first matching `CacheBackend` wins; one with a nil or empty `EngineSelector` is skipped (a "match-everything" selector would silently claim every pod in the namespace). | -| Injection | Resolves the runtime adapter via `runtime.Registry.Select(runtimeID, cache)`, resolves `spec.EffectiveRemoteStorage()` independently, and constructs a structured provider `Binding{Protocol, Endpoint}`. Managed ownership uses `status.endpoint` from the live Service; External ownership uses the trimmed, provider-validated `spec.remoteStorage.endpoint` (or legacy `spec.endpoint`) with no fallback to stale status; omitted canonical `remoteStorage` produces a nil host-only binding. The webhook calls `runtime.InjectEngineConfigWithBinding`, so the adapter selects the LMCache, RESP, or Mooncake engine wire from the binding protocol instead of inferring storage from `spec.type`. A missing endpoint fails open only when the selected adapter and binding require one. Events-only skips engine injection because it wires no KV connector and appends only the kvevent-subscriber sidecar. Adapters preserve existing user args/env and make repeat injection idempotent. | +| Injection | Resolves the runtime adapter via `runtime.Registry.Select(runtimeID, cache)`, resolves `spec.EffectiveRemoteStorage()` independently, and constructs a structured provider binding. Managed ownership uses `status.endpoint` from the live Service; External network providers use the trimmed, provider-validated `spec.remoteStorage.endpoint` (or legacy `spec.endpoint`) with no fallback to stale status; External NFS instead binds its typed server/export/mount fields and needs no endpoint; omitted canonical `remoteStorage` produces a nil host-only binding. The webhook calls `runtime.InjectEngineConfigWithBinding`, so the adapter selects the LMCache, RESP, Mooncake, or file/NFS engine wire from the binding protocol instead of inferring storage from `spec.type`. A missing endpoint fails open only when the selected adapter and binding require one. Events-only skips engine injection because it wires no KV connector and appends only the kvevent-subscriber sidecar. Adapters preserve existing user args/env and make repeat injection idempotent. | | Annotations | Stamps TWO annotations on every successfully mutated pod: `inferencecache.io/injected-by: /` (operator-readable identity, shows in `kubectl describe pod`) AND `inferencecache.io/injected-by-uid: ` (the matched CR's metadata.uid). Successful injection also clears any stale `inferencecache.io/inject-skipped` marker. Reads `inferencecache.io/skip-inject: ` as an opt-out: the webhook returns Allowed, skips engine wiring, clears any stale injected-by/injected-by-uid pair, and stamps `inferencecache.io/inject-skipped: skip-inject-annotation` so explicit operator opt-out is distinguishable from selector drift. On all other fail-open returns after the pod is decoded (list/no match/missing endpoint/adapter errors), the webhook strips stale injected-by/injected-by-uid and inject-skipped annotations so a user cannot trick the events controller by pre-stamping a pod template. Decode failures fail open before a Pod exists to patch, so stale annotations cannot be cleared on that path. | | Events | The webhook itself does NOT record events (the apiserver assigns `metadata.uid` after mutating admission, so a webhook-recorded event would carry `involvedObject.uid=""` and be invisible to `kubectl describe pod`). Instead, the pod-watching `engine-pod-events` controller reads the persisted decision annotations after CREATE. For injected pods, it validates `inferencecache.io/injected-by-uid` against the live CR's `metadata.uid` and records a `Normal InjectedByCacheBackend` event on the now-persisted pod. For explicitly skipped pods carrying both a truthy `inferencecache.io/skip-inject` and `inferencecache.io/inject-skipped: skip-inject-annotation`, it records a `Normal SkippedByOperator` event on that pod. The skip marker is not authenticated, and `skipInjection` treats a pre-existing correct marker as already converged; `SkippedByOperator` therefore means the persisted pod carries the explicit opt-out plus skipped marker, not proof that the webhook authored the marker. The UID match REDUCES — but does NOT eliminate — the failurePolicy=Ignore forgery surface for injected pods: a casual copy-paste of an injected pod's annotations into a fresh template won't match the live CR's UID, but `metadata.uid` is not secret, so a pod creator with `get` RBAC on CacheBackends can read it and stamp the pair correctly. The injected Event signals "the webhook claims this pod was injected and the claim is consistent with the live CR," not "the webhook was cryptographically authenticated." The controller skips the injected event when the CR is missing, the UID annotation is absent, or the UID does not match — see the controller godoc for the full skip table. controller-runtime's EventBroadcaster aggregates duplicates on the apiserver side, so a re-enqueue across controller restarts upserts the existing event rather than spamming. | -| Idempotency | The handler calls the adapter unconditionally on every admission and trusts the adapter to converge the full injected contract. For LMCache this is env plus the engine-specific required surface — `--kv-transfer-config` for vLLM; for SGLang `--enable-lmcache` + `--lmcache-config-file` **plus** the MP-worker native sidecar and the shared config / `/dev/shm` volumes + mounts. Its merge primitives (`upsertEnv` / `upsertArgPair` / `upsertFlag`, and for SGLang `adoptContainer` / `adoptVolume` / `upsertMountByName`) converge on the desired value rather than appending a duplicate. The SGLang `adopt*` pair additionally distinguishes the adapter's own prior injection (converge) from an operator's object squatting a reserved name (reject → fail-open admit) — see [Names the MP wire reserves](#sglang-engine-support). Native HiCache validates all reserved arguments against the original pod before mutation, preserves one matching or well-formed operator-supplied value, appends each missing canonical argument once, and rejects conflicts, malformed values, or duplicates without partially changing the pod. Re-admission of a fully-injected pod therefore produces an empty JSON-patch set. Trusting the adapter rather than a handler-side env-presence shortcut avoids the trap where a partially-injected pod is admitted permanently missing the rest of the contract. | +| Idempotency | The handler calls the adapter unconditionally on every admission and trusts the adapter to converge the full injected contract. For LMCache this is env plus the engine-specific required surface — `--kv-transfer-config` for vLLM; for SGLang `--enable-lmcache` + `--lmcache-config-file` **plus** the MP-worker native sidecar and the shared config / `/dev/shm` volumes + mounts. Its merge primitives (`upsertEnv` / `upsertArgPair` / `upsertFlag`, and for SGLang `adoptContainer` / `adoptVolume` / `upsertMountByName`) converge on the desired value rather than appending a duplicate. The SGLang `adopt*` pair additionally distinguishes the adapter's own prior injection (converge) from an operator's object squatting a reserved name (reject → fail-open admit) — see [Names the MP wire reserves](#sglang-engine-support). Native HiCache validates all reserved arguments against the original pod before mutation, preserves one matching value, appends each missing canonical argument once, and rejects conflicts, malformed values, or duplicates without partially changing the pod. Its NFS path applies the same all-or-nothing rule to the storage environment variable, volume, and mount. Re-admission of a fully-injected pod therefore produces an empty JSON-patch set. Trusting the adapter rather than a handler-side env-presence shortcut avoids the trap where a partially-injected pod is admitted permanently missing the rest of the contract. | | Fail-open | Every error path (decode failure, list error, no matching backend, missing `status.endpoint`, no registered adapter, adapter rejection, re-encode failure) returns `admission.Allowed(...)` with a reason — webhook errors MUST NOT block engine admission. `MutatingWebhookConfiguration.failurePolicy` is also pinned to `Ignore` as a belt-and-suspenders second layer. | | Verbs | `CREATE` only. UPDATE re-admissions to a running pod don't re-inject (and the engine container can't pick up env changes without a restart anyway); UPDATEs to engine pods are rare in this fleet. | diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 8dec0846..bc3e47ba 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -97,8 +97,10 @@ # is observed by the controller without rendering a Deployment, Service, or # HPA; status.endpoint stays empty and no synthetic Ready condition is # published. A matching engine Pod sent through real server-side dry-run -# admission receives the complete native --hicache-* argument contract, -# proving the endpoint-free Pod webhook path without starting SGLang. +# admission receives the complete native --hicache-* argument contract. +# Updating the CR from the committed NFS sample remains endpoint-free and +# dry-run admission additionally proves the file-storage args, env, inline +# NFS volume, and mount without starting SGLang or mounting the fake export. # 10. The /snapshot endpoint rejects unauthenticated callers AT THE NETWORK # LAYER: a side curl pod outside the controller's SA identity (and outside # the NetworkPolicy allowlist) has its connection to :8081 DROPPED by the @@ -2460,8 +2462,8 @@ if ! grep -q "spec.endpoint is only valid when spec.type=External" <<<"$reject_o fi # Canonical runtime/cache adapters must explicitly accept their remote binding. -# Native SGLang HiCache is engine-local and accepts only a nil binding, so a -# Redis provider must be rejected before the controller could provision an +# Native SGLang HiCache accepts host-only and file/NFS bindings; a RESP/Redis +# provider must still be rejected before the controller could provision an # unused remote tier. reject_output="$(kubectl apply -f - <&1 || true apiVersion: inferencecache.io/v1alpha1 @@ -2770,6 +2772,109 @@ if [ "$hc_injected_by" != "$HICACHE_SMOKE_NS/$HICACHE_SMOKE_CB_NAME" ]; then fi log "native HiCache Pod webhook injected the complete CLI contract and backend identity" +# Update the same CacheBackend to the committed canonical NFS L3 shape. NFS is +# endpoint-free and controller-unmanaged, just like host-only native HiCache. +hc_l3_sample_tmp="$(mktemp "$tmpdir/sample-sglang-hicache-l3-nfs.XXXXXX")" +sed "s|^ name: sglang-hicache-l3-nfs\$| name: $HICACHE_SMOKE_CB_NAME|" \ + config/samples/cachebackend-sglang-hicache-l3-nfs.yaml > "$hc_l3_sample_tmp" +kubectl -n "$HICACHE_SMOKE_NS" apply -f "$hc_l3_sample_tmp" >/dev/null \ + || fail "kubectl apply native SGLang HiCache NFS L3 sample failed" + +hc_l3_generation="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ + -o jsonpath='{.metadata.generation}')" +if [ "$hc_l3_generation" = "$hc_generation" ]; then + fail "native HiCache NFS update did not advance metadata.generation" +fi +deadline=$(($(date +%s) + HICACHE_SMOKE_TIMEOUT)) +hc_l3_observed_generation="" +until [ "$hc_l3_observed_generation" = "$hc_l3_generation" ]; do + hc_l3_observed_generation="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ + -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true)" + if [ "$(date +%s)" -ge "$deadline" ]; then + kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" -o yaml || true + fail "controller did not observe native HiCache NFS generation $hc_l3_generation within ${HICACHE_SMOKE_TIMEOUT}s (observed $hc_l3_observed_generation)" + fi + sleep 1 +done +hc_l3_endpoint="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ + -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" +hc_l3_ready="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" +if [ -n "$hc_l3_endpoint" ] || [ -n "$hc_l3_ready" ]; then + kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" -o yaml || true + fail "native HiCache NFS published server-backed status (endpoint=$hc_l3_endpoint Ready=$hc_l3_ready, want both absent)" +fi + +hc_l3_dep_count="$(kubectl -n "$HICACHE_SMOKE_NS" get deploy -o name 2>/dev/null | wc -l | tr -d ' ')" +hc_l3_svc_count="$(kubectl -n "$HICACHE_SMOKE_NS" get svc -o name 2>/dev/null | wc -l | tr -d ' ')" +hc_l3_hpa_count="$(kubectl -n "$HICACHE_SMOKE_NS" get hpa -o name 2>/dev/null | wc -l | tr -d ' ')" +if [ "$hc_l3_dep_count" != "0" ] || [ "$hc_l3_svc_count" != "0" ] || [ "$hc_l3_hpa_count" != "0" ]; then + kubectl -n "$HICACHE_SMOKE_NS" get deploy,svc,hpa || true + fail "native HiCache NFS rendered controller-owned workload (deploy=$hc_l3_dep_count svc=$hc_l3_svc_count hpa=$hc_l3_hpa_count, want 0/0/0)" +fi + +# Render an engine Pod through real server-side CREATE admission, but do not +# persist it: the documentation-only NFS server must never be mounted by the +# kind smoke node. This covers the real CRD and webhook wiring without claiming +# an SGLang or NFS data-plane test. +hc_l3_engine_fixture="$(mktemp "$tmpdir/pod-sglang-hicache-l3.XXXXXX.yaml")" +cat > "$hc_l3_engine_fixture" </dev/null)"; then + fail "matching SGLang Pod did not pass NFS L3 server-side dry-run admission" +fi +hc_l3_expected_args=$'sleep\n3600\n--enable-hierarchical-cache\n--hicache-ratio\n2.0\n--hicache-storage-backend\nfile\n--hicache-storage-prefetch-policy\nwait_complete' +if [ "$hc_l3_pod_args" != "$hc_l3_expected_args" ]; then + printf '[install-smoke] admitted SGLang NFS L3 args:\n%s\n' "$hc_l3_pod_args" >&2 + fail "native HiCache NFS L3 mutation did not produce the expected complete argument contract" +fi + +hc_l3_storage_dir="$(kubectl create --dry-run=server --request-timeout=30s \ + -f "$hc_l3_engine_fixture" \ + -o go-template='{{range (index .spec.containers 0).env}}{{if eq .name "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR"}}{{.value}}{{end}}{{end}}' 2>/dev/null)" \ + || fail "could not read native HiCache NFS storage-directory env" +hc_l3_nfs_server="$(kubectl create --dry-run=server --request-timeout=30s \ + -f "$hc_l3_engine_fixture" \ + -o jsonpath='{.spec.volumes[?(@.name=="inferencecache-hicache-l3")].nfs.server}' 2>/dev/null)" \ + || fail "could not read native HiCache NFS volume server" +hc_l3_nfs_path="$(kubectl create --dry-run=server --request-timeout=30s \ + -f "$hc_l3_engine_fixture" \ + -o jsonpath='{.spec.volumes[?(@.name=="inferencecache-hicache-l3")].nfs.path}' 2>/dev/null)" \ + || fail "could not read native HiCache NFS volume path" +hc_l3_mount_path="$(kubectl create --dry-run=server --request-timeout=30s \ + -f "$hc_l3_engine_fixture" \ + -o jsonpath='{.spec.containers[0].volumeMounts[?(@.name=="inferencecache-hicache-l3")].mountPath}' 2>/dev/null)" \ + || fail "could not read native HiCache NFS mount path" +hc_l3_injected_by="$(kubectl create --dry-run=server --request-timeout=30s \ + -f "$hc_l3_engine_fixture" \ + -o go-template='{{index .metadata.annotations "inferencecache.io/injected-by"}}' 2>/dev/null)" \ + || fail "could not read native HiCache NFS injection annotation" +if [ "$hc_l3_storage_dir" != "/mnt/hicache" ] || \ + [ "$hc_l3_nfs_server" != "192.0.2.10" ] || \ + [ "$hc_l3_nfs_path" != "/hicache" ] || \ + [ "$hc_l3_mount_path" != "/mnt/hicache" ] || \ + [ "$hc_l3_injected_by" != "$HICACHE_SMOKE_NS/$HICACHE_SMOKE_CB_NAME" ]; then + fail "native HiCache NFS render env=$hc_l3_storage_dir server=$hc_l3_nfs_server export=$hc_l3_nfs_path mount=$hc_l3_mount_path injected-by=$hc_l3_injected_by; want /mnt/hicache, 192.0.2.10, /hicache, /mnt/hicache, $HICACHE_SMOKE_NS/$HICACHE_SMOKE_CB_NAME" +fi +log "native HiCache NFS L3 stayed endpoint-free, rendered no managed workload, and dry-run admission injected the complete file/NFS contract" + kubectl delete cb -n "$HICACHE_SMOKE_NS" "$HICACHE_SMOKE_CB_NAME" --ignore-not-found --wait=false >/dev/null || true kubectl delete namespace "$HICACHE_SMOKE_NS" --ignore-not-found --wait=false >/dev/null || true diff --git a/internal/controller/cachebackend_controller.go b/internal/controller/cachebackend_controller.go index e5a6ebba..68d2da98 100644 --- a/internal/controller/cachebackend_controller.go +++ b/internal/controller/cachebackend_controller.go @@ -33,13 +33,14 @@ import ( sglangadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime/sglang" ) -// Status condition types published on a managed CacheBackend. +// Status condition types published on a CacheBackend. // // Ready reports whether the managed backend workload is currently serving // (gated by the KV-event readiness gate — see evaluateKVEventReadiness). // Progressing reports whether the controller is still driving the live state // toward the desired state (template render, child apply, rollout in flight, -// awaiting first KV event). Degraded reports a terminal unhealthy state. +// awaiting first KV event). Degraded reports a terminal unhealthy state, +// including a stored configuration the registered capabilities reject. // Ready + Progressing together tell a still-converging backend // (Ready=False, Progressing=True) apart from a stuck/degraded one // (Ready=False, Progressing=False); Degraded names the specific failure. @@ -47,9 +48,9 @@ const ( conditionTypeReady = "Ready" conditionTypeProgressing = "Progressing" // Degraded is published alongside Ready. It is True only when the - // backend is in a genuinely degraded terminal state (rolled out but - // replicas unavailable, or the workload is Available but no KV events - // observed within firstEventTimeout). + // backend is in a genuinely degraded terminal state (unsupported + // configuration, rolled out but replicas unavailable, or the workload is + // Available but no KV events observed within firstEventTimeout). conditionTypeDegraded = "Degraded" ) @@ -138,13 +139,12 @@ const DefaultMatchedEnginePodsChurnRequeueInterval = 5 * time.Second // Event reasons emitted on a CacheBackend. // -// The cache is an optimization, never a serving dependency: BackendDegraded -// and BackendRecovered narrate transitions of the managed workload's -// availability so operators see backend readiness changes in -// `kubectl describe`. The FailClosedEnabled / FailOpenRestored pair -// narrates transitions of the spec.integration.failOpen toggle — -// explicitly fail-closed is loud because the cache then becomes a serving -// dependency. +// By default the cache is an optimization, not a serving dependency: +// BackendDegraded and BackendRecovered narrate transitions of the managed +// workload's availability or an unsupported stored configuration so operators +// see changes in `kubectl describe`. The FailClosedEnabled / FailOpenRestored +// pair narrates transitions of the spec.integration.failOpen toggle — explicitly +// fail-closed is loud because that backend is intentionally a serving dependency. const ( eventReasonBackendDegraded = "BackendDegraded" eventReasonBackendRecovered = "BackendRecovered" @@ -181,6 +181,13 @@ const ( // LMCACHE_REMOTE_URL the engine connector refuses at startup — // turning a cache misconfiguration into a serving outage). conditionReasonExternalEndpointInvalid = "ExternalEndpointInvalid" + // Unsupported* reasons are defensive controller backstops for resources + // stored before admission installed the corresponding capability check, or + // written while admission was bypassed. The adapter/provider registries remain + // the source of truth; these reasons only publish their rejection on status. + conditionReasonUnsupportedRuntimeBackend = "UnsupportedRuntimeBackend" + conditionReasonUnsupportedRemoteStorage = "UnsupportedRemoteStorage" + conditionReasonUnsupportedRemoteBinding = "UnsupportedRemoteBinding" ) // CacheBackendReconciler reconciles a CacheBackend object. @@ -298,12 +305,14 @@ func (r *CacheBackendReconciler) matchedEnginePodsChurnRequeueInterval() time.Du // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch -// Reconcile drives a CacheBackend toward its desired state. External backends -// only mirror their configured endpoint to status; managed backends (LMCache -// in Phase 1) ask the registered runtime adapter for the cache-server pod -// spec + service spec, wrap them into a Deployment + Service the controller -// owns, optionally reconcile an HPA from spec.autoscaling, and publish the -// resolved endpoint. +// Reconcile drives a CacheBackend toward its desired state. External network +// backends mirror their configured endpoint to status; endpoint-free External +// bindings such as NFS are consumed directly by selected engine Pods and are +// acknowledged without publishing an endpoint or synthetic Ready condition. +// Managed backends ask the registered runtime adapter for the cache-server pod spec + +// service spec, wrap them into a Deployment + Service the controller owns, +// optionally reconcile an HPA from spec.autoscaling, and publish the resolved +// endpoint. // // On every reconcile — including ones that return an apply error — transitions // in the observed Ready condition (entering/leaving Ready=False/ @@ -392,10 +401,11 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request // dispatch routes a CacheBackend by integration mode and effective remote // storage ownership. EventsOnly and canonical host-only configurations shed -// managed provider workloads; External storage mirrors its configured endpoint -// to status; Managed Redis, LMCacheServer, and Mooncake storage is rendered by -// the selected runtime/provider adapter. Unsupported combinations also shed any -// previously managed workload. +// managed provider workloads; External network storage mirrors its configured +// endpoint to status; endpoint-free External storage is acknowledged without +// rendering a workload or publishing endpoint readiness; Managed Redis, +// LMCacheServer, and Mooncake storage is rendered by the selected runtime/provider +// adapter. Unsupported combinations also shed any previously managed workload. func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend) (ctrl.Result, error) { registry := r.Registry if registry == nil { @@ -436,14 +446,16 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge // the pod webhook can't select an adapter for it (so it can never inject the // subscriber → no events ever flow). Treat the no-adapter case the same as // the Offload no-adapter path below: shed any workload and reconcile as - // unmanaged (no Ready/Progressing published), so the CR isn't advertised as - // a working routing tier the substrate can never feed. + // unsupported with Ready=False, so the CR isn't advertised as a working + // routing tier and kubectl/doctor can explain the adapter rejection. if backend.Spec.IsEventsOnly() { if _, err := registry.Select(runtimeID, backend); err != nil { - logger.V(1).Info("no runtime adapter for events-only backend; treating as unmanaged", + logger.V(1).Info("no runtime adapter for events-only backend; marking unsupported", "runtime", runtimeID, "type", backend.Spec.Type, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("runtime %s with cache type %s is unsupported: %v", + runtimeID, backend.Spec.EffectiveCacheType(), err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRuntimeBackend, message) } if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { return ctrl.Result{}, err @@ -460,22 +472,58 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge logger.V(1).Info("no runtime adapter for backend", "runtime", runtimeID, "type", backend.Spec.Type, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("runtime %s with cache type %s is unsupported: %v", + runtimeID, backend.Spec.EffectiveCacheType(), err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRuntimeBackend, message) } if storage != nil && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + backendRegistry := r.BackendRegistry + if backendRegistry == nil { + backendRegistry = backendprovider.DefaultRegistry() + } + if _, err := backendRegistry.Select(storage); err != nil { + logger.V(1).Info("no external remote-storage provider for backend; marking unsupported", + "provider", storage.Provider, "ownership", storage.Ownership, + "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + message := fmt.Sprintf("remote storage provider %s with ownership %s is unsupported: %v", + storage.Provider, storage.Ownership, err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteStorage, message) + } + // The provider registry decides whether External NFS is a supported + // capability. This separate validator checks only today's inline + // server/path binding; a future Managed NFS/PVC provider will bypass it + // and supply its own binding contract. + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS { + if err := backendadapter.ValidateInlineNFSBinding(storage); err != nil { + logger.V(1).Info("external NFS binding is invalid; marking unsupported", + "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + message := fmt.Sprintf("inline NFS binding is invalid: %v", err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteStorage, message) + } + } protocol, err := backendadapter.ProtocolFor(storage) if err != nil { - logger.V(1).Info("external storage has no supported binding protocol; treating as unmanaged", + logger.V(1).Info("external storage has no supported binding protocol; marking unsupported", "runtime", runtimeID, "type", backend.Spec.EffectiveCacheType(), "provider", storage.Provider, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("remote storage provider %s is unsupported: %v", storage.Provider, err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteStorage, message) } binding := backendadapter.BindingFor(storage, protocol, storage.Endpoint) if err := adapterruntime.ValidateRemoteBinding(adapter, binding, backend); err != nil { - logger.V(1).Info("runtime adapter does not accept external-storage binding; treating as unmanaged", + logger.V(1).Info("runtime adapter does not accept external-storage binding; marking unsupported", "runtime", runtimeID, "type", backend.Spec.EffectiveCacheType(), "protocol", protocol, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + message := fmt.Sprintf("runtime %s with cache type %s does not accept remote-storage binding %s: %v", + runtimeID, backend.Spec.EffectiveCacheType(), protocol, err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteBinding, message) + } + // Endpoint-free external bindings are mounted into or otherwise consumed + // directly by engine Pods. They have no provider workload or endpoint to + // reconcile. Adapter compatibility above decides which runtime/cache pairs + // currently support such a binding; controller dispatch stays generic. + if !backendadapter.BindingRequiresEndpoint(binding) { return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) } // A backend switched from a managed type to External must shed its workload. @@ -500,10 +548,12 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge return ctrl.Result{}, err } if bindingAware, ok := adapter.(adapterruntime.RemoteBindingAdapter); !ok || !bindingAware.SupportsRemoteBinding(nil) { - logger.V(1).Info("runtime adapter does not support host-only caching; treating as unmanaged", + logger.V(1).Info("runtime adapter does not support host-only caching; marking unsupported", "runtime", runtimeID, "type", backend.Spec.EffectiveCacheType(), "namespace", backend.Namespace, "name", backend.Name) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("runtime %s with cache type %s does not accept host-only caching", + runtimeID, backend.Spec.EffectiveCacheType()) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteBinding, message) } // Native HiCache remains endpoint-free and intentionally publishes no // Ready condition until its separate readiness contract is implemented. @@ -519,10 +569,12 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge } provider, err := backendRegistry.Select(storage) if err != nil { - logger.V(1).Info("no remote-storage provider for backend; treating as unmanaged", + logger.V(1).Info("no remote-storage provider for backend; marking unsupported", "provider", storage.Provider, "ownership", storage.Ownership, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("remote storage provider %s with ownership %s is unsupported: %v", + storage.Provider, storage.Ownership, err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteStorage, message) } rendered, err := provider.Render(backend) if err != nil { @@ -530,10 +582,12 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge } binding := &backendadapter.Binding{Protocol: rendered.Protocol} if err := adapterruntime.ValidateRemoteBinding(adapter, binding, backend); err != nil { - logger.V(1).Info("runtime adapter does not accept remote-storage binding; treating as unmanaged", + logger.V(1).Info("runtime adapter does not accept remote-storage binding; marking unsupported", "runtime", runtimeID, "type", backend.Spec.EffectiveCacheType(), "protocol", rendered.Protocol, "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) + message := fmt.Sprintf("runtime %s with cache type %s does not accept remote-storage binding %s: %v", + runtimeID, backend.Spec.EffectiveCacheType(), rendered.Protocol, err) + return ctrl.Result{}, r.reconcileUnsupported(ctx, backend, conditionReasonUnsupportedRemoteBinding, message) } return r.reconcileManaged(ctx, logger, backend, rendered) @@ -849,15 +903,39 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen return ctrl.Result{RequeueAfter: gate.requeueAfter}, err } -// reconcileUnmanaged sheds any previously owned workload and clears managed status -// for a backend this module no longer provisions (unsupported runtime/backend or -// deferred kind). The managed conditions are removed; firstKVEventObservedAt and -// status.indexParticipation are left as-is (see reconcileExternal's comment — the -// latch is a monotonic marker and indexParticipation is poller-owned). The +// reconcileUnmanaged sheds any previously owned workload and clears managed +// status for a valid backend that intentionally has no controller-provisioned +// workload (for example endpoint-free HiCache) or uses a deferred kind. The +// managed conditions are removed; firstKVEventObservedAt and +// status.indexParticipation are left as-is (see reconcileExternal's comment — +// the latch is a monotonic marker and indexParticipation is poller-owned). The // firstAvailableAt gate anchor IS reset, so a later managed/events-only re-entry // starts a fresh first-event window instead of reusing a pre-unmanaged // availability time. func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { + return r.reconcileWithoutManagedWorkload(ctx, backend, "", "") +} + +// reconcileUnsupported is the defensive controller-side counterpart to +// admission's adapter/provider capability checks. It sheds any stale managed +// workload but, unlike reconcileUnmanaged, publishes why the stored or +// admission-bypassed resource cannot be provisioned. The registry error is the +// source of truth; this method does not maintain a second compatibility table. +func (r *CacheBackendReconciler) reconcileUnsupported( + ctx context.Context, + backend *cachev1alpha1.CacheBackend, + reason string, + message string, +) error { + return r.reconcileWithoutManagedWorkload(ctx, backend, reason, message) +} + +func (r *CacheBackendReconciler) reconcileWithoutManagedWorkload( + ctx context.Context, + backend *cachev1alpha1.CacheBackend, + unsupportedReason string, + unsupportedMessage string, +) error { if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { return err } @@ -879,9 +957,33 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend // a stale identifier. backend.Status.ObservedServerInstance = "" backend.Status.ObservedGeneration = backend.Generation - meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeReady) - meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeProgressing) - meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeDegraded) + if unsupportedReason == "" { + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeReady) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeProgressing) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeDegraded) + } else { + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeReady, + Status: metav1.ConditionFalse, + Reason: unsupportedReason, + Message: unsupportedMessage, + ObservedGeneration: backend.Generation, + }) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: unsupportedReason, + Message: unsupportedMessage, + ObservedGeneration: backend.Generation, + }) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeDegraded, + Status: metav1.ConditionTrue, + Reason: unsupportedReason, + Message: unsupportedMessage, + ObservedGeneration: backend.Generation, + }) + } meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeFunctionalProbeOK) // EngineKernelsHealthy is a managed-path-only condition; clear any left // over so an unmanaged CR doesn't carry a stale kernel verdict. @@ -2582,7 +2684,7 @@ func (r *CacheBackendReconciler) emitTransitionEvents(cb *cachev1alpha1.CacheBac if before.failOpen && !after.failOpen { r.Recorder.Eventf(cb, nil, corev1.EventTypeWarning, eventReasonFailClosedEnabled, eventReasonFailClosedEnabled, - "fail-closed mode enabled — cache is now a serving dependency; engine requests will fail when the cache is unreachable") + "fail-closed mode enabled — cache is now a serving dependency; serving may be unavailable when the cache cannot be reached") } if !before.failOpen && after.failOpen { r.Recorder.Eventf(cb, nil, corev1.EventTypeNormal, eventReasonFailOpenRestored, eventReasonFailOpenRestored, diff --git a/internal/controller/cachebackend_controller_test.go b/internal/controller/cachebackend_controller_test.go index 9f8aaa33..8b386ab5 100644 --- a/internal/controller/cachebackend_controller_test.go +++ b/internal/controller/cachebackend_controller_test.go @@ -214,7 +214,7 @@ func TestReconcileCanonicalHostOnlyCacheCreatesNoProviderWorkload(t *testing.T) } } -func TestReconcileCanonicalSGLangHiCacheWithRemoteStorageIsUnmanaged(t *testing.T) { +func TestReconcileCanonicalSGLangHiCacheWithUnsupportedRemoteStorageIsDegraded(t *testing.T) { scheme := newScheme(t) cb := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "hicache-remote", Namespace: "ns1", Generation: 1}, @@ -240,8 +240,244 @@ func TestReconcileCanonicalSGLangHiCacheWithRemoteStorageIsUnmanaged(t *testing. if got.Status.Endpoint != "" { t.Fatalf("status.endpoint = %q, want empty for unsupported binding", got.Status.Endpoint) } - if ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("unsupported binding published Ready condition: %+v", ready) + ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRemoteBinding { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRemoteBinding) + } + if degraded := meta.FindStatusCondition(got.Status.Conditions, conditionTypeDegraded); degraded == nil || degraded.Status != metav1.ConditionTrue || degraded.Reason != conditionReasonUnsupportedRemoteBinding { + t.Fatalf("Degraded = %+v, want True/%s", degraded, conditionReasonUnsupportedRemoteBinding) + } +} + +func TestReconcileCanonicalSGLangHiCacheExternalNFSIsEndpointFree(t *testing.T) { + scheme := newScheme(t) + falseValue := false + cb := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "hicache-nfs", Namespace: "ns1", Generation: 3}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ + FailOpen: &falseValue, + }, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ + MatchLabels: map[string]string{"app": "sglang"}, + }, + HiCache: &cachev1alpha1.SGLangHiCacheSpec{ + Ratio: "2.0", + StoragePrefetchPolicy: cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + }, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + }, + }, + Status: cachev1alpha1.CacheBackendStatus{Conditions: []metav1.Condition{ + {Type: conditionTypeReady, Status: metav1.ConditionFalse, Reason: conditionReasonUnsupportedRemoteBinding}, + {Type: conditionTypeProgressing, Status: metav1.ConditionFalse, Reason: conditionReasonUnsupportedRemoteBinding}, + {Type: conditionTypeDegraded, Status: metav1.ConditionTrue, Reason: conditionReasonUnsupportedRemoteBinding}, + }}, + } + r := newReconciler(scheme, cb) + + reconcile(t, r, cb.Name, cb.Namespace) + + var deployments appsv1.DeploymentList + if err := r.List(context.Background(), &deployments, client.InNamespace(cb.Namespace)); err != nil { + t.Fatalf("list Deployments: %v", err) + } + var services corev1.ServiceList + if err := r.List(context.Background(), &services, client.InNamespace(cb.Namespace)); err != nil { + t.Fatalf("list Services: %v", err) + } + var hpas autoscalingv2.HorizontalPodAutoscalerList + if err := r.List(context.Background(), &hpas, client.InNamespace(cb.Namespace)); err != nil { + t.Fatalf("list HPAs: %v", err) + } + if len(deployments.Items) != 0 || len(services.Items) != 0 || len(hpas.Items) != 0 { + t.Fatalf("endpoint-free NFS rendered managed workload: deployments=%d services=%d hpas=%d", + len(deployments.Items), len(services.Items), len(hpas.Items)) + } + + got := getBackend(t, r, cb.Name, cb.Namespace) + if got.Status.Endpoint != "" { + t.Fatalf("status.endpoint = %q, want empty for NFS", got.Status.Endpoint) + } + if len(got.Status.Conditions) != 0 { + t.Fatalf("endpoint-free NFS published conditions: %v", got.Status.Conditions) + } + if got.Status.ObservedGeneration != got.Generation { + t.Fatalf("observedGeneration = %d, want generation %d", got.Status.ObservedGeneration, got.Generation) + } +} + +func TestReconcileCanonicalSGLangHiCacheManagedNFSIsUnsupported(t *testing.T) { + scheme := newScheme(t) + falseValue := false + cb := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "hicache-managed-nfs", Namespace: "ns1", Generation: 5}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ + FailOpen: &falseValue, + }, + HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + }, + }, + } + r := newReconciler(scheme, cb) + + reconcile(t, r, cb.Name, cb.Namespace) + + got := getBackend(t, r, cb.Name, cb.Namespace) + ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRemoteStorage { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRemoteStorage) + } + if !strings.Contains(ready.Message, "ownership Managed is unsupported") { + t.Fatalf("Ready message = %q, want unsupported Managed ownership", ready.Message) + } +} + +func TestReconcileCanonicalSGLangHiCacheInvalidStoredNFSIsUnsupported(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackendRemoteStorageSpec) + wantMessage string + }{ + { + name: "invalid server", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.Server = "@" + }, + wantMessage: "remoteStorage.nfs.server", + }, + { + name: "relative export path", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.Path = "hicache" + }, + wantMessage: "remoteStorage.nfs.path", + }, + { + name: "root mount path", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.MountPath = "/" + }, + wantMessage: "remoteStorage.nfs.mountPath", + }, + { + name: "forbidden endpoint", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.Endpoint = "nfs.example.com:2049" + }, + wantMessage: "remoteStorage.endpoint", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newScheme(t) + falseValue := false + cb := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-hicache-nfs", Namespace: "ns1", Generation: 7}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ + FailOpen: &falseValue, + }, + HiCache: &cachev1alpha1.SGLangHiCacheSpec{ + Ratio: "2.0", + StoragePrefetchPolicy: cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + }, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + }, + }, + } + tt.mutate(cb.Spec.RemoteStorage) + r := newReconciler(scheme, cb) + + reconcile(t, r, cb.Name, cb.Namespace) + + got := getBackend(t, r, cb.Name, cb.Namespace) + ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRemoteStorage { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRemoteStorage) + } + if !strings.Contains(ready.Message, tt.wantMessage) { + t.Fatalf("Ready message = %q, want %q", ready.Message, tt.wantMessage) + } + degraded := meta.FindStatusCondition(got.Status.Conditions, conditionTypeDegraded) + if degraded == nil || degraded.Status != metav1.ConditionTrue || degraded.Reason != conditionReasonUnsupportedRemoteStorage { + t.Fatalf("Degraded = %+v, want True/%s", degraded, conditionReasonUnsupportedRemoteStorage) + } + if got.Status.ObservedGeneration != got.Generation { + t.Fatalf("observedGeneration = %d, want generation %d", got.Status.ObservedGeneration, got.Generation) + } + if _, err := getOptionalDeployment(t, r, cb.Name, cb.Namespace); !apierrors.IsNotFound(err) { + t.Fatalf("invalid stored NFS rendered Deployment: %v", err) + } + }) + } +} + +func TestReconcileCanonicalVLLMLMCacheExternalNFSIsUnsupported(t *testing.T) { + scheme := newScheme(t) + cb := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "vllm-lmcache-nfs", Namespace: "ns1", Generation: 4}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/lmcache", + MountPath: "/mnt/lmcache", + }, + }, + }, + } + r := newReconciler(scheme, cb) + + reconcile(t, r, cb.Name, cb.Namespace) + + got := getBackend(t, r, cb.Name, cb.Namespace) + ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRemoteBinding { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRemoteBinding) + } + if !strings.Contains(ready.Message, "does not accept remote-storage binding file") { + t.Fatalf("Ready message = %q, want rejected file binding", ready.Message) + } + if got.Status.Endpoint != "" { + t.Fatalf("status.endpoint = %q, want empty for unsupported NFS binding", got.Status.Endpoint) + } + if got.Status.ObservedGeneration != got.Generation { + t.Fatalf("observedGeneration = %d, want generation %d", got.Status.ObservedGeneration, got.Generation) } } @@ -1140,10 +1376,10 @@ func TestReconcileExternalAdvancesObservedGeneration(t *testing.T) { } } -func TestReconcileUnmanagedTypeNoop(t *testing.T) { +func TestReconcileUnsupportedTypePublishesStatus(t *testing.T) { scheme := newScheme(t) // AIBrix has no registered runtime adapter, so it exercises the - // "unsupported managed type → reconcileUnmanaged" path. (Mooncake is no + // "unsupported managed type → reconcileUnsupported" path. (Mooncake is no // longer a stand-in for an unsupported type — it has an adapter now and // reconciles managed; see TestReconcileManagedMooncake.) cb := &cachev1alpha1.CacheBackend{ @@ -1159,11 +1395,16 @@ func TestReconcileUnmanagedTypeNoop(t *testing.T) { t.Fatalf("list deployments: %v", err) } if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 for unmanaged type", len(deps.Items)) + t.Fatalf("deployments = %d, want 0 for unsupported type", len(deps.Items)) + } + got := getBackend(t, r, "cache", "ns1") + ready := findCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRuntimeBackend { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRuntimeBackend) } } -func TestReconcileEventsOnlyUnsupportedPairIsUnmanaged(t *testing.T) { +func TestReconcileEventsOnlyUnsupportedPairPublishesStatus(t *testing.T) { // An EventsOnly backend whose (engine, type) pair has no registered // adapter must reconcile as UNMANAGED, NOT as active events-only. // Admission rejects an unsupported pair at write time, but a @@ -1172,7 +1413,7 @@ func TestReconcileEventsOnlyUnsupportedPairIsUnmanaged(t *testing.T) { // adapter for an unsupported pair, so it could never inject the // kvevent-subscriber and no KV event would ever flow. dispatch confirms an // adapter is selectable before routing to reconcileEventsOnly; on failure it - // falls to reconcileUnmanaged. AIBrix has no shipping adapter (the default + // publishes an unsupported status. AIBrix has no shipping adapter (the default // registry supports (vllm, LMCache) + (vllm, Mooncake) + External), so it // is the unsupported-type fixture here — Mooncake is no longer unsupported. scheme := newScheme(t) @@ -1191,15 +1432,13 @@ func TestReconcileEventsOnlyUnsupportedPairIsUnmanaged(t *testing.T) { reconcile(t, r, "cache", "ns1") got := getBackend(t, r, "cache", "ns1") - // reconcileUnmanaged removes the Ready / Progressing conditions; the - // events-only path (reconcileEventsOnly) would have PUBLISHED them. Their - // absence is the discriminator between "reconciled as unmanaged" and - // "reconciled as active events-only". - if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("unsupported-pair events-only must NOT publish Ready (unmanaged path); got %+v", ready) + // The unsupported path publishes a terminal condition; the events-only path + // would instead publish its normal AwaitingFirstKVEvent/active status. + if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRuntimeBackend { + t.Fatalf("unsupported-pair Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRuntimeBackend) } - if prog := findCondition(got.Status.Conditions, conditionTypeProgressing); prog != nil { - t.Fatalf("unsupported-pair events-only must NOT publish Progressing (unmanaged path); got %+v", prog) + if prog := findCondition(got.Status.Conditions, conditionTypeProgressing); prog == nil || prog.Status != metav1.ConditionFalse || prog.Reason != conditionReasonUnsupportedRuntimeBackend { + t.Fatalf("unsupported-pair Progressing = %+v, want False/%s", prog, conditionReasonUnsupportedRuntimeBackend) } // And no workload is provisioned (unmanaged sheds everything). var deps appsv1.DeploymentList @@ -1471,7 +1710,7 @@ func TestReconcileCanonicalExternalEndpointUsesProviderProtocol(t *testing.T) { } } -func TestReconcileCanonicalExternalUnsupportedBindingStaysUnmanaged(t *testing.T) { +func TestReconcileCanonicalExternalUnsupportedBindingPublishesStatus(t *testing.T) { scheme := newScheme(t) cb := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "external-redis", Namespace: "default", Generation: 2}, @@ -1501,8 +1740,12 @@ func TestReconcileCanonicalExternalUnsupportedBindingStaysUnmanaged(t *testing.T if got.Status.Endpoint != "" { t.Fatalf("status.endpoint = %q, want cleared for unsupported external binding", got.Status.Endpoint) } - if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("Ready = %+v, want absent for unmanaged unsupported external binding", ready) + ready := findCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonUnsupportedRemoteBinding { + t.Fatalf("Ready = %+v, want False/%s", ready, conditionReasonUnsupportedRemoteBinding) + } + if !strings.Contains(ready.Message, "does not accept remote-storage binding resp") { + t.Fatalf("Ready message = %q, want rejected RESP binding", ready.Message) } if got.Status.ObservedGeneration != cb.Generation { t.Fatalf("status.observedGeneration = %d, want %d", got.Status.ObservedGeneration, cb.Generation) diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index 88f903d4..f4c86f68 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -17,6 +17,7 @@ import ( cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" + backendprovider "github.com/cachebox-project/inference-cache/pkg/adapters/backend/provider" adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" externaladapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime/external" sglangadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime/sglang" @@ -85,7 +86,9 @@ const InjectSkippedReasonSkipAnnotation = "skip-inject-annotation" // MutatingWebhookConfiguration AND a fail-open posture in the handler give a // belt-and-suspenders guarantee: even if the controller is unreachable or // the handler returns an error response, pod admission is never blocked. -// The cache is always an optimization, never a serving dependency. +// This describes admission failure only. A successfully injected backend can +// still be an intentional data-plane serving dependency when its contract +// requires spec.integration.failOpen=false, as External NFS does. type EngineInjector struct { // Reader lists CacheBackends in the pod's namespace. Production wiring // passes the manager's APIReader (an uncached live client) — pod @@ -106,6 +109,13 @@ type EngineInjector struct { // would have wired. Registry *adapterruntime.Registry + // BackendRegistry resolves the supported (remote-storage provider, + // ownership) capabilities. nil uses the shipping provider registry. The + // pod webhook consults the same capability boundary as the controller so + // an admission-bypassed object cannot be unsupported in status but still + // mutate engine Pods. + BackendRegistry *backendadapter.Registry + // Log is the handler's logger. nil falls back to logf.FromContext at // call time; tests typically inject logr.Discard(). Log logr.Logger @@ -179,8 +189,32 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi runtimeID, cache.Spec.Type, err)) } - endpoint := effectiveEndpoint(cache) storage := cache.Spec.EffectiveRemoteStorage() + if storage != nil { + backendRegistry := h.BackendRegistry + if backendRegistry == nil { + backendRegistry = backendprovider.DefaultRegistry() + } + if _, err := backendRegistry.Select(storage); err != nil { + log.V(1).Info("fail-open: remote-storage capability is unsupported", + "provider", storage.Provider, "ownership", storage.Ownership, "error", err.Error()) + return failOpen(req, &pod, fmt.Sprintf( + "unsupported remote-storage provider=%q ownership=%q (fail-open): %v", + storage.Provider, storage.Ownership, err)) + } + // Inline NFS is the current External binding shape. A future Managed + // NFS provider will expose a separate PVC-backed binding and must not + // inherit these server/path rules. + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS && + storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + if err := backendadapter.ValidateInlineNFSBinding(storage); err != nil { + log.V(1).Info("fail-open: inline NFS binding is invalid", "error", err.Error()) + return failOpen(req, &pod, fmt.Sprintf("invalid inline NFS binding (fail-open): %v", err)) + } + } + } + + endpoint := effectiveEndpoint(cache) protocol, protocolErr := backendadapter.ProtocolFor(storage) if protocolErr != nil { log.V(1).Info("fail-open: remote-storage protocol is unsupported", "error", protocolErr.Error()) @@ -575,9 +609,9 @@ func skipInjection(req admission.Request, pod *corev1.Pod) admission.Response { // effectiveEndpoint returns the address the engine pod should be wired // to for the given CacheBackend. The source is type-scoped: // -// - External: spec.endpoint is authoritative — the operator owns it, -// admission validates it, status.endpoint is just a reconciler -// mirror that may briefly lag during an update. If a new pod +// - External network providers: spec.endpoint is authoritative — the +// operator owns it, admission validates it, and status.endpoint is just a +// reconciler mirror that may briefly lag during an update. If a new pod // admits between an operator's spec.endpoint update and the // status patch, status would still hold the OLD value and the // pod would boot wired to the stale address; pod admission is @@ -589,9 +623,10 @@ func skipInjection(req admission.Request, pod *corev1.Pod) admission.Response { // provisions, and spec.endpoint is admission-rejected for these // types (see rejectEndpointOnNonExternal), so there's nothing // else to fall back on. The webhook must wait for status. -// - Engine-local types (SGLangHiCache): no endpoint is required. The -// selected adapter's EndpointRequirement capability bypasses the gate -// before this empty result is consumed. +// - Engine-local types (SGLangHiCache): no endpoint is required. External +// NFS carries its server/export/mount contract in a structured file +// binding rather than an endpoint. The selected adapter and concrete +// binding bypass the endpoint gate before this empty result is consumed. // // Returns "" when no endpoint is currently usable; callers fail-open. // @@ -618,6 +653,9 @@ func effectiveEndpoint(cache *cachev1alpha1.CacheBackend) string { } if storage := cache.Spec.EffectiveRemoteStorage(); storage != nil && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS { + return "" + } // For External, re-apply the provider-specific admission-time shape // check on the stored spec endpoint. The validating webhook already // rejects malformed values at write time, but a pre-existing diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index f55999b8..0b2c54fa 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -339,6 +339,106 @@ func TestHandle_MatchAndInject_SGLangHiCacheWithoutEndpoint(t *testing.T) { } } +func TestHandle_MatchAndInject_CanonicalSGLangHiCacheNFS(t *testing.T) { + const ns = "engines" + cb := readyCacheBackend("hicache-nfs", ns, map[string]string{"app": "sglang"}) + falseValue := false + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.Integration.Engine = "" + cb.Spec.Integration.FailOpen = &falseValue + cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{ + Ratio: "2.0", + StoragePrefetchPolicy: cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + } + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + cb.Status.Endpoint = "" + + pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) + req := newRequest(t, pod, ns) + resp := newHandler(t, cb).Handle(context.Background(), req) + if !resp.Allowed || len(resp.Patches) == 0 { + t.Fatalf("HiCache NFS injection = Allowed %v, patches %d", resp.Allowed, len(resp.Patches)) + } + mutated := applyPatches(t, req.Object.Raw, resp) + mustHaveArgPair(t, mutated, "--hicache-storage-backend", "file") + mustHaveArgPair(t, mutated, "--hicache-storage-prefetch-policy", "wait_complete") + mustHaveEnv(t, mutated, "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR", "/mnt/hicache") + if len(mutated.Spec.Volumes) != 1 || mutated.Spec.Volumes[0].NFS == nil || + mutated.Spec.Volumes[0].NFS.Server != "10.0.0.25" || mutated.Spec.Volumes[0].NFS.Path != "/hicache" { + t.Fatalf("NFS volume = %+v", mutated.Spec.Volumes) + } + mounts := mutated.Spec.Containers[0].VolumeMounts + if len(mounts) != 1 || mounts[0].Name != "inferencecache-hicache-l3" || mounts[0].MountPath != "/mnt/hicache" { + t.Fatalf("HiCache NFS mount = %+v", mounts) + } +} + +func TestHandle_CanonicalSGLangHiCacheNFSInvalidStoredSourceFailsOpen(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackendRemoteStorageSpec) + }{ + { + name: "unsupported managed ownership", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.Ownership = cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged + }, + }, + { + name: "forbidden endpoint", + mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.Endpoint = "nfs.example.com:2049" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const ns = "engines" + falseValue := false + cb := readyCacheBackend("hicache-nfs", ns, map[string]string{"app": "sglang"}) + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.Integration.Engine = "" + cb.Spec.Integration.FailOpen = &falseValue + cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{ + Ratio: "2.0", + StoragePrefetchPolicy: cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + } + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + tt.mutate(cb.Spec.RemoteStorage) + cb.Status.Endpoint = "" + + pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) + req := newRequest(t, pod, ns) + resp := newHandler(t, cb).Handle(context.Background(), req) + if !resp.Allowed { + t.Fatalf("invalid stored NFS source must fail open: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Fatalf("invalid stored NFS source produced %d patches, want original Pod unchanged", len(resp.Patches)) + } + }) + } +} + func TestHandle_CanonicalSGLangHiCacheWithRemoteStorageFailsOpen(t *testing.T) { const ns = "engines" cb := readyCacheBackend("hicache-remote", ns, map[string]string{"app": "sglang"}) diff --git a/internal/webhook/v1alpha1/cachebackend_webhook.go b/internal/webhook/v1alpha1/cachebackend_webhook.go index d4b858df..b6492839 100644 --- a/internal/webhook/v1alpha1/cachebackend_webhook.go +++ b/internal/webhook/v1alpha1/cachebackend_webhook.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + pathpkg "path" "reflect" "sort" "strconv" @@ -98,8 +99,9 @@ const ( type CacheBackendDefaulter struct{} // CacheBackendValidator rejects CacheBackend specs that are structurally -// broken — External without an endpoint, cross-namespace endpoints without -// explicit opt-in, runtime/backend pairs no installed adapter supports — +// broken — External network storage without an endpoint, NFS without a typed +// mount declaration, cross-namespace endpoints without explicit opt-in, +// runtime/backend pairs no installed adapter supports — // before the reconciler ever sees them. It implements [admission.Validator] // over CacheBackend. // @@ -328,7 +330,12 @@ func validateCanonicalCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.Error "managed providers publish their observed endpoint in status.endpoint")) } case cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal: - if strings.TrimSpace(storage.Endpoint) == "" { + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS { + if strings.TrimSpace(storage.Endpoint) != "" { + errs = append(errs, field.Forbidden(storagePath.Child("endpoint"), + "NFS uses remoteStorage.nfs.server and path instead of an endpoint")) + } + } else if strings.TrimSpace(storage.Endpoint) == "" { errs = append(errs, field.Required(storagePath.Child("endpoint"), "required when remoteStorage.ownership=External")) } else if err := adapterruntime.ValidateExternalEndpoint(storage.Provider, storage.Endpoint); err != nil { @@ -337,25 +344,41 @@ func validateCanonicalCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.Error } type providerConfig struct { - provider cachev1alpha1.CacheBackendRemoteStorageProvider - set bool - path *field.Path + provider cachev1alpha1.CacheBackendRemoteStorageProvider + set bool + path *field.Path + allowsExternal bool } configs := []providerConfig{ - {cachev1alpha1.CacheBackendRemoteStorageProviderRedis, storage.Redis != nil, storagePath.Child("redis")}, - {cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, storage.LMCacheServer != nil, storagePath.Child("lmCacheServer")}, - {cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, storage.Mooncake != nil, storagePath.Child("mooncake")}, + {cachev1alpha1.CacheBackendRemoteStorageProviderRedis, storage.Redis != nil, storagePath.Child("redis"), false}, + {cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, storage.LMCacheServer != nil, storagePath.Child("lmCacheServer"), false}, + {cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, storage.Mooncake != nil, storagePath.Child("mooncake"), false}, + {cachev1alpha1.CacheBackendRemoteStorageProviderNFS, storage.NFS != nil, storagePath.Child("nfs"), true}, } for _, config := range configs { if config.set && storage.Provider != config.provider { errs = append(errs, field.Forbidden(config.path, fmt.Sprintf("configuration belongs to provider %s, but remoteStorage.provider=%s", config.provider, storage.Provider))) } - if config.set && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + if config.set && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal && !config.allowsExternal { errs = append(errs, field.Forbidden(config.path, "provider workload configuration is valid only with Managed ownership")) } } + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS { + if storage.Ownership != cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + errs = append(errs, field.NotSupported( + storagePath.Child("ownership"), storage.Ownership, + []string{string(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal)}, + )) + } + if storage.NFS == nil { + errs = append(errs, field.Required(storagePath.Child("nfs"), + "required when remoteStorage.provider=NFS")) + } else { + errs = append(errs, validateNFSRemoteStorage(storage.NFS, storagePath.Child("nfs"))...) + } + } if storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged { switch storage.Provider { case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: @@ -378,6 +401,40 @@ func validateCanonicalCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.Error return errs } +func validateNFSRemoteStorage(storage *cachev1alpha1.NFSRemoteStorageSpec, storagePath *field.Path) field.ErrorList { + var errs field.ErrorList + server := storage.Server + switch { + case strings.TrimSpace(server) == "": + errs = append(errs, field.Required(storagePath.Child("server"), + "NFS mount-target hostname or IP address is required")) + case server != strings.TrimSpace(server): + errs = append(errs, field.Invalid(storagePath.Child("server"), server, + "must not contain surrounding whitespace")) + case backendadapter.ValidateNFSServer(server) != nil: + errs = append(errs, field.Invalid(storagePath.Child("server"), server, + "must be a valid IPv4 address, IPv6 address, or DNS-1123 hostname")) + } + errs = append(errs, validateCleanAbsolutePath(storage.Path, storagePath.Child("path"), true)...) + errs = append(errs, validateCleanAbsolutePath(storage.MountPath, storagePath.Child("mountPath"), false)...) + return errs +} + +func validateCleanAbsolutePath(value string, valuePath *field.Path, allowRoot bool) field.ErrorList { + if strings.TrimSpace(value) == "" { + return field.ErrorList{field.Required(valuePath, "absolute path is required")} + } + if value != strings.TrimSpace(value) || !pathpkg.IsAbs(value) || pathpkg.Clean(value) != value { + return field.ErrorList{field.Invalid(valuePath, value, + "must be a clean absolute path without surrounding whitespace")} + } + if !allowRoot && value == "/" { + return field.ErrorList{field.Invalid(valuePath, value, + "must not mount remote storage over the container root")} + } + return nil +} + func validateManagedProviderCommand(path *field.Path, command []string) field.ErrorList { if command == nil { return nil @@ -408,6 +465,8 @@ func validateSGLangHiCache(cb *cachev1alpha1.CacheBackend) field.ErrorList { } var errs field.ErrorList + nfsStorage := cb.Spec.RemoteStorage != nil && + cb.Spec.RemoteStorage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS if cb.Spec.HiCache == nil { errs = append(errs, field.Required(hiCachePath, fmt.Sprintf("required when spec.type=%q", cachev1alpha1.CacheBackendTypeSGLangHiCache))) @@ -465,6 +524,26 @@ func validateSGLangHiCache(cb *cachev1alpha1.CacheBackend) field.ErrorList { }, )) } + if !validHiCacheStoragePrefetchPolicy(spec.StoragePrefetchPolicy) { + errs = append(errs, field.NotSupported( + hiCachePath.Child("storagePrefetchPolicy"), spec.StoragePrefetchPolicy, + []string{ + string(cachev1alpha1.SGLangHiCacheStoragePrefetchBestEffort), + string(cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete), + string(cachev1alpha1.SGLangHiCacheStoragePrefetchTimeout), + }, + )) + } else if nfsStorage && spec.StoragePrefetchPolicy == "" { + errs = append(errs, field.Required( + hiCachePath.Child("storagePrefetchPolicy"), + "required when remoteStorage.provider=NFS", + )) + } else if !nfsStorage && spec.StoragePrefetchPolicy != "" { + errs = append(errs, field.Forbidden( + hiCachePath.Child("storagePrefetchPolicy"), + "valid only when remoteStorage.provider=NFS", + )) + } } if adapterruntime.ResolveRuntimeID(cb) != adapterruntime.RuntimeSGLang { @@ -505,13 +584,20 @@ func validateSGLangHiCache(cb *cachev1alpha1.CacheBackend) field.ErrorList { []string{string(cachev1alpha1.CacheBackendIntegrationRoleReadWrite)}, )) } - if !cachev1alpha1.IntegrationFailOpen(cb.Spec.Integration) { - errs = append(errs, field.NotSupported( - field.NewPath("spec", "integration", "failOpen"), - false, - []string{"true"}, - )) - } + } + failOpen := cachev1alpha1.IntegrationFailOpen(cb.Spec.Integration) + switch { + case nfsStorage && failOpen: + errs = append(errs, field.Invalid( + field.NewPath("spec", "integration", "failOpen"), true, + "must be false with remoteStorage.provider=NFS because the inline NFS volume is a Pod startup dependency", + )) + case !nfsStorage && !failOpen: + errs = append(errs, field.NotSupported( + field.NewPath("spec", "integration", "failOpen"), + false, + []string{"true"}, + )) } for key := range cb.Spec.BackendConfig { if key != "model" { @@ -563,6 +649,18 @@ func validHiCacheMemoryLayout(value cachev1alpha1.SGLangHiCacheMemoryLayout) boo } } +func validHiCacheStoragePrefetchPolicy(value cachev1alpha1.SGLangHiCacheStoragePrefetchPolicy) bool { + switch value { + case "", + cachev1alpha1.SGLangHiCacheStoragePrefetchBestEffort, + cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + cachev1alpha1.SGLangHiCacheStoragePrefetchTimeout: + return true + default: + return false + } +} + // rejectUnsupportedSGLangRole rejects a non-ReadWrite spec.integration.role on a // (sglang, LMCache) backend. SGLang's --enable-lmcache integration has no // kv_role split equivalent to vLLM's LMCache connector — it always both stores diff --git a/internal/webhook/v1alpha1/cachebackend_webhook_test.go b/internal/webhook/v1alpha1/cachebackend_webhook_test.go index 3981a062..e0693b7e 100644 --- a/internal/webhook/v1alpha1/cachebackend_webhook_test.go +++ b/internal/webhook/v1alpha1/cachebackend_webhook_test.go @@ -56,6 +56,27 @@ func newHiCacheBackend() *cachev1alpha1.CacheBackend { } } +func newCanonicalHiCacheNFSBackend() *cachev1alpha1.CacheBackend { + cb := newHiCacheBackend() + falseValue := false + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.Integration.Engine = "" + cb.Spec.Integration.FailOpen = &falseValue + cb.Spec.BackendConfig = nil + cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "model-a"} + cb.Spec.HiCache.StoragePrefetchPolicy = cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + return cb +} + func TestValidator_SGLangHiCacheAccepted(t *testing.T) { if _, err := (&CacheBackendValidator{}).ValidateCreate(context.Background(), newHiCacheBackend()); err != nil { t.Fatalf("valid SGLangHiCache rejected: %v", err) @@ -77,6 +98,78 @@ func TestValidator_CanonicalSGLangHiCacheRejectsRemoteStorage(t *testing.T) { "does not accept remote binding protocol") } +func TestValidator_CanonicalSGLangHiCacheAcceptsExternalNFS(t *testing.T) { + if _, err := (&CacheBackendValidator{}).ValidateCreate(context.Background(), newCanonicalHiCacheNFSBackend()); err != nil { + t.Fatalf("valid canonical HiCache+NFS rejected: %v", err) + } +} + +func TestValidator_CanonicalHiCacheNFSServerAccepted(t *testing.T) { + servers := []struct { + name string + server string + }{ + {"IPv4", "10.0.0.25"}, + // NFSVolumeSource.Server stores raw IPv6; kubelet brackets it when + // constructing the mount source. See backend.ValidateNFSServer. + {"raw IPv6 (kubelet formats mount source)", "2001:db8::25"}, + {"DNS hostname", "nfs.example.com"}, + {"single-label DNS hostname", "nfs-server"}, + } + validator := &CacheBackendValidator{} + for _, tc := range servers { + t.Run(tc.name, func(t *testing.T) { + cb := newCanonicalHiCacheNFSBackend() + cb.Spec.RemoteStorage.NFS.Server = tc.server + if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("valid NFS server %q rejected: %v", tc.server, err) + } + }) + } +} + +func TestValidator_CanonicalHiCacheNFSContract(t *testing.T) { + trueValue := true + cases := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend) + want string + }{ + {"managed ownership", func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.RemoteStorage.Ownership = cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged + }, "remoteStorage.ownership"}, + {"fail open", func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.Integration.FailOpen = &trueValue + }, "integration.failOpen"}, + {"omitted failOpen", func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.Integration.FailOpen = nil + }, "integration.failOpen"}, + {"endpoint", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.Endpoint = "10.0.0.25:2049" }, "remoteStorage.endpoint"}, + {"missing NFS block", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS = nil }, "remoteStorage.nfs"}, + {"server with scheme", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "nfs://10.0.0.25" }, "remoteStorage.nfs.server"}, + {"server with invalid character", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "@" }, "remoteStorage.nfs.server"}, + {"server with query", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "host?query" }, "remoteStorage.nfs.server"}, + {"option-like server", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "-option" }, "remoteStorage.nfs.server"}, + {"server with port", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "nfs.example.com:2049" }, "remoteStorage.nfs.server"}, + {"bracketed IPv6 server", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Server = "[2001:db8::25]" }, "remoteStorage.nfs.server"}, + {"relative export path", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.Path = "hicache" }, "remoteStorage.nfs.path"}, + {"root mount", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.RemoteStorage.NFS.MountPath = "/" }, "remoteStorage.nfs.mountPath"}, + {"missing prefetch policy", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache.StoragePrefetchPolicy = "" }, "hiCache.storagePrefetchPolicy"}, + {"invalid prefetch policy", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache.StoragePrefetchPolicy = "forever" }, "hiCache.storagePrefetchPolicy"}, + } + validator := &CacheBackendValidator{} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cb := newCanonicalHiCacheNFSBackend() + tc.mutate(cb) + _, err := validator.ValidateCreate(context.Background(), cb) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("ValidateCreate error = %v, want field %q", err, tc.want) + } + }) + } +} + func TestValidator_CanonicalCacheHierarchy(t *testing.T) { validator := &CacheBackendValidator{} @@ -415,6 +508,8 @@ func TestValidator_SGLangHiCacheArgsAreReserved(t *testing.T) { "--hicache-write-policy", "--hicache-io-backend", "--hicache-mem-layout", + "--hicache-storage-backend", + "--hicache-storage-prefetch-policy", } { t.Run(flag, func(t *testing.T) { cb := newHiCacheBackend() @@ -429,6 +524,21 @@ func TestValidator_SGLangHiCacheArgsAreReserved(t *testing.T) { } } +func TestValidator_SGLangHiCacheStorageDirectoryEnvIsReserved(t *testing.T) { + cb := newCanonicalHiCacheNFSBackend() + cb.Spec.Integration.EngineOverrides = &cachev1alpha1.EngineInjectionOverrides{ + Env: []corev1.EnvVar{{ + Name: "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR", + Value: "/other", + }}, + } + _, err := (&CacheBackendValidator{}).ValidateCreate(context.Background(), cb) + if err == nil || !strings.Contains(err.Error(), "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR") || + !strings.Contains(err.Error(), "reserved") { + t.Fatalf("ValidateCreate error = %v, want reserved storage env", err) + } +} + func TestDefaulter_MaterialisesIntegrationForFirstEventTimeout(t *testing.T) { // The webhook materialises spec.integration solely to persist // firstEventTimeout: the CRD-schema default for firstEventTimeout only diff --git a/pkg/adapters/backend/backend.go b/pkg/adapters/backend/backend.go index 33369ca0..0e9c9b68 100644 --- a/pkg/adapters/backend/backend.go +++ b/pkg/adapters/backend/backend.go @@ -6,8 +6,12 @@ package backend import ( "errors" "fmt" + "net" + "path" + "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" ) @@ -19,13 +23,90 @@ const ( ProtocolLMCache Protocol = "lm" ProtocolRESP Protocol = "resp" ProtocolMooncakeStore Protocol = "mooncakestore" + ProtocolFile Protocol = "file" ) +// NFSBinding is the mount contract exposed by an externally owned NFS +// provider. Runtime adapters translate it into their engine-specific file +// storage wiring. +type NFSBinding struct { + Server string + Path string + MountPath string +} + +// ValidateNFSServer enforces the hostname/IP portion of the shared NFS +// binding contract. Admission and runtime adapters both call this helper so a +// stored or admission-bypassed CacheBackend cannot reach pod injection with a +// server value the validating webhook would reject. +// +// IPv6 is intentionally accepted only as a raw literal (for example +// "2001:db8::25"), not an already-bracketed URI authority. Kubernetes stores +// that raw value in corev1.NFSVolumeSource.Server; kubelet's in-tree NFS plugin +// detects it with netutil.IsIPv6String and adds brackets in +// getServerFromSource before constructing the mount source as +// "[2001:db8::25]:/export". Keeping that boundary explicit avoids rejecting a +// Kubernetes-supported input or moving kubelet-owned formatting into this +// controller. +func ValidateNFSServer(server string) error { + switch { + case strings.TrimSpace(server) == "": + return errors.New("NFS server must not be empty") + case server != strings.TrimSpace(server): + return errors.New("NFS server must not contain surrounding whitespace") + case net.ParseIP(server) == nil && len(validation.IsDNS1123Subdomain(server)) != 0: + return errors.New("NFS server must be a valid IPv4 address, IPv6 address, or DNS-1123 hostname") + default: + return nil + } +} + +// ValidateInlineNFSBinding validates the source fields used by today's +// externally owned, inline NFS Pod volume. Managed NFS must use a distinct +// provider/binding contract (for example a controller-managed PVC) rather +// than reusing this server/path shape. +func ValidateInlineNFSBinding(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) error { + if storage == nil { + return errors.New("remoteStorage must not be nil for an inline NFS binding") + } + if storage.Provider != cachev1alpha1.CacheBackendRemoteStorageProviderNFS { + return fmt.Errorf("remoteStorage.provider must be NFS for an inline NFS binding, got %q", storage.Provider) + } + if storage.Ownership != cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + return fmt.Errorf("remoteStorage.ownership must be External for an inline NFS binding, got %q", storage.Ownership) + } + if strings.TrimSpace(storage.Endpoint) != "" { + return errors.New("remoteStorage.endpoint must be empty for an inline NFS binding") + } + if storage.NFS == nil { + return errors.New("remoteStorage.nfs is required for an inline NFS binding") + } + if err := ValidateNFSServer(storage.NFS.Server); err != nil { + return fmt.Errorf("remoteStorage.nfs.server: %w", err) + } + if err := validateNFSPath(storage.NFS.Path, "remoteStorage.nfs.path", true); err != nil { + return err + } + return validateNFSPath(storage.NFS.MountPath, "remoteStorage.nfs.mountPath", false) +} + +func validateNFSPath(value, fieldName string, allowRoot bool) error { + if strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) || + !path.IsAbs(value) || path.Clean(value) != value { + return fmt.Errorf("%s must be a clean absolute path without surrounding whitespace", fieldName) + } + if !allowRoot && value == "/" { + return fmt.Errorf("%s must not mount remote storage over the container root", fieldName) + } + return nil +} + // Binding is the structured connection information an engine adapter accepts. // A nil binding means the requested hierarchy is host-only. type Binding struct { Protocol Protocol Endpoint string + NFS *NFSBinding } // RenderedStorage is the provider-owned workload shape. PodSpec and Service are @@ -83,7 +164,22 @@ func BindingFor(storage *cachev1alpha1.CacheBackendRemoteStorageSpec, protocol P if storage == nil { return nil } - return &Binding{Protocol: protocol, Endpoint: resolvedEndpoint} + binding := &Binding{Protocol: protocol, Endpoint: resolvedEndpoint} + if storage.NFS != nil { + binding.NFS = &NFSBinding{ + Server: storage.NFS.Server, + Path: storage.NFS.Path, + MountPath: storage.NFS.MountPath, + } + } + return binding +} + +// BindingRequiresEndpoint reports whether the engine wire dials a network +// endpoint. File-backed NFS is mounted into the Pod and therefore carries no +// status/spec endpoint. +func BindingRequiresEndpoint(binding *Binding) bool { + return binding != nil && binding.Protocol != ProtocolFile } // ProtocolFor returns the connection protocol associated with a provider. @@ -98,6 +194,8 @@ func ProtocolFor(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) (Protocol return ProtocolLMCache, nil case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: return ProtocolMooncakeStore, nil + case cachev1alpha1.CacheBackendRemoteStorageProviderNFS: + return ProtocolFile, nil default: return "", fmt.Errorf("%w: unknown provider=%q", ErrNoProvider, storage.Provider) } diff --git a/pkg/adapters/backend/backend_test.go b/pkg/adapters/backend/backend_test.go index 79b23b2e..65eedce1 100644 --- a/pkg/adapters/backend/backend_test.go +++ b/pkg/adapters/backend/backend_test.go @@ -1,11 +1,83 @@ package backend import ( + "strings" "testing" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" ) +func TestValidateNFSServerIPv6UsesKubeletLiteralContract(t *testing.T) { + // corev1.NFSVolumeSource.Server carries the raw literal. Kubelet's in-tree + // NFS plugin recognizes it with netutil.IsIPv6String, adds brackets in + // getServerFromSource, and only then builds "[server]:path" for mount.nfs. + if err := ValidateNFSServer("2001:db8::25"); err != nil { + t.Fatalf("raw IPv6 literal rejected: %v", err) + } + if err := ValidateNFSServer("[2001:db8::25]"); err == nil { + t.Fatal("bracketed IPv6 must be rejected at the NFSVolumeSource.Server boundary") + } +} + +func TestValidateInlineNFSBinding(t *testing.T) { + valid := func() *cachev1alpha1.CacheBackendRemoteStorageSpec { + return &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + } + + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackendRemoteStorageSpec) + wantErr string + }{ + {name: "valid"}, + {name: "managed ownership", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.Ownership = cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged + }, wantErr: "ownership must be External"}, + {name: "endpoint", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.Endpoint = "nfs.example.com:2049" + }, wantErr: "endpoint must be empty"}, + {name: "missing NFS", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS = nil + }, wantErr: "remoteStorage.nfs is required"}, + {name: "invalid server", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.Server = "@" + }, wantErr: "remoteStorage.nfs.server"}, + {name: "relative export path", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.Path = "hicache" + }, wantErr: "remoteStorage.nfs.path"}, + {name: "root mount", mutate: func(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { + storage.NFS.MountPath = "/" + }, wantErr: "must not mount remote storage over the container root"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + storage := valid() + if tt.mutate != nil { + tt.mutate(storage) + } + err := ValidateInlineNFSBinding(storage) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("ValidateInlineNFSBinding: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ValidateInlineNFSBinding error = %v, want %q", err, tt.wantErr) + } + }) + } +} + func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, @@ -21,3 +93,29 @@ func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { t.Fatalf("binding endpoint = %q, want caller-resolved endpoint", got.Endpoint) } } + +func TestBindingForNFSKeepsStructuredMountWithoutEndpoint(t *testing.T) { + storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + protocol, err := ProtocolFor(storage) + if err != nil { + t.Fatalf("ProtocolFor: %v", err) + } + got := BindingFor(storage, protocol, "") + if got == nil || got.Protocol != ProtocolFile || got.NFS == nil { + t.Fatalf("BindingFor = %+v, want structured file/NFS binding", got) + } + if got.NFS.Server != "10.0.0.25" || got.NFS.Path != "/hicache" || got.NFS.MountPath != "/mnt/hicache" { + t.Fatalf("NFS binding = %+v", got.NFS) + } + if BindingRequiresEndpoint(got) { + t.Fatal("file/NFS binding unexpectedly requires an endpoint") + } +} diff --git a/pkg/adapters/backend/provider/provider.go b/pkg/adapters/backend/provider/provider.go index 55002ebe..7f845393 100644 --- a/pkg/adapters/backend/provider/provider.go +++ b/pkg/adapters/backend/provider/provider.go @@ -72,6 +72,7 @@ func DefaultRegistry() *backendadapter.Registry { {provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, protocol: backendadapter.ProtocolRESP}, {provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, protocol: backendadapter.ProtocolLMCache}, {provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, protocol: backendadapter.ProtocolMooncakeStore}, + {provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, protocol: backendadapter.ProtocolFile}, } { registry.Register(provider) } diff --git a/pkg/adapters/backend/provider/provider_test.go b/pkg/adapters/backend/provider/provider_test.go index 45f41376..d61bc71d 100644 --- a/pkg/adapters/backend/provider/provider_test.go +++ b/pkg/adapters/backend/provider/provider_test.go @@ -48,6 +48,29 @@ func TestManagedRedisProviderOwnsTypedWorkloadConfig(t *testing.T) { } } +func TestExternalNFSProviderHasFileProtocolAndNoWorkload(t *testing.T) { + cache := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", Path: "/hicache", MountPath: "/mnt/hicache", + }, + }, + }} + selected, err := DefaultRegistry().Select(cache.Spec.RemoteStorage) + if err != nil { + t.Fatalf("Select: %v", err) + } + rendered, err := selected.Render(cache) + if err != nil { + t.Fatalf("Render: %v", err) + } + if rendered.Protocol != "file" || rendered.PodSpec != nil || rendered.Service != nil { + t.Fatalf("rendered NFS storage = %+v, want file protocol without workload", rendered) + } +} + func TestCanonicalProviderDoesNotInheritLegacyWorkloadConfig(t *testing.T) { cache := &cachev1alpha1.CacheBackend{ Spec: cachev1alpha1.CacheBackendSpec{ diff --git a/pkg/adapters/runtime/adapter.go b/pkg/adapters/runtime/adapter.go index 70e071b2..e823175e 100644 --- a/pkg/adapters/runtime/adapter.go +++ b/pkg/adapters/runtime/adapter.go @@ -146,7 +146,7 @@ func AdapterRequiresEndpoint(adapter KVCacheRuntimeAdapter) bool { // hierarchy. Binding-aware adapters accept nil for host-only operation. func AdapterRequiresEndpointFor(adapter KVCacheRuntimeAdapter, binding *backendadapter.Binding) bool { if bindingAware, ok := adapter.(RemoteBindingAdapter); ok { - return !bindingAware.SupportsRemoteBinding(binding) || binding != nil + return !bindingAware.SupportsRemoteBinding(binding) || backendadapter.BindingRequiresEndpoint(binding) } return AdapterRequiresEndpoint(adapter) } diff --git a/pkg/adapters/runtime/sglang/hicache.go b/pkg/adapters/runtime/sglang/hicache.go index cedebcce..31d856a8 100644 --- a/pkg/adapters/runtime/sglang/hicache.go +++ b/pkg/adapters/runtime/sglang/hicache.go @@ -3,6 +3,8 @@ package sglang import ( "fmt" "math" + "path" + "reflect" "strconv" "strings" @@ -15,12 +17,17 @@ import ( ) const ( - SGLangEnableHiCacheArg = "--enable-hierarchical-cache" - SGLangHiCacheSizeArg = "--hicache-size" - SGLangHiCacheRatioArg = "--hicache-ratio" - SGLangHiCacheWritePolicyArg = "--hicache-write-policy" - SGLangHiCacheIOBackendArg = "--hicache-io-backend" - SGLangHiCacheMemoryLayoutArg = "--hicache-mem-layout" + SGLangEnableHiCacheArg = "--enable-hierarchical-cache" + SGLangHiCacheSizeArg = "--hicache-size" + SGLangHiCacheRatioArg = "--hicache-ratio" + SGLangHiCacheWritePolicyArg = "--hicache-write-policy" + SGLangHiCacheIOBackendArg = "--hicache-io-backend" + SGLangHiCacheMemoryLayoutArg = "--hicache-mem-layout" + SGLangHiCacheStorageBackendArg = "--hicache-storage-backend" + SGLangHiCacheStoragePrefetchPolicyArg = "--hicache-storage-prefetch-policy" + + SGLangHiCacheFileStorageDirectoryEnv = "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR" + SGLangHiCacheStorageVolumeName = "inferencecache-hicache-l3" ) type hiCacheAdapter struct { @@ -29,7 +36,7 @@ type hiCacheAdapter struct { } // NewHiCacheAdapter returns the endpoint-free adapter for SGLang's native -// host-memory hierarchical cache. +// hierarchical cache, including its optional file-backed NFS storage tier. func NewHiCacheAdapter(opts ...runtimeadapter.Option) runtimeadapter.KVCacheRuntimeAdapter { var cfg runtimeadapter.Options for _, option := range opts { @@ -57,14 +64,15 @@ func (hiCacheAdapter) SupportedPairs() []runtimeadapter.SupportedPair { func (hiCacheAdapter) RequiresEndpoint() bool { return false } func (hiCacheAdapter) SupportsRemoteBinding(binding *backendadapter.Binding) bool { - return binding == nil + return binding == nil || + (binding.Protocol == backendadapter.ProtocolFile && binding.NFS != nil) } func (a hiCacheAdapter) InjectEngineConfigWithBinding(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { - if binding != nil { + if !a.SupportsRemoteBinding(binding) { return fmt.Errorf("SGLang HiCache adapter does not support remote binding protocol %q", binding.Protocol) } - return a.InjectEngineConfig(pod, "", cache) + return injectHiCacheEngineConfig(pod, binding, cache) } func (hiCacheAdapter) ResolveCacheServer(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { @@ -75,10 +83,19 @@ func (hiCacheAdapter) ResolveCacheServer(cache *cachev1alpha1.CacheBackend) (*co } func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *cachev1alpha1.CacheBackend) error { + return injectHiCacheEngineConfig(pod, nil, cache) +} + +func injectHiCacheEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { cfg, err := resolveHiCacheConfig(cache) if err != nil { return err } + nfsConfigured := cache.Spec.RemoteStorage != nil && + cache.Spec.RemoteStorage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS + if nfsConfigured != (binding != nil) { + return fmt.Errorf("inject SGLang HiCache config: remoteStorage.provider=NFS and the file binding must be configured together") + } if pod == nil { return fmt.Errorf("inject SGLang HiCache config: pod is nil") } @@ -104,6 +121,8 @@ func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *c SGLangHiCacheWritePolicyArg, SGLangHiCacheIOBackendArg, SGLangHiCacheMemoryLayoutArg, + SGLangHiCacheStorageBackendArg, + SGLangHiCacheStoragePrefetchPolicyArg, } { values, malformed := argValues(args, flag) if malformed || len(values) > 1 { @@ -116,7 +135,7 @@ func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *c value string equivalent func(string, string) bool } - desired := make([]desiredArg, 0, 4) + desired := make([]desiredArg, 0, 6) if cfg.sizeGB != nil { desired = append(desired, desiredArg{ flag: SGLangHiCacheSizeArg, @@ -157,6 +176,22 @@ func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *c equivalent: equivalentExact, }) } + if binding == nil { + for _, flag := range []string{SGLangHiCacheStorageBackendArg, SGLangHiCacheStoragePrefetchPolicyArg} { + if err := rejectPresentArg(args, flag, "requires remoteStorage.provider=NFS"); err != nil { + return err + } + } + } else { + desired = append(desired, + desiredArg{flag: SGLangHiCacheStorageBackendArg, value: "file", equivalent: equivalentExact}, + desiredArg{ + flag: SGLangHiCacheStoragePrefetchPolicyArg, + value: string(cfg.storagePrefetchPolicy), + equivalent: equivalentExact, + }, + ) + } present := make(map[string]bool, len(desired)) for _, want := range desired { @@ -174,6 +209,11 @@ func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *c present[want.flag] = true } + storagePlan, err := planHiCacheNFSWiring(pod, engineIndex, binding) + if err != nil { + return err + } + work := pod.DeepCopy() updated := append([]string(nil), work.Containers[engineIndex].Args...) if !hasExactArg(updated, SGLangEnableHiCacheArg) { @@ -185,6 +225,15 @@ func (hiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, _ string, cache *c } } work.Containers[engineIndex].Args = updated + if storagePlan.addEnv { + work.Containers[engineIndex].Env = append(work.Containers[engineIndex].Env, storagePlan.env) + } + if storagePlan.addVolumeMount { + work.Containers[engineIndex].VolumeMounts = append(work.Containers[engineIndex].VolumeMounts, storagePlan.volumeMount) + } + if storagePlan.addVolume { + work.Volumes = append(work.Volumes, storagePlan.volume) + } *pod = *work return nil } @@ -216,21 +265,26 @@ func hiCacheReservedArgs() []string { SGLangHiCacheWritePolicyArg, SGLangHiCacheIOBackendArg, SGLangHiCacheMemoryLayoutArg, + SGLangHiCacheStorageBackendArg, + SGLangHiCacheStoragePrefetchPolicyArg, } } -func (hiCacheAdapter) ReservedEnv() []string { return nil } +func (hiCacheAdapter) ReservedEnv() []string { + return []string{SGLangHiCacheFileStorageDirectoryEnv} +} func (hiCacheAdapter) EngineContainerName() string { return enginewire.SGLangEngineContainerName } type resolvedHiCacheConfig struct { - sizeGB *int32 - ratio string - writePolicy cachev1alpha1.SGLangHiCacheWritePolicy - ioBackend cachev1alpha1.SGLangHiCacheIOBackend - memoryLayout cachev1alpha1.SGLangHiCacheMemoryLayout + sizeGB *int32 + ratio string + writePolicy cachev1alpha1.SGLangHiCacheWritePolicy + ioBackend cachev1alpha1.SGLangHiCacheIOBackend + memoryLayout cachev1alpha1.SGLangHiCacheMemoryLayout + storagePrefetchPolicy cachev1alpha1.SGLangHiCacheStoragePrefetchPolicy } // ValidateHiCacheBackend validates the contract again at the adapter boundary. @@ -255,14 +309,20 @@ func resolveHiCacheConfig(cache *cachev1alpha1.CacheBackend) (resolvedHiCacheCon if cachev1alpha1.IntegrationMode(cache.Spec.Integration) != cachev1alpha1.CacheBackendIntegrationModeOffload { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.mode must be Offload") } + nfsStorage := cache.Spec.RemoteStorage != nil && + cache.Spec.RemoteStorage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS if cache.Spec.Integration != nil { role := cache.Spec.Integration.Role if role != "" && role != cachev1alpha1.CacheBackendIntegrationRoleReadWrite { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.role must be ReadWrite") } - if !cachev1alpha1.IntegrationFailOpen(cache.Spec.Integration) { - return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.failOpen must be true") - } + } + failOpen := cachev1alpha1.IntegrationFailOpen(cache.Spec.Integration) + if nfsStorage && failOpen { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.failOpen must be false with remoteStorage.provider=NFS because the inline NFS volume is a Pod startup dependency") + } + if !nfsStorage && !failOpen { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.failOpen must be true") } if cache.Spec.Autoscaling != nil { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: autoscaling is unsupported for an engine-local backend") @@ -290,6 +350,16 @@ func resolveHiCacheConfig(cache *cachev1alpha1.CacheBackend) (resolvedHiCacheCon return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: suppression of reserved argument %q is unsupported", flag) } } + for _, env := range overrides.Env { + if env.Name == SGLangHiCacheFileStorageDirectoryEnv { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: engine override for reserved environment variable %q is unsupported", env.Name) + } + } + for _, name := range overrides.SuppressEnv { + if name == SGLangHiCacheFileStorageDirectoryEnv { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: suppression of reserved environment variable %q is unsupported", name) + } + } } if cache.Spec.HiCache == nil { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: spec.hiCache is required") @@ -316,15 +386,154 @@ func resolveHiCacheConfig(cache *cachev1alpha1.CacheBackend) (resolvedHiCacheCon if !validMemoryLayout(spec.MemoryLayout) { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: unsupported memoryLayout %q", spec.MemoryLayout) } + if !validStoragePrefetchPolicy(spec.StoragePrefetchPolicy) { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: unsupported storagePrefetchPolicy %q", spec.StoragePrefetchPolicy) + } + if nfsStorage && spec.StoragePrefetchPolicy == "" { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: storagePrefetchPolicy is required with remoteStorage.provider=NFS") + } + if !nfsStorage && spec.StoragePrefetchPolicy != "" { + return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: storagePrefetchPolicy requires remoteStorage.provider=NFS") + } return resolvedHiCacheConfig{ - sizeGB: spec.SizeGB, - ratio: spec.Ratio, - writePolicy: spec.WritePolicy, - ioBackend: spec.IOBackend, - memoryLayout: spec.MemoryLayout, + sizeGB: spec.SizeGB, + ratio: spec.Ratio, + writePolicy: spec.WritePolicy, + ioBackend: spec.IOBackend, + memoryLayout: spec.MemoryLayout, + storagePrefetchPolicy: spec.StoragePrefetchPolicy, }, nil } +func validStoragePrefetchPolicy(value cachev1alpha1.SGLangHiCacheStoragePrefetchPolicy) bool { + switch value { + case "", + cachev1alpha1.SGLangHiCacheStoragePrefetchBestEffort, + cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete, + cachev1alpha1.SGLangHiCacheStoragePrefetchTimeout: + return true + default: + return false + } +} + +type hiCacheNFSWiringPlan struct { + addEnv bool + env corev1.EnvVar + addVolume bool + volume corev1.Volume + addVolumeMount bool + volumeMount corev1.VolumeMount +} + +func planHiCacheNFSWiring( + pod *corev1.PodSpec, + engineIndex int, + binding *backendadapter.Binding, +) (hiCacheNFSWiringPlan, error) { + if binding == nil { + return hiCacheNFSWiringPlan{}, nil + } + if binding.Protocol != backendadapter.ProtocolFile || binding.NFS == nil { + return hiCacheNFSWiringPlan{}, fmt.Errorf("inject SGLang HiCache config: file binding requires NFS mount configuration") + } + nfs := binding.NFS + if err := backendadapter.ValidateNFSServer(nfs.Server); err != nil { + return hiCacheNFSWiringPlan{}, fmt.Errorf("inject SGLang HiCache config: %w", err) + } + if err := validateHiCachePath(nfs.Path, "remoteStorage.nfs.path", true); err != nil { + return hiCacheNFSWiringPlan{}, err + } + if err := validateHiCachePath(nfs.MountPath, "remoteStorage.nfs.mountPath", false); err != nil { + return hiCacheNFSWiringPlan{}, err + } + + plan := hiCacheNFSWiringPlan{ + env: corev1.EnvVar{ + Name: SGLangHiCacheFileStorageDirectoryEnv, + Value: nfs.MountPath, + }, + volume: corev1.Volume{ + Name: SGLangHiCacheStorageVolumeName, + VolumeSource: corev1.VolumeSource{NFS: &corev1.NFSVolumeSource{ + Server: nfs.Server, + Path: nfs.Path, + }}, + }, + volumeMount: corev1.VolumeMount{ + Name: SGLangHiCacheStorageVolumeName, + MountPath: nfs.MountPath, + }, + } + + envCount := 0 + for _, env := range pod.Containers[engineIndex].Env { + if env.Name != plan.env.Name { + continue + } + envCount++ + if !reflect.DeepEqual(env, plan.env) { + return hiCacheNFSWiringPlan{}, fmt.Errorf( + "inject SGLang HiCache config: existing env %s conflicts with remoteStorage.nfs.mountPath", + plan.env.Name, + ) + } + } + if envCount > 1 { + return hiCacheNFSWiringPlan{}, fmt.Errorf("inject SGLang HiCache config: env %s is duplicated", plan.env.Name) + } + plan.addEnv = envCount == 0 + + volumeCount := 0 + for _, volume := range pod.Volumes { + if volume.Name != plan.volume.Name { + continue + } + volumeCount++ + if !reflect.DeepEqual(volume, plan.volume) { + return hiCacheNFSWiringPlan{}, fmt.Errorf( + "inject SGLang HiCache config: volume %q conflicts with remoteStorage.nfs", + plan.volume.Name, + ) + } + } + if volumeCount > 1 { + return hiCacheNFSWiringPlan{}, fmt.Errorf("inject SGLang HiCache config: volume %q is duplicated", plan.volume.Name) + } + plan.addVolume = volumeCount == 0 + + mountCount := 0 + for _, mount := range pod.Containers[engineIndex].VolumeMounts { + if mount.Name != plan.volumeMount.Name && mount.MountPath != plan.volumeMount.MountPath { + continue + } + mountCount++ + if !reflect.DeepEqual(mount, plan.volumeMount) { + return hiCacheNFSWiringPlan{}, fmt.Errorf( + "inject SGLang HiCache config: volume mount name %q or path %q conflicts with remoteStorage.nfs", + plan.volumeMount.Name, + plan.volumeMount.MountPath, + ) + } + } + if mountCount > 1 { + return hiCacheNFSWiringPlan{}, fmt.Errorf("inject SGLang HiCache config: volume mount %q is duplicated", plan.volumeMount.Name) + } + plan.addVolumeMount = mountCount == 0 + return plan, nil +} + +func validateHiCachePath(value, fieldName string, allowRoot bool) error { + if strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) || + !path.IsAbs(value) || path.Clean(value) != value { + return fmt.Errorf("inject SGLang HiCache config: %s must be a clean absolute path", fieldName) + } + if !allowRoot && value == "/" { + return fmt.Errorf("inject SGLang HiCache config: %s must not be the container root", fieldName) + } + return nil +} + func validWritePolicy(value cachev1alpha1.SGLangHiCacheWritePolicy) bool { switch value { case "", diff --git a/pkg/adapters/runtime/sglang/hicache_test.go b/pkg/adapters/runtime/sglang/hicache_test.go index 01232eb8..85b32a93 100644 --- a/pkg/adapters/runtime/sglang/hicache_test.go +++ b/pkg/adapters/runtime/sglang/hicache_test.go @@ -32,6 +32,23 @@ func newHiCacheBackend(spec *cachev1alpha1.SGLangHiCacheSpec) *cachev1alpha1.Cac } } +func addHiCacheNFSBinding(cache *cachev1alpha1.CacheBackend) *backendadapter.Binding { + falseValue := false + cache.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cache.Spec.Integration.FailOpen = &falseValue + cache.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "10.0.0.25", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + cache.Spec.HiCache.StoragePrefetchPolicy = cachev1alpha1.SGLangHiCacheStoragePrefetchWaitComplete + return backendadapter.BindingFor(cache.Spec.RemoteStorage, backendadapter.ProtocolFile, "") +} + func TestHiCacheAdapterContract(t *testing.T) { adapter := NewHiCacheAdapter() cache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}) @@ -64,7 +81,129 @@ func TestHiCacheAdapterContract(t *testing.T) { t.Fatal("SGLangHiCache adapter must accept a nil host-only binding") } if bindingAware.SupportsRemoteBinding(&backendadapter.Binding{Protocol: backendadapter.ProtocolRESP}) { - t.Fatal("SGLangHiCache adapter unexpectedly accepts remote storage") + t.Fatal("SGLangHiCache adapter unexpectedly accepts a RESP binding") + } + nfsCache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}) + if binding := addHiCacheNFSBinding(nfsCache); !bindingAware.SupportsRemoteBinding(binding) { + t.Fatal("SGLangHiCache adapter must accept a file/NFS binding") + } +} + +func TestHiCacheInjectsFileNFSBindingAtomicallyAndIdempotently(t *testing.T) { + cache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{ + Ratio: "2", + WritePolicy: cachev1alpha1.SGLangHiCacheWriteThrough, + }) + binding := addHiCacheNFSBinding(cache) + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: "sglang", + Args: []string{"--model-path", "model"}, + }}} + adapter := NewHiCacheAdapter().(runtimeadapter.RemoteBindingAdapter) + if err := adapter.InjectEngineConfigWithBinding(pod, binding, cache); err != nil { + t.Fatalf("InjectEngineConfigWithBinding: %v", err) + } + + for flag, want := range map[string]string{ + SGLangHiCacheStorageBackendArg: "file", + SGLangHiCacheStoragePrefetchPolicyArg: "wait_complete", + } { + if got, ok := testArgValue(pod.Containers[0].Args, flag); !ok || got != want { + t.Errorf("%s = (%q, %v), want %q", flag, got, ok, want) + } + } + if got := pod.Containers[0].Env; len(got) != 1 || got[0].Name != SGLangHiCacheFileStorageDirectoryEnv || got[0].Value != "/mnt/hicache" { + t.Fatalf("storage env = %+v", got) + } + if got := pod.Volumes; len(got) != 1 || got[0].Name != SGLangHiCacheStorageVolumeName || + got[0].NFS == nil || got[0].NFS.Server != "10.0.0.25" || got[0].NFS.Path != "/hicache" { + t.Fatalf("NFS volume = %+v", got) + } + if got := pod.Containers[0].VolumeMounts; len(got) != 1 || got[0].Name != SGLangHiCacheStorageVolumeName || got[0].MountPath != "/mnt/hicache" { + t.Fatalf("storage mount = %+v", got) + } + afterFirst := pod.DeepCopy() + if err := adapter.InjectEngineConfigWithBinding(pod, binding, cache); err != nil { + t.Fatalf("second InjectEngineConfigWithBinding: %v", err) + } + if !reflect.DeepEqual(pod, afterFirst) { + t.Fatalf("second injection changed pod:\nfirst=%+v\nsecond=%+v", afterFirst, pod) + } +} + +func TestHiCacheFileNFSBindingConflictsFailAtomically(t *testing.T) { + cache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}) + binding := addHiCacheNFSBinding(cache) + adapter := NewHiCacheAdapter().(runtimeadapter.RemoteBindingAdapter) + cases := []struct { + name string + mutate func(*corev1.PodSpec) + }{ + {"storage arg", func(p *corev1.PodSpec) { p.Containers[0].Args = []string{SGLangHiCacheStorageBackendArg, "mooncake"} }}, + {"storage env", func(p *corev1.PodSpec) { + p.Containers[0].Env = []corev1.EnvVar{{Name: SGLangHiCacheFileStorageDirectoryEnv, Value: "/other"}} + }}, + {"storage volume", func(p *corev1.PodSpec) { p.Volumes = []corev1.Volume{{Name: SGLangHiCacheStorageVolumeName}} }}, + {"storage mount", func(p *corev1.PodSpec) { + p.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: SGLangHiCacheStorageVolumeName, MountPath: "/other"}} + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "sglang"}}} + tc.mutate(pod) + before := pod.DeepCopy() + if err := adapter.InjectEngineConfigWithBinding(pod, binding, cache); err == nil { + t.Fatal("InjectEngineConfigWithBinding returned no error") + } + if !reflect.DeepEqual(pod, before) { + t.Fatalf("failed injection partially mutated pod:\nbefore=%+v\nafter=%+v", before, pod) + } + }) + } +} + +func TestHiCacheFileNFSBindingRejectsInvalidServerAtomically(t *testing.T) { + adapter := NewHiCacheAdapter().(runtimeadapter.RemoteBindingAdapter) + for _, server := range []string{"@", "host?query", "-option"} { + t.Run(server, func(t *testing.T) { + cache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}) + binding := addHiCacheNFSBinding(cache) + binding.NFS.Server = server + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: enginewire.SGLangEngineContainerName, + Args: []string{"--model-path", "model"}, + }}} + before := pod.DeepCopy() + + err := adapter.InjectEngineConfigWithBinding(pod, binding, cache) + if err == nil || !strings.Contains(err.Error(), "NFS server") { + t.Fatalf("InjectEngineConfigWithBinding error = %v, want NFS server validation error", err) + } + if !reflect.DeepEqual(pod, before) { + t.Fatalf("failed injection partially mutated pod:\nbefore=%+v\nafter=%+v", before, pod) + } + }) + } +} + +func TestHiCacheFileNFSBindingRequiresFailClosedAtomically(t *testing.T) { + cache := newHiCacheBackend(&cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}) + binding := addHiCacheNFSBinding(cache) + trueValue := true + cache.Spec.Integration.FailOpen = &trueValue + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: enginewire.SGLangEngineContainerName, + Args: []string{"--model-path", "model"}, + }}} + before := pod.DeepCopy() + + err := NewHiCacheAdapter().(runtimeadapter.RemoteBindingAdapter).InjectEngineConfigWithBinding(pod, binding, cache) + if err == nil || !strings.Contains(err.Error(), "integration.failOpen must be false") { + t.Fatalf("InjectEngineConfigWithBinding error = %v, want NFS fail-closed validation error", err) + } + if !reflect.DeepEqual(pod, before) { + t.Fatalf("failed injection partially mutated pod:\nbefore=%+v\nafter=%+v", before, pod) } } @@ -326,12 +465,14 @@ func TestHiCacheReservedArgs(t *testing.T) { SGLangHiCacheWritePolicyArg, SGLangHiCacheIOBackendArg, SGLangHiCacheMemoryLayoutArg, + SGLangHiCacheStorageBackendArg, + SGLangHiCacheStoragePrefetchPolicyArg, } if !reflect.DeepEqual(got, want) { t.Fatalf("ReservedArgs = %v, want %v", got, want) } - if got := NewHiCacheAdapter().ReservedEnv(); len(got) != 0 { - t.Fatalf("ReservedEnv = %v, want empty", got) + if got := NewHiCacheAdapter().ReservedEnv(); !reflect.DeepEqual(got, []string{SGLangHiCacheFileStorageDirectoryEnv}) { + t.Fatalf("ReservedEnv = %v, want storage directory env", got) } } diff --git a/pkg/cli/doctor/checks/cachebackend.go b/pkg/cli/doctor/checks/cachebackend.go index 9abde0db..cf5ffe3e 100644 --- a/pkg/cli/doctor/checks/cachebackend.go +++ b/pkg/cli/doctor/checks/cachebackend.go @@ -16,24 +16,29 @@ import ( const checkCacheBackendHealth = "CacheBackendHealth" // CacheBackendHealth inspects each CacheBackend in scope across the dimensions -// the controller surfaces on status: Ready, matched engine pods, index -// participation (KV-event freshness), and endpoint reachability. Every failing -// axis emits its own finding so the operator sees exactly what is wrong; a -// backend that passes every applicable axis emits a single OK. +// the controller surfaces on status: Ready (where published), matched engine +// pods, index participation (KV-event freshness), and endpoint reachability. +// Every failing axis emits its own finding so the operator sees exactly what is +// wrong. A backend that passes every applicable axis emits a single OK, except +// NFS, whose unprobed mount and L3 data path keep the aggregate result at INFO. // -// Three health models coexist, derived from EffectiveRemoteStorage so canonical +// Four health models coexist, derived from EffectiveRemoteStorage so canonical // and legacy resources are classified identically: // - Managed remote storage: every axis applies. -// - External remote storage: only Ready + endpoint reachability apply. +// - External network storage: only Ready + endpoint reachability apply. // - Host-only caching: engine/index axes apply, but endpoint checks do not. +// - External NFS: engine/index axes apply like host-only caching, but Ready +// and endpoint checks do not because the controller publishes neither for +// endpoint-free mounted storage. Passing those observable axes yields INFO, +// not a synthetic healthy verdict for the unprobed NFS data path. // // The matched-engine-pod axis prefers the controller-written // status.matchedEnginePods (its authoritative snapshot) but falls back to a -// live label match when status is absent — so doctor flags a selector mismatch -// even before the controller has reconciled, which is exactly the freshly- -// misconfigured state operators run doctor in. It fires only when the backend -// actually declares an engineSelector: a selectorless backend has nothing to -// mismatch. +// live label match when status is absent or belongs to an older generation — +// so doctor flags a selector mismatch even before the controller has reconciled, +// which is exactly the freshly-misconfigured state operators run doctor in. It +// fires only when the backend actually declares an engineSelector: a +// selectorless backend has nothing to mismatch. // // The index-participation axis keys off lastEventAt, not the prefix count: zero // warm prefixes is a VALID state for an up-but-idle backend (PROJECT_CONTEXT), @@ -55,21 +60,60 @@ func CacheBackendHealth(ctx context.Context, c client.Client, ns string, now tim findings = append(findings, f) } - // Ready condition (all backends). - if ready := findCondition(cb.Status.Conditions, conditionReady); ready == nil || ready.Status != metav1.ConditionTrue { + // Status-derived health axes are usable only after the controller has + // processed the current spec generation; otherwise an old Ready=True, + // endpoint, matched-pod count, or index observation could make a just-edited + // backend look healthy. Real API objects start at generation 1, so + // generation 0 is also unobserved (it occurs only in synthetic clients/tests). + statusCurrent := cb.Generation > 0 && cb.Status.ObservedGeneration == cb.Generation + ready := findCondition(cb.Status.Conditions, conditionReady) + readyCurrent := ready == nil || ready.ObservedGeneration == cb.Generation + switch { + case !statusCurrent: note(doctor.Finding{ Code: doctor.CodeBackendNotReady, Status: doctor.StatusWarn, - Check: checkCacheBackendHealth, Resource: ref, Message: notReadyMessage(ready), + Check: checkCacheBackendHealth, Resource: ref, + Message: fmt.Sprintf( + "controller has not observed the current CacheBackend generation: metadata.generation=%d, status.observedGeneration=%d", + cb.Generation, cb.Status.ObservedGeneration, + ), + }) + case !readyCurrent: + note(doctor.Finding{ + Code: doctor.CodeBackendNotReady, Status: doctor.StatusWarn, + Check: checkCacheBackendHealth, Resource: ref, + Message: fmt.Sprintf( + "Ready condition is stale for the current CacheBackend generation: metadata.generation=%d, Ready.observedGeneration=%d", + cb.Generation, ready.ObservedGeneration, + ), }) } + statusUsable := statusCurrent && readyCurrent storage := cb.Spec.EffectiveRemoteStorage() - external := storage != nil && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal - managed := !external - if managed { + nfsBacked := storage != nil && storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderNFS + externalEndpoint := storage != nil && + storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal && + !nfsBacked + endpointBacked := storage != nil && !nfsBacked + + // A supported endpoint-free NFS binding publishes no Ready condition + // because the controller cannot verify its data path. An explicit + // non-True Ready verdict still applies: the controller uses it to surface + // adapter/provider capability failures for stored or admission-bypassed + // resources, and doctor consumes that status instead of duplicating the + // compatibility table. + if statusUsable && ((ready != nil && ready.Status != metav1.ConditionTrue) || (ready == nil && !nfsBacked)) { + note(doctor.Finding{ + Code: doctor.CodeBackendNotReady, Status: doctor.StatusWarn, + Check: checkCacheBackendHealth, Resource: ref, Message: notReadyMessage(ready), + }) + } + + if !externalEndpoint { // Matched engine pods — only meaningful when a selector is declared. if hasEngineSelector(cb) { - count, err := matchedEnginePodCount(ctx, c, cb) + count, err := matchedEnginePodCount(ctx, c, cb, statusUsable) switch { case err != nil: note(doctor.Finding{ @@ -86,39 +130,50 @@ func CacheBackendHealth(ctx context.Context, c client.Client, ns string, now tim } } - // Index participation: keyed off KV-event observation, not prefix count. - if f, ok := participationFinding(cb.Status.IndexParticipation, cb.Status.FirstKVEventObservedAt, ref, now, staleWindow); ok { - note(f) - } + if statusUsable { + // Index participation: keyed off KV-event observation, not prefix count. + if f, ok := participationFinding(cb.Status.IndexParticipation, cb.Status.FirstKVEventObservedAt, ref, now, staleWindow); ok { + note(f) + } - // Functional self-test gate: the controller writes FunctionalProbeOK - // only once the backend is otherwise eligible, so its absence is not a - // problem (Ready/CB003 already cover the not-yet-ready state). A - // present-but-not-True condition is the silent-failure signal the bare - // Ready bit does not explain — surface its reason/message. - if fp := findCondition(cb.Status.Conditions, conditionFunctionalProbeOK); fp != nil && fp.Status != metav1.ConditionTrue { - note(doctor.Finding{ - Code: doctor.CodeBackendFunctionalProbeFailing, Status: doctor.StatusWarn, - Check: checkCacheBackendHealth, Resource: ref, - Message: fmt.Sprintf("FunctionalProbeOK=%s (reason %s): the controller's end-to-end functional self-test is failing for this backend — %s", fp.Status, fp.Reason, fp.Message), - }) + // Functional self-test gate: the controller writes FunctionalProbeOK + // only once the backend is otherwise eligible, so its absence is not a + // problem (Ready/CB003 already cover the not-yet-ready state). A + // present-but-not-True condition is the silent-failure signal the bare + // Ready bit does not explain — surface its reason/message. + if fp := findCondition(cb.Status.Conditions, conditionFunctionalProbeOK); fp != nil && fp.Status != metav1.ConditionTrue { + note(doctor.Finding{ + Code: doctor.CodeBackendFunctionalProbeFailing, Status: doctor.StatusWarn, + Check: checkCacheBackendHealth, Resource: ref, + Message: fmt.Sprintf("FunctionalProbeOK=%s (reason %s): the controller's end-to-end functional self-test is failing for this backend — %s", fp.Status, fp.Reason, fp.Message), + }) + } } } // Endpoint presence + reachability applies only when a remote provider // exists. Host-only caches intentionally publish no endpoint. - if storage != nil { + if statusUsable && endpointBacked { if f, ok := endpointFinding(ctx, cb, ref, dial); ok { note(f) } } - if healthy { + if !healthy { + continue + } + if nfsBacked { findings = append(findings, doctor.Finding{ - Code: doctor.CodeBackendHealthy, Status: doctor.StatusOK, - Check: checkCacheBackendHealth, Resource: ref, Message: healthyMessage(cb, managed, storage != nil, dial != nil), + Code: doctor.CodeBackendNFSUnverified, Status: doctor.StatusInfo, + Check: checkCacheBackendHealth, Resource: ref, + Message: "NFS-backed backend passed all observable engine and index checks, but the NFS mount and HiCache L3 store/read data path were not verified", }) + continue } + findings = append(findings, doctor.Finding{ + Code: doctor.CodeBackendHealthy, Status: doctor.StatusOK, + Check: checkCacheBackendHealth, Resource: ref, Message: healthyMessage(cb, externalEndpoint, endpointBacked, dial != nil), + }) } return findings } @@ -163,8 +218,8 @@ func participationFinding(ip *cachev1alpha1.CacheBackendIndexParticipation, firs // when present, else a live label-match count against the backend's namespace. // A pod-list failure is returned as an error (not silently coerced to 0) so the // caller can report it as inconclusive rather than as a selector mismatch. -func matchedEnginePodCount(ctx context.Context, c client.Client, cb *cachev1alpha1.CacheBackend) (int, error) { - if cb.Status.MatchedEnginePods != nil { +func matchedEnginePodCount(ctx context.Context, c client.Client, cb *cachev1alpha1.CacheBackend, statusUsable bool) (int, error) { + if statusUsable && cb.Status.MatchedEnginePods != nil { return int(*cb.Status.MatchedEnginePods), nil } var pods corev1.PodList @@ -206,7 +261,7 @@ func endpointFinding(ctx context.Context, cb *cachev1alpha1.CacheBackend, ref st func notReadyMessage(ready *metav1.Condition) string { if ready == nil { - return "Ready condition is not set yet — the controller has not reconciled this backend, or it has never observed a KV event" + return "Ready condition is not set for the current observed generation" } return fmt.Sprintf("Ready=%s (reason %s): %s", ready.Status, ready.Reason, ready.Message) } @@ -216,7 +271,7 @@ func staleMessage(lastEventAt *metav1.Time, now time.Time, staleWindow time.Dura return fmt.Sprintf("status.indexParticipation.lastEventAt is %s old (EngineStale): exceeds the %s freshness window — KV events have stopped flowing", age, staleWindow) } -func healthyMessage(cb *cachev1alpha1.CacheBackend, managed, hasRemoteStorage, dialed bool) string { +func healthyMessage(cb *cachev1alpha1.CacheBackend, externalEndpoint, endpointBacked, dialed bool) string { // Only claim "reachable" when an actual TCP probe ran; with no dialer // (e.g. --config-only) doctor verified the endpoint is published, not that // it answers. @@ -224,10 +279,10 @@ func healthyMessage(cb *cachev1alpha1.CacheBackend, managed, hasRemoteStorage, d if dialed { endpoint = "endpoint reachable" } - if !managed { + if externalEndpoint { return "External backend: Ready, " + endpoint } - if !hasRemoteStorage { + if !endpointBacked { endpoint = "host-only (endpoint not applicable)" } matched := "engine pods matched" diff --git a/pkg/cli/doctor/checks/checks_test.go b/pkg/cli/doctor/checks/checks_test.go index f472d50a..d1f19c46 100644 --- a/pkg/cli/doctor/checks/checks_test.go +++ b/pkg/cli/doctor/checks/checks_test.go @@ -84,19 +84,23 @@ func ptr[T any](v T) *T { return &v } // --- fixtures --------------------------------------------------------------- func readyCond(status metav1.ConditionStatus, reason, msg string) metav1.Condition { - return metav1.Condition{Type: conditionReady, Status: status, Reason: reason, Message: msg} + return metav1.Condition{ + Type: conditionReady, Status: status, Reason: reason, Message: msg, + ObservedGeneration: 1, + } } func healthyBackend(now time.Time) *cachev1alpha1.CacheBackend { return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "good", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "good", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "10.0.0.5:8200", - MatchedEnginePods: ptr(int32(2)), - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "KVEventObserved", "ready")}, + ObservedGeneration: 1, + Endpoint: "10.0.0.5:8200", + MatchedEnginePods: ptr(int32(2)), + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "KVEventObserved", "ready")}, IndexParticipation: &cachev1alpha1.CacheBackendIndexParticipation{ PrefixCount: 5, LastEventAt: &metav1.Time{Time: now.Add(-10 * time.Second)}, @@ -105,6 +109,30 @@ func healthyBackend(now time.Time) *cachev1alpha1.CacheBackend { } } +func observedNFSBackend(now time.Time) *cachev1alpha1.CacheBackend { + cb := healthyBackend(now) + falseValue := false + cb.Name = "hicache-nfs" + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: &falseValue} + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/hicache", + MountPath: "/mnt/hicache", + }, + } + cb.Status.Endpoint = "" + // Valid endpoint-free NFS is reconciled as unmanaged: the controller + // acknowledges the current generation but intentionally publishes no Ready + // condition because it cannot verify the mounted L3 data path. + cb.Status.Conditions = nil + return cb +} + // --- ServerReachability ----------------------------------------------------- type stubHealth struct { @@ -386,22 +414,58 @@ func TestCacheBackendHealth(t *testing.T) { } }) + t.Run("stale controller status cannot preserve old healthy verdict", func(t *testing.T) { + cb := healthyBackend(now) + cb.Generation = 2 + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "engine", Namespace: cb.Namespace, Labels: map[string]string{"app": "engine"}, + }} + fs := CacheBackendHealth(ctx, fakeClient(t, cb, pod), "", now, DefaultStaleWindow, okDial) + if len(fs) != 1 || fs[0].Code != doctor.CodeBackendNotReady || + !strings.Contains(fs[0].Message, "status.observedGeneration=1") { + t.Fatalf("stale controller status should produce one generation-specific CB001, got %v", fs) + } + if hasCode(fs, doctor.CodeBackendHealthy) != nil { + t.Fatalf("stale Ready=True must not preserve CB006, got %v", codesOf(fs)) + } + }) + + t.Run("stale Ready condition cannot preserve old healthy verdict", func(t *testing.T) { + cb := healthyBackend(now) + cb.Generation = 2 + cb.Status.ObservedGeneration = 2 + // The overall status claims generation 2, but Ready still describes + // generation 1. Treat the specific verdict as stale as well. + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "engine", Namespace: cb.Namespace, Labels: map[string]string{"app": "engine"}, + }} + fs := CacheBackendHealth(ctx, fakeClient(t, cb, pod), "", now, DefaultStaleWindow, okDial) + if len(fs) != 1 || fs[0].Code != doctor.CodeBackendNotReady || + !strings.Contains(fs[0].Message, "Ready.observedGeneration=1") { + t.Fatalf("stale Ready condition should produce one generation-specific CB001, got %v", fs) + } + }) + t.Run("fresh misconfigured backend, no status", func(t *testing.T) { cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "bad", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "bad", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "missing"}}, }, } c := fakeClient(t, cb) fs := CacheBackendHealth(ctx, c, "ns1", now, DefaultStaleWindow, okDial) - for _, want := range []string{doctor.CodeBackendNotReady, doctor.CodeBackendSelectorMismatch, doctor.CodeBackendNotReportingState, doctor.CodeBackendEndpointUnreachable} { - if hasCode(fs, want) == nil { - t.Errorf("want %s, got %v", want, codesOf(fs)) - } + if hasCode(fs, doctor.CodeBackendNotReady) == nil || + !strings.Contains(fs[0].Message, "has not observed the current") { + t.Fatalf("fresh unreconciled backend should report generation-specific CB001, got %v", fs) } - if hasCode(fs, doctor.CodeBackendHealthy) != nil { - t.Errorf("did not expect CB006 for a broken backend") + if hasCode(fs, doctor.CodeBackendSelectorMismatch) == nil { + t.Fatalf("fresh unreconciled backend should retain live selector diagnostics, got %v", fs) + } + for _, staleCode := range []string{doctor.CodeBackendNotReportingState, doctor.CodeBackendEndpointUnreachable, doctor.CodeBackendHealthy} { + if hasCode(fs, staleCode) != nil { + t.Fatalf("fresh unreconciled backend must not interpret stale status as %s, got %v", staleCode, codesOf(fs)) + } } }) @@ -409,10 +473,14 @@ func TestCacheBackendHealth(t *testing.T) { // No status.matchedEnginePods, but a live pod matches the selector => // no mismatch finding. cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "live", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "live", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}, }, + Status: cachev1alpha1.CacheBackendStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "Ready", "ready")}, + }, } pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns1", Labels: map[string]string{"app": "engine"}}} c := fakeClient(t, cb, pod) @@ -509,11 +577,12 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { t.Run("FunctionalProbeOK absent on External backend is ignored", func(t *testing.T) { // External backends skip the managed axes entirely, including the probe. cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{Type: cachev1alpha1.CacheBackendTypeExternal, Endpoint: "h:1"}, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "h:1", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ok")}, + ObservedGeneration: 1, + Endpoint: "h:1", + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ok")}, }, } c := fakeClient(t, cb) @@ -568,8 +637,12 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { t.Run("pod-list error is inconclusive (API001), not a selector mismatch", func(t *testing.T) { cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "x", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "x", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}}, + Status: cachev1alpha1.CacheBackendStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "Ready", "ready")}, + }, } c := listErrClient{Client: fakeClient(t, cb), failOn: func(l client.ObjectList) bool { _, ok := l.(*corev1.PodList) @@ -586,14 +659,15 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { t.Run("External backend skips pod-match and index-participation axes", func(t *testing.T) { cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ Type: cachev1alpha1.CacheBackendTypeExternal, Endpoint: "cache.example.com:8200", }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "cache.example.com:8200", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, + ObservedGeneration: 1, + Endpoint: "cache.example.com:8200", + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, }, } c := fakeClient(t, cb) @@ -605,7 +679,7 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { t.Run("canonical External backend skips managed axes", func(t *testing.T) { cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "canonical-ext", Namespace: "ns1"}, + ObjectMeta: metav1.ObjectMeta{Name: "canonical-ext", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, @@ -616,8 +690,9 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { }, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "cache.example.com:8200", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, + ObservedGeneration: 1, + Endpoint: "cache.example.com:8200", + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, }, } fs := CacheBackendHealth(ctx, fakeClient(t, cb), "", now, DefaultStaleWindow, okDial) @@ -646,6 +721,70 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { } }) + t.Run("canonical NFS-backed HiCache reports unverified after observable axes pass", func(t *testing.T) { + cb := observedNFSBackend(now) + fs := CacheBackendHealth(ctx, fakeClient(t, cb), "", now, DefaultStaleWindow, badDial) + if len(fs) != 1 || fs[0].Code != doctor.CodeBackendNFSUnverified || fs[0].Status != doctor.StatusInfo { + t.Fatalf("canonical NFS-backed HiCache should be CB008 INFO, got %v", fs) + } + if !strings.Contains(fs[0].Message, "not verified") { + t.Fatalf("CB008 message = %q, want explicit unverified data-path scope", fs[0].Message) + } + if hasCode(fs, doctor.CodeBackendHealthy) != nil { + t.Fatalf("NFS-backed HiCache must not report synthetic CB006 health, got %v", codesOf(fs)) + } + if hasCode(fs, doctor.CodeBackendNotReady) != nil { + t.Fatalf("NFS-backed HiCache must not report CB001 when Ready is absent, got %v", codesOf(fs)) + } + if hasCode(fs, doctor.CodeBackendEndpointUnreachable) != nil { + t.Fatalf("NFS-backed HiCache must not report CB005, got %v", codesOf(fs)) + } + }) + + t.Run("unobserved NFS generation reports not ready instead of unverified", func(t *testing.T) { + cb := observedNFSBackend(now) + cb.Generation = 2 + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "engine", Namespace: cb.Namespace, Labels: map[string]string{"app": "engine"}, + }} + fs := CacheBackendHealth(ctx, fakeClient(t, cb, pod), "", now, DefaultStaleWindow, badDial) + if len(fs) != 1 || fs[0].Code != doctor.CodeBackendNotReady || + !strings.Contains(fs[0].Message, "status.observedGeneration=1") { + t.Fatalf("unobserved NFS generation should produce generation-specific CB001, got %v", fs) + } + if hasCode(fs, doctor.CodeBackendNFSUnverified) != nil { + t.Fatalf("unobserved NFS generation must not produce CB008, got %v", codesOf(fs)) + } + }) + + t.Run("controller-rejected NFS binding reports Ready failure instead of unverified", func(t *testing.T) { + cb := healthyBackend(now) + cb.Name = "unsupported-nfs" + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM + cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderNFS, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + NFS: &cachev1alpha1.NFSRemoteStorageSpec{ + Server: "192.0.2.10", + Path: "/lmcache", + MountPath: "/mnt/lmcache", + }, + } + cb.Status.Endpoint = "" + cb.Status.Conditions = []metav1.Condition{ + readyCond(metav1.ConditionFalse, "UnsupportedRemoteBinding", "vLLM LMCache does not accept file binding"), + } + + fs := CacheBackendHealth(ctx, fakeClient(t, cb), "", now, DefaultStaleWindow, badDial) + if f := hasCode(fs, doctor.CodeBackendNotReady); f == nil || !strings.Contains(f.Message, "UnsupportedRemoteBinding") { + t.Fatalf("controller-rejected NFS should surface CB001 with controller reason, got %v", fs) + } + if hasCode(fs, doctor.CodeBackendNFSUnverified) != nil { + t.Fatalf("controller-rejected NFS must not report CB008, got %v", codesOf(fs)) + } + }) + t.Run("selectorless managed backend skips the matched-pods axis", func(t *testing.T) { cb := healthyBackend(now) cb.Spec.EngineSelector = nil diff --git a/pkg/cli/doctor/finding.go b/pkg/cli/doctor/finding.go index 82f1fb91..af1024d0 100644 --- a/pkg/cli/doctor/finding.go +++ b/pkg/cli/doctor/finding.go @@ -159,6 +159,10 @@ const ( // why Ready may be downgraded. Surfaces the underlying reason the bare Ready // bit does not. CodeBackendFunctionalProbeFailing = "CB007" + // CodeBackendNFSUnverified: the NFS-backed CacheBackend passed every + // observable engine and index check, but inference-cache does not verify the + // NFS mount or HiCache L3 store/read data path, so overall health is unknown. + CodeBackendNFSUnverified = "CB008" // CodeEnginePodNotInjected: a pod matching a CacheBackend's engineSelector // carries no injection marker — neither a validated