diff --git a/Containerfile.gkm-operator b/Containerfile.gkm-operator index af593693..4af25c9f 100644 --- a/Containerfile.gkm-operator +++ b/Containerfile.gkm-operator @@ -24,7 +24,6 @@ COPY cmd/main.go cmd/main.go COPY api/ api/ COPY pkg/ pkg/ COPY internal/controller/ internal/controller/ -COPY internal/webhook/ internal/webhook/ COPY vendor/ vendor/ COPY Makefile Makefile diff --git a/Makefile b/Makefile index cdf8f2a7..eb285e70 100644 --- a/Makefile +++ b/Makefile @@ -304,10 +304,19 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified prepare-deploy: cd config/operator && $(KUSTOMIZE) edit set image quay.io/gkm/operator=${OPERATOR_IMG} cd config/agent && $(KUSTOMIZE) edit set image quay.io/gkm/agent=${AGENT_IMG} -ifdef NO_GPU +ifeq ($(KIND_CLUSTER),true) cd config/configMap && \ $(SED) \ -e '/literals:/a\ - gkm.nogpu=true' \ + -e '/literals:/a\ - gkm.kindcluster=true' \ + -e 's@gkm\.agent\.image=.*@gkm.agent.image=$(AGENT_IMG)@' \ + -e 's@gkm\.extract\.image=.*@gkm.extract.image=$(EXTRACT_IMG)@' \ + kustomization.yaml.env > kustomization.yaml +else ifeq ($(NO_GPU),true) + cd config/configMap && \ + $(SED) \ + -e '/literals:/a\ - gkm.nogpu=true' \ + -e '/literals:/a\ - gkm.kindcluster=false' \ -e 's@gkm\.agent\.image=.*@gkm.agent.image=$(AGENT_IMG)@' \ -e 's@gkm\.extract\.image=.*@gkm.extract.image=$(EXTRACT_IMG)@' \ kustomization.yaml.env > kustomization.yaml @@ -591,11 +600,11 @@ deploy-on-kind: kind-load-images tmp-cleanup @echo "Add label gkm-test-node=false to node kind-gpu-sim-worker2." $(KUBECTL) label node kind-gpu-sim-worker2 gkm-test-node=false --overwrite ## NOTE: config/kind-gpu is an overlay of config/default - $(MAKE) deploy DEPLOY_PATH=config/kind-gpu NO_GPU=true + $(MAKE) deploy DEPLOY_PATH=config/kind-gpu NO_GPU=true KIND_CLUSTER=true .PHONY: redeploy-on-kind redeploy-on-kind: ## Redeploy controller and agent to Kind GPU cluster after run-on-kind and undeploy-on-kind have been called. Skips some onetime steps in deploy. - $(MAKE) redeploy DEPLOY_PATH=config/kind-gpu NO_GPU=true + $(MAKE) redeploy DEPLOY_PATH=config/kind-gpu NO_GPU=true KIND_CLUSTER=true @echo "Deployment to $(DEPLOY_PATH) completed." .PHONY: undeploy-on-kind diff --git a/agent/main.go b/agent/main.go index e05880dc..0b2333c0 100644 --- a/agent/main.go +++ b/agent/main.go @@ -49,11 +49,24 @@ func main() { setupLog.Info("No-GPU set to true") } + extractLogLevel := os.Getenv("EXTRACT_GO_LOG") + if extractLogLevel == "" { + extractLogLevel = "info" + setupLog.Info("Extract Job Log Level set to info") + } + + kindCluster := false + if os.Getenv("KIND_CLUSTER") == "true" { + kindCluster = true + setupLog.Info("KIND Cluster set to true") + } + nodeName := os.Getenv("KUBE_NODE_NAME") if nodeName == "" { setupLog.Error(fmt.Errorf("KUBE_NODE_NAME env var not set"), "Couldn't determine current node") os.Exit(1) } + setupLog.Info("KUBE_NODE_NAME processing", "Node", nodeName) extractImage := utils.JobExtractImage tmpExtractImage := os.Getenv("EXTRACT_IMAGE") @@ -175,12 +188,13 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("GKM-Agent-NS"), - CacheDir: utils.DefaultCacheDir, NodeName: nodeName, NoGpu: noGpu, + KindCluster: kindCluster, + ExtractLogLevel: extractLogLevel, ExtractImage: extractImage, - CrdCacheStr: "GKMCache", - CrdCacheNodeStr: "GKMCacheNode", + CrdCacheStr: utils.CrdGKMCache, + CrdCacheNodeStr: utils.CrdGKMCacheNode, } if err = (&gkmAgent.GKMCacheAgentReconciler{ ReconcilerCommonAgent: commonNs, @@ -198,12 +212,13 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("GKM-Agent-CL"), - CacheDir: utils.DefaultCacheDir, NodeName: nodeName, NoGpu: noGpu, + KindCluster: kindCluster, + ExtractLogLevel: extractLogLevel, ExtractImage: extractImage, - CrdCacheStr: "ClusterGKMCache", - CrdCacheNodeStr: "ClusterGKMCacheNode", + CrdCacheStr: utils.CrdClusterGKMCache, + CrdCacheNodeStr: utils.CrdClusterGKMCacheNode, } if err = (&gkmAgent.ClusterGKMCacheAgentReconciler{ ReconcilerCommonAgent: commonCl, diff --git a/api/v1alpha1/clustergkmcachenode_types.go b/api/v1alpha1/clustergkmcachenode_types.go index 116003dd..56057459 100644 --- a/api/v1alpha1/clustergkmcachenode_types.go +++ b/api/v1alpha1/clustergkmcachenode_types.go @@ -33,7 +33,6 @@ import ( // created, GKM ensures that one ClusterGKMCacheNode instance is created per // Kubernetes Node. Cluster GKMCacheNode cannot be edited by an application or // user, only by GKM. -// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".status.nodeName" // +kubebuilder:printcolumn:name="Node-In-Use",type=string,JSONPath=`.status.counts.nodeInUseCnt` // +kubebuilder:printcolumn:name="Node-Not-In-Use",type=string,JSONPath=`.status.counts.nodeNotInUseCnt` // +kubebuilder:printcolumn:name="Node-Error",type=string,JSONPath=`.status.counts.nodeErrorCnt` diff --git a/api/v1alpha1/gkmcachenode_types.go b/api/v1alpha1/gkmcachenode_types.go index 743c2dac..2ed5f57e 100644 --- a/api/v1alpha1/gkmcachenode_types.go +++ b/api/v1alpha1/gkmcachenode_types.go @@ -32,7 +32,6 @@ import ( // instance are created in a namespace, GKM ensures that one GKMCacheNode // instance is created per Kubernetes Node. GKMCacheNode cannot be edited by an // application or user, only by GKM. -// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".status.nodeName" // +kubebuilder:printcolumn:name="Node-In-Use",type=string,JSONPath=`.status.counts.nodeInUseCnt` // +kubebuilder:printcolumn:name="Node-Not-In-Use",type=string,JSONPath=`.status.counts.nodeNotInUseCnt` // +kubebuilder:printcolumn:name="Node-Error",type=string,JSONPath=`.status.counts.nodeErrorCnt` diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index ee74419e..016c9a58 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -79,10 +79,9 @@ type GKMCacheSpec struct { // storageClassName contains the name of the Kubernetes Storage Class, which // will be used for the PersistentVolume and PersistentVolumeClaim the GKM will - // create in order to store the extract GPU Kernel Cache. - // +required - // +kubebuilder:validation:Required - StorageClassName string `json:"storageClassName"` + // create in order to store the extract GPU Kernel Cache. If not provided, then + // default Storage Class will be used. + StorageClassName string `json:"storageClassName,omitempty"` // accessMode is the set of capabilities being requested by the generated PVC. // This field is optional. If not provided, it will default to "ReadWriteOnce". diff --git a/cmd/main.go b/cmd/main.go index 4c29b646..f2b1260e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -36,12 +36,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" gkmv1alpha1 "github.com/redhat-et/GKM/api/v1alpha1" gkmOperator "github.com/redhat-et/GKM/internal/controller/gkm-operator" - gkmWebhook "github.com/redhat-et/GKM/internal/webhook" "github.com/redhat-et/GKM/pkg/utils" // +kubebuilder:scaffold:imports ) @@ -71,6 +69,18 @@ func main() { setupLog.Info("No-GPU set to true", "noGpu", noGpu) } + extractLogLevel := os.Getenv("EXTRACT_GO_LOG") + if extractLogLevel == "" { + extractLogLevel = "info" + setupLog.Info("Extract Job Log Level set to info") + } + + kindCluster := false + if os.Getenv("KIND_CLUSTER") == "true" { + kindCluster = true + setupLog.Info("KIND Cluster set to true") + } + extractImage := utils.JobExtractImage tmpExtractImage := os.Getenv("EXTRACT_IMAGE") if tmpExtractImage != "" { @@ -184,16 +194,6 @@ func main() { os.Exit(1) } - mutator := &gkmWebhook.PodMutator{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Decoder: admission.NewDecoder(mgr.GetScheme()), - } - mgr.GetWebhookServer().Register( - "/mutate-v1-pod", - &admission.Webhook{Handler: mutator}, - ) - if err = (&gkmOperator.GKMConfigMapReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -211,9 +211,11 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), NoGpu: noGpu, + KindCluster: kindCluster, + ExtractLogLevel: extractLogLevel, ExtractImage: extractImage, - CrdCacheStr: "GKMCache", - CrdCacheNodeStr: "GKMCacheNode", + CrdCacheStr: utils.CrdGKMCache, + CrdCacheNodeStr: utils.CrdGKMCacheNode, } if err = (&gkmOperator.GKMCacheOperatorReconciler{ ReconcilerCommonOperator: commonNs, @@ -231,9 +233,10 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), NoGpu: noGpu, + KindCluster: kindCluster, ExtractImage: extractImage, - CrdCacheStr: "ClusterGKMCache", - CrdCacheNodeStr: "ClusterGKMCacheNode", + CrdCacheStr: utils.CrdClusterGKMCache, + CrdCacheNodeStr: utils.CrdClusterGKMCacheNode, } if err = (&gkmOperator.ClusterGKMCacheOperatorReconciler{ ReconcilerCommonOperator: commonCl, diff --git a/config/agent/gkm-agent.yaml b/config/agent/gkm-agent.yaml index b108bbcd..d6ea1978 100644 --- a/config/agent/gkm-agent.yaml +++ b/config/agent/gkm-agent.yaml @@ -32,6 +32,11 @@ spec: configMapKeyRef: name: gkm-config key: gkm.nogpu + - name: KIND_CLUSTER + valueFrom: + configMapKeyRef: + name: gkm-config + key: gkm.kindcluster - name: GO_LOG valueFrom: configMapKeyRef: @@ -51,36 +56,12 @@ spec: memory: "128Mi" cpu: "100m" volumeMounts: - - name: gkm-state - mountPath: /var/lib/gkm - mountPropagation: Bidirectional - - name: gkm-runtime - mountPath: /run/gkm - mountPropagation: Bidirectional - - name: sys - mountPath: /sys - readOnly: true - - name: dev - mountPath: /dev + - mountPath: /mnt/kernel-caches + name: kernel-caches + readOnly: false volumes: - # This volume is the GKM State directory. This is where GPU Kernel Cache - # will be extracted. - - name: gkm-state - hostPath: - path: /var/lib/gkm - type: DirectoryOrCreate - # This volume is the GKM Runtime directory. This is where the Usage data - # will tracked which pods are using which cache. - - name: gkm-runtime - hostPath: - path: /run/gkm - type: DirectoryOrCreate - - name: sys - hostPath: - path: /sys - type: Directory - - name: dev + - name: kernel-caches hostPath: - path: /dev - type: Directory + path: /kernel-caches + type: DirectoryOrCreate \ No newline at end of file diff --git a/config/configMap/configMap.yaml b/config/configMap/configMap.yaml index 12d3b863..a342f65d 100644 --- a/config/configMap/configMap.yaml +++ b/config/configMap/configMap.yaml @@ -7,9 +7,11 @@ data: ## Can be set to "info", "debug", or "trace". Not processed at runtime. gkm.operator.log.level: info gkm.agent.log.level: info + gkm.extract.log.level: info ## Can be configured at runtime gkm.agent.image: quay.io/gkm/agent:latest gkm.extract.image: quay.io/gkm/gkm-extract:latest gkm.nogpu: false + gkm.kindcluster: false ## Enable/disable Kyverno image signature verification (defaults to true/enabled) gkm.kyverno.enabled: "true" diff --git a/config/configMap/kustomization.yaml b/config/configMap/kustomization.yaml index b46ed2b7..30305fe2 100644 --- a/config/configMap/kustomization.yaml +++ b/config/configMap/kustomization.yaml @@ -8,7 +8,8 @@ kind: Kustomization configMapGenerator: - behavior: merge literals: - - gkm.nogpu=true + - gkm.nogpu=false + - gkm.kindcluster=false - gkm.agent.image=quay.io/gkm/agent:latest - gkm.extract.image=quay.io/gkm/gkm-extract:latest name: config diff --git a/config/crd/bases/gkm.io_clustergkmcachenodes.yaml b/config/crd/bases/gkm.io_clustergkmcachenodes.yaml index 83c5f6da..5906a958 100644 --- a/config/crd/bases/gkm.io_clustergkmcachenodes.yaml +++ b/config/crd/bases/gkm.io_clustergkmcachenodes.yaml @@ -15,9 +15,6 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .status.nodeName - name: Node - type: string - jsonPath: .status.counts.nodeInUseCnt name: Node-In-Use type: string diff --git a/config/crd/bases/gkm.io_clustergkmcaches.yaml b/config/crd/bases/gkm.io_clustergkmcaches.yaml index 014e63f9..dad7298c 100644 --- a/config/crd/bases/gkm.io_clustergkmcaches.yaml +++ b/config/crd/bases/gkm.io_clustergkmcaches.yaml @@ -1147,7 +1147,8 @@ spec: description: |- storageClassName contains the name of the Kubernetes Storage Class, which will be used for the PersistentVolume and PersistentVolumeClaim the GKM will - create in order to store the extract GPU Kernel Cache. + create in order to store the extract GPU Kernel Cache. If not provided, then + default Storage Class will be used. type: string workloadNamespaces: description: |- @@ -1161,7 +1162,6 @@ spec: type: array required: - image - - storageClassName type: object status: description: |- diff --git a/config/crd/bases/gkm.io_gkmcachenodes.yaml b/config/crd/bases/gkm.io_gkmcachenodes.yaml index 91104196..c7d5fb56 100644 --- a/config/crd/bases/gkm.io_gkmcachenodes.yaml +++ b/config/crd/bases/gkm.io_gkmcachenodes.yaml @@ -15,9 +15,6 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .status.nodeName - name: Node - type: string - jsonPath: .status.counts.nodeInUseCnt name: Node-In-Use type: string diff --git a/config/crd/bases/gkm.io_gkmcaches.yaml b/config/crd/bases/gkm.io_gkmcaches.yaml index 1e345421..7f7c3999 100644 --- a/config/crd/bases/gkm.io_gkmcaches.yaml +++ b/config/crd/bases/gkm.io_gkmcaches.yaml @@ -1145,7 +1145,8 @@ spec: description: |- storageClassName contains the name of the Kubernetes Storage Class, which will be used for the PersistentVolume and PersistentVolumeClaim the GKM will - create in order to store the extract GPU Kernel Cache. + create in order to store the extract GPU Kernel Cache. If not provided, then + default Storage Class will be used. type: string workloadNamespaces: description: |- @@ -1159,7 +1160,6 @@ spec: type: array required: - image - - storageClassName type: object status: description: |- diff --git a/config/operator/operator.yaml b/config/operator/operator.yaml index d694ee2e..59d481d5 100644 --- a/config/operator/operator.yaml +++ b/config/operator/operator.yaml @@ -84,6 +84,11 @@ spec: configMapKeyRef: name: gkm-config key: gkm.nogpu + - name: KIND_CLUSTER + valueFrom: + configMapKeyRef: + name: gkm-config + key: gkm.kindcluster - name: GO_LOG valueFrom: configMapKeyRef: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index de823acd..e61b200c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -4,25 +4,6 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-v1-pod - failurePolicy: Fail - name: mpod.kb.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/examples/.gitignore b/examples/.gitignore index 8950abb6..07333523 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,11 +1,10 @@ # Ignore generated yaml files base/common/namespace-1.yaml base/scope/cluster/namespace-2.yaml -overlays/access/*.yaml +overlays/pods/*.yaml overlays/scope/*.yaml output/*.yaml -variants/access/rox/*.yaml -variants/access/rwo/*.yaml +variants/pods/*.yaml variants/scope/cluster/*.yaml variants/scope/namespace/*.yaml .gkm-generate-files.exclusivelock diff --git a/examples/base/access/rwo/ds-1.yaml b/examples/base/access/rwo/ds-1.yaml deleted file mode 100644 index 091bbd25..00000000 --- a/examples/base/access/rwo/ds-1.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: gkm-test-ds-1 - namespace: gkm-test-ns-1 - labels: - gkm.io/pvc-mutation: "true" -spec: - selector: - matchLabels: - name: gkm-test-ds-1 - template: - metadata: - labels: - name: gkm-test-ds-1 - gkm.io/pvc-mutation: "true" - spec: - securityContext: - fsGroup: 1000 - containers: - - name: test - image: quay.io/fedora/fedora-minimal - imagePullPolicy: IfNotPresent - command: [sleep, 365d] - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - name: kernel-volume - mountPath: /cache - readOnly: true - volumes: - - name: kernel-volume - persistentVolumeClaim: - claimName: gkm-test-obj diff --git a/examples/base/access/rwo/ds-2.yaml b/examples/base/access/rwo/ds-2.yaml deleted file mode 100644 index 414b8b88..00000000 --- a/examples/base/access/rwo/ds-2.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: gkm-test-ds-2 - namespace: gkm-test-ns-1 - labels: - gkm.io/pvc-mutation: "true" -spec: - selector: - matchLabels: - name: gkm-test-ds-2 - template: - metadata: - labels: - name: gkm-test-ds-2 - gkm.io/pvc-mutation: "true" - spec: - securityContext: - fsGroup: 1000 - containers: - - name: test - image: quay.io/fedora/fedora-minimal - imagePullPolicy: IfNotPresent - command: [sleep, 365d] - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - name: kernel-volume - mountPath: /cache - volumes: - - name: kernel-volume - persistentVolumeClaim: - claimName: gkm-test-obj diff --git a/examples/base/access/rwo/ds-3.yaml b/examples/base/access/rwo/ds-3.yaml deleted file mode 100644 index 71689b15..00000000 --- a/examples/base/access/rwo/ds-3.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: gkm-test-ds-3 - namespace: gkm-test-ns-1 - labels: - gkm.io/pvc-mutation: "true" -spec: - selector: - matchLabels: - name: gkm-test-ds-3 - template: - metadata: - labels: - name: gkm-test-ds-3 - gkm.io/pvc-mutation: "true" - spec: - securityContext: - fsGroup: 1000 - containers: - - name: test - image: quay.io/fedora/fedora-minimal - imagePullPolicy: IfNotPresent - command: [sleep, 365d] - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - name: kernel-volume - mountPath: /cache - volumes: - - name: kernel-volume - persistentVolumeClaim: - claimName: gkm-test-obj diff --git a/examples/base/access/rwo/kustomization.yaml b/examples/base/access/rwo/kustomization.yaml deleted file mode 100644 index d2b79dcc..00000000 --- a/examples/base/access/rwo/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ds-1.yaml - - ds-2.yaml - - ds-3.yaml diff --git a/examples/base/access/rox/kustomization.yaml b/examples/base/pods/kustomization.yaml similarity index 100% rename from examples/base/access/rox/kustomization.yaml rename to examples/base/pods/kustomization.yaml diff --git a/examples/base/access/rox/pod-1.yaml b/examples/base/pods/pod-1.yaml similarity index 93% rename from examples/base/access/rox/pod-1.yaml rename to examples/base/pods/pod-1.yaml index 09928da7..b80d1809 100644 --- a/examples/base/access/rox/pod-1.yaml +++ b/examples/base/pods/pod-1.yaml @@ -18,7 +18,7 @@ spec: runAsUser: 1000 volumeMounts: - name: kernel-volume - mountPath: /cache + mountPath: /kernel-caches volumes: - name: kernel-volume persistentVolumeClaim: diff --git a/examples/base/access/rox/pod-2.yaml b/examples/base/pods/pod-2.yaml similarity index 93% rename from examples/base/access/rox/pod-2.yaml rename to examples/base/pods/pod-2.yaml index e7629638..ba32b99d 100644 --- a/examples/base/access/rox/pod-2.yaml +++ b/examples/base/pods/pod-2.yaml @@ -18,7 +18,7 @@ spec: runAsUser: 1000 volumeMounts: - name: kernel-volume - mountPath: /cache + mountPath: /kernel-caches volumes: - name: kernel-volume persistentVolumeClaim: diff --git a/examples/base/access/rox/pod-3.yaml b/examples/base/pods/pod-3.yaml similarity index 93% rename from examples/base/access/rox/pod-3.yaml rename to examples/base/pods/pod-3.yaml index 37e2c04c..693b5164 100644 --- a/examples/base/access/rox/pod-3.yaml +++ b/examples/base/pods/pod-3.yaml @@ -18,7 +18,7 @@ spec: runAsUser: 1000 volumeMounts: - name: kernel-volume - mountPath: /cache + mountPath: /kernel-caches volumes: - name: kernel-volume persistentVolumeClaim: diff --git a/examples/cleanup-files.sh b/examples/cleanup-files.sh index f05693ef..4b0da3fe 100755 --- a/examples/cleanup-files.sh +++ b/examples/cleanup-files.sh @@ -12,11 +12,10 @@ fi rm -f base/common/namespace-1.yaml rm -f base/scope/cluster/namespace-2.yaml -rm -f overlays/access/*.yaml +rm -f overlays/pods/*.yaml rm -f overlays/scope/*.yaml rm -f output/*.yaml -rm -f variants/access/rox/*.yaml -rm -f variants/access/rwo/*.yaml +rm -f variants/pods/*.yaml rm -f variants/scope/cluster/*.yaml rm -f variants/scope/namespace/*.yaml diff --git a/examples/generate-files.sh b/examples/generate-files.sh index 189b0a3b..e003160b 100755 --- a/examples/generate-files.sh +++ b/examples/generate-files.sh @@ -82,9 +82,9 @@ CUSTOM_AFFINITY=${CUSTOM_AFFINITY:-""} CUSTOM_TOLERATION=${CUSTOM_TOLERATION:-""} DEBUG=${DEBUG:-false} # Node Selector: -# CUSTOM_NODE_SELECTOR_1 is for Pod 1 or DaemonSet 1 -# CUSTOM_NODE_SELECTOR_2 is for Pod 2 or DaemonSet 2 -# CUSTOM_NODE_SELECTOR_3 is for Pod 3 or DaemonSet 3 +# CUSTOM_NODE_SELECTOR_1 is for Pod 1 +# CUSTOM_NODE_SELECTOR_2 is for Pod 2 +# CUSTOM_NODE_SELECTOR_3 is for Pod 3 CUSTOM_NODE_SELECTOR_1=${CUSTOM_NODE_SELECTOR_1:-""} CUSTOM_NODE_SELECTOR_2=${CUSTOM_NODE_SELECTOR_2:-""} CUSTOM_NODE_SELECTOR_3=${CUSTOM_NODE_SELECTOR_3:-""} @@ -92,7 +92,7 @@ CUSTOM_NODE_SELECTOR_3=${CUSTOM_NODE_SELECTOR_3:-""} # Constants BASE_DIR_COMMON="base/common" OUTPUT_DIR="output" -OVERLAY_DIR_ACCESS="overlays/access" +OVERLAY_DIR_PODS="overlays/pods" OVERLAY_DIR_SCOPE="overlays/scope" AFFINITY_NFD_CUDA_FILE="patch/affinity-nfd-cuda.txt" AFFINITY_NFD_ROCM_FILE="patch/affinity-nfd-rocm.txt" @@ -101,14 +101,12 @@ NODE_SELECTOR_KIND_FALSE_FILE="patch/node-selector-kind-false.txt" TOLERATION_KIND_FILE="patch/toleration-kind.txt" TOLERATION_NFD_CUDA_FILE="patch/toleration-nfd-cuda.txt" +BASE_DIR_PODS="base/pods" +VARIANTS_DIR_PODS="variants/pods" + + # AccessMode of the PVC, valid values: rox (ReadOnlyMany) or rwo (ReadWriteOnce) -if [[ "$ACCESS" == "rox" ]]; then - BASE_DIR_ACCESS="base/access/rox" - VARIANTS_DIR_ACCESS="variants/access/rox" -elif [[ "$ACCESS" == "rwo" ]]; then - BASE_DIR_ACCESS="base/access/rwo" - VARIANTS_DIR_ACCESS="variants/access/rwo" -else +if [[ "$ACCESS" != "rox" && "$ACCESS" != "rwo" ]]; then echo "ERROR: Parameter 1 (ACCESS) must be \"rox\" or \"rwo\"." exit 1 fi @@ -174,7 +172,7 @@ fi # # Build overlays/scope/kustomization.yaml file with Namespace and GKMCache or ClusterGKMCache -# Build overlays/access/kustomization.yaml file with Pods or DaemonSets +# Build overlays/pods/kustomization.yaml file with Pods. # Broken into two files to control ordering of objects. # mkdir -p "${OVERLAY_DIR_SCOPE}" @@ -191,15 +189,15 @@ components: nameSuffix: -${NAME_SUFFIX} EOF -mkdir -p "${OVERLAY_DIR_ACCESS}" -cat < ${OVERLAY_DIR_ACCESS}/kustomization.yaml +mkdir -p "${OVERLAY_DIR_PODS}" +cat < ${OVERLAY_DIR_PODS}/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../${BASE_DIR_ACCESS} +- ../../${BASE_DIR_PODS} components: -- ../../${VARIANTS_DIR_ACCESS} +- ../../${VARIANTS_DIR_PODS} nameSuffix: -${NAME_SUFFIX} EOF @@ -212,27 +210,15 @@ EOF # if [[ "$ENVIRONMENT" == "kind" ]]; then - # Where in a POD or DaemonSet Spec an initContainer is inserted - KIND_INIT_CONTAINER_PATH_POD="/spec/initContainers" - KIND_INIT_CONTAINER_PATH_DAEMON_SET="/spec/template/spec/initContainers" - - if [[ "$ACCESS" == "rox" ]]; then - # ReadOnlyMany (rox) implies a Pod is being deployed, so set the path properly - KIND_INIT_CONTAINER_PATH=${KIND_INIT_CONTAINER_PATH_POD} - elif [[ "$ACCESS" == "rwo" ]]; then - # ReadWriteOnce (rwo) implies a DaemonSet is being deployed, so set the path properly - KIND_INIT_CONTAINER_PATH=${KIND_INIT_CONTAINER_PATH_DAEMON_SET} - fi - - # KIND_INIT_CONTAINER holds a patch that is used to add an initContainer to Pods or DaemonSets - # when KIND Cluster is being used. It sets the permissions of the PVC VolumeMount so that the + # KIND_INIT_CONTAINER holds a patch that is used to add an initContainer to Pods when + # KIND Cluster is being used. It sets the permissions of the PVC VolumeMount so that the # Pod can access it. Only needed in KIND. KIND_INIT_CONTAINER=" # For KIND Cluster, add initContainer that sets the permissions on the PVC VolumeMount - op: add - path: ${KIND_INIT_CONTAINER_PATH} + path: /spec/initContainers value: [] - op: add - path: ${KIND_INIT_CONTAINER_PATH}/- + path: /spec/initContainers/- value: name: fix-permissions image: quay.io/fedora/fedora-minimal @@ -258,7 +244,7 @@ fi if [[ "$ACCESS" == "rox" ]]; then # ACCESS_ROX_ACCESS_MODE holds a patch that is used to add ReadOnlyMany to the AccessMode field # in a GKMCache or ClusterGKMCache. Kubernetes does not have a way to be queried to determine if - # ReadOnlyMany is supported by a StorageClass so GKM Operator/Agent need tobe told. + # ReadOnlyMany is supported by a StorageClass so GKM Operator/Agent needs to be told. ACCESS_ROX_ACCESS_MODE=" # Append ReadOnlyMany to the spec.accessModes slice in the GKMCache or ClusterGKMCache - op: add path: /spec/accessModes/- @@ -337,70 +323,41 @@ ${TOLERATION_INSTANCE}" | sed -e 's/[\/&]/\\&/g' -e ':a;N;$!ba;s/\n/\\n/g') - # Where in a POD or DaemonSet Spec affinity/toleration/is inserted - AFFINITY_PATH_POD="/spec/affinity" - AFFINITY_PATH_DAEMON_SET="/spec/template/spec/affinity" - TOLERATION_PATH_POD="/spec/tolerations" - TOLERATION_PATH_DAEMON_SET="/spec/template/spec/tolerations" - - if [[ "$ACCESS" == "rox" ]]; then - # ReadOnlyMany (rox) implies a Pod is being deployed, so set the path properly - AFFINITY_PATH=${AFFINITY_PATH_POD} - TOLERATION_PATH=${TOLERATION_PATH_POD} - elif [[ "$ACCESS" == "rwo" ]]; then - # ReadWriteOnce (rwo) implies a DaemonSet is being deployed, so set the path properly - AFFINITY_PATH=${AFFINITY_PATH_DAEMON_SET} - TOLERATION_PATH=${TOLERATION_PATH_DAEMON_SET} - fi - if [[ "$AFFINITY_INSTANCE" != "" ]]; then - AFFINITY_ADD_POD_DS=" # Add a Affinity to Pod or DaemonSet + AFFINITY_ADD_POD=" # Add a Affinity to Pod - op: add - path: ${AFFINITY_PATH} + path: /spec/affinity value: ${AFFINITY_INSTANCE}" - # AFFINITY_ADD_POD_DS contains a multiline string, so special characters need + # AFFINITY_ADD_POD contains a multiline string, so special characters need # to be stripped for sed to process properly. - ESCAPED_AFFINITY_ADD_POD_DS=$(printf '%s\n' "$AFFINITY_ADD_POD_DS" \ + ESCAPED_AFFINITY_ADD_POD=$(printf '%s\n' "$AFFINITY_ADD_POD" \ | sed -e 's/[\/&]/\\&/g' -e ':a;N;$!ba;s/\n/\\n/g') fi if [[ "$TOLERATION_INSTANCE" != "" ]]; then - TOLERATION_ADD_POD_DS=" # Add a Toleration to Pod or DaemonSet + TOLERATION_ADD_POD=" # Add a Toleration to Pod - op: add - path: ${TOLERATION_PATH} + path: /spec/tolerations value: [] - op: add - path: ${TOLERATION_PATH}/- + path: /spec/tolerations/- value: ${TOLERATION_INSTANCE}" - # TOLERATION_ADD_POD_DS contains a multiline string, so special characters need + # TOLERATION_ADD_POD contains a multiline string, so special characters need # to be stripped for sed to process properly. - ESCAPED_TOLERATION_ADD_POD_DS=$(printf '%s\n' "$TOLERATION_ADD_POD_DS" \ + ESCAPED_TOLERATION_ADD_POD=$(printf '%s\n' "$TOLERATION_ADD_POD" \ | sed -e 's/[\/&]/\\&/g' -e ':a;N;$!ba;s/\n/\\n/g') fi fi # Node Selector: -# .._SELECTOR_1 is for Pod 1 or DaemonSet 1 -# .._SELECTOR_2 is for Pod 2 or DaemonSet 2 -# .._SELECTOR_3 is for Pod 3 or DaemonSet 3 - -# Where in a POD or DaemonSet Spec a Node Selector is inserted -NODE_SELECTOR_PATH_POD="/spec/nodeSelector" -NODE_SELECTOR_PATH_DAEMON_SET="/spec/template/spec/nodeSelector" - -if [[ "$ACCESS" == "rox" ]]; then - # ReadOnlyMany (rox) implies a Pod is being deployed, so set the path properly - NODE_SELECTOR_PATH=${NODE_SELECTOR_PATH_POD} -elif [[ "$ACCESS" == "rwo" ]]; then - # ReadWriteOnce (rwo) implies a DaemonSet is being deployed, so set the path properly - NODE_SELECTOR_PATH=${NODE_SELECTOR_PATH_DAEMON_SET} -fi - +# .._SELECTOR_1 is for Pod 1 +# .._SELECTOR_2 is for Pod 2 +# .._SELECTOR_3 is for Pod 3 if [[ "$CUSTOM_NODE_SELECTOR_1" != "" ]]; then if [[ -r "${CUSTOM_NODE_SELECTOR_1}" ]]; then NODE_SELECTOR_INSTANCE_1=$(cat "${CUSTOM_NODE_SELECTOR_1}") || { echo "Error: Failed to read file - CUSTOM_NODE_SELECTOR_1=${CUSTOM_NODE_SELECTOR_1}" >&2; exit 1; } @@ -411,9 +368,9 @@ if [[ "$CUSTOM_NODE_SELECTOR_1" != "" ]]; then fi if [[ "$NODE_SELECTOR_INSTANCE_1" != "" ]]; then - NODE_SELECTOR_1=" # Add NodeSelector to Pod/DaemonSet 1 + NODE_SELECTOR_1=" # Add NodeSelector to Pod 1 - op: add - path: ${NODE_SELECTOR_PATH} + path: /spec/nodeSelector value: ${NODE_SELECTOR_INSTANCE_1}" @@ -436,9 +393,9 @@ elif [[ "$ENVIRONMENT" == "kind" && "$ACCESS" == "rwo" ]]; then fi if [[ "$NODE_SELECTOR_INSTANCE_2" != "" ]]; then - NODE_SELECTOR_2=" # Add NodeSelector to Pod/DaemonSet 2 + NODE_SELECTOR_2=" # Add NodeSelector to Pod 2 - op: add - path: ${NODE_SELECTOR_PATH} + path: /spec/nodeSelector value: ${NODE_SELECTOR_INSTANCE_2}" @@ -461,9 +418,9 @@ elif [[ "$ENVIRONMENT" == "kind" && "$ACCESS" == "rwo" ]]; then fi if [[ "$NODE_SELECTOR_INSTANCE_3" != "" ]]; then - NODE_SELECTOR_3=" # Add NodeSelector to Pod/DaemonSet 3 + NODE_SELECTOR_3=" # Add NodeSelector to Pod 3 - op: add - path: ${NODE_SELECTOR_PATH} + path: /spec/nodeSelector value: ${NODE_SELECTOR_INSTANCE_3}" @@ -494,19 +451,19 @@ if [[ "$SCOPE" == "cl" ]]; then popd > /dev/null fi -# UPDATE Pod or DaemonSet -# For both rox and rwo, for each Pod or DaemonSet: +# UPDATE Pod +# For both rox and rwo, for each Pod # - set the Namespace # - set the PVC Claim in the Volume to the generated GKMCache or ClusterGKMCache name # - insert the KIND Init Container if KIND Cluster, otherwise remove the placeholder -pushd ${VARIANTS_DIR_ACCESS} > /dev/null +pushd ${VARIANTS_DIR_PODS} > /dev/null ${SED} \ -e "s/NAMESPACE_1/${NAMESPACE_1}/g" \ -e "s/NAMESPACE_2/${NAMESPACE_2}/g" \ -e "s/OBJECT_NAME/${OBJECT_NAME}/g" \ -e "s@KIND_INIT_CONTAINER@${ESCAPED_KIND_INIT_CONTAINER}@g" \ - -e "s@TOLERATION_ADD_POD_DS@${ESCAPED_TOLERATION_ADD_POD_DS}@g" \ - -e "s@AFFINITY_ADD_POD_DS@${ESCAPED_AFFINITY_ADD_POD_DS}@g" \ + -e "s@TOLERATION_ADD_POD@${ESCAPED_TOLERATION_ADD_POD}@g" \ + -e "s@AFFINITY_ADD_POD@${ESCAPED_AFFINITY_ADD_POD}@g" \ -e "s@NODE_SELECTOR_1@${ESCAPED_NODE_SELECTOR_1}@g" \ -e "s@NODE_SELECTOR_2@${ESCAPED_NODE_SELECTOR_2}@g" \ -e "s@NODE_SELECTOR_3@${ESCAPED_NODE_SELECTOR_3}@g" \ @@ -545,9 +502,9 @@ popd > /dev/null OUTPUT_FILENAME=${OUTPUT_DIR}/${NAME_SUFFIX}${ENV_FILENAME_SUFFIX}.yaml mkdir -p "${OUTPUT_DIR}" -${KUSTOMIZE} build overlays/scope > ${OUTPUT_FILENAME} || exit 1 +"${KUSTOMIZE}" build "${OVERLAY_DIR_SCOPE}" > "${OUTPUT_FILENAME}" || exit 1 echo "---" >> ${OUTPUT_FILENAME} -${KUSTOMIZE} build overlays/access >> ${OUTPUT_FILENAME} || exit 1 +"${KUSTOMIZE}" build "${OVERLAY_DIR_PODS}" >> "${OUTPUT_FILENAME}" || exit 1 if [[ "${DEBUG}" == true ]]; then cat ${OUTPUT_FILENAME} diff --git a/examples/variants/access/rwo/kustomization.env b/examples/variants/access/rwo/kustomization.env deleted file mode 100644 index 8ca51c0a..00000000 --- a/examples/variants/access/rwo/kustomization.env +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component - -patches: -# For each DaemonSet: -# - Set the Namespace of the DaemonSet -# - Overwrite the PVC Name in the DaemonSet Volume -- target: - kind: DaemonSet - name: gkm-test-ds-1 - patch: |- - - op: replace - path: /metadata/namespace - value: NAMESPACE_1 - - op: replace - path: /spec/template/spec/volumes/0/persistentVolumeClaim/claimName - value: OBJECT_NAME -KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS -NODE_SELECTOR_1 -- target: - kind: DaemonSet - name: gkm-test-ds-2 - patch: |- - - op: replace - path: /metadata/namespace - value: NAMESPACE_1 - - op: replace - path: /spec/template/spec/volumes/0/persistentVolumeClaim/claimName - value: OBJECT_NAME -KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS -NODE_SELECTOR_2 -- target: - kind: DaemonSet - name: gkm-test-ds-3 - patch: |- - - op: replace - path: /metadata/namespace - value: NAMESPACE_2 - - op: replace - path: /spec/template/spec/volumes/0/persistentVolumeClaim/claimName - value: OBJECT_NAME -KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS -NODE_SELECTOR_3 diff --git a/examples/variants/access/rox/kustomization.env b/examples/variants/pods/kustomization.env similarity index 88% rename from examples/variants/access/rox/kustomization.env rename to examples/variants/pods/kustomization.env index 340e9827..512b454d 100644 --- a/examples/variants/access/rox/kustomization.env +++ b/examples/variants/pods/kustomization.env @@ -16,8 +16,8 @@ patches: path: /spec/volumes/0/persistentVolumeClaim/claimName value: OBJECT_NAME KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS +AFFINITY_ADD_POD +TOLERATION_ADD_POD NODE_SELECTOR_1 - target: kind: Pod @@ -30,8 +30,8 @@ NODE_SELECTOR_1 path: /spec/volumes/0/persistentVolumeClaim/claimName value: OBJECT_NAME KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS +AFFINITY_ADD_POD +TOLERATION_ADD_POD NODE_SELECTOR_2 - target: kind: Pod @@ -44,6 +44,6 @@ NODE_SELECTOR_2 path: /spec/volumes/0/persistentVolumeClaim/claimName value: OBJECT_NAME KIND_INIT_CONTAINER -AFFINITY_ADD_POD_DS -TOLERATION_ADD_POD_DS +AFFINITY_ADD_POD +TOLERATION_ADD_POD NODE_SELECTOR_3 diff --git a/gkm-extract/main.go b/gkm-extract/main.go index 7fc94f9b..e23f15ee 100644 --- a/gkm-extract/main.go +++ b/gkm-extract/main.go @@ -60,9 +60,11 @@ func ExtractCache(cacheDir, imageURL string, noGpu bool, log logr.Logger) (err e log.Info("unable to chown", "err", err) } - if err := os.Chmod(cacheDir, 0755); err != nil { - log.Info("unable to chmod", "err", err) - } + /* + if err := os.Chmod(cacheDir, 0755); err != nil { + log.Info("unable to chmod", "err", err) + } + */ // Only one initialization should occur initFile := filepath.Join(cacheDir, ".initialized") diff --git a/internal/controller/gkm-agent/cluster_gkmcache_controller.go b/internal/controller/gkm-agent/cluster_gkmcache_controller.go index e6ecf5d4..1cf7f636 100644 --- a/internal/controller/gkm-agent/cluster_gkmcache_controller.go +++ b/internal/controller/gkm-agent/cluster_gkmcache_controller.go @@ -167,8 +167,8 @@ func (r *ClusterGKMCacheAgentReconciler) getCacheNode( cacheNodeList := &gkmv1alpha1.ClusterGKMCacheNodeList{} labelSelector := map[string]string{ - utils.GKMClusterCacheNodeLabelCache: cacheName, - utils.GKMCacheLabelHostname: r.NodeName, + //utils.GKMClusterCacheNodeLabelCache: cacheName, + utils.GKMCacheLabelHostname: r.NodeName, } err := r.List(ctx, cacheNodeList, client.InNamespace(cacheNamespace), @@ -206,7 +206,8 @@ func (r *ClusterGKMCacheAgentReconciler) createCacheNode(ctx context.Context, ca // Build up GKMCacheNode gkmCacheNode := &gkmv1alpha1.ClusterGKMCacheNode{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.GenerateUniqueName(cacheName), + //Name: utils.GenerateUniqueName(cacheName), + Name: r.NodeName, Finalizers: []string{}, Labels: map[string]string{ utils.GKMCacheLabelHostname: r.NodeName, diff --git a/internal/controller/gkm-agent/common.go b/internal/controller/gkm-agent/common.go index fd9c8c71..611492f3 100644 --- a/internal/controller/gkm-agent/common.go +++ b/internal/controller/gkm-agent/common.go @@ -36,19 +36,6 @@ import ( "github.com/redhat-et/GKM/pkg/utils" ) -var defaultCacheDir string - -func init() { - initializeCachePath(utils.DefaultCacheDir) -} - -// Allow overriding UsageDir location for Testing -var ExportForTestInitializeCachePath = initializeCachePath - -func initializeCachePath(value string) { - defaultCacheDir = value -} - // GKMInstance is a generic interface that can either be a gkmv1alpha1.GKMCache or // a gkmv1alpha1.ClusterGKMCache. This is used to allow both a GKMCache and a ClusterGKMCache // to be processed by the same code. @@ -101,9 +88,10 @@ type ReconcilerCommonAgent[C GKMInstance, CL GKMInstanceList[C], N GKMNodeInstan Scheme *runtime.Scheme Logger logr.Logger Recorder record.EventRecorder - CacheDir string NodeName string NoGpu bool + KindCluster bool + ExtractLogLevel string ExtractImage string CrdCacheStr string // For logging/errors: GKMCache or ClusterGKMCache CrdCacheNodeStr string // For logging/errors: GKMCacheNode or ClusterGKMCacheNode @@ -438,6 +426,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) reconcileCommonAgent( } else { // DELETE pvcInUse := false + pvcDeleting := false // Get the PVC Status, which is the Per Namespace PV and PVC information. // If it doesn't exist for this Namespace, then move on to the next Namespace. @@ -463,7 +452,6 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) reconcileCommonAgent( // If Owner is Agent, then attempt to delete Job, PVC and PV. Otherwise, // there is nothing to do here. if gkmCache.GetPvcOwner() == gkmv1alpha1.PvcOwnerAgent { - var pvcDeleting bool if updated, updateReason, pvcInUse, pvcDeleting, err = common.ManagePvcStatusDelete( ctx, r.Client, @@ -510,7 +498,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) reconcileCommonAgent( } // If nothing was updated, then this PVC Status can be removed. - if !updated && !pvcInUse { + if !updated && !pvcInUse && !pvcDeleting { delete(cacheStatus.PvcStatus, pvcNamespace) updated = true skipPvcCopy = true @@ -825,7 +813,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) managePVandPVC( // The preferred method for creating a PV is to create the PVC and Kubelet auto-creates the PV. // In a KIND cluster, there is not a true CSI driver for storage management, so the PV must be // manually created. - if r.NoGpu { + if r.KindCluster { _, found, updatedName, err := common.PvExists( ctx, r.Client, @@ -965,6 +953,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) manageJob( stillPending := false var err error + //jobNamespace = utils.GKMDefaultNamespace if (*gkmCache).GetPvcOwner() == gkmv1alpha1.PvcOwnerAgent { // If the condition on the PVC Status is Pending, then a Job to extract the cache has not been // launched. Build up and launch the job. @@ -973,7 +962,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) manageJob( // // For both GKMCache and ClusterGKMCache, just use the PVC name, because Jobs are // always created per Node per Namespace. Name is already unique. - jobName := pvcStatus.PvcName + jobName := pvcStatus.PvcName + "-job-download-" r.Logger.Info("Cache NOT Extracted, extract now", "Namespace", (*gkmCache).GetNamespace(), @@ -981,7 +970,8 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) manageJob( "Job Name", jobName, "Name", (*gkmCache).GetName(), "digest", resolvedDigest, - "NoGpu", r.NoGpu) + "NoGpu", r.NoGpu, + "KIND", r.KindCluster) err = common.LaunchJob( ctx, @@ -994,10 +984,12 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) manageJob( (*gkmCache).GetImage(), resolvedDigest, r.NoGpu, + r.KindCluster, r.ExtractImage, pvcStatus, (*gkmCache).GetPodTemplate(), r.Logger, + r.ExtractLogLevel, ) if err != nil { @@ -1461,7 +1453,7 @@ func (r *ReconcilerCommonAgent[C, CL, N, NL]) addCounts( r.Client, r.NodeName, pvcNamespace, - pvcStatus.PvcName, + cacheName, /* PvcName: Serving PVC has same name as Cache */ r.Logger, ) } diff --git a/internal/controller/gkm-agent/namespace_gkmcache_controller.go b/internal/controller/gkm-agent/namespace_gkmcache_controller.go index 167e454f..f9200ec7 100644 --- a/internal/controller/gkm-agent/namespace_gkmcache_controller.go +++ b/internal/controller/gkm-agent/namespace_gkmcache_controller.go @@ -168,8 +168,8 @@ func (r *GKMCacheAgentReconciler) getCacheNode( cacheNodeList := &gkmv1alpha1.GKMCacheNodeList{} labelSelector := map[string]string{ - utils.GKMCacheNodeLabelCache: cacheName, - utils.GKMCacheLabelHostname: r.NodeName, + //utils.GKMCacheNodeLabelCache: cacheName, + utils.GKMCacheLabelHostname: r.NodeName, } err := r.List(ctx, cacheNodeList, client.InNamespace(cacheNamespace), @@ -207,12 +207,13 @@ func (r *GKMCacheAgentReconciler) createCacheNode(ctx context.Context, cacheName // Build up GKMCacheNode gkmCacheNode := &gkmv1alpha1.GKMCacheNode{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.GenerateUniqueName(cacheName), + //Name: utils.GenerateUniqueName(cacheName), + Name: r.NodeName, Namespace: cacheNamespace, Finalizers: []string{}, Labels: map[string]string{ - utils.GKMCacheLabelHostname: r.NodeName, - utils.GKMCacheNodeLabelCache: cacheName, + utils.GKMCacheLabelHostname: r.NodeName, + //utils.GKMCacheNodeLabelCache: cacheName, }, }, } diff --git a/internal/controller/gkm-operator/cluster_gkmcache_controller.go b/internal/controller/gkm-operator/cluster_gkmcache_controller.go index edc94423..7b80b257 100644 --- a/internal/controller/gkm-operator/cluster_gkmcache_controller.go +++ b/internal/controller/gkm-operator/cluster_gkmcache_controller.go @@ -91,17 +91,19 @@ func (r *ClusterGKMCacheOperatorReconciler) SetupWithManager(mgr ctrl.Manager) e } func (r *ClusterGKMCacheOperatorReconciler) enqueueClusterGKMCache(ctx context.Context, obj client.Object) []reconcile.Request { + var name string crList := &gkmv1alpha1.ClusterGKMCacheList{} if err := r.List(ctx, crList); err != nil || len(crList.Items) == 0 { - return nil + name = "EmptyClusterCache" + } else { + cr := crList.Items[0] + name = cr.Name } - cr := crList.Items[0] - return []reconcile.Request{ { NamespacedName: types.NamespacedName{ - Name: cr.Name, + Name: name, }, }, } diff --git a/internal/controller/gkm-operator/common.go b/internal/controller/gkm-operator/common.go index 402f047b..ce1be2a2 100644 --- a/internal/controller/gkm-operator/common.go +++ b/internal/controller/gkm-operator/common.go @@ -92,6 +92,8 @@ type ReconcilerCommonOperator[ Scheme *runtime.Scheme Logger logr.Logger NoGpu bool + KindCluster bool + ExtractLogLevel string ExtractImage string CrdCacheStr string // For logging/errors: GKMCache or ClusterGKMCache CrdCacheNodeStr string // For logging/errors: GKMCacheNode or ClusterGKMCacheNode @@ -150,7 +152,13 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) reconcileCommonOperator( r.Logger.V(1).Info("Start reconcileCommonOperator()") - inUseGkmCacheList := make(map[string]bool) + // This is a Map indexed by the Cache Namespace and Name. If a GKMCache or + // ClusterGKMCache instance is deleted while the associated Serving PVC is + // still being used by a Pod, then that PVC is "stranded" and needs to be + // cleaned up. This map tracks what caches were processed in the main loop + // so that when walking PVCs the code can determine which PVCs are stranded + // and need to be checked if they are still in use and can be deleted. + inUseGkmCacheList := make(map[string]map[string]bool) // Get the list of existing GKMCache or ClusterGKMCache objects from KubeAPI Server. gkmCacheList, err := reconciler.getCacheList(ctx, []client.ListOption{}) @@ -177,7 +185,10 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) reconcileCommonOperator( ) cacheDeleting := reconciler.isBeingDeleted(&gkmCache) - inUseGkmCacheList[gkmCache.GetName()] = cacheDeleting + if _, ok := inUseGkmCacheList[gkmCache.GetNamespace()]; !ok { + inUseGkmCacheList[gkmCache.GetNamespace()] = make(map[string]bool) + } + inUseGkmCacheList[gkmCache.GetNamespace()][gkmCache.GetName()] = cacheDeleting // See if Digest has been set (Webhook validated and image is allowed to be used). annotations := gkmCache.GetAnnotations() @@ -214,259 +225,263 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) reconcileCommonOperator( gkmCacheStatus.Counts = gkmv1alpha1.CacheCounts{} gkmCacheStatus.ResolvedDigest = resolvedDigest + // The PvcOwner is the controller that creates and manages the PV/PVC/Job. + // * If AccessMode is ReadOnlyMany (ROX), then only one PV/PVC/Job is needed for + // the Cluster so the Operator creates and manages the resources. The Job downloads + // and extracts the content from the OCI Image once to the PVC and the Storage + // backend is responsible to distributing the content to each Node. Not all + // Clusters have a Storage instance that supports this, so RWO must also be supported. + // * If AccessMode is ReadWriteOnce (RWO), then: + // * Agent on each Node creates and manages a PV/PVC/Job per Node. The Job downloads + // and extracts the content from the OCI Image on each Node. These are the download + // PV/PVC/Job instances. + // * Once the download has occurred, the Operator creates one Serving PV/PVC that + // refeneces the path used in the download PVC. This Serving PVC is what is given to + // the ISVC to mount in the workload. Note, no Serving Job is needed. if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerUnknown || gkmCacheStatus.PvcOwner == "" { // Initialize the condition to pending. r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondPending.Condition()) - gkmCacheStatus.PvcOwner = gkmv1alpha1.PvcOwnerAgent - accessMode := gkmCache.GetAccessMode() - for _, mode := range accessMode { - if mode == corev1.ReadOnlyMany { - gkmCacheStatus.PvcOwner = gkmv1alpha1.PvcOwnerOperator - } - } + gkmCacheStatus.PvcOwner = determineOwner(gkmCache.GetAccessMode()) r.Logger.Info("Owner not set, setting now", "Updated Value", gkmCacheStatus.PvcOwner) } - // If the PVC AccessMode is ReadOnlyMany, then only one PVC per Namespace needs to be created - // and the storage backend will handle propagating the extracted cache to each node. Since - // there is only one, the Operator handles the creation here. - if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerOperator { - updated := false - updateReason := "" - - // Loop through the list of Namespaces. For GKMCache, it's just the namespace - // GKMCache is created in. For ClusterGKMCache, it's the Workload Namespace list - // that was provided in ClusterGKMCache. - namespaceList := gkmCache.GetWorkloadNamespaces() - if len(namespaceList) == 0 { - if gkmCache.GetNamespace() == "" { - r.Logger.Info("No namespaces in ClusterGKMCache Spec.WorkloadNamespaces, so no PVCs created", - "Namespace", gkmCache.GetNamespace(), - "Name", gkmCache.GetName(), - ) - } + updated := false + updateReason := "" + + // pvcInUse is used to indicate on a delete of the GKMCache or ClusterGKMCache that a + // PVC is still in use. The must go ahead and delete the GKMCache or ClusterGKMCache + // and will use manageStrandedPvcs() to clean up the PV and PVCs once they are no longer + // being used by a pod. + pvcInUse := false + + // pvcDeleting is used to indicate that KubeAPI has been called to delete the PVC but + // the delete is still being processed. + pvcDeleting := false + + // Map index by Namespace. Contains the collection of counts per Namespace + // so the Operator created Serving PVC can provide a summary State of the + // Download PVCs from each Node. Per Namespace is needed for the ClusterGKMCache, + // because there is a PVC per namespace. Collected for the GKMCache just to + // simplify code. + namespaceCnts := make(map[string]*gkmv1alpha1.CacheCounts) + + // If Agent managed, then collect the counts now. For Operator managed, + // delay the work to collect until there is no more work to do. + if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerAgent { + if err := r.collectNodeCounts( + ctx, + reconciler, + &gkmCache, + gkmCacheStatus, + namespaceCnts, + ); err != nil { + errorHit = true + continue } - for _, pvcNamespace := range namespaceList { - var pvcStatus gkmv1alpha1.PvcStatus - skipPvcCopy := false + } - namespaceExists, namespaceDeleting, err := r.namespaceExists(ctx, pvcNamespace) - if err != nil { - errorHit = true - continue - } + // Loop through the list of Namespaces. For GKMCache, it's just the namespace + // GKMCache is created in. For ClusterGKMCache, it's the Workload Namespace list + // that was provided in ClusterGKMCache. + namespaceList := gkmCache.GetWorkloadNamespaces() + if len(namespaceList) == 0 { + if gkmCache.GetNamespace() == "" { + r.Logger.Info("No namespaces in ClusterGKMCache Spec.WorkloadNamespaces, so no PVCs created", + "Namespace", gkmCache.GetNamespace(), + "Name", gkmCache.GetName(), + ) + } + } + for _, pvcNamespace := range namespaceList { + var pvcStatus gkmv1alpha1.PvcStatus + skipPvcCopy := false - // CREATE or UPDATE - if !cacheDeleting && !namespaceDeleting { + namespaceExists, namespaceDeleting, err := r.namespaceExists(ctx, pvcNamespace) + if err != nil { + errorHit = true + continue + } - // Get the PVC Status, which is the Per Namespace PV and PVC information. - if gkmCacheStatus.PvcStatus == nil { - gkmCacheStatus.PvcStatus = make(map[string]gkmv1alpha1.PvcStatus) - updated = true - updateReason = "PvcStatus Allocation" - } + // CREATE or UPDATE + if !cacheDeleting && !namespaceDeleting { - var pvcStatusExisted bool - pvcStatus, pvcStatusExisted = gkmCacheStatus.PvcStatus[pvcNamespace] - if !pvcStatusExisted { - pvcStatus = gkmv1alpha1.PvcStatus{} - gkmv1alpha1.SetPvcStatusConditions(&pvcStatus, gkmv1alpha1.GkmCondPending.Condition()) - pvcStatus.PvcOwner = gkmv1alpha1.PvcOwnerOperator - updated = true - updateReason = "PvcStatus Initialization" - } + // Get the PVC Status, which is the Per Namespace PV and PVC information. + if gkmCacheStatus.PvcStatus == nil { + gkmCacheStatus.PvcStatus = make(map[string]gkmv1alpha1.PvcStatus) + updated = true + updateReason = "PvcStatus Allocation" + } - // Manage PV, PVC and Job used for extracted GPU Kernel Cache - if pvcUpdated, pvcUpdateReason, pending, err := r.managePvcStatusModify( - ctx, - reconciler, - &gkmCache, - gkmCacheStatus, - &pvcStatus, - pvcNamespace, - resolvedDigest, - capacity, - namespaceExists, - ); err != nil { - errorHit = true - continue - } else if pvcUpdated { - updated = true - updateReason = pvcUpdateReason - } else if pending { - stillInUse = true - } - } else { - // DELETE - pvcInUse := false + var pvcStatusExisted bool + pvcStatus, pvcStatusExisted = gkmCacheStatus.PvcStatus[pvcNamespace] + if !pvcStatusExisted { + pvcStatus = gkmv1alpha1.PvcStatus{} + gkmv1alpha1.SetPvcStatusConditions(&pvcStatus, gkmv1alpha1.GkmCondPending.Condition()) + pvcStatus.PvcOwner = gkmv1alpha1.PvcOwnerOperator + updated = true + updateReason = "PvcStatus Initialization" + } - // Get the PVC Status, which is the Per Namespace PV and PVC information. - // If it doesn't exist for this Namespace, then move on to the next Namespace. - if gkmCacheStatus.PvcStatus == nil { - continue - } + // Manage PV, PVC and Job used for extracted GPU Kernel Cache + if pvcUpdated, pvcUpdateReason, pending, err := r.managePvcStatusModify( + ctx, + reconciler, + &gkmCache, + gkmCacheStatus, + &pvcStatus, + pvcNamespace, + resolvedDigest, + capacity, + namespaceExists, + namespaceCnts, + ); err != nil { + errorHit = true + continue + } else if pvcUpdated { + updated = true + updateReason = pvcUpdateReason + } else if pending { + stillInUse = true + } + } else { + // DELETE - var pvcStatusExisted bool - pvcStatus, pvcStatusExisted = gkmCacheStatus.PvcStatus[pvcNamespace] - if !pvcStatusExisted { - continue - } + // Get the PVC Status, which is the Per Namespace PV and PVC information. + // If it doesn't exist for this Namespace, then move on to the next Namespace. + if gkmCacheStatus.PvcStatus == nil { + continue + } - var pvcDeleting bool - if updated, updateReason, pvcInUse, pvcDeleting, err = common.ManagePvcStatusDelete( - ctx, - r.Client, - gkmCache.GetNamespace(), - gkmCache.GetName(), - "", // NodeName - &pvcStatus, - gkmv1alpha1.PvcOwnerOperator, - pvcNamespace, - resolvedDigest, - r.Logger, - ); err != nil { - errorHit = true - continue - } else if pvcInUse || pvcDeleting { - stillInUse = true - if !gkmv1alpha1.GkmCondDeleting.IsConditionSet(pvcStatus.Conditions) { - gkmv1alpha1.SetPvcStatusConditions(&pvcStatus, gkmv1alpha1.GkmCondDeleting.Condition()) - updated = true - updateReason = "Update Condition to Deleting" - } - } + var pvcStatusExisted bool + pvcStatus, pvcStatusExisted = gkmCacheStatus.PvcStatus[pvcNamespace] + if !pvcStatusExisted { + continue + } - // If nothing was updated, then this PVC Status can be removed. - if !updated && !pvcInUse { - delete(gkmCacheStatus.PvcStatus, pvcNamespace) + if updated, updateReason, pvcInUse, pvcDeleting, err = common.ManagePvcStatusDelete( + ctx, + r.Client, + gkmCache.GetNamespace(), + gkmCache.GetName(), + "", // NodeName + &pvcStatus, + gkmv1alpha1.PvcOwnerOperator, + pvcNamespace, + resolvedDigest, + r.Logger, + ); err != nil { + errorHit = true + continue + } else if pvcInUse || pvcDeleting { + stillInUse = true + if !gkmv1alpha1.GkmCondDeleting.IsConditionSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(&pvcStatus, gkmv1alpha1.GkmCondDeleting.Condition()) updated = true - skipPvcCopy = true - updateReason = "Remove PVC Namespace entry" + updateReason = "Update Condition to Deleting" } } - if updated { - if !skipPvcCopy { - // Update the Cache Status copy of the PVC Status before writing the data below. - gkmCacheStatus.PvcStatus[pvcNamespace] = pvcStatus - } - break + // If nothing was updated, then this PVC Status can be removed. + if !updated && !pvcInUse && !pvcDeleting { + delete(gkmCacheStatus.PvcStatus, pvcNamespace) + updated = true + skipPvcCopy = true + updateReason = "Remove PVC Namespace entry" } - } // For each Namespace + } - // Call KubeAPI to update the Status for the GKMCache (or ClusterGKMCache) that was - // modified above. if updated { - gkmCacheStatus.LastUpdated = metav1.Now() - changed, err := reconciler.cacheUpdateStatus(ctx, &gkmCache, gkmCacheStatus, updateReason) - if err != nil { - errorHit = true - continue - } else { - // GKMCache Object was updated successfully. - // Return and Reconcile will be retriggered with the GKMCache Object. - r.Logger.V(1).Info("Return after CacheStatus Write", "Reason", updateReason, "changed", changed) - if changed { - return ctrl.Result{Requeue: false}, nil - } else { - return ctrl.Result{Requeue: true, RequeueAfter: utils.RetryAgentNodeStatusUpdate}, nil - } + if !skipPvcCopy { + // Update the Cache Status copy of the PVC Status before writing the data below. + gkmCacheStatus.PvcStatus[pvcNamespace] = pvcStatus } + break } - } - - // Call KubeAPI to Retrieve the list of GKMCacheNodes for this Namespace. - // Should be one per Node if GKMCache was created in the Namespace. - opts := []client.ListOption{ - client.InNamespace(gkmCache.GetNamespace()), - } - gkmCacheNodeList, err := reconciler.getCacheNodeList(ctx, opts) - if err != nil { - // Error returned if unable to call KubeAPI. Don't block Reconcile on one instance, - // log and go to next GKMCache. - r.Logger.Error(err, "failed to get GKMCacheNode List", "Namespace", gkmCache.GetNamespace(), "Name", gkmCache.GetName()) - errorHit = true - continue - } + } // For each Namespace - // Loop through each GKMCacheNode (i.e. each Node) - for _, gkmCacheNode := range (*gkmCacheNodeList).GetItems() { - nodeStatus := gkmCacheNode.GetStatus() - if nodeStatus != nil { - // See if this GKMCache has been added to the GKMCacheNode - if _, ok := nodeStatus.CacheStatuses[gkmCache.GetName()]; ok { - // This Cache was found in a GKMCacheNode instance - gkmCacheStatus.Counts.NodeCnt += nodeStatus.Counts.NodeCnt - gkmCacheStatus.Counts.NodeInUseCnt += nodeStatus.Counts.NodeInUseCnt - gkmCacheStatus.Counts.NodeNotInUseCnt += nodeStatus.Counts.NodeNotInUseCnt - gkmCacheStatus.Counts.NodeErrorCnt += nodeStatus.Counts.NodeErrorCnt - gkmCacheStatus.Counts.PodRunningCnt += nodeStatus.Counts.PodRunningCnt - gkmCacheStatus.Counts.PodDeletingCnt += nodeStatus.Counts.PodDeletingCnt - gkmCacheStatus.Counts.PodOutdatedCnt += nodeStatus.Counts.PodOutdatedCnt + // Call KubeAPI to update the Status for the GKMCache (or ClusterGKMCache) that was + // modified above. + if updated { + gkmCacheStatus.LastUpdated = metav1.Now() + changed, err := reconciler.cacheUpdateStatus(ctx, &gkmCache, gkmCacheStatus, updateReason) + if err != nil { + errorHit = true + continue + } else { + // GKMCache Object was updated successfully. + // Return and Reconcile will be retriggered with the GKMCache Object. + r.Logger.V(1).Info("Return after CacheStatus Write", "Reason", updateReason, "changed", changed) + if changed { + return ctrl.Result{Requeue: false}, nil + } else { + return ctrl.Result{Requeue: true, RequeueAfter: utils.RetryAgentNodeStatusUpdate}, nil } } } - r.Logger.V(1).Info("Processed GKMCache", - "Namespace", gkmCache.GetNamespace(), - "CacheName", gkmCache.GetName(), - "NodeCnt", gkmCacheStatus.Counts.NodeCnt, - "NodeInUse", gkmCacheStatus.Counts.NodeInUseCnt, - "NodeNotInUse", gkmCacheStatus.Counts.NodeNotInUseCnt, - "NodeError", gkmCacheStatus.Counts.NodeErrorCnt, - "PodRunning", gkmCacheStatus.Counts.PodRunningCnt, - "PodDeleting", gkmCacheStatus.Counts.PodDeletingCnt, - "PodOutdated", gkmCacheStatus.Counts.PodOutdatedCnt, - "Conditions", gkmCacheStatus.Conditions, - ) + // If Agent managed, the counts were collected above, so collect for + // Operator managed now. + if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerOperator { + if err := r.collectNodeCounts( + ctx, + reconciler, + &gkmCache, + gkmCacheStatus, + namespaceCnts, + ); err != nil { + errorHit = true + continue + } + } - if !cacheDeleting { - reason := "Update Counts" - // Adjust Condition if need. If Operator owns the PVC extraction, then - // wait for Pending and Downloading state to clear before adjusting based - // on GKMCacheNode or ClusterGKMCacheNode counts. - if gkmCacheStatus.Counts.NodeErrorCnt != 0 { - if !gkmv1alpha1.GkmCondError.IsConditionSet(gkmCacheStatus.Conditions) { - r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondError.Condition()) - reason = "Set Error Condition" - } - } else if gkmCacheStatus.Counts.PodOutdatedCnt != 0 { - if !gkmv1alpha1.GkmCondOutdated.IsConditionSet(gkmCacheStatus.Conditions) { - r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondOutdated.Condition()) - reason = "Set Outdated Condition" - } - } else if gkmCacheStatus.Counts.NodeInUseCnt != 0 { - if !gkmv1alpha1.GkmCondRunning.IsConditionSet(gkmCacheStatus.Conditions) { - r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondRunning.Condition()) - reason = "Set Running Condition" - } - } else if gkmCacheStatus.Counts.NodeNotInUseCnt != 0 { - if !gkmv1alpha1.GkmCondExtracted.IsConditionSet(gkmCacheStatus.Conditions) { - r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondExtracted.Condition()) - reason = "Set Extracted Condition" - } + // Adjust the Cache Condition if need. This is a summary of all the Nodes. + if gkmCacheStatus.Counts.NodeErrorCnt != 0 { + if !gkmv1alpha1.GkmCondError.IsConditionSet(gkmCacheStatus.Conditions) { + r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondError.Condition()) + updated = true + updateReason = "Set Error Cache Condition" + } + } else if gkmCacheStatus.Counts.PodOutdatedCnt != 0 { + if !gkmv1alpha1.GkmCondOutdated.IsConditionSet(gkmCacheStatus.Conditions) { + r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondOutdated.Condition()) + updated = true + updateReason = "Set Outdated Cache Condition" + } + } else if gkmCacheStatus.Counts.NodeInUseCnt != 0 { + if !gkmv1alpha1.GkmCondRunning.IsConditionSet(gkmCacheStatus.Conditions) { + r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondRunning.Condition()) + updated = true + updateReason = "Set Running Cache Condition" } + } else if gkmCacheStatus.Counts.NodeNotInUseCnt != 0 { + if !gkmv1alpha1.GkmCondExtracted.IsConditionSet(gkmCacheStatus.Conditions) { + r.setCacheConditions(gkmCacheStatus, gkmv1alpha1.GkmCondExtracted.Condition()) + updated = true + updateReason = "Set Extracted Cache Condition" + } + } - if !reflect.DeepEqual(gkmCache.GetStatus(), gkmCacheStatus) { - gkmCacheStatus.LastUpdated = metav1.Now() + if updated || !reflect.DeepEqual(gkmCache.GetStatus(), gkmCacheStatus) { + gkmCacheStatus.LastUpdated = metav1.Now() - if changed, err := reconciler.cacheUpdateStatus(ctx, &gkmCache, gkmCacheStatus, reason); err != nil { - errorHit = true - continue + if changed, err := reconciler.cacheUpdateStatus(ctx, &gkmCache, gkmCacheStatus, updateReason); err != nil { + errorHit = true + continue + } else { + // GKMCache Object was updated successfully. + // Return and Reconcile will be retriggered with the GKMCache Object. + r.Logger.V(1).Info("Return after CacheStatus Write", "Reason", updateReason, "changed", changed) + if changed { + return ctrl.Result{Requeue: false}, nil } else { - // GKMCache Object was updated successfully. - // Return and Reconcile will be retriggered with the GKMCache Object. - r.Logger.V(1).Info("Return after CacheStatus Write", "Reason", reason, "changed", changed) - if changed { - return ctrl.Result{Requeue: false}, nil - } else { - return ctrl.Result{Requeue: true, RequeueAfter: utils.RetryAgentNodeStatusUpdate}, nil - } + return ctrl.Result{Requeue: true, RequeueAfter: utils.RetryAgentNodeStatusUpdate}, nil } } - } else { - if gkmCacheStatus.Counts.NodeCnt == 0 { + } + + if cacheDeleting { + if gkmCacheStatus.Counts.NodeCnt == 0 && !pvcDeleting { // Everything should be cleaned up, so delete the GKMCacheNode specific // finalizer from the GKMCache. changed, err := reconciler.cacheRemoveFinalizer(ctx, &gkmCache) @@ -486,7 +501,7 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) reconcileCommonOperator( stillInUse = true } } - } + } // FOR EACH GKMCache } // Walk the GKMCacheNode or ClusterGKMCacheNode and determine if any PVCs are stranded. @@ -524,6 +539,7 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) managePvcStatusModify( resolvedDigest string, capacity string, namespaceExists bool, + namespaceCnts map[string]*gkmv1alpha1.CacheCounts, ) (bool, string, bool, error) { updated := false updateReason := "" @@ -534,7 +550,7 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) managePvcStatusModify( // If updated is already true, still manage PV and PVCs, because up to this // point, it's just been initialization and allocation of structures, no // actual work on kube objects. - if updated, updateReason, err := r.managePVandPVC( + if updated, updateReason, pending, err := r.managePVandPVC( ctx, reconciler, gkmCache, @@ -543,18 +559,22 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) managePvcStatusModify( pvcNamespace, capacity, namespaceExists, - ); err != nil || updated { + namespaceCnts, + ); err != nil || updated || pending { return updated, updateReason, pending, err } // Launch Job to Extract Cache - updated, updateReason, pending, err = r.manageJob( - ctx, - gkmCache, - pvcStatus, - pvcNamespace, - resolvedDigest, - ) + if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerOperator { + updated, updateReason, pending, err = r.manageJob( + ctx, + gkmCache, + pvcStatus, + pvcNamespace, + resolvedDigest, + ) + } + return updated, updateReason, pending, err } @@ -570,9 +590,12 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) managePVandPVC( pvcNamespace string, capacity string, namespaceExists bool, -) (bool, string, error) { + namespaceCnts map[string]*gkmv1alpha1.CacheCounts, +) (bool, string, bool, error) { updated := false updateReason := "" + pending := false + pvCreated := false if gkmv1alpha1.GkmCondNoNamespace.IsConditionSet(pvcStatus.Conditions) { @@ -588,131 +611,172 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) managePVandPVC( // launched for this Namespace. Make sure the PV and PVC are in a valid state to handle the extraction. if gkmv1alpha1.GkmCondPending.IsConditionSet(pvcStatus.Conditions) { r.Logger.Info("Condition is Pending so managing PV/PVC") - if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerOperator { - if !namespaceExists { - gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondNoNamespace.Condition()) + + if !namespaceExists { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondNoNamespace.Condition()) + updated = true + updateReason = "Update Condition to No Namespace" + return updated, updateReason, pending, nil + } + // Hold off creating the Serving PV/PVC until at least one Download PVC has completed + if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerAgent && + gkmCacheStatus.Counts.NodeInUseCnt == 0 && gkmCacheStatus.Counts.NodeNotInUseCnt == 0 { + pending = true + return updated, updateReason, pending, nil + } + + // The preferred method for creating a PV is to create the PVC and Kubelet auto-creates the PV. + // In a KIND cluster, there is not a true CSI driver for storage management, so the PV must be + // manually created. + if r.KindCluster { + _, found, updatedName, err := common.PvExists( + ctx, + r.Client, + (*gkmCache).GetName(), + "", // NodeName + pvcStatus.PvName, + pvcNamespace, + gkmCacheStatus.ResolvedDigest, + r.Logger, + ) + if err != nil { + return updated, updateReason, pending, err + } else if updatedName != "" { + pvcStatus.PvName = updatedName updated = true - updateReason = "Update Condition to No Namespace" - return updated, updateReason, nil - } + updateReason = "Writing PV Name" + pvCreated = true + } else if !found { + // Call KubeAPI to create the PV. + pvcStatus.PvName = utils.GenerateUniqueName((*gkmCache).GetName()) + + accessModes := []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + } - // The preferred method for creating a PV is to create the PVC and Kubelet auto-creates the PV. - // In a KIND cluster, there is not a true CSI driver for storage management, so the PV must be - // manually created. - if r.NoGpu { - _, found, updatedName, err := common.PvExists( + err := common.CreatePv( ctx, r.Client, + r.Scheme, + (*gkmCache).GetClientObject(), + (*gkmCache).GetNamespace(), (*gkmCache).GetName(), "", // NodeName pvcStatus.PvName, pvcNamespace, + accessModes, + (*gkmCache).GetStorageClassName(), + capacity, gkmCacheStatus.ResolvedDigest, r.Logger, ) - if err != nil { - return updated, updateReason, err - } else if updatedName != "" { - pvcStatus.PvName = updatedName - updated = true - updateReason = "Writing PV Name" - pvCreated = true - } else if !found { - // Call KubeAPI to create the PV. - pvcStatus.PvName = utils.GenerateUniqueName((*gkmCache).GetName()) - - accessModes := []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany, - } - err := common.CreatePv( - ctx, - r.Client, - r.Scheme, - (*gkmCache).GetClientObject(), - (*gkmCache).GetNamespace(), - (*gkmCache).GetName(), - "", // NodeName - pvcStatus.PvName, - pvcNamespace, - accessModes, - (*gkmCache).GetStorageClassName(), - capacity, - gkmCacheStatus.ResolvedDigest, - r.Logger, - ) + if err != nil { + return false, updateReason, pending, err + } - if err != nil { - return false, updateReason, err - } + updated = true + updateReason = "Create PV" + pvCreated = true + } + } - updated = true - updateReason = "Create PV" - pvCreated = true + // If PV was not written above, then determine if PVC needs to be created. + if !pvCreated { + _ /* pvc */, found, updatedName, err := common.PvcExists( + ctx, + r.Client, + (*gkmCache).GetName(), + "", // NodeName + pvcStatus.PvcName, + pvcNamespace, + gkmCacheStatus.ResolvedDigest, + r.Logger, + ) + if err != nil { + return updated, updateReason, pending, err + } else if updatedName != "" { + pvcStatus.PvcName = updatedName + updated = true + updateReason = "Writing PVC Name" + } else if !found { + // Call KubeAPI to create the PVC. + // + // For both GKMCache and ClusterGKMCache, just use the cache name, because PVCs + // always created in a Namespace. For GKMCache, it's the same namespaces as the + // GKMCache. For ClusterGKMCache, name is unique at cluster level and will be + // created in GKMDefaultNamespace. + pvcStatus.PvcName = (*gkmCache).GetName() + + accessModes := []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, } - } - // If PV was not written above, then determine if PVC needs to be created. - if !pvCreated { - _ /* pvc */, found, updatedName, err := common.PvcExists( + err := common.CreatePvc( ctx, r.Client, + r.Scheme, + (*gkmCache).GetClientObject(), + (*gkmCache).GetNamespace(), (*gkmCache).GetName(), - "", // NodeName + "", + pvcStatus.PvName, pvcStatus.PvcName, pvcNamespace, + accessModes, + (*gkmCache).GetStorageClassName(), + capacity, gkmCacheStatus.ResolvedDigest, r.Logger, ) - if err != nil { - return updated, updateReason, err - } else if updatedName != "" { - pvcStatus.PvcName = updatedName - updated = true - updateReason = "Writing PVC Name" - } else if !found { - // Call KubeAPI to create the PVC. - // - // For both GKMCache and ClusterGKMCache, just use the cache name, because PVCs - // always created in a Namespace. For GKMCache, it's the same namespaces as the - // GKMCache. For ClusterGKMCache, name is unique at cluster level and will be - // created in GKMDefaultNamespace. - pvcStatus.PvcName = (*gkmCache).GetName() - - accessModes := []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany, - } - err := common.CreatePvc( - ctx, - r.Client, - r.Scheme, - (*gkmCache).GetClientObject(), - (*gkmCache).GetNamespace(), - (*gkmCache).GetName(), - "", - pvcStatus.PvName, - pvcStatus.PvcName, - pvcNamespace, - accessModes, - (*gkmCache).GetStorageClassName(), - capacity, - gkmCacheStatus.ResolvedDigest, - r.Logger, - ) + if err != nil { + return updated, updateReason, pending, err + } - if err != nil { - return updated, updateReason, err - } + updated = true + updateReason = "Create PVC" + // This is the Serving PVC so OCI Image was extracted via the Download PVC, so set + // condition to extracted. + if gkmCacheStatus.PvcOwner == gkmv1alpha1.PvcOwnerAgent && + !gkmv1alpha1.IsConditionDownloadSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondExtracted.Condition()) + } + } + } + } else { + if cnts, exists := namespaceCnts[pvcNamespace]; exists { + // Adjust the Cache Condition if need. This is a summary of all the Nodes. + if cnts.NodeErrorCnt != 0 { + if !gkmv1alpha1.GkmCondError.IsConditionSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondExtracted.Condition()) + updated = true + updateReason = "Set Error PVC Condition" + } + } else if cnts.PodOutdatedCnt != 0 { + if !gkmv1alpha1.GkmCondOutdated.IsConditionSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondOutdated.Condition()) + updated = true + updateReason = "Set Outdated PVC Condition" + } + } else if cnts.NodeInUseCnt != 0 { + if !gkmv1alpha1.GkmCondRunning.IsConditionSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondRunning.Condition()) updated = true - updateReason = "Create PVC" + updateReason = "Set Running PVC Condition" + } + } else if cnts.NodeNotInUseCnt != 0 { + if !gkmv1alpha1.GkmCondExtracted.IsConditionSet(pvcStatus.Conditions) { + gkmv1alpha1.SetPvcStatusConditions(pvcStatus, gkmv1alpha1.GkmCondExtracted.Condition()) + updated = true + updateReason = "Set Extracted PVC Condition" } } } } - return updated, updateReason, nil + return updated, updateReason, pending, nil } // manageJob determines if the GPU Kernel Cache has been extracted. If not, checks the condition and either @@ -748,6 +812,7 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) manageJob( "Name", (*gkmCache).GetName(), "digest", resolvedDigest, "NoGpu", r.NoGpu, + "KIND", r.KindCluster, "ExtractImage", r.ExtractImage, ) @@ -762,10 +827,12 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) manageJob( (*gkmCache).GetImage(), resolvedDigest, r.NoGpu, + r.KindCluster, r.ExtractImage, pvcStatus, (*gkmCache).GetPodTemplate(), r.Logger, + r.ExtractLogLevel, ) if err != nil { @@ -872,103 +939,306 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) manageJob( return updated, updateReason, stillPending, err } +func (r *ReconcilerCommonOperator[C, CL, N, NL]) collectNodeCounts( + ctx context.Context, + reconciler OperatorReconciler[C, CL, N, NL], + gkmCache *C, + gkmCacheStatus *gkmv1alpha1.GKMCacheStatus, + namespaceCnts map[string]*gkmv1alpha1.CacheCounts, +) error { + // Call KubeAPI to Retrieve the list of GKMCacheNodes for a Namespace or + // ClusterGKMCacheNodes. Should be one per Node. + opts := []client.ListOption{ + client.InNamespace((*gkmCache).GetNamespace()), + } + gkmCacheNodeList, err := reconciler.getCacheNodeList(ctx, opts) + if err != nil { + // Error returned if unable to call KubeAPI. Don't block Reconcile on one instance, + // log and go to next GKMCache. + r.Logger.Error(err, "failed to get GKMCacheNode List", + "Namespace", (*gkmCache).GetNamespace(), + "Name", (*gkmCache).GetName()) + return err + } + + // Loop through each GKMCacheNode (i.e. each Node) + for _, gkmCacheNode := range (*gkmCacheNodeList).GetItems() { + nodeStatus := gkmCacheNode.GetStatus() + if nodeStatus != nil { + // See if this GKMCache has been added to the GKMCacheNode + if _, ok := nodeStatus.CacheStatuses[(*gkmCache).GetName()]; ok { + // This Cache was found in a GKMCacheNode instance so collect a summary + // of the counts for all Namespaces (cluster scoped may have more than + // one namespace). + gkmCacheStatus.Counts.NodeCnt += nodeStatus.Counts.NodeCnt + gkmCacheStatus.Counts.NodeInUseCnt += nodeStatus.Counts.NodeInUseCnt + gkmCacheStatus.Counts.NodeNotInUseCnt += nodeStatus.Counts.NodeNotInUseCnt + gkmCacheStatus.Counts.NodeErrorCnt += nodeStatus.Counts.NodeErrorCnt + gkmCacheStatus.Counts.PodRunningCnt += nodeStatus.Counts.PodRunningCnt + gkmCacheStatus.Counts.PodDeletingCnt += nodeStatus.Counts.PodDeletingCnt + gkmCacheStatus.Counts.PodOutdatedCnt += nodeStatus.Counts.PodOutdatedCnt + + for cacheName, digestList := range nodeStatus.CacheStatuses { + for digest, cacheStatus := range digestList { + for namespace, pvcStatus := range cacheStatus.PvcStatus { + r.Logger.Info("Looping through namespaces", + "Name", cacheName, + "Namespace", namespace, + "Digest", digest, + "pvcStatus", pvcStatus) + + // Check if namespace exists + cnts, exists := namespaceCnts[namespace] + + // Allocate if missing + if !exists { + cnts = &gkmv1alpha1.CacheCounts{} + namespaceCnts[namespace] = cnts + } + + switch gkmv1alpha1.GetLatestConditionType(pvcStatus.Conditions).Type { + case string(gkmv1alpha1.GkmCondPending): + // Temp state, ignore + case string(gkmv1alpha1.GkmCondExtracted): + cnts.NodeNotInUseCnt++ + case string(gkmv1alpha1.GkmCondRunning): + cnts.NodeInUseCnt++ + case string(gkmv1alpha1.GkmCondDeleting): + cnts.PodDeletingCnt++ + case string(gkmv1alpha1.GkmCondError): + cnts.NodeErrorCnt++ + case string(gkmv1alpha1.GkmCondUnloadError): + cnts.NodeErrorCnt++ + case string(gkmv1alpha1.GkmCondOutdated): + cnts.PodOutdatedCnt++ + } + } + } + } + } + } + } + + r.Logger.V(1).Info("Processed GKMCache", + "Namespace", (*gkmCache).GetNamespace(), + "CacheName", (*gkmCache).GetName(), + "NodeCnt", gkmCacheStatus.Counts.NodeCnt, + "NodeInUse", gkmCacheStatus.Counts.NodeInUseCnt, + "NodeNotInUse", gkmCacheStatus.Counts.NodeNotInUseCnt, + "NodeError", gkmCacheStatus.Counts.NodeErrorCnt, + "PodRunning", gkmCacheStatus.Counts.PodRunningCnt, + "PodDeleting", gkmCacheStatus.Counts.PodDeletingCnt, + "PodOutdated", gkmCacheStatus.Counts.PodOutdatedCnt, + "Conditions", gkmCacheStatus.Conditions, + ) + + return nil +} + // manageStrandedPvcs walks the GKMCacheNode or ClusterGKMCacheNode and determines if any PVCs are // stranded (GKMCache or ClusterGKMCache was deleted but Pod was still using PVC). If so, see if the // Pod using them is still active. If not, clean them up. func (r *ReconcilerCommonOperator[C, CL, N, NL]) manageStrandedPvcs( ctx context.Context, reconciler OperatorReconciler[C, CL, N, NL], - inUseGkmCacheList map[string]bool, + inUseGkmCacheList map[string]map[string]bool, ) (bool, bool) { pending := false errorHit := false r.Logger.V(1).Info("ENTER manageStrandedPvcs()") - gkmCacheNodeList, err := reconciler.getCacheNodeList(ctx, []client.ListOption{}) + pvcList, err := common.GetGkmPvcList(ctx, r.Client, r.Logger) if err != nil { errorHit = true return pending, errorHit } - if (*gkmCacheNodeList).GetItemsLen() == 0 { - // KubeAPI doesn't have any GKMCacheNode instances - r.Logger.Info("No GKMCacheNode entries found") - } else { - // There are GKMCacheNode instances created, so loop through each and any check if PVCs are stranded. - for _, gkmCacheNode := range (*gkmCacheNodeList).GetItems() { - r.Logger.V(1).Info("Verifying Cache Node", - "Object", r.CrdCacheStr, - "Namespace", gkmCacheNode.GetNamespace(), - "Name", gkmCacheNode.GetName(), + for _, pvc := range pvcList { + // Retrieve the Cache Name from the labels in the PVC. + labels := pvc.GetLabels() + + gkmCacheNamespace, found := labels[utils.PvcLabelCacheNamespace] + if !found { + continue + } else { + // If the Cache Namespace is empty, then this is a ClusterGKMCache created + // PVC, make sure the Object is the same. Else handles the inverse. + if gkmCacheNamespace == "" && r.CrdCacheStr != utils.CrdClusterGKMCache { + continue + } else if gkmCacheNamespace != "" && r.CrdCacheStr != utils.CrdGKMCache { + continue + } + } + + gkmCacheName, found := labels[utils.PvcLabelCache] + if !found { + continue + } + pvName, found := labels[utils.PvcLabelPvName] + if !found { + continue + } + nodeName, found := labels[utils.PvcLabelNode] + if !found { + continue + } + digest, found := labels[utils.PvcLabelDigest] + if !found { + continue + } + + if cl, ok := inUseGkmCacheList[gkmCacheNamespace]; ok { + if _, ok := cl[gkmCacheName]; ok { + r.Logger.V(1).Info("For PVC Skipping Cache because Cache still exists", + "Object", r.CrdCacheStr, + "Namespace", gkmCacheNamespace, + "Name", gkmCacheName, + ) + continue + } + } + + pvcStatus := gkmv1alpha1.PvcStatus{ + PvName: pvName, + PvcName: pvc.Name, + PvcOwner: gkmv1alpha1.PvcOwnerOperator, + } + + pvcUpdated, pvcUpdateReason, pvcInUse, pvcDeleting, err := common.ManagePvcStatusDelete( + ctx, + r.Client, + gkmCacheNamespace, + gkmCacheName, + nodeName, + &pvcStatus, + gkmv1alpha1.PvcOwnerOperator, + pvc.GetNamespace(), // pvcNamespace + digest, + r.Logger, + ) + if err != nil { + errorHit = true + continue + } + // This PVC Status is associated with the GKMCacheNode or ClusterGKMCacheNode. + // Operator cannot update it. The CacheNode is being used to store the PV and PVC + // names. So if something was updated (PVC or PV was deleted), just mark pending so + // this code checks again to see if anything needs to be deleted. + if pvcUpdated || pvcDeleting { + pending = true + r.Logger.Info("PVC still In Use or was Updated", + "Object", r.CrdCacheNodeStr, + "Name", gkmCacheName, + "Digest", digest, + "Namespace", gkmCacheNamespace, + "PVC Namespace", pvc.GetNamespace(), + "PVC Name", pvc.GetName(), + "Still In Use", pvcInUse, + "Deleting", pvcDeleting, + "Updated", pvcUpdated, + "Reason", pvcUpdateReason, ) - nodeStatus := gkmCacheNode.GetStatus() - if nodeStatus != nil { - for cacheName, digestList := range nodeStatus.CacheStatuses { - // Check to see if GKMCache was processed above in main loop. If so, skip over. - if _, ok := inUseGkmCacheList[cacheName]; ok { - r.Logger.V(1).Info("Skipping Cache because Cache still exists", - "Object", r.CrdCacheStr, - "Namespace", gkmCacheNode.GetNamespace(), - "Name", cacheName, - ) - continue - } + } + } - for digest, cacheStatus := range digestList { - for namespace, pvcStatus := range cacheStatus.PvcStatus { - if pvcStatus.PvcOwner == gkmv1alpha1.PvcOwnerOperator && - gkmv1alpha1.GkmCondDeleting.IsConditionSet(pvcStatus.Conditions) { - r.Logger.Info("PVC is in Deleting State", - "Object", r.CrdCacheNodeStr, - "Name", cacheName, - "Digest", digest, - "Namespace", namespace, - "PVC", pvcStatus.PvcName, - "PV", pvcStatus.PvName, - ) - pvcUpdated, pvcUpdateReason, pvcInUse, pvcDeleting, err := common.ManagePvcStatusDelete( - ctx, - r.Client, - namespace, - cacheName, - "", // NodeName - &pvcStatus, - gkmv1alpha1.PvcOwnerOperator, - namespace, - digest, - r.Logger, - ) - if err != nil { - errorHit = true - continue - } - // This PVC Status is associated with the GKMCacheNode or ClusterGKMCacheNode. - // Operator cannot update it. The CacheNode is being used to store the PV and PVC - // names. So if something was updated (PVC or PV was deleted), just mark pending so - // this code checks again to see if anything needs to be deleted. - if pvcUpdated || pvcDeleting { - pending = true - r.Logger.Info("PVC still In Use or was Updated", - "Object", r.CrdCacheNodeStr, - "Name", cacheName, - "Digest", digest, - "Namespace", namespace, - "PVC", pvcStatus.PvcName, - "PV", pvcStatus.PvName, - "Still In Use", pvcInUse, - "Deleting", pvcDeleting, - "Updated", pvcUpdated, - "Reason", pvcUpdateReason, - ) - } - } - } - } - } + // Process PVs. Only need to look for PVs in a Phase of Failed (no PVC). + // If they have a PVC, they will be in a Phase od Bonded. + pvList, err := common.GetGkmPvFailedList(ctx, r.Client, r.Logger) + if err != nil { + errorHit = true + return pending, errorHit + } + + for _, pv := range pvList { + // Retrieve the Cache Name from the labels in the PV. + labels := pv.GetLabels() + + gkmCacheNamespace, found := labels[utils.PvLabelCacheNamespace] + if !found { + continue + } else { + // If the Cache Namespace is empty, then this is a ClusterGKMCache created + // PV, make sure the Object is the same. Else handles the inverse. + if gkmCacheNamespace == "" && r.CrdCacheStr != utils.CrdClusterGKMCache { + continue + } else if gkmCacheNamespace != "" && r.CrdCacheStr != utils.CrdGKMCache { + continue + } + } + + gkmCacheName, found := labels[utils.PvLabelCache] + if !found { + continue + } + nodeName, found := labels[utils.PvLabelNode] + if !found { + continue + } + digest, found := labels[utils.PvLabelDigest] + if !found { + continue + } + pvcNamespace, found := labels[utils.PvLabelPvcNamespace] + if !found { + continue + } + + if cl, ok := inUseGkmCacheList[gkmCacheNamespace]; ok { + if _, ok := cl[gkmCacheName]; ok { + r.Logger.V(1).Info("For PV Skipping Cache because Cache still exists", + "Object", r.CrdCacheStr, + "Namespace", gkmCacheNamespace, + "Name", gkmCacheName, + ) + continue } } + + pvcStatus := gkmv1alpha1.PvcStatus{ + PvName: pv.Name, + PvcOwner: gkmv1alpha1.PvcOwnerOperator, + } + + pvUpdated, pvUpdateReason, pvInUse, pvDeleting, err := common.ManagePvcStatusDelete( + ctx, + r.Client, + gkmCacheNamespace, + gkmCacheName, + nodeName, + &pvcStatus, + gkmv1alpha1.PvcOwnerOperator, + pvcNamespace, + digest, + r.Logger, + ) + if err != nil { + errorHit = true + continue + } + // This PV Status is associated with the GKMCacheNode or ClusterGKMCacheNode. + // Operator cannot update it. The CacheNode is being used to store the PV and PVC + // names. So if something was updated (PVC or PV was deleted), just mark pending so + // this code checks again to see if anything needs to be deleted. + if pvUpdated || pvDeleting { + pending = true + r.Logger.Info("PV still In Use or was Updated", + "Object", r.CrdCacheNodeStr, + "Name", gkmCacheName, + "Digest", digest, + "Namespace", gkmCacheNamespace, + "PVC Namespace", pvcNamespace, + "PV Name", pv.GetName(), + "Still In Use", pvInUse, + "Deleting", pvDeleting, + "Updated", pvUpdated, + "Reason", pvUpdateReason, + ) + } } + + r.Logger.V(1).Info("EXIT manageStrandedPvcs()") + return pending, errorHit } @@ -991,3 +1261,15 @@ func (r *ReconcilerCommonOperator[C, CL, N, NL]) namespaceExists(ctx context.Con return namespaceExists, namespaceDeleting, nil } + +// determineOwner walks the list of AccessMode values and if any of the values are +// ReadOnlyMany then the owner is PvcOwnerOperator, otherwise PvcOwnerAgent. +func determineOwner(accessMode []corev1.PersistentVolumeAccessMode) gkmv1alpha1.PvcOwner { + pvcOwner := gkmv1alpha1.PvcOwnerAgent + for _, mode := range accessMode { + if mode == corev1.ReadOnlyMany { + pvcOwner = gkmv1alpha1.PvcOwnerOperator + } + } + return pvcOwner +} diff --git a/internal/controller/gkm-operator/namespace_gkmcache_controller.go b/internal/controller/gkm-operator/namespace_gkmcache_controller.go index b5d99034..42ccc203 100644 --- a/internal/controller/gkm-operator/namespace_gkmcache_controller.go +++ b/internal/controller/gkm-operator/namespace_gkmcache_controller.go @@ -91,17 +91,19 @@ func (r *GKMCacheOperatorReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *GKMCacheOperatorReconciler) enqueueGKMCache(ctx context.Context, obj client.Object) []reconcile.Request { + var name string crList := &gkmv1alpha1.GKMCacheList{} if err := r.List(ctx, crList); err != nil || len(crList.Items) == 0 { - return nil + name = "EmptyCache" + } else { + cr := crList.Items[0] + name = cr.Name } - cr := crList.Items[0] - return []reconcile.Request{ { NamespacedName: types.NamespacedName{ - Name: cr.Name, + Name: name, }, }, } diff --git a/internal/webhook/pod_pvc_webhook.go b/internal/webhook/pod_pvc_webhook.go deleted file mode 100644 index 7e419bf8..00000000 --- a/internal/webhook/pod_pvc_webhook.go +++ /dev/null @@ -1,379 +0,0 @@ -package webhook - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - - admissionv1 "k8s.io/api/admission/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - gkmv1alpha1 "github.com/redhat-et/GKM/api/v1alpha1" - "github.com/redhat-et/GKM/pkg/utils" -) - -const ( - TargetLabelValue = "true" - - VolumeNameToMutate = "mydata" // optional filter by volume name -) - -var ( - podWebhookLog = logf.Log.WithName("webhook-pod") -) - -type PodMutator struct { - client.Client - Scheme *runtime.Scheme - Decoder admission.Decoder -} - -// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,verbs=create,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1 - -func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { - // Only mutate on CREATE - if req.Operation != admissionv1.Create { - return admission.Allowed("Skipping non-create operation") - } - - pod := &corev1.Pod{} - if err := m.Decoder.Decode(req, pod); err != nil { - podWebhookLog.Error(err, "Error Decoding Pod object", "req", req) - return admission.Errored(http.StatusBadRequest, err) - } - podName := pod.Name - if podName == "" { - podName = pod.GenerateName - } - - // Filter by label - if pod.Labels[utils.GKMCachePvcMutation] != TargetLabelValue { - podWebhookLog.V(1).Info("Label on Pod Not Found", - "PodName", podName, - "Namespace", pod.Namespace, - "Label", utils.GKMCachePvcMutation, - ) - return admission.Allowed("pod does not match target label") - } - - // Need to know the Node - nodeName := m.getNodeName(pod) - if nodeName == "" { - err := fmt.Errorf("Node not known") - podWebhookLog.Error(err, "Node not known", "PodName", podName, "Namespace", pod.Namespace) - return admission.Errored(http.StatusInternalServerError, err) - } - - // Track if we mutate - mutated := false - - // Iterate volumes - for i, vol := range pod.Spec.Volumes { - if vol.PersistentVolumeClaim == nil { - continue - } - - podWebhookLog.Info("Examining Vol", - "PodName", podName, - "Namespace", pod.Namespace, - "Vol Name", vol.Name, - "Claim Name", vol.PersistentVolumeClaim.ClaimName, - ) - - pvcName, err := m.getPvcName(ctx, podName, pod.Namespace, vol.PersistentVolumeClaim.ClaimName, nodeName) - if err == nil { - pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = pvcName - mutated = true - } - } - - if !mutated { - //return admission.Allowed("no pvc volumes mutated") - podWebhookLog.Info("Denied: Dependency not ready yet", - "PodName", podName, - "Namespace", pod.Namespace, - ) - return admission.Denied("dependency not ready yet") - } - - mutatedBytes, err := json.Marshal(pod) - if err != nil { - podWebhookLog.Info("Error: Marshal error", "err", err) - return admission.Errored(http.StatusInternalServerError, err) - } - - // Return patch response - return admission.PatchResponseFromRaw(req.Object.Raw, mutatedBytes) -} - -func (m *PodMutator) getNodeName(pod *corev1.Pod) string { - if pod.Spec.Affinity == nil || - pod.Spec.Affinity.NodeAffinity == nil || - pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - return "" - } - - r := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution - for _, t := range r.NodeSelectorTerms { - for _, mf := range t.MatchFields { - if mf.Key != "metadata.name" { - continue - } - - if mf.Operator != corev1.NodeSelectorOpIn { - continue - } - - if len(mf.Values) == 0 { - continue - } - - // DaemonSet controller sets exactly one node value - return mf.Values[0] - } - } - - return "" -} - -func (m *PodMutator) getPvcName(ctx context.Context, podName, namespace, cacheName, nodeName string) (string, error) { - // Look for GKMCache first (namespace based). If not found, then look for ClusterGKMCache. - gkmCacheNodeList := &gkmv1alpha1.GKMCacheNodeList{} - labelSelector := map[string]string{ - utils.GKMCacheLabelHostname: nodeName, - utils.GKMCacheNodeLabelCache: cacheName, - } - err := m.Client.List( - ctx, - gkmCacheNodeList, - client.InNamespace(namespace), - client.MatchingLabels(labelSelector), - ) - if err == nil { - if gkmCacheNodeList.GetItemsLen() == 1 { - gkmCacheNode := gkmCacheNodeList.Items[0] - - podWebhookLog.Info("GKMCacheNode Found", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - ) - - resolvedDigest := "" - var cacheStatus gkmv1alpha1.CacheStatus - for tmpCacheName, cacheList := range gkmCacheNode.Status.CacheStatuses { - if tmpCacheName == cacheName { - for digest, tmpCacheStatus := range cacheList { - // Use the first Digest found. Structure is setup to support multiple digests, - // but there is no way at the moment to add additional digests. - // TODO: resolvedDigest should be at a high level in the structure and the GKMCacheNode - // only supports one GKMCache. - resolvedDigest = digest - cacheStatus = tmpCacheStatus - break - } - } - } - - if resolvedDigest == "" { - podWebhookLog.Info("ResolvedDigest NOT set", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - ) - err = fmt.Errorf("ResolvedDigest not set yet") - return "", err - } - - podWebhookLog.Info("ResolvedDigest Found", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - - pvcStatus, pvcStatusExisted := cacheStatus.PvcStatus[namespace] - if !pvcStatusExisted { - podWebhookLog.Info("Can't get PVC Status", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - err = fmt.Errorf("PVC Status not set yet") - return "", err - } - - if pvcStatus.PvcName == "" { - podWebhookLog.Info("PVC Name not set yet", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - err = fmt.Errorf("PVC Name not set yet") - return "", err - } - - return pvcStatus.PvcName, nil - } else { - if len(gkmCacheNodeList.GetItems()) == 0 { - podWebhookLog.Info("List returned NO GKMCache ", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "Node", nodeName, - "err", err, - ) - } else { - for _, gkmCacheNode := range gkmCacheNodeList.GetItems() { - podWebhookLog.Info("ERROR: More than one GKMCacheNode Found", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - ) - } - } - } - } else { - podWebhookLog.Info("Error reading GKMCacheNode List", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "Node", nodeName, - "Error", err, - ) - } - - // Look for ClusterGKMCache - clusterGkmCacheNodeList := &gkmv1alpha1.ClusterGKMCacheNodeList{} - clusterLabelSelector := map[string]string{ - utils.GKMCacheLabelHostname: nodeName, - utils.GKMClusterCacheNodeLabelCache: cacheName, - } - err = m.Client.List( - ctx, - clusterGkmCacheNodeList, - client.MatchingLabels(clusterLabelSelector), - ) - if err == nil { - if clusterGkmCacheNodeList.GetItemsLen() == 1 { - clusterGkmCacheNode := clusterGkmCacheNodeList.Items[0] - podWebhookLog.Info("ClusterGKMCacheNode Found", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "ClusterCacheNodeName", clusterGkmCacheNode.GetName(), - "Node", nodeName, - ) - - resolvedDigest := "" - var cacheStatus gkmv1alpha1.CacheStatus - for tmpCacheName, cacheList := range clusterGkmCacheNode.Status.CacheStatuses { - if tmpCacheName == cacheName { - for digest, tmpCacheStatus := range cacheList { - resolvedDigest = digest - cacheStatus = tmpCacheStatus - } - } - } - - if resolvedDigest == "" { - podWebhookLog.Info("ResolvedDigest NOT set", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "ClusterCacheNodeName", clusterGkmCacheNode.GetName(), - "Node", nodeName, - ) - err = fmt.Errorf("ResolvedDigest not set yet") - return "", err - } - - podWebhookLog.Info("ResolvedDigest Found", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "ClusterCacheNodeName", clusterGkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - - pvcStatus, pvcStatusExisted := cacheStatus.PvcStatus[namespace] - if !pvcStatusExisted { - podWebhookLog.Info("Can't get PVC Status", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "ClusterCacheNodeName", clusterGkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - err = fmt.Errorf("PVC Status not set yet") - return "", err - } - - if pvcStatus.PvcName == "" { - podWebhookLog.Info("PVC Name not set yet", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "ClusterCacheNodeName", clusterGkmCacheNode.GetName(), - "Node", nodeName, - "ResolvedDigest", resolvedDigest, - ) - err = fmt.Errorf("PVC Name not set yet") - return "", err - } - - return pvcStatus.PvcName, nil - } else { - if len(gkmCacheNodeList.GetItems()) == 0 { - podWebhookLog.Info("List returned NO ClusterGKMCache ", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "Node", nodeName, - "err", err, - ) - } else { - for _, gkmCacheNode := range gkmCacheNodeList.GetItems() { - podWebhookLog.Info("ERROR: More than one ClusterGKMCacheNode Found", - "PodName", podName, - "Namespace", namespace, - "CacheName", cacheName, - "CacheNodeName", gkmCacheNode.GetName(), - "Node", nodeName, - ) - } - } - } - } else { - podWebhookLog.Info("Error reading ClusterGKMCacheNode List", - "PodName", podName, - "Namespace", namespace, - "ClusterCacheName", cacheName, - "Node", nodeName, - "Error", err, - ) - } - err = fmt.Errorf("unable to determine PVC") - return "", err -} diff --git a/pkg/common/k8s.go b/pkg/common/k8s.go index 2cf47189..da20033c 100644 --- a/pkg/common/k8s.go +++ b/pkg/common/k8s.go @@ -11,8 +11,10 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" @@ -63,8 +65,21 @@ func PodPredicate(nodeName string) predicate.Funcs { DeleteFunc: func(e event.DeleteEvent) bool { pod := e.Object.(*corev1.Pod) - // Pod deleted → definitely stopped using PVC - return pod.Spec.NodeName == nodeName && hasPVC(pod) + logger := log.Log.WithName("pod-predicate") + logger.Info("Delete:", + "Pod Name", pod.Name, + "Pod Namespace", pod.Namespace, + "Pod Phase", pod.Status.Phase, + "Pod PVC", hasPVC(pod), + ) + + // Pod deleted, stopped using PVC + nodeMatch := false + if nodeName == "" || pod.Spec.NodeName == nodeName { + nodeMatch = true + } + + return nodeMatch && hasPVC(pod) }, } } @@ -209,10 +224,11 @@ func CreatePv( ObjectMeta: metav1.ObjectMeta{ Name: pvName, Labels: map[string]string{ - utils.PvLabelCache: gkmCacheName, - utils.PvLabelPvcNamespace: pvcNamespace, - utils.PvLabelNode: nodeName, - utils.PvLabelDigest: trimDigest[:utils.MaxLabelValueLength], + utils.PvLabelCache: gkmCacheName, + utils.PvLabelCacheNamespace: gkmCacheNamespace, + utils.PvLabelPvcNamespace: pvcNamespace, + utils.PvLabelNode: nodeName, + utils.PvLabelDigest: trimDigest[:utils.MaxLabelValueLength], }, }, Spec: corev1.PersistentVolumeSpec{ @@ -220,15 +236,12 @@ func CreatePv( corev1.ResourceStorage: resource.MustParse(capacity), }, AccessModes: accessModes, - PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, - StorageClassName: storageClass, - VolumeMode: func() *corev1.PersistentVolumeMode { - m := corev1.PersistentVolumeFilesystem - return &m - }(), + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + VolumeMode: ptr.To(corev1.PersistentVolumeFilesystem), PersistentVolumeSource: corev1.PersistentVolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/tmp/gkm", + Path: "/kernel-caches", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), }, }, }, @@ -251,6 +264,11 @@ func CreatePv( } } + // Since optional, only apply if set. + if storageClass != "" { + pv.Spec.StorageClassName = storageClass + } + if err := client.Create(ctx, pv); err != nil { log.Error(err, "Failed to create PV.", "namespace", gkmCacheNamespace, @@ -349,6 +367,46 @@ func PvExists( return retPv, found, updatedName, nil } +// GetGkmPvFailedList retrieves all the PVs created by GKM based on labels on the PV. +// It will liimt the returned list to just those PVs in a Phase of Failed. +func GetGkmPvFailedList( + ctx context.Context, + objClient client.Client, + log logr.Logger, +) ([]corev1.PersistentVolume, error) { + + labelSelector, err := labels.Parse( + utils.PvLabelCache + "," + utils.PvLabelPvcNamespace, + ) + if err != nil { + return nil, err + } + + pvList := &corev1.PersistentVolumeList{} + + if err := objClient.List( + ctx, + pvList, + client.MatchingLabelsSelector{Selector: labelSelector}, + ); err != nil { + return nil, err + } + + var failed []corev1.PersistentVolume + + for _, pv := range pvList.Items { + if pv.Status.Phase == corev1.VolumeFailed { + failed = append(failed, pv) + } + } + log.Info("GKM PV List", + "NumPVs", len(pvList.Items), + "NumFailedPVs", len(failed), + ) + + return failed, nil +} + // DeletePv tries to delete PV created by GKM func DeletePv( ctx context.Context, @@ -492,10 +550,12 @@ func CreatePvc( Name: pvcName, Namespace: pvcNamespace, Labels: map[string]string{ - utils.PvcLabelCache: gkmCacheName, - utils.PvcLabelPvcNamespace: pvcNamespace, - utils.PvcLabelNode: nodeName, - utils.PvcLabelDigest: trimDigest[:utils.MaxLabelValueLength], + utils.PvcLabelCache: gkmCacheName, + utils.PvcLabelCacheNamespace: gkmCacheNamespace, + utils.PvcLabelPvcNamespace: pvcNamespace, + utils.PvcLabelPvName: pvName, + utils.PvcLabelNode: nodeName, + utils.PvcLabelDigest: trimDigest[:utils.MaxLabelValueLength], }, }, Spec: corev1.PersistentVolumeClaimSpec{ @@ -505,7 +565,6 @@ func CreatePvc( corev1.ResourceStorage: resource.MustParse(capacity), }, }, - StorageClassName: &storageClass, VolumeMode: func() *corev1.PersistentVolumeMode { m := corev1.PersistentVolumeFilesystem return &m @@ -518,6 +577,16 @@ func CreatePvc( pvc.Spec.VolumeName = pvName } + // StorageClass must match PV, or PVC won't bind. + // Empty string prevents Kubernetes from auto-filling default StorageClass. + if storageClass != "" { + pvc.Spec.StorageClassName = &storageClass + } else if pvName != "" { + // Manual PV binding requires explicit empty storageClassName + emptyClass := "" + pvc.Spec.StorageClassName = &emptyClass + } + if err := client.Create(ctx, pvc); err != nil { log.Error(err, "Failed to create PVC.", "namespace", gkmCacheNamespace, @@ -619,6 +688,37 @@ func PvcExists( return retPvc, found, updatedName, nil } +// GetGkmPvcList retrieves all the PVCs created by GKM based on labels on the PVC. +func GetGkmPvcList( + ctx context.Context, + objClient client.Client, + log logr.Logger, +) ([]corev1.PersistentVolumeClaim, error) { + + labelSelector, err := labels.Parse( + utils.PvcLabelCache + "," + utils.PvcLabelPvcNamespace, + ) + if err != nil { + return nil, err + } + + pvcList := &corev1.PersistentVolumeClaimList{} + + if err := objClient.List( + ctx, + pvcList, + client.MatchingLabelsSelector{Selector: labelSelector}, + ); err != nil { + return nil, err + } + + log.Info("GKM PVC List", + "NumPVCs", len(pvcList.Items), + ) + + return pvcList.Items, nil +} + // GetPvcUsedByList walks the Pods in the Namespace and tries to determine which // Pods are using the given PVC. func GetPvcUsedByList( @@ -838,10 +938,12 @@ func LaunchJob( cacheImage string, resolvedDigest string, noGpu bool, + kindCluster bool, extractImage string, pvcStatus *gkmv1alpha1.PvcStatus, podTemplate *gkmv1alpha1.PodTemplate, log logr.Logger, + jobLogLevel string, ) error { log.Info("Creating download job", "jobName", jobName, "pvcName", pvcStatus.PvcName) @@ -890,6 +992,7 @@ func LaunchJob( {Name: utils.JobExtractEnvCacheDir, Value: utils.MountPath}, {Name: utils.JobExtractEnvImageUrl, Value: updatedImage}, {Name: utils.JobExtractEnvNoGpu, Value: noGpuString}, + {Name: utils.JobExtractEnvGoLog, Value: jobLogLevel}, } container.VolumeMounts = []corev1.VolumeMount{ @@ -982,10 +1085,10 @@ func LaunchJob( } } - // For KIND Clusters, currently identified by NoGpu, Kubelet can't change the ownership + // For KIND Clusters, currently identified by kindCluster, Kubelet can't change the ownership // of the directory of a Volume Mount. So an InitContainer is added to the job the manage // the ownership. - if noGpu { + if kindCluster { var rootUser int64 = 0 commandString := diff --git a/pkg/utils/contants.go b/pkg/utils/contants.go index 435ecd1a..4c59e592 100644 --- a/pkg/utils/contants.go +++ b/pkg/utils/contants.go @@ -5,34 +5,18 @@ import ( ) const ( - // DefaultSocketFilename is the location of the Unix domain socket for this CSI driver - // for Kubelet to send requests. - DefaultSocketFilename string = "unix:///var/lib/kubelet/plugins/csi-gkm/csi.sock" - - // DefaultImagePort is the location of port the Image Server will listen on for GKM - // to send requests. - DefaultImagePort string = ":50051" - - // DefaultCacheDir is the default root directory to store the expanded the GPU Kernel - // images. - DefaultCacheDir = "/var/lib/gkm/caches" - CacheFilename = "cache.json" - - // DefaultUsageDir is the default root directory to store the usage data for the GPU Kernel - // images. - DefaultUsageDir = "/run/gkm/usage" - UsageFilename = "usage.json" - - // DefaultCacheDir is the default root directory to store the expanded the GPU Kernel - // images. - ClusterScopedSubDir = "cluster-scoped" - // Name of the GKM ConfigMap that is used to control how GKM is Deployed and Functions. GKMConfigName = "gkm-config" // Name of the GKM Namespace where Operator and Agent run. GKMDefaultNamespace = "gkm-system" + // Name of the CRDs. Used in each reconciler as the CrdCacheStr. + CrdGKMCache = "GKMCache" + CrdGKMCacheNode = "GKMCacheNode" + CrdClusterGKMCache = "ClusterGKMCache" + CrdClusterGKMCacheNode = "ClusterGKMCacheNode" + // GKMCache and ClusterGKMCache Annotations GKMCacheAnnotationResolvedDigest = "gkm.io/resolvedDigest" GKMCacheAnnotationCacheSizeBytes = "gkm.io/cache-size-bytes" @@ -43,17 +27,19 @@ const ( GKMCacheLabelHostname = "kubernetes.io/hostname" GKMCacheNodeLabelCache = "gkm.io/gkm-cache" GKMClusterCacheNodeLabelCache = "gkm.io/cluster-gkm-cache" - GKMCachePvcMutation = "gkm.io/pvc-mutation" // PV and PVC Labels - PvLabelCache = "cache-name" - PvLabelPvcNamespace = "pvc-namespace" - PvLabelNode = "node" - PvLabelDigest = "digest" - PvcLabelCache = "cache-name" - PvcLabelPvcNamespace = "pvc-namespace" - PvcLabelNode = "node" - PvcLabelDigest = "digest" + PvLabelCache = "cache-name" + PvLabelCacheNamespace = "cache-namespace" + PvLabelPvcNamespace = "pvc-namespace" + PvLabelNode = "node" + PvLabelDigest = "digest" + PvcLabelCache = "cache-name" + PvcLabelCacheNamespace = "cache-namespace" + PvcLabelPvcNamespace = "pvc-namespace" + PvcLabelNode = "node" + PvcLabelDigest = "digest" + PvcLabelPvName = "pv-name" // OCI Image Label ImageLabelCacheSizeBytesSubstring = "cache-size-bytes" @@ -65,6 +51,7 @@ const ( JobExtractEnvCacheDir = "GKM_CACHE_DIR" JobExtractEnvImageUrl = "GKM_IMAGE_URL" JobExtractEnvNoGpu = "NO_GPU" + JobExtractEnvGoLog = "GO_LOG" JobExtractPvcSourceMountName = "gkm-pvc-source" JobExtractLabelPvc = "pvc" JobExtractLabelDigest = "digest" @@ -73,7 +60,7 @@ const ( JobFSGroup = 1000 MaxLabelValueLength = 60 // Labels can only be 63 characters DigestPrefix = "sha256:" - MountPath = "/cache" + MountPath = "/kernel-caches" // Kyverno Annotations KyvernoVerifyImagesAnnotation = "kyverno.io/verify-images" @@ -103,7 +90,7 @@ const ( ConfigMapIndexExtractImage = "gkm.extract.image" ConfigMapIndexExtractLogLevel = "gkm.extract.log.level" ConfigMapIndexNoGpu = "gkm.nogpu" - ConfigMapIndexKindCluster = "gkm.kind.cluster" + ConfigMapIndexKindCluster = "gkm.kindcluster" ConfigMapIndexKyvernoEnabled = "gkm.kyverno.enabled" // Duration for Kubernetes to Retry a failed request