From fd078e3831cfcfef274b3d0c731b75b03bcfb930 Mon Sep 17 00:00:00 2001 From: Alexandre Proulx Date: Tue, 14 Oct 2025 11:32:23 -0400 Subject: [PATCH 1/4] Added safe-start support Added safe-start support --- .github/workflows/chainsaw-e2e-test-1.31.yaml | 7 +- ...-1.28.yaml => chainsaw-e2e-test-1.32.yaml} | 17 +- ...-1.29.yaml => chainsaw-e2e-test-1.33.yaml} | 17 +- ...-1.30.yaml => chainsaw-e2e-test-1.34.yaml} | 17 +- Makefile | 139 +++++--- build | 2 +- cluster/local/integration_tests.sh | 193 ---------- cmd/provider/main.go | 334 ++++++++++++------ e2e/kind/kind-config-1.31.yaml | 2 +- ...config-1.28.yaml => kind-config-1.32.yaml} | 2 +- ...config-1.29.yaml => kind-config-1.33.yaml} | 2 +- ...config-1.30.yaml => kind-config-1.34.yaml} | 2 +- e2e/tests/safe-start/.chainsaw.yaml | 13 + e2e/tests/safe-start/chainsaw-test.yaml | 101 ++++++ e2e/tests/stable/chainsaw-test.yaml | 7 +- go.mod | 13 +- go.sum | 26 +- hack/deploy-provider.sh | 49 --- hack/expect_bucket.sh | 7 +- hack/generate-tests.sh | 7 +- .../APIVERSION/KIND_LOWER_types.go.tmpl | 2 +- .../controller/KIND_LOWER/KIND_LOWER.go.tmpl | 12 +- .../KIND_LOWER/KIND_LOWER_test.go.tmpl | 6 +- internal/controller/bucket/setup.go | 11 + package/crossplane.yaml | 4 +- 25 files changed, 535 insertions(+), 457 deletions(-) rename .github/workflows/{chainsaw-e2e-test-1.28.yaml => chainsaw-e2e-test-1.32.yaml} (70%) rename .github/workflows/{chainsaw-e2e-test-1.29.yaml => chainsaw-e2e-test-1.33.yaml} (70%) rename .github/workflows/{chainsaw-e2e-test-1.30.yaml => chainsaw-e2e-test-1.34.yaml} (70%) delete mode 100755 cluster/local/integration_tests.sh rename e2e/kind/{kind-config-1.28.yaml => kind-config-1.32.yaml} (94%) rename e2e/kind/{kind-config-1.29.yaml => kind-config-1.33.yaml} (94%) rename e2e/kind/{kind-config-1.30.yaml => kind-config-1.34.yaml} (94%) create mode 100644 e2e/tests/safe-start/.chainsaw.yaml create mode 100644 e2e/tests/safe-start/chainsaw-test.yaml delete mode 100755 hack/deploy-provider.sh diff --git a/.github/workflows/chainsaw-e2e-test-1.31.yaml b/.github/workflows/chainsaw-e2e-test-1.31.yaml index a80f1947..53142053 100644 --- a/.github/workflows/chainsaw-e2e-test-1.31.yaml +++ b/.github/workflows/chainsaw-e2e-test-1.31.yaml @@ -30,7 +30,7 @@ jobs: run: make vendor vendor.check - name: Docker cache - uses: ScribeMD/docker-cache@0.3.7 + uses: ScribeMD/docker-cache@0.5.0 with: key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} @@ -41,3 +41,8 @@ jobs: AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' AWS_DEFAULT_REGION: 'us-east-1' + + - name: Run chainsaw-safe-start tests 1.31 + run: make chainsaw-safe-start + env: + LATEST_KUBE_VERSION: '1.31' diff --git a/.github/workflows/chainsaw-e2e-test-1.28.yaml b/.github/workflows/chainsaw-e2e-test-1.32.yaml similarity index 70% rename from .github/workflows/chainsaw-e2e-test-1.28.yaml rename to .github/workflows/chainsaw-e2e-test-1.32.yaml index 6d5444e3..04d776d6 100644 --- a/.github/workflows/chainsaw-e2e-test-1.28.yaml +++ b/.github/workflows/chainsaw-e2e-test-1.32.yaml @@ -1,14 +1,14 @@ # This file was auto-generated by hack/generate-tests.sh -name: chainsaw e2e test 1.28 +name: chainsaw e2e test 1.32 on: [push] concurrency: - group: chainsaw-1.28-${{ github.ref }}-1 + group: chainsaw-1.32-${{ github.ref }}-1 cancel-in-progress: true permissions: contents: read jobs: test: - name: chainsaw e2e test 1.28 + name: chainsaw e2e test 1.32 runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -30,14 +30,19 @@ jobs: run: make vendor vendor.check - name: Docker cache - uses: ScribeMD/docker-cache@0.3.7 + uses: ScribeMD/docker-cache@0.5.0 with: key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} - - name: Run chainsaw tests 1.28 + - name: Run chainsaw tests 1.32 run: make chainsaw env: - LATEST_KUBE_VERSION: '1.28' + LATEST_KUBE_VERSION: '1.32' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' AWS_DEFAULT_REGION: 'us-east-1' + + - name: Run chainsaw-safe-start tests 1.32 + run: make chainsaw-safe-start + env: + LATEST_KUBE_VERSION: '1.32' diff --git a/.github/workflows/chainsaw-e2e-test-1.29.yaml b/.github/workflows/chainsaw-e2e-test-1.33.yaml similarity index 70% rename from .github/workflows/chainsaw-e2e-test-1.29.yaml rename to .github/workflows/chainsaw-e2e-test-1.33.yaml index 54d1b9e8..19985e94 100644 --- a/.github/workflows/chainsaw-e2e-test-1.29.yaml +++ b/.github/workflows/chainsaw-e2e-test-1.33.yaml @@ -1,14 +1,14 @@ # This file was auto-generated by hack/generate-tests.sh -name: chainsaw e2e test 1.29 +name: chainsaw e2e test 1.33 on: [push] concurrency: - group: chainsaw-1.29-${{ github.ref }}-1 + group: chainsaw-1.33-${{ github.ref }}-1 cancel-in-progress: true permissions: contents: read jobs: test: - name: chainsaw e2e test 1.29 + name: chainsaw e2e test 1.33 runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -30,14 +30,19 @@ jobs: run: make vendor vendor.check - name: Docker cache - uses: ScribeMD/docker-cache@0.3.7 + uses: ScribeMD/docker-cache@0.5.0 with: key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} - - name: Run chainsaw tests 1.29 + - name: Run chainsaw tests 1.33 run: make chainsaw env: - LATEST_KUBE_VERSION: '1.29' + LATEST_KUBE_VERSION: '1.33' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' AWS_DEFAULT_REGION: 'us-east-1' + + - name: Run chainsaw-safe-start tests 1.33 + run: make chainsaw-safe-start + env: + LATEST_KUBE_VERSION: '1.33' diff --git a/.github/workflows/chainsaw-e2e-test-1.30.yaml b/.github/workflows/chainsaw-e2e-test-1.34.yaml similarity index 70% rename from .github/workflows/chainsaw-e2e-test-1.30.yaml rename to .github/workflows/chainsaw-e2e-test-1.34.yaml index cabda14d..00fb6f07 100644 --- a/.github/workflows/chainsaw-e2e-test-1.30.yaml +++ b/.github/workflows/chainsaw-e2e-test-1.34.yaml @@ -1,14 +1,14 @@ # This file was auto-generated by hack/generate-tests.sh -name: chainsaw e2e test 1.30 +name: chainsaw e2e test 1.34 on: [push] concurrency: - group: chainsaw-1.30-${{ github.ref }}-1 + group: chainsaw-1.34-${{ github.ref }}-1 cancel-in-progress: true permissions: contents: read jobs: test: - name: chainsaw e2e test 1.30 + name: chainsaw e2e test 1.34 runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -30,14 +30,19 @@ jobs: run: make vendor vendor.check - name: Docker cache - uses: ScribeMD/docker-cache@0.3.7 + uses: ScribeMD/docker-cache@0.5.0 with: key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} - - name: Run chainsaw tests 1.30 + - name: Run chainsaw tests 1.34 run: make chainsaw env: - LATEST_KUBE_VERSION: '1.30' + LATEST_KUBE_VERSION: '1.34' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' AWS_DEFAULT_REGION: 'us-east-1' + + - name: Run chainsaw-safe-start tests 1.34 + run: make chainsaw-safe-start + env: + LATEST_KUBE_VERSION: '1.34' diff --git a/Makefile b/Makefile index e732ab6b..d003fc45 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,16 @@ PLATFORMS ?= linux_amd64 linux_arm64 # Generate chainsaw e2e tests for the following kind node versions # TEST_KIND_NODES is not intended to be updated manually. # Please edit LATEST_KIND_NODE instead and run 'make update-kind-nodes'. -TEST_KIND_NODES ?= 1.28.7,1.29.2,1.30.8,1.31.4 - -LATEST_KUBE_VERSION ?= 1.31 -LATEST_KIND_NODE ?= 1.31.4 +TEST_KIND_NODES ?= 1.31.13,1.32.9,1.33.5,1.34.1 +KIND_VERSION ?= v0.30.0 +LATEST_KUBE_VERSION ?= 1.34 +LATEST_KIND_NODE ?= 1.34.1 +KUBECTL_VERSION ?= v1.34.0 REPO ?= provider-ceph -CROSSPLANE_VERSION ?= 1.20.0 +CROSSPLANE_CLI_VERSION ?= v2.0.2 +CROSSPLANE_NAMESPACE ?= crossplane-system +CROSSPLANE_VERSION ?= 2.0.2 LOCALSTACK_VERSION ?= 4.7 CERT_MANAGER_VERSION ?= 1.14.0 @@ -69,13 +72,18 @@ IMAGES = provider-ceph XPKG_REG_ORGS ?= xpkg.upbound.io/linode # NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are # inferred. -XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/linode +XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/linode xpkg.crossplane.internal/dev XPKGS = provider-ceph -include build/makelib/xpkg.mk -include build/makelib/local.xpkg.mk VAL_WBHK_STAGE ?= $(ROOT_DIR)/staging/validatingwebhookconfiguration +# ==================================================================================== +# Setup Controlplane and E2E Testing (Crossplane v2) + +-include build/makelib/controlplane.mk + # NOTE(hasheddan): we force image building to happen prior to xpkg build so that # we ensure image is present in daemon. xpkg.build.provider-ceph: do.build.images @@ -84,9 +92,6 @@ fallthrough: submodules @echo Initial setup complete. Running make again . . . @make -# integration tests -e2e.run: test-integration - # Update kind node versions to be tested. update-kind-nodes: LATEST_KIND_NODE=$(LATEST_KIND_NODE) ./hack/update-kind-nodes.sh @@ -95,12 +100,6 @@ update-kind-nodes: generate-tests: TEST_KIND_NODES=$(TEST_KIND_NODES) REPO=$(REPO) LOCALSTACK_VERSION=$(LOCALSTACK_VERSION) CERT_MANAGER_VERSION=$(CERT_MANAGER_VERSION) ./hack/generate-tests.sh -# Run integration tests. -test-integration: $(KIND) $(KUBECTL) $(UP) $(HELM) - @$(INFO) running integration tests using kind $(KIND_VERSION) - @KIND_NODE_IMAGE_TAG=${KIND_NODE_IMAGE_TAG} $(ROOT_DIR)/cluster/local/integration_tests.sh || $(FAIL) - @$(OK) integration tests passed - # Update the submodules, such as the common build scripts. submodules: @git submodule sync @@ -117,6 +116,16 @@ go.cachedir: build.init: $(CROSSPLANE_CLI) +# Export build variables for integration tests +build.vars: common.buildvars k8s_tools.buildvars + @echo CROSSPLANE_VERSION=$(CROSSPLANE_VERSION) + @echo CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) + @echo CROSSPLANE_CLI_VERSION=$(CROSSPLANE_CLI_VERSION) + @echo KIND_VERSION=$(KIND_VERSION) + @echo KUBECTL_VERSION=$(KUBECTL_VERSION) + @echo LOCALSTACK_VERSION=$(LOCALSTACK_VERSION) + @echo CERT_MANAGER_VERSION=$(CERT_MANAGER_VERSION) + # This is for running out-of-cluster locally, and is for convenience. Running # this make target will print out the command which was used. For more control, # try running the binary directly with different arguments. @@ -136,14 +145,22 @@ localstack-cluster: $(KIND) $(KUBECTL) @KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) SOURCE="file://$(PWD)/e2e/localstack/localstack-deployment.yaml" ./hack/load-images.sh @$(KUBECTL) apply -R -f e2e/localstack/localstack-deployment.yaml -# Spin up a Kind cluster and install Crossplane via Helm. -crossplane-cluster: $(HELM) cluster - @$(INFO) Installing Crossplane - @$(HELM) repo add crossplane-stable https://charts.crossplane.io/stable +# Internal helper target to install Crossplane via Helm with configurable activations +# Usage: make _install-crossplane CROSSPLANE_ACTIVATIONS="" +# CROSSPLANE_ACTIVATIONS can be empty (for default) or "provider.defaultActivations={}" +.PHONY: _install-crossplane +_install-crossplane: $(HELM) + @$(INFO) Installing Crossplane $(CROSSPLANE_VERSION) + @$(HELM) repo add crossplane-stable https://charts.crossplane.io/stable --force-update @$(HELM) repo update + @$(HELM) get notes -n $(CROSSPLANE_NAMESPACE) crossplane >/dev/null 2>&1 || $(HELM) install crossplane --create-namespace --namespace=$(CROSSPLANE_NAMESPACE) --version $(CROSSPLANE_VERSION) $(CROSSPLANE_ACTIVATIONS) --set packageCache.sizeLimit=128Mi crossplane-stable/crossplane + @$(OK) Crossplane installed + +# Spin up a Kind cluster and install Crossplane 2.0+ with safe-start (empty activations) +crossplane-cluster: $(HELM) cluster + @$(INFO) Installing Crossplane with safe-start support @KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) SOURCE="helm template crossplane --namespace crossplane-system --version $(CROSSPLANE_VERSION) crossplane-stable/crossplane" ./hack/load-images.sh - @$(HELM) install crossplane --namespace crossplane-system --create-namespace --version $(CROSSPLANE_VERSION) crossplane-stable/crossplane - @$(OK) Installing Crossplane + @$(MAKE) _install-crossplane CROSSPLANE_ACTIVATIONS="--set provider.defaultActivations={}" ## Deploy cert manager to the K8s cluster specified in ~/.kube/config. cert-manager: $(KUBECTL) @@ -164,38 +181,56 @@ kustomize-webhook: $(KUSTOMIZE) @cp -f $(VAL_WBHK_STAGE)/service-patch-$(WEBHOOK_TYPE).yaml $(VAL_WBHK_STAGE)/service-patch.yaml $(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml -# Build the controller image and the provider package. -# Load the controller image to the Kind cluster and add the provider package -# to the Provider. -# The following is taken from local.xpkg.deploy.provider. -# However, it is modified to use the "--zap-devel" flag instead of "-d" which does -# not exist in this project and would therefore cause the controller to CrashLoop. -load-package: $(KIND) build kustomize-webhook - @$(MAKE) local.xpkg.sync - @$(INFO) deploying provider package $(PROJECT_NAME) - @$(KIND) load docker-image $(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH) -n $(KIND_CLUSTER_NAME) - @BUILD_REGISTRY=$(BUILD_REGISTRY) PROJECT_NAME=$(PROJECT_NAME) ARCH=$(ARCH) VERSION=$(VERSION) ./hack/deploy-provider.sh - @$(OK) deploying provider package $(PROJECT_NAME) $(VERSION) - -# Spin up a Kind cluster and localstack and install Crossplane via Helm. -# Build the controller image and the provider package. -# Load the controller image to the Kind cluster and add the provider package -# to the Provider. -# Run Chainsaw test suite on newly built controller image. -# Destroy Kind and localstack. +# Setup Crossplane with default activations for standard testing +.PHONY: crossplane-standard +crossplane-standard: $(KIND) $(KUBECTL) + @$(INFO) Setting up kind cluster for standard testing + @$(KIND) get kubeconfig --name $(KIND_CLUSTER_NAME) >/dev/null 2>&1 || $(KIND) create cluster --name=$(KIND_CLUSTER_NAME) + @$(INFO) Setting kubectl context to kind-$(KIND_CLUSTER_NAME) + @$(KUBECTL) config use-context "kind-$(KIND_CLUSTER_NAME)" + @$(MAKE) _install-crossplane CROSSPLANE_ACTIVATIONS="" + +# Setup Crossplane for safe-start testing +.PHONY: crossplane-safe-start +crossplane-safe-start: $(KIND) $(KUBECTL) + @$(INFO) Setting up kind cluster for safe-start testing + @$(KIND) get kubeconfig --name $(KIND_CLUSTER_NAME) >/dev/null 2>&1 || $(KIND) create cluster --name=$(KIND_CLUSTER_NAME) + @$(INFO) Setting kubectl context to kind-$(KIND_CLUSTER_NAME) + @$(KUBECTL) config use-context "kind-$(KIND_CLUSTER_NAME)" + @$(MAKE) _install-crossplane CROSSPLANE_ACTIVATIONS="--set provider.defaultActivations={}" + +# Run Chainsaw test suite for safe-start testing +.PHONY: chainsaw-safe-start +chainsaw-safe-start: $(CHAINSAW) build generate-pkg generate-tests + @$(INFO) Running chainsaw safe-start test suite with Crossplane v2 + @$(MAKE) crossplane-safe-start + @$(MAKE) localstack-cluster + @$(MAKE) local.xpkg.deploy.provider.$(PROJECT_NAME) + @$(CHAINSAW) test e2e/tests/safe-start --config e2e/tests/safe-start/.chainsaw.yaml +# || ($(MAKE) controlplane.down && $(FAIL)) + @$(OK) Running chainsaw safe-start test suite + @$(MAKE) controlplane.down + +# Run Chainsaw test suite using build submodule's controlplane infrastructure +# This is the standardized approach for Crossplane v2 testing .PHONY: chainsaw -chainsaw: $(CHAINSAW) generate-pkg generate-tests crossplane-cluster localstack-cluster load-package - @$(INFO) Running chainsaw test suite - $(CHAINSAW) test e2e/tests/stable --config e2e/tests/stable/.chainsaw.yaml +chainsaw: $(CHAINSAW) build generate-pkg generate-tests + @$(INFO) Running chainsaw test suite with Crossplane v2 + @$(MAKE) crossplane-standard + @$(MAKE) localstack-cluster + @$(MAKE) local.xpkg.deploy.provider.$(PROJECT_NAME) + @$(CHAINSAW) test e2e/tests/stable --config e2e/tests/stable/.chainsaw.yaml @$(OK) Running chainsaw test suite - @$(MAKE) cluster-clean + @$(MAKE) controlplane.down .PHONY: ceph-chainsaw -ceph-chainsaw: $(CHAINSAW) crossplane-cluster load-package +ceph-chainsaw: $(CHAINSAW) build generate-pkg @$(INFO) Running chainsaw test suite against ceph cluster - $(CHAINSAW) test e2e/tests/ceph + @$(MAKE) controlplane.up + @$(MAKE) local.xpkg.deploy.provider.$(PROJECT_NAME) + @$(CHAINSAW) test e2e/tests/ceph || ($(MAKE) controlplane.down && $(FAIL)) @$(OK) Running chainsaw test suite against ceph cluster - @$(MAKE) cluster-clean + @$(MAKE) controlplane.down # Spin up a Kind cluster and localstack and install Crossplane CRDs (not # containerised Crossplane componenets). @@ -280,7 +315,7 @@ cluster-clean: $(KIND) $(KUBECTL) @$(KIND) delete cluster --name=$(KIND_CLUSTER_NAME) @$(OK) Deleting kind cluster -.PHONY: submodules fallthrough test-integration run cluster dev-cluster dev cluster-clean +.PHONY: submodules fallthrough run cluster dev-cluster dev cluster-clean # ==================================================================================== # Special Targets @@ -346,9 +381,9 @@ help-special: crossplane.help AWS ?= /usr/local/bin/aws aws: ifeq (,$(wildcard $(AWS))) - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o $PWD/bin/awscliv2.zip - unzip $PWD/bin/awscliv2.zip -d $PWD/bin/ - $PWD/bin/aws/install + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o $(PWD)/bin/awscliv2.zip + unzip $(PWD)/bin/awscliv2.zip -d $(PWD)/bin/ + $(PWD)/bin/aws/install endif NILAWAY_VERSION ?= latest @@ -360,7 +395,7 @@ nilcheck: $(NILAWAY) ## Run nil check against codemake. @# Backendstore contains mostly nil safe generated files. @# Lifecycleconfig_helper has false positive reports: https://github.com/uber-go/nilaway/issues/207 go list ./... | xargs -I {} -d '\n' $(NILAWAY) \ - -exclude-errors-in-files $(PWD)/internal/controller/bucket/bucket_backends.go,$(PWD)/internal/rgw/lifecycleconfig_helpers.go,$(PWD)/internal/rgw/objectlockconfiguration_helpers.go \ + -exclude-errors-in-files $(PWD)/cmd/provider/main.go,$(PWD)/internal/controller/bucket/bucket_backends.go,$(PWD)/internal/rgw/lifecycleconfig_helpers.go,$(PWD)/internal/rgw/objectlockconfiguration_helpers.go \ -exclude-pkgs github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1,github.com/linode/provider-ceph/internal/backendstore \ -include-pkgs {} ./... diff --git a/build b/build index b964dbe0..0a8b8840 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit b964dbe0ff0856a762f1a06fe554c647d22af7f0 +Subproject commit 0a8b8840306079292b7a3746ce1d5382b6868af8 diff --git a/cluster/local/integration_tests.sh b/cluster/local/integration_tests.sh deleted file mode 100755 index ee2d0a89..00000000 --- a/cluster/local/integration_tests.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env bash -set -e - -# setting up colors -BLU='\033[0;34m' -YLW='\033[0;33m' -GRN='\033[0;32m' -RED='\033[0;31m' -NOC='\033[0m' # No Color -echo_info(){ - printf "\n${BLU}%s${NOC}" "$1" -} -echo_step(){ - printf "\n${BLU}>>>>>>> %s${NOC}\n" "$1" -} -echo_sub_step(){ - printf "\n${BLU}>>> %s${NOC}\n" "$1" -} - -echo_step_completed(){ - printf "${GRN} [✔]${NOC}" -} - -echo_success(){ - printf "\n${GRN}%s${NOC}\n" "$1" -} -echo_warn(){ - printf "\n${YLW}%s${NOC}" "$1" -} -echo_error(){ - printf "\n${RED}%s${NOC}" "$1" - exit 1 -} - - -# The name of your provider. Many provider Makefiles override this value. -PACKAGE_NAME="provider-ceph" - - -# ------------------------------ -projectdir="$( cd "$( dirname "${BASH_SOURCE[0]}")"/../.. && pwd )" - -# get the build environment variables from the special build.vars target in the main makefile -eval $(make --no-print-directory -C ${projectdir} build.vars) - -# ------------------------------ - -SAFEHOSTARCH="${SAFEHOSTARCH:-amd64}" -BUILD_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-${SAFEHOSTARCH}" -PACKAGE_IMAGE="crossplane.io/inttests/${PROJECT_NAME}:${VERSION}" -CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-controller-${SAFEHOSTARCH}" - -version_tag="$(cat ${projectdir}/_output/version)" -# tag as latest version to load into kind cluster -PACKAGE_CONTROLLER_IMAGE="${DOCKER_REGISTRY}/${PROJECT_NAME}-controller:${VERSION}" -K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}" - -CROSSPLANE_NAMESPACE="crossplane-system" - -# cleanup on exit -if [ "$skipcleanup" != true ]; then - function cleanup { - echo_step "Cleaning up..." - export KUBECONFIG= - "${KIND}" delete cluster --name="${K8S_CLUSTER}" - } - - trap cleanup EXIT -fi - -# setup package cache -echo_step "setting up local package cache" -CACHE_PATH="${projectdir}/.work/inttest-package-cache" -mkdir -p "${CACHE_PATH}" -echo "created cache dir at ${CACHE_PATH}" -docker tag "${BUILD_IMAGE}" "${PACKAGE_IMAGE}" -"${UP}" xpkg xp-extract --from-daemon "${PACKAGE_IMAGE}" -o "${CACHE_PATH}/${PACKAGE_NAME}.gz" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.gz" - -# create kind cluster with extra mounts -KIND_NODE_IMAGE="kindest/node:${KIND_NODE_IMAGE_TAG}" -echo_step "creating k8s cluster using kind ${KIND_VERSION} and node image ${KIND_NODE_IMAGE}" -KIND_CONFIG="$( cat < $current ]]; then - echo_error "timeout of ${timeout}s has been reached" - fi - sleep $step; -done - -echo_success "Integration tests succeeded!" diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 7f0e5fbe..047782b4 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -18,6 +18,8 @@ package main //go:generate go get github.com/maxbrunsfeld/counterfeiter/v6 +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch + import ( "context" "flag" @@ -27,10 +29,14 @@ import ( "time" "github.com/alecthomas/kingpin/v2" + "github.com/go-logr/logr" "github.com/linode/provider-ceph/internal/otel/traces" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.uber.org/zap/zapcore" + authv1 "k8s.io/api/authorization/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -40,14 +46,18 @@ import ( kcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/crossplane/crossplane-runtime/v2/pkg/controller" + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/feature" + "github.com/crossplane/crossplane-runtime/v2/pkg/gate" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" "github.com/crossplane/crossplane-runtime/v2/pkg/meta" "github.com/crossplane/crossplane-runtime/v2/pkg/ratelimiter" + "github.com/crossplane/crossplane-runtime/v2/pkg/reconciler/customresourcesgate" "github.com/crossplane/crossplane-runtime/v2/pkg/reconciler/managed" "github.com/crossplane/crossplane-runtime/v2/pkg/resource" "github.com/crossplane/crossplane-runtime/v2/pkg/statemetrics" @@ -71,50 +81,36 @@ var defaultZapConfig = map[string]string{ "zap-time-encoding": "rfc3339nano", } -//nolint:maintidx // Function requires a lot of setup operations. -func main() { - var ( - app = kingpin.New(filepath.Base(os.Args[0]), "Ceph support for Crossplane.").DefaultEnvars() - leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool() - leaderRenew = app.Flag("leader-renew", "Set leader election renewal.").Short('r').Default("10s").OverrideDefaultFromEnvar("LEADER_ELECTION_RENEW").Duration() - - syncInterval = app.Flag("sync", "How often all resources will be double-checked for drift from the desired state.").Short('s').Default("1h").Duration() - syncTimeout = app.Flag("sync-timeout", "Cache sync timeout.").Default("10s").Duration() - backendMonitorInterval = app.Flag("backend-monitor-interval", "Interval between backend monitor controller reconciliations.").Default("60s").Duration() - pollInterval = app.Flag("poll", "How often individual resources will be checked for drift from the desired state").Short('p').Default("30m").Duration() - pollStateMetricInterval = app.Flag("poll-state-metric", "State metric recording interval").Default("5s").Duration() - reconcileConcurrency = app.Flag("reconcile-concurrency", "Set number of reconciliation loops.").Default("100").Int() - maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("1000").Int() - reconcileTimeout = app.Flag("reconcile-timeout", "Object reconciliation timeout").Short('t').Default("3s").Duration() - s3Timeout = app.Flag("s3-timeout", "S3 API operations timeout").Default("10s").Duration() - creationGracePeriod = app.Flag("creation-grace-period", "Duration to wait for the external API to report that a newly created external resource exists.").Default("10s").Duration() - tracesEnabled = app.Flag("otel-enable-tracing", "").Default("false").Bool() - tracesExportTimeout = app.Flag("otel-traces-export-timeout", "Timeout when exporting traces").Default("2s").Duration() - tracesExportInterval = app.Flag("otel-traces-export-interval", "Interval at which traces are exported").Default("5s").Duration() - tracesExportAddress = app.Flag("otel-traces-export-address", "Address of otel collector").Default("opentelemetry-collector.opentelemetry:4317").String() - - kubeClientRate = app.Flag("kube-client-rate", "The global maximum rate per second at how many requests the client can do.").Default("1000").Int() - - namespace = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String() - enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("false").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() - - autoPauseBucket = app.Flag("auto-pause-bucket", "Enable auto pause of reconciliation of ready buckets").Default("false").Envar("AUTO_PAUSE_BUCKET").Bool() - minReplicas = app.Flag("minimum-replicas", "Minimum number of replicas of a bucket before it is considered Ready").Default("1").Envar("MINIMUM_REPLICAS").Uint() - recreateMissingBucket = app.Flag("recreate-missing-bucket", "Recreates existing bucket if missing").Default("true").Envar("RECREATE_MISSING_BUCKET").Bool() - - assumeRoleArn = app.Flag("assume-role-arn", "Assume role ARN to be used for STS authentication").Default("").Envar("ASSUME_ROLE_ARN").String() +// canWatchCRD checks if the provider has the necessary RBAC permissions to +// watch CustomResourceDefinitions. This is required for safe-start to function. +func canWatchCRD(ctx context.Context, mgr manager.Manager) (bool, error) { + if err := authv1.AddToScheme(mgr.GetScheme()); err != nil { + return false, err + } + verbs := []string{"get", "list", "watch"} + for _, verb := range verbs { + sar := &authv1.SelfSubjectAccessReview{ + Spec: authv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authv1.ResourceAttributes{ + Group: "apiextensions.k8s.io", + Resource: "customresourcedefinitions", + Verb: verb, + }, + }, + } + if err := mgr.GetClient().Create(ctx, sar); err != nil { + return false, errors.Wrapf(err, "unable to perform RBAC check for verb %s on CustomResourceDefinitions", verb) + } + if !sar.Status.Allowed { + return false, nil + } + } - webhookHost = app.Flag("webhook-host", "The host of the webhook server.").Default("0.0.0.0").Envar("WEBHOOK_HOST").String() - webhookTLSCertDir = app.Flag("webhook-tls-cert-dir", "The directory of TLS certificate that will be used by the webhook server. There should be tls.crt and tls.key files.").Default("/").Envar("WEBHOOK_TLS_CERT_DIR").String() - _ = app.Flag("enable-validation-webhooks", "Enable support for Webhooks. [Deprecated, has no effect]").Default("false").Bool() - // Subresource Client Flags. - disableACLReconcile = app.Flag("disable-acl-reconcile", "Disable reconciliation of Bucket ACLs.").Default("false").Envar("DISABLE_ACL_RECONCILE").Bool() - disablePolicyReconcile = app.Flag("disable-policy-reconcile", "Disable reconciliation of Bucket Policies.").Default("false").Envar("DISABLE_POLICY_RECONCILE").Bool() - disableLifecycleConfigReconcile = app.Flag("disable-lifecycle-config-reconcile", "Disable reconciliation of Bucket Lifecycle Configurations.").Default("false").Envar("DISABLE_LIFECYCLE_CONFIG_RECONCILE").Bool() - disableVersioningConfigReconcile = app.Flag("disable-versioning-config-reconcile", "Disable reconciliation of Bucket Versioning Configurations.").Default("false").Envar("DISABLE_VERSIONING_CONFIG_RECONCILE").Bool() - disableObjectLockConfigReconcile = app.Flag("disable-object-lock-config-reconcile", "Disable reconciliation of Object Lock Configurations.").Default("false").Envar("DISABLE_OBJECT_LOCK_CONFIG_RECONCILE").Bool() - ) + return true, nil +} +// setupZapLogging configures zap logging flags and returns configured options. +func setupZapLogging(app *kingpin.Application, debugFlag *bool) []zap.Opts { var zo zap.Options var zapDevel *bool @@ -131,9 +127,6 @@ func main() { switch f.Name { case "zap-devel": - // Store the value for zap-devel for use when we come to zap-log-level so that we - // do not overwrite the level. VisitAll visits flags in lexicographical order so it - // is safe to assume "zap-devel" will always be visited before "zap-log-level". zapDevel = kf.Bool() zapOpts = append(zapOpts, func(o *zap.Options) { o.Development = *zapDevel @@ -156,8 +149,7 @@ func main() { zapOpts = append(zapOpts, func(o *zap.Options) { l := zapcore.Level(0) app.FatalIfError(l.Set(*ll), "Unable to unmarshal zap-log-level") - // if zap-devel is enabled, the log level should be debug (-1). - if *zapDevel { + if *zapDevel || *debugFlag { l = zapcore.Level(-1) } o.Level = l @@ -178,13 +170,187 @@ func main() { } }) + return zapOpts +} + +// configureSafeStart configures the safe-start gate if permissions allow. +func configureSafeStart(o *controller.Options, canSafeStart bool) { + if canSafeStart { + o.Gate = new(gate.Gate[schema.GroupVersionKind]) + } +} + +// configureManagementPolicies enables management policies if requested. +func configureManagementPolicies(o *controller.Options, enableManagementPolicies bool, log logr.Logger) { + if enableManagementPolicies { + o.Features.Enable(features.EnableAlphaManagementPolicies) + log.Info("Alpha feature enabled", "flag", features.EnableAlphaManagementPolicies) + } +} + +// createBucketConnector creates a bucket connector with all required options. +func createBucketConnector( + mgr manager.Manager, + backendStore *backendstore.BackendStore, + s3ClientHandler *s3clienthandler.Handler, + log logr.Logger, + autoPauseBucket *bool, + minReplicas *uint, + recreateMissingBucket *bool, + reconcileTimeout *time.Duration, + creationGracePeriod *time.Duration, + pollInterval *time.Duration, + disableACLReconcile *bool, + disablePolicyReconcile *bool, + disableLifecycleConfigReconcile *bool, + disableVersioningConfigReconcile *bool, + disableObjectLockConfigReconcile *bool, +) *bucket.Connector { + return bucket.NewConnector( + bucket.WithAutoPause(autoPauseBucket), + bucket.WithMinimumReplicas(minReplicas), + bucket.WithRecreateMissingBucket(recreateMissingBucket), + bucket.WithBackendStore(backendStore), + bucket.WithKubeClient(mgr.GetClient()), + bucket.WithKubeReader(mgr.GetAPIReader()), + bucket.WithOperationTimeout(*reconcileTimeout), + bucket.WithCreationGracePeriod(*creationGracePeriod), + bucket.WithPollInterval(*pollInterval), + bucket.WithLog(log), + bucket.WithSubresourceClients( + bucket.NewSubresourceClients( + backendStore, + s3ClientHandler, + bucket.SubresourceClientConfig{ + LifecycleConfigurationClientDisabled: *disableLifecycleConfigReconcile, + ACLClientDisabled: *disableACLReconcile, + PolicyClientDisabled: *disablePolicyReconcile, + VersioningConfigurationClientDisabled: *disableVersioningConfigReconcile, + ObjectLockConfigurationClientDisabled: *disableObjectLockConfigReconcile}, + log)), + bucket.WithS3ClientHandler(s3ClientHandler), + bucket.WithUsage(resource.NewLegacyProviderConfigUsageTracker(mgr.GetClient(), &v1alpha1.ProviderConfigUsage{})), + bucket.WithNewServiceFn(bucket.NewNoOpService)) +} + +// setupControllers sets up bucket controllers with or without safe-start gating. +func setupControllers(mgr manager.Manager, o controller.Options, connector *bucket.Connector, canSafeStart bool, log logr.Logger) { + if canSafeStart { + // Setup the CRD gate controller first + kingpin.FatalIfError(customresourcesgate.Setup(mgr, o), "Cannot setup CRD gate") + // Setup controllers with gated versions + kingpin.FatalIfError(bucket.SetupGated(mgr, o, connector), "Cannot setup gated Bucket controller") + } else { + log.Info("Provider has missing RBAC permissions for watching CRDs, controller SafeStart capability will be disabled") + // Setup controllers directly without gating + kingpin.FatalIfError(bucket.Setup(mgr, o, connector), "Cannot setup Bucket controller") + } +} + +// setupBucketWebhook sets up the bucket validating webhook. +func setupBucketWebhook(mgr manager.Manager, backendStore *backendstore.BackendStore) { + kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr). + For(&providercephv1alpha1.Bucket{}). + WithValidator(bucket.NewBucketValidator(backendStore)). + Complete(), "Cannot setup bucket validating webhook") +} + +// setupProviderConfigControllers sets up the provider config, backend monitor, and health check controllers. +func setupProviderConfigControllers( + mgr manager.Manager, + o controller.Options, + backendStore *backendstore.BackendStore, + kubeClientUncached client.Client, + log logr.Logger, + s3Timeout time.Duration, + backendMonitorInterval time.Duration, + autoPauseBucket *bool, +) { + kingpin.FatalIfError(providerconfig.Setup(mgr, o, + backendmonitor.NewController( + backendmonitor.WithKubeClient(mgr.GetClient()), + backendmonitor.WithBackendStore(backendStore), + backendmonitor.WithS3Timeout(s3Timeout), + backendmonitor.WithRequeueInterval(backendMonitorInterval), + backendmonitor.WithLogger(log)), + healthcheck.NewController( + healthcheck.WithAutoPause(autoPauseBucket), + healthcheck.WithBackendStore(backendStore), + healthcheck.WithKubeClientUncached(kubeClientUncached), + healthcheck.WithKubeClientCached(mgr.GetClient()), + healthcheck.WithHttpClient(&http.Client{Timeout: s3Timeout}), + healthcheck.WithLogger(log))), + "Cannot setup ProviderConfig controllers") +} + +// createS3ClientHandler creates an S3 client handler with all required options. +func createS3ClientHandler( + assumeRoleArn *string, + backendStore *backendstore.BackendStore, + kubeClient client.Client, + s3Timeout time.Duration, + log logr.Logger, +) *s3clienthandler.Handler { + return s3clienthandler.NewHandler( + s3clienthandler.WithAssumeRoleArn(assumeRoleArn), + s3clienthandler.WithBackendStore(backendStore), + s3clienthandler.WithKubeClient(kubeClient), + s3clienthandler.WithS3Timeout(s3Timeout), + s3clienthandler.WithLog(log)) +} + +func main() { + var ( + app = kingpin.New(filepath.Base(os.Args[0]), "Ceph support for Crossplane.").DefaultEnvars() + leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool() + leaderRenew = app.Flag("leader-renew", "Set leader election renewal.").Short('r').Default("10s").OverrideDefaultFromEnvar("LEADER_ELECTION_RENEW").Duration() + + syncInterval = app.Flag("sync", "How often all resources will be double-checked for drift from the desired state.").Short('s').Default("1h").Duration() + syncTimeout = app.Flag("sync-timeout", "Cache sync timeout.").Default("10s").Duration() + backendMonitorInterval = app.Flag("backend-monitor-interval", "Interval between backend monitor controller reconciliations.").Default("60s").Duration() + pollInterval = app.Flag("poll", "How often individual resources will be checked for drift from the desired state").Short('p').Default("30m").Duration() + pollStateMetricInterval = app.Flag("poll-state-metric", "State metric recording interval").Default("5s").Duration() + reconcileConcurrency = app.Flag("reconcile-concurrency", "Set number of reconciliation loops.").Default("100").Int() + maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("1000").Int() + reconcileTimeout = app.Flag("reconcile-timeout", "Object reconciliation timeout").Short('t').Default("3s").Duration() + s3Timeout = app.Flag("s3-timeout", "S3 API operations timeout").Default("10s").Duration() + creationGracePeriod = app.Flag("creation-grace-period", "Duration to wait for the external API to report that a newly created external resource exists.").Default("10s").Duration() + tracesEnabled = app.Flag("otel-enable-tracing", "").Default("false").Bool() + tracesExportTimeout = app.Flag("otel-traces-export-timeout", "Timeout when exporting traces").Default("2s").Duration() + tracesExportInterval = app.Flag("otel-traces-export-interval", "Interval at which traces are exported").Default("5s").Duration() + tracesExportAddress = app.Flag("otel-traces-export-address", "Address of otel collector").Default("opentelemetry-collector.opentelemetry:4317").String() + + kubeClientRate = app.Flag("kube-client-rate", "The global maximum rate per second at how many requests the client can do.").Default("1000").Int() + + namespace = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String() + enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("false").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() + + autoPauseBucket = app.Flag("auto-pause-bucket", "Enable auto pause of reconciliation of ready buckets").Default("false").Envar("AUTO_PAUSE_BUCKET").Bool() + minReplicas = app.Flag("minimum-replicas", "Minimum number of replicas of a bucket before it is considered Ready").Default("1").Envar("MINIMUM_REPLICAS").Uint() + recreateMissingBucket = app.Flag("recreate-missing-bucket", "Recreates existing bucket if missing").Default("true").Envar("RECREATE_MISSING_BUCKET").Bool() + + assumeRoleArn = app.Flag("assume-role-arn", "Assume role ARN to be used for STS authentication").Default("").Envar("ASSUME_ROLE_ARN").String() + + webhookHost = app.Flag("webhook-host", "The host of the webhook server.").Default("0.0.0.0").Envar("WEBHOOK_HOST").String() + webhookTLSCertDir = app.Flag("webhook-tls-cert-dir", "The directory of TLS certificate that will be used by the webhook server. There should be tls.crt and tls.key files.").Default("/").Envar("WEBHOOK_TLS_CERT_DIR").String() + _ = app.Flag("enable-validation-webhooks", "Enable support for Webhooks. [Deprecated, has no effect]").Default("false").Bool() + // Subresource Client Flags. + disableACLReconcile = app.Flag("disable-acl-reconcile", "Disable reconciliation of Bucket ACLs.").Default("false").Envar("DISABLE_ACL_RECONCILE").Bool() + disablePolicyReconcile = app.Flag("disable-policy-reconcile", "Disable reconciliation of Bucket Policies.").Default("false").Envar("DISABLE_POLICY_RECONCILE").Bool() + disableLifecycleConfigReconcile = app.Flag("disable-lifecycle-config-reconcile", "Disable reconciliation of Bucket Lifecycle Configurations.").Default("false").Envar("DISABLE_LIFECYCLE_CONFIG_RECONCILE").Bool() + disableVersioningConfigReconcile = app.Flag("disable-versioning-config-reconcile", "Disable reconciliation of Bucket Versioning Configurations.").Default("false").Envar("DISABLE_VERSIONING_CONFIG_RECONCILE").Bool() + disableObjectLockConfigReconcile = app.Flag("disable-object-lock-config-reconcile", "Disable reconciliation of Object Lock Configurations.").Default("false").Envar("DISABLE_OBJECT_LOCK_CONFIG_RECONCILE").Bool() + ) + + debugFlag := app.Flag("debug", "Enable debug logging (sets zap-log-level to debug)").Default("false").Bool() + zapOpts := setupZapLogging(app, debugFlag) + kingpin.MustParse(app.Parse(os.Args[1:])) log := zap.New(zapOpts...).WithName("provider-ceph") ctrl.SetLogger(log) klog.SetLogger(log) - // Init otel tracer provider if the user sets the flag if *tracesEnabled { flush, err := traces.InitTracerProvider(log, *tracesExportAddress, *tracesExportTimeout, *tracesExportInterval) kingpin.FatalIfError(err, "Cannot start tracer provider") @@ -265,6 +431,14 @@ func main() { }) kingpin.FatalIfError(err, "Cannot create controller manager") + ctx := context.Background() + + kingpin.FatalIfError(apiextensionsv1.AddToScheme(mgr.GetScheme()), "Cannot add apiextensionsv1 to scheme") + + // Check if provider has permissions to watch CRDs for safe-start capability + canSafeStart, err := canWatchCRD(ctx, mgr) + kingpin.FatalIfError(err, "SafeStart precheck failed") + o := controller.Options{ Logger: logging.NewLogrLogger(log), MaxConcurrentReconciles: *reconcileConcurrency, @@ -274,10 +448,8 @@ func main() { MetricOptions: &mo, } - if *enableManagementPolicies { - o.Features.Enable(features.EnableAlphaManagementPolicies) - log.Info("Alpha feature enabled", "flag", features.EnableAlphaManagementPolicies) - } + configureSafeStart(&o, canSafeStart) + configureManagementPolicies(&o, *enableManagementPolicies, log) backendStore := backendstore.NewBackendStore() @@ -289,59 +461,15 @@ func main() { }) kingpin.FatalIfError(err, "Cannot create Kube client") - kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr). - For(&providercephv1alpha1.Bucket{}). - WithValidator(bucket.NewBucketValidator(backendStore)). - Complete(), "Cannot setup bucket validating webhook") + setupBucketWebhook(mgr, backendStore) + setupProviderConfigControllers(mgr, o, backendStore, kubeClientUncached, log, *s3Timeout, *backendMonitorInterval, autoPauseBucket) + s3ClientHandler := createS3ClientHandler(assumeRoleArn, backendStore, mgr.GetClient(), *s3Timeout, log) - kingpin.FatalIfError(providerconfig.Setup(mgr, o, - backendmonitor.NewController( - backendmonitor.WithKubeClient(mgr.GetClient()), - backendmonitor.WithBackendStore(backendStore), - backendmonitor.WithS3Timeout(*s3Timeout), - backendmonitor.WithRequeueInterval(*backendMonitorInterval), - backendmonitor.WithLogger(log)), - healthcheck.NewController( - healthcheck.WithAutoPause(autoPauseBucket), - healthcheck.WithBackendStore(backendStore), - healthcheck.WithKubeClientUncached(kubeClientUncached), - healthcheck.WithKubeClientCached(mgr.GetClient()), - healthcheck.WithHttpClient(&http.Client{Timeout: *s3Timeout}), - healthcheck.WithLogger(log))), - "Cannot setup ProviderConfig controllers") + connector := createBucketConnector(mgr, backendStore, s3ClientHandler, log, autoPauseBucket, minReplicas, + recreateMissingBucket, reconcileTimeout, creationGracePeriod, pollInterval, disableACLReconcile, + disablePolicyReconcile, disableLifecycleConfigReconcile, disableVersioningConfigReconcile, disableObjectLockConfigReconcile) - s3ClientHandler := s3clienthandler.NewHandler( - s3clienthandler.WithAssumeRoleArn(assumeRoleArn), - s3clienthandler.WithBackendStore(backendStore), - s3clienthandler.WithKubeClient(mgr.GetClient()), - s3clienthandler.WithS3Timeout(*s3Timeout), - s3clienthandler.WithLog(log)) - - kingpin.FatalIfError(bucket.Setup(mgr, o, bucket.NewConnector( - bucket.WithAutoPause(autoPauseBucket), - bucket.WithMinimumReplicas(minReplicas), - bucket.WithRecreateMissingBucket(recreateMissingBucket), - bucket.WithBackendStore(backendStore), - bucket.WithKubeClient(mgr.GetClient()), - bucket.WithKubeReader(mgr.GetAPIReader()), - bucket.WithOperationTimeout(*reconcileTimeout), - bucket.WithCreationGracePeriod(*creationGracePeriod), - bucket.WithPollInterval(*pollInterval), - bucket.WithLog(log), - bucket.WithSubresourceClients( - bucket.NewSubresourceClients( - backendStore, - s3ClientHandler, - bucket.SubresourceClientConfig{ - LifecycleConfigurationClientDisabled: *disableLifecycleConfigReconcile, - ACLClientDisabled: *disableACLReconcile, - PolicyClientDisabled: *disablePolicyReconcile, - VersioningConfigurationClientDisabled: *disableVersioningConfigReconcile, - ObjectLockConfigurationClientDisabled: *disableObjectLockConfigReconcile}, - log)), - bucket.WithS3ClientHandler(s3ClientHandler), - bucket.WithUsage(resource.NewLegacyProviderConfigUsageTracker(mgr.GetClient(), &v1alpha1.ProviderConfigUsage{})), - bucket.WithNewServiceFn(bucket.NewNoOpService))), "Cannot setup Bucket controller") + setupControllers(mgr, o, connector, canSafeStart, log) kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") } diff --git a/e2e/kind/kind-config-1.31.yaml b/e2e/kind/kind-config-1.31.yaml index daec3e48..294566e4 100644 --- a/e2e/kind/kind-config-1.31.yaml +++ b/e2e/kind/kind-config-1.31.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.31.4 + image: kindest/node:v1.31.13 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.28.yaml b/e2e/kind/kind-config-1.32.yaml similarity index 94% rename from e2e/kind/kind-config-1.28.yaml rename to e2e/kind/kind-config-1.32.yaml index 694060a4..6471fae8 100644 --- a/e2e/kind/kind-config-1.28.yaml +++ b/e2e/kind/kind-config-1.32.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.28.7 + image: kindest/node:v1.32.9 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.29.yaml b/e2e/kind/kind-config-1.33.yaml similarity index 94% rename from e2e/kind/kind-config-1.29.yaml rename to e2e/kind/kind-config-1.33.yaml index 20ab0892..a54f7675 100644 --- a/e2e/kind/kind-config-1.29.yaml +++ b/e2e/kind/kind-config-1.33.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.29.2 + image: kindest/node:v1.33.5 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.30.yaml b/e2e/kind/kind-config-1.34.yaml similarity index 94% rename from e2e/kind/kind-config-1.30.yaml rename to e2e/kind/kind-config-1.34.yaml index 04f92b2c..73dbc6b4 100644 --- a/e2e/kind/kind-config-1.30.yaml +++ b/e2e/kind/kind-config-1.34.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.30.8 + image: kindest/node:v1.34.1 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/tests/safe-start/.chainsaw.yaml b/e2e/tests/safe-start/.chainsaw.yaml new file mode 100644 index 00000000..bc93f6c4 --- /dev/null +++ b/e2e/tests/safe-start/.chainsaw.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: safe-start-test-configuration +spec: + timeouts: + apply: 30s + assert: 60s + cleanup: 30s + delete: 30s + error: 30s + exec: 30s \ No newline at end of file diff --git a/e2e/tests/safe-start/chainsaw-test.yaml b/e2e/tests/safe-start/chainsaw-test.yaml new file mode 100644 index 00000000..850cd772 --- /dev/null +++ b/e2e/tests/safe-start/chainsaw-test.yaml @@ -0,0 +1,101 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + creationTimestamp: null + name: provider-ceph-safe-start +spec: + steps: + - name: Assert that Crossplane 2.0+ has been installed. + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crossplane + namespace: crossplane-system + status: + readyReplicas: 1 + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crossplane-rbac-manager + namespace: crossplane-system + status: + readyReplicas: 1 + + - name: Verify MRDs are not created. + try: + # MRD exists but is Inactive + - assert: + resource: + apiVersion: apiextensions.crossplane.io/v1alpha1 + kind: ManagedResourceDefinition + metadata: + name: buckets.provider-ceph.ceph.crossplane.io + spec: + state: Inactive + # ProviderConfig CRD should exist (not gated) + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: providerconfigs.ceph.crossplane.io + + - name: Verify Bucket CRD does not exist before activation. + try: + - script: + content: | + if kubectl get crd buckets.provider-ceph.ceph.crossplane.io 2>/dev/null; then + echo "ERROR: Bucket CRD should not exist before activation" + exit 1 + fi + echo "SUCCESS: Bucket CRD does not exist as expected" + exit 0 + + - name: Create ManagedResourceActivationPolicy to activate Bucket MRD. + try: + - apply: + resource: + apiVersion: apiextensions.crossplane.io/v1alpha1 + kind: ManagedResourceActivationPolicy + metadata: + name: activate-provider-ceph-buckets + spec: + activate: + - "buckets.provider-ceph.ceph.crossplane.io" + + - name: Verify MRD is now Active and CRD exists. + try: + - assert: + resource: + apiVersion: apiextensions.crossplane.io/v1alpha1 + kind: ManagedResourceDefinition + metadata: + name: buckets.provider-ceph.ceph.crossplane.io + spec: + state: Active + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: buckets.provider-ceph.ceph.crossplane.io + + - name: Clean up ManagedResourceActivationPolicy. + try: + - delete: + ref: + apiVersion: apiextensions.crossplane.io/v1alpha1 + kind: ManagedResourceActivationPolicy + name: activate-provider-ceph-buckets + - error: + resource: + apiVersion: apiextensions.crossplane.io/v1alpha1 + kind: ManagedResourceActivationPolicy + metadata: + name: activate-provider-ceph-buckets \ No newline at end of file diff --git a/e2e/tests/stable/chainsaw-test.yaml b/e2e/tests/stable/chainsaw-test.yaml index b8cac72c..035de2a3 100755 --- a/e2e/tests/stable/chainsaw-test.yaml +++ b/e2e/tests/stable/chainsaw-test.yaml @@ -366,7 +366,7 @@ spec: # This method of iterative assertions is necessary here because # these conditions do not always appear in the same order. # It's safe to perform this assertions because we have already - # asserted that these conditios exist. + # asserted that these conditions exist. ~.(conditions[?reason == 'Available']): status: "True" type: Ready @@ -376,6 +376,9 @@ spec: - name: Check for buckets on Localstack backends. try: + # Wait to ensure buckets are fully created on backends and avoid racing conditions + - sleep: + duration: 2s # Check for subresource-configs on all backends. - command: args: @@ -1209,7 +1212,7 @@ spec: - scale - -n - crossplane-system - - deploy/provider-ceph-provider-cep + - deploy/provider-ceph-xpkg-crosspl - --replicas=0 entrypoint: kubectl diff --git a/go.mod b/go.mod index 7a35c538..d2ad0f37 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 github.com/aws/smithy-go v1.20.1 github.com/crossplane/crossplane-runtime/v2 v2.0.0 - github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 + github.com/crossplane/crossplane-tools v0.0.0-20250731192036-00d407d8b7ec github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 github.com/pkg/errors v0.9.1 @@ -27,18 +27,18 @@ require ( golang.org/x/sync v0.17.0 google.golang.org/grpc v1.68.1 k8s.io/api v0.33.0 + k8s.io/apiextensions-apiserver v0.33.0 k8s.io/apimachinery v0.33.0 k8s.io/client-go v0.33.0 k8s.io/klog/v2 v2.130.1 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e sigs.k8s.io/controller-runtime v0.19.0 sigs.k8s.io/controller-tools v0.18.0 ) require ( dario.cat/mergo v1.0.1 // indirect - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect @@ -55,7 +55,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dave/jennifer v1.7.0 // indirect + github.com/dave/jennifer v1.7.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect @@ -119,12 +119,11 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.33.0 // indirect k8s.io/code-generator v0.33.0 // indirect k8s.io/component-base v0.33.0 // indirect k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 06baed01..08e611ef 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,8 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= @@ -56,10 +54,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crossplane/crossplane-runtime/v2 v2.0.0 h1:PK2pTKfshdDZ5IfoiMRiCi0PBnIjqbS0KGXEJgRdrb4= github.com/crossplane/crossplane-runtime/v2 v2.0.0/go.mod h1:pkd5UzmE8esaZAApevMutR832GjJ1Qgc5Ngr78ByxrI= -github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 h1:HigXs5tEQxWz0fcj8hzbU2UAZgEM7wPe0XRFOsrtF8Y= -github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79/go.mod h1:+e4OaFlOcmr0JvINHl/yvEYBrZawzTgj6pQumOH1SS0= -github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE= -github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= +github.com/crossplane/crossplane-tools v0.0.0-20250731192036-00d407d8b7ec h1:+51Et4UW8XrvGne8RAqn9qEIfhoqPXYqIp/kQvpMaAo= +github.com/crossplane/crossplane-tools v0.0.0-20250731192036-00d407d8b7ec/go.mod h1:8etxwmP4cZwJDwen4+PQlnc1tggltAhEfyyigmdHulQ= +github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= +github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -184,10 +182,11 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -296,7 +295,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -320,14 +318,14 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/controller-tools v0.18.0 h1:rGxGZCZTV2wJreeRgqVoWab/mfcumTMmSwKzoM9xrsE= sigs.k8s.io/controller-tools v0.18.0/go.mod h1:gLKoiGBriyNh+x1rWtUQnakUYEujErjXs9pf+x/8n1U= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/hack/deploy-provider.sh b/hack/deploy-provider.sh deleted file mode 100755 index 424aec05..00000000 --- a/hack/deploy-provider.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -: "${BUILD_REGISTRY?= required}" -: "${PROJECT_NAME?= required}" -: "${ARCH?= required}" -: "${VERSION?= required}" - -# Apply a configuration for the provider deployment. -kubectl apply -f - </dev/null; then address="$(kubectl get no ${address%:*} -o jsonpath='{.status.addresses[0].address}'):${address#*:}" fi -# Check whether the bucket already exists. +# Set AWS credentials for localstack +export AWS_ACCESS_KEY_ID=Dummy +export AWS_SECRET_ACCESS_KEY=Dummy +export AWS_DEFAULT_REGION=us-east-1 + +# Check whether the bucket already exists. # We suppress all output - we're interested only in the return code. bucket_exists() { diff --git a/hack/generate-tests.sh b/hack/generate-tests.sh index 17faa92f..ee34a3b6 100755 --- a/hack/generate-tests.sh +++ b/hack/generate-tests.sh @@ -82,7 +82,7 @@ jobs: run: make vendor vendor.check - name: Docker cache - uses: ScribeMD/docker-cache@0.3.7 + uses: ScribeMD/docker-cache@0.5.0 with: key: docker-\${{ runner.os }}-\${{ hashFiles('go.sum') }}} @@ -93,6 +93,11 @@ jobs: AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' AWS_DEFAULT_REGION: 'us-east-1' + + - name: Run chainsaw-safe-start tests ${major} + run: make chainsaw-safe-start + env: + LATEST_KUBE_VERSION: '${major}' EOF done diff --git a/hack/helpers/apis/GROUP_LOWER/APIVERSION/KIND_LOWER_types.go.tmpl b/hack/helpers/apis/GROUP_LOWER/APIVERSION/KIND_LOWER_types.go.tmpl index 755cdef0..d0a8c4a3 100644 --- a/hack/helpers/apis/GROUP_LOWER/APIVERSION/KIND_LOWER_types.go.tmpl +++ b/hack/helpers/apis/GROUP_LOWER/APIVERSION/KIND_LOWER_types.go.tmpl @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + xpv1 "github.com/crossplane/crossplane-runtime/v2/apis/common/v1" ) // {{ .Env.KIND }}Parameters are the configurable fields of a {{ .Env.KIND }}. diff --git a/hack/helpers/controller/KIND_LOWER/KIND_LOWER.go.tmpl b/hack/helpers/controller/KIND_LOWER/KIND_LOWER.go.tmpl index abbbb7f3..34ee2d00 100644 --- a/hack/helpers/controller/KIND_LOWER/KIND_LOWER.go.tmpl +++ b/hack/helpers/controller/KIND_LOWER/KIND_LOWER.go.tmpl @@ -25,12 +25,12 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/crossplane/crossplane-runtime/pkg/connection" - "github.com/crossplane/crossplane-runtime/pkg/controller" - "github.com/crossplane/crossplane-runtime/pkg/event" - "github.com/crossplane/crossplane-runtime/pkg/ratelimiter" - "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" - "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/crossplane-runtime/v2/pkg/connection" + "github.com/crossplane/crossplane-runtime/v2/pkg/controller" + "github.com/crossplane/crossplane-runtime/v2/pkg/event" + "github.com/crossplane/crossplane-runtime/v2/pkg/ratelimiter" + "github.com/crossplane/crossplane-runtime/v2/pkg/reconciler/managed" + "github.com/crossplane/crossplane-runtime/v2/pkg/resource" "github.com/crossplane/provider-{{ .Env.PROVIDER | strings.ToLower }}/apis/{{ .Env.GROUP | strings.ToLower }}/{{ .Env.APIVERSION | strings.ToLower }}" apisv1alpha1 "github.com/crossplane/provider-{{ .Env.PROVIDER | strings.ToLower }}/apis/v1alpha1" diff --git a/hack/helpers/controller/KIND_LOWER/KIND_LOWER_test.go.tmpl b/hack/helpers/controller/KIND_LOWER/KIND_LOWER_test.go.tmpl index 40f5c689..0beb2082 100644 --- a/hack/helpers/controller/KIND_LOWER/KIND_LOWER_test.go.tmpl +++ b/hack/helpers/controller/KIND_LOWER/KIND_LOWER_test.go.tmpl @@ -22,9 +22,9 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" - "github.com/crossplane/crossplane-runtime/pkg/resource" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/reconciler/managed" + "github.com/crossplane/crossplane-runtime/v2/pkg/resource" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" ) // Unlike many Kubernetes projects Crossplane does not use third party testing diff --git a/internal/controller/bucket/setup.go b/internal/controller/bucket/setup.go index 25cd2889..662e8bc6 100644 --- a/internal/controller/bucket/setup.go +++ b/internal/controller/bucket/setup.go @@ -39,6 +39,17 @@ var ( NewNoOpService = func(_ []byte) (interface{}, error) { return &NoOpService{}, nil } ) +// SetupGated registers controller setup with the gate, waiting for the required CRD. +func SetupGated(mgr ctrl.Manager, o controller.Options, c *Connector) error { + o.Gate.Register(func() { + if err := Setup(mgr, o, c); err != nil { + panic(err) + } + }, v1alpha1.BucketGroupVersionKind) + + return nil +} + // Setup adds a controller that reconciles Bucket managed resources. func Setup(mgr ctrl.Manager, o controller.Options, c *Connector) error { name := managed.ControllerName(v1alpha1.BucketGroupKind) diff --git a/package/crossplane.yaml b/package/crossplane.yaml index 2cd67dd8..fe3c1eb5 100644 --- a/package/crossplane.yaml +++ b/package/crossplane.yaml @@ -1,4 +1,4 @@ -apiVersion: meta.pkg.crossplane.io/v1alpha1 +apiVersion: meta.pkg.crossplane.io/v1 kind: Provider metadata: name: provider-ceph @@ -11,5 +11,7 @@ metadata: A Crossplane provider that performs CRUD operations on s3 buckets. friendly-name.meta.crossplane.io: Provider Ceph spec: + capabilities: + - safe-start controller: image: xpkg.upbound.io/linode/provider-ceph:v1.1.1-rc.0.1.g7f906ff From 79370eaf2251024b1ac6aa4a2ffe4c0ff3935534 Mon Sep 17 00:00:00 2001 From: Alexandre Proulx Date: Wed, 15 Oct 2025 10:43:04 -0400 Subject: [PATCH 2/4] fix: Remove rbac annotation, the crossplane v2 tooling handles it. --- cmd/provider/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 047782b4..1c065360 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -18,8 +18,6 @@ package main //go:generate go get github.com/maxbrunsfeld/counterfeiter/v6 -//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch - import ( "context" "flag" From 9f3064ccafcf17f8a4dac91778d7c64bb41fe647 Mon Sep 17 00:00:00 2001 From: Alexandre Proulx Date: Thu, 16 Oct 2025 10:19:25 -0400 Subject: [PATCH 3/4] chore: improve code formatting and use correct kind node versions - Improves code readability in main.go by formatting function calls with multiple parameters - Reverts kind node versions to slightly older versions (1.31.12, 1.32.8, 1.33.4, 1.34.0) - Fixes grammatical error in log message for RBAC permissions - Removes commented out code in Makefile --- Makefile | 3 +-- cmd/provider/main.go | 41 +++++++++++++++++++++++++++++----- e2e/kind/kind-config-1.31.yaml | 2 +- e2e/kind/kind-config-1.32.yaml | 2 +- e2e/kind/kind-config-1.33.yaml | 2 +- e2e/kind/kind-config-1.34.yaml | 2 +- 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index d003fc45..5f3415df 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ PLATFORMS ?= linux_amd64 linux_arm64 # Generate chainsaw e2e tests for the following kind node versions # TEST_KIND_NODES is not intended to be updated manually. # Please edit LATEST_KIND_NODE instead and run 'make update-kind-nodes'. -TEST_KIND_NODES ?= 1.31.13,1.32.9,1.33.5,1.34.1 +TEST_KIND_NODES ?= 1.31.12,1.32.8,1.33.4,1.34.0 KIND_VERSION ?= v0.30.0 LATEST_KUBE_VERSION ?= 1.34 LATEST_KIND_NODE ?= 1.34.1 @@ -207,7 +207,6 @@ chainsaw-safe-start: $(CHAINSAW) build generate-pkg generate-tests @$(MAKE) localstack-cluster @$(MAKE) local.xpkg.deploy.provider.$(PROJECT_NAME) @$(CHAINSAW) test e2e/tests/safe-start --config e2e/tests/safe-start/.chainsaw.yaml -# || ($(MAKE) controlplane.down && $(FAIL)) @$(OK) Running chainsaw safe-start test suite @$(MAKE) controlplane.down diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 1c065360..0f5c48fb 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -239,7 +239,7 @@ func setupControllers(mgr manager.Manager, o controller.Options, connector *buck // Setup controllers with gated versions kingpin.FatalIfError(bucket.SetupGated(mgr, o, connector), "Cannot setup gated Bucket controller") } else { - log.Info("Provider has missing RBAC permissions for watching CRDs, controller SafeStart capability will be disabled") + log.Info("Provider is missing RBAC permissions for watching CRDs, controller SafeStart capability will be disabled") // Setup controllers directly without gating kingpin.FatalIfError(bucket.Setup(mgr, o, connector), "Cannot setup Bucket controller") } @@ -460,12 +460,41 @@ func main() { kingpin.FatalIfError(err, "Cannot create Kube client") setupBucketWebhook(mgr, backendStore) - setupProviderConfigControllers(mgr, o, backendStore, kubeClientUncached, log, *s3Timeout, *backendMonitorInterval, autoPauseBucket) - s3ClientHandler := createS3ClientHandler(assumeRoleArn, backendStore, mgr.GetClient(), *s3Timeout, log) + setupProviderConfigControllers( + mgr, + o, + backendStore, + kubeClientUncached, + log, + *s3Timeout, + *backendMonitorInterval, + autoPauseBucket, + ) + s3ClientHandler := createS3ClientHandler( + assumeRoleArn, + backendStore, + mgr.GetClient(), + *s3Timeout, + log, + ) - connector := createBucketConnector(mgr, backendStore, s3ClientHandler, log, autoPauseBucket, minReplicas, - recreateMissingBucket, reconcileTimeout, creationGracePeriod, pollInterval, disableACLReconcile, - disablePolicyReconcile, disableLifecycleConfigReconcile, disableVersioningConfigReconcile, disableObjectLockConfigReconcile) + connector := createBucketConnector( + mgr, + backendStore, + s3ClientHandler, + log, + autoPauseBucket, + minReplicas, + recreateMissingBucket, + reconcileTimeout, + creationGracePeriod, + pollInterval, + disableACLReconcile, + disablePolicyReconcile, + disableLifecycleConfigReconcile, + disableVersioningConfigReconcile, + disableObjectLockConfigReconcile, + ) setupControllers(mgr, o, connector, canSafeStart, log) diff --git a/e2e/kind/kind-config-1.31.yaml b/e2e/kind/kind-config-1.31.yaml index 294566e4..3051793d 100644 --- a/e2e/kind/kind-config-1.31.yaml +++ b/e2e/kind/kind-config-1.31.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.31.13 + image: kindest/node:v1.31.12 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.32.yaml b/e2e/kind/kind-config-1.32.yaml index 6471fae8..4c95f2e6 100644 --- a/e2e/kind/kind-config-1.32.yaml +++ b/e2e/kind/kind-config-1.32.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.32.9 + image: kindest/node:v1.32.8 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.33.yaml b/e2e/kind/kind-config-1.33.yaml index a54f7675..fe3fdb04 100644 --- a/e2e/kind/kind-config-1.33.yaml +++ b/e2e/kind/kind-config-1.33.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.33.5 + image: kindest/node:v1.33.4 extraPortMappings: - containerPort: 32566 hostPort: 32566 diff --git a/e2e/kind/kind-config-1.34.yaml b/e2e/kind/kind-config-1.34.yaml index 73dbc6b4..ec2e0c7b 100644 --- a/e2e/kind/kind-config-1.34.yaml +++ b/e2e/kind/kind-config-1.34.yaml @@ -3,7 +3,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.34.1 + image: kindest/node:v1.34.0 extraPortMappings: - containerPort: 32566 hostPort: 32566 From b1faa8ff99487368a7fd5183e484f70dbac99759 Mon Sep 17 00:00:00 2001 From: Alexandre Proulx Date: Thu, 16 Oct 2025 10:25:24 -0400 Subject: [PATCH 4/4] chore: ran tests after rebase --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index d2ad0f37..61f14541 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/protobuf v1.36.7 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 08e611ef..70e1c608 100644 --- a/go.sum +++ b/go.sum @@ -284,8 +284,6 @@ google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=