From cce9ed7c01c124c5e81f2cfaeba72c81a2bf3b35 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 3 Dec 2024 18:42:46 -0800 Subject: [PATCH 1/4] ci: allow pushing caren image for e2e tests to an external registry --- .goreleaser.yml | 4 ++-- make/go.mk | 6 ++++++ test/e2e/config/caren.yaml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 9132e9a7d..3bb2159a4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -45,7 +45,7 @@ before: --namespace caren-system \ --set-string image.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }}{{ end }} \ --set-string helmRepository.images.bundleInitializer.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \ - --set-string image.repository=ko.local/{{ .ProjectName }}{{ end }} \ + --set-string image.repository={{ .Env.LOCAL_IMAGE_REGISTRY }}/{{ .ProjectName }}{{ end }} \ ) EOF' - sed -i -e 's/\${/$${/g' -e 's/v0.0.0-dev/v{{ trimprefix .Version "v" }}/g' runtime-extensions-components.yaml @@ -90,7 +90,7 @@ builds: sh -ec 'if [ {{ .IsSnapshot }} == true ] && [ {{ .Runtime.Goarch }} == {{ .Arch }} ]; then env SOURCE_DATE_EPOCH=$(date +%s) \ KO_DATA_DATE_EPOCH=$(date +%s) \ - KO_DOCKER_REPO=ko.local/{{ .ProjectName }} \ + KO_DOCKER_REPO={{ .Env.LOCAL_IMAGE_REGISTRY }}/{{ .ProjectName }} \ ko build \ --bare \ --platform linux/{{ .Arch }} \ diff --git a/make/go.mk b/make/go.mk index d0cb3897c..d5ac6b9f9 100644 --- a/make/go.mk +++ b/make/go.mk @@ -13,6 +13,12 @@ override undefine GOARCH ALL_GO_SUBMODULES := $(shell find -mindepth 2 -maxdepth 2 -name go.mod -printf '%P\n' | sort) GO_SUBMODULES_NO_DOCS := $(filter-out $(addsuffix /go.mod,docs),$(ALL_GO_SUBMODULES)) THIRD_PARTY_GO_SUBMODULES := $(shell find hack/third-party -mindepth 2 -name go.mod -printf 'hack/third-party/%P\n' | sort) +# self hosted tests requires the local caren images to be available to the workload clusters. +# export LOCAL_IMAGE_REGISTRY to an accessible registry when running self hosted tests. +# When e2e tests builds the project using goreleaser, the images are pushed to the registry and available for the workload clusters. +# by default, the CAREN image is stored in local container engine store as ko.local/cluster-api-runtime-extensions-nutanix:${TAG} +LOCAL_IMAGE_REGISTRY ?= ko.local +export LOCAL_IMAGE_REGISTRY define go_test source <(setup-envtest use -p env $(ENVTEST_VERSION)) && \ diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index e976bbf3f..4a1f54963 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -4,7 +4,7 @@ managementClusterName: caren-e2e images: - - name: ko.local/cluster-api-runtime-extensions-nutanix:${E2E_IMAGE_TAG} + - name: ${LOCAL_IMAGE_REGISTRY}/cluster-api-runtime-extensions-nutanix:${E2E_IMAGE_TAG} loadBehavior: mustLoad - name: ghcr.io/nutanix-cloud-native/cluster-api-runtime-extensions-helm-chart-bundle-initializer:${E2E_IMAGE_TAG} loadBehavior: mustLoad From 6c0c3e41ef969ee3c4a0d1147e28ec8986132406 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 3 Dec 2024 18:43:08 -0800 Subject: [PATCH 2/4] ci: enable running nutanix self hosted e2e tests --- .github/workflows/checks.yml | 10 +++++----- .github/workflows/e2e.yml | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 00eb50ed2..7db12b090 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -96,10 +96,9 @@ jobs: matrix: config: - {"provider": "Docker", "kubernetesVersion": "v1.29.9"} - - {"provider": "Docker", "kubernetesVersion": "v1.30.6"} - - {"provider": "Docker", "kubernetesVersion": "v1.31.2"} - # Uncomment below once we have the ability to run e2e tests on other providers from GHA. - # - {"provider": "Nutanix", "kubernetesVersion": "v1.29.6"} + # - {"provider": "Docker", "kubernetesVersion": "v1.30.6"} + # - {"provider": "Docker", "kubernetesVersion": "v1.31.2"} + - {"provider": "Nutanix", "kubernetesVersion": "v1.30.5", "osImage": "nkp-rocky-9.4-release-1.30.5-20240930171619"} # - {"provider": "AWS", "kubernetesVersion": "v1.29.6"} fail-fast: false uses: ./.github/workflows/e2e.yml @@ -107,7 +106,8 @@ jobs: focus: Self-hosted provider: ${{ matrix.config.provider }} kubernetes-version: ${{ matrix.config.kubernetesVersion }} - runs-on: ${{ matrix.config.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} + runs-on: 'self-hosted-ncn-dind' + os-image: ${{ matrix.config.provider == 'Nutanix' && matrix.config.osImage || '' }} secrets: inherit permissions: contents: read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cac019bbf..86421bd51 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -63,6 +63,13 @@ jobs: restore-keys: | ${{ runner.os }}-go- + - name: Login to Internal Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.LOCAL_IMAGE_REGISTRY }} + username: ${{ secrets.LOCAL_IMAGE_REGISTRY_USERNAME }} + password: ${{ secrets.LOCAL_IMAGE_REGISTRY_TOKEN }} + # The default disk size of Github hosted runners is ~14GB, this is not enough to run the e2e tests. # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840. - name: Cleanup Disk Space @@ -94,6 +101,7 @@ jobs: NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ inputs.os-image }} KINDEST_IMAGE_TAG: ${{ inputs.kubernetes-version }} E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes-version }} + LOCAL_IMAGE_REGISTRY: ${{ secrets.LOCAL_IMAGE_REGISTRY }} - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report From ef19cee332be2c668810857144c8a3de24c468ff Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 3 Dec 2024 19:38:20 -0800 Subject: [PATCH 3/4] test: set control plane ip for nutanix e2e tests --- .github/workflows/e2e.yml | 3 ++- test/e2e/framework/self_hosted.go | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 86421bd51..a873b74cf 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -64,6 +64,7 @@ jobs: ${{ runner.os }}-go- - name: Login to Internal Container Registry + if: inputs.focus == 'Self-hosted' uses: docker/login-action@v3 with: registry: ${{ secrets.LOCAL_IMAGE_REGISTRY }} @@ -101,7 +102,7 @@ jobs: NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ inputs.os-image }} KINDEST_IMAGE_TAG: ${{ inputs.kubernetes-version }} E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes-version }} - LOCAL_IMAGE_REGISTRY: ${{ secrets.LOCAL_IMAGE_REGISTRY }} + LOCAL_IMAGE_REGISTRY: ${{ inputs.focus == 'Self-hosted' && secrets.LOCAL_IMAGE_REGISTRY || 'ko.local' }} - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report diff --git a/test/e2e/framework/self_hosted.go b/test/e2e/framework/self_hosted.go index 6b8993179..0b0bc6205 100644 --- a/test/e2e/framework/self_hosted.go +++ b/test/e2e/framework/self_hosted.go @@ -28,6 +28,8 @@ import ( "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/e2e/framework/nutanix" ) // SelfHostedSpecInput is the input for SelfHostedSpec. @@ -180,6 +182,30 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) if input.InfrastructureProvider != nil { infrastructureProvider = *input.InfrastructureProvider } + + // For Nutanix provider, reserve an IP address for the workload cluster control plane endpoint - + // remember to unreserve it! + if infrastructureProvider == "nutanix" { + By( + "Reserving an IP address for the workload cluster control plane endpoint", + ) + nutanixClient, err := nutanix.NewV4Client( + nutanix.CredentialsFromCAPIE2EConfig(input.E2EConfig), + ) + Expect(err).ToNot(HaveOccurred()) + //nolint:contextcheck // ReserverIP function does not accept context. Its okay to ignore the context check in tests. + controlPlaneEndpointIP, unreserveControlPlaneEndpointIP, err := nutanix.ReserveIP( + input.E2EConfig.GetVariable("NUTANIX_SUBNET_NAME"), + input.E2EConfig.GetVariable( + "NUTANIX_PRISM_ELEMENT_CLUSTER_NAME", + ), + nutanixClient, + ) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(unreserveControlPlaneEndpointIP) + clusterctlVariables["CONTROL_PLANE_ENDPOINT_IP"] = controlPlaneEndpointIP + } + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: input.BootstrapClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ From 891c6a88197952e79d2d995885673398da22074c Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Wed, 4 Dec 2024 16:18:53 -0800 Subject: [PATCH 4/4] test: create helm-chart-bundle-initializer image for e2e test --- .github/workflows/checks.yml | 4 ++-- .goreleaser.yml | 15 ++++++++++++--- test/e2e/config/caren.yaml | 4 ++-- test/e2e/self_hosted_test.go | 7 ++++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7db12b090..c691d2f8f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -96,8 +96,8 @@ jobs: matrix: config: - {"provider": "Docker", "kubernetesVersion": "v1.29.9"} - # - {"provider": "Docker", "kubernetesVersion": "v1.30.6"} - # - {"provider": "Docker", "kubernetesVersion": "v1.31.2"} + - {"provider": "Docker", "kubernetesVersion": "v1.30.6"} + - {"provider": "Docker", "kubernetesVersion": "v1.31.2"} - {"provider": "Nutanix", "kubernetesVersion": "v1.30.5", "osImage": "nkp-rocky-9.4-release-1.30.5-20240930171619"} # - {"provider": "AWS", "kubernetesVersion": "v1.29.6"} fail-fast: false diff --git a/.goreleaser.yml b/.goreleaser.yml index 3bb2159a4..36cd933ff 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -44,8 +44,9 @@ before: $(helm template {{ .ProjectName }} ./charts/{{ .ProjectName }} \ --namespace caren-system \ --set-string image.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }}{{ end }} \ - --set-string helmRepository.images.bundleInitializer.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \ - --set-string image.repository={{ .Env.LOCAL_IMAGE_REGISTRY }}/{{ .ProjectName }}{{ end }} \ + --set-string helmRepository.images.bundleInitializer.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }}{{ end }} \ + {{ if .IsSnapshot }}--set-string image.repository={{ .Env.LOCAL_IMAGE_REGISTRY }}/{{ .ProjectName }}{{ end }} \ + {{ if .IsSnapshot }}--set-string helmRepository.images.bundleInitializer.repository={{ .Env.LOCAL_IMAGE_REGISTRY }}/cluster-api-runtime-extensions-helm-chart-bundle-initializer{{ end }} \ ) EOF' - sed -i -e 's/\${/$${/g' -e 's/v0.0.0-dev/v{{ trimprefix .Version "v" }}/g' runtime-extensions-components.yaml @@ -96,8 +97,16 @@ builds: --platform linux/{{ .Arch }} \ -t v{{ trimprefix .Version "v" }}-{{ .Arch }} \ ./cmd + docker buildx build \ + --platform linux/{{ .Arch }} \ + -t {{ .Env.LOCAL_IMAGE_REGISTRY }}/cluster-api-runtime-extensions-helm-chart-bundle-initializer:v{{ trimprefix .Version "v" }}-{{ .Arch }} \ + -f ./hack/addons/helm-chart-bundler/Dockerfile \ + --load \ + . + if [ {{ .Env.LOCAL_IMAGE_REGISTRY }} != "ko.local" ]; then + docker push {{ .Env.LOCAL_IMAGE_REGISTRY }}/cluster-api-runtime-extensions-helm-chart-bundle-initializer:v{{ trimprefix .Version "v" }}-{{ .Arch }} + fi fi' - archives: - name_template: '{{ .ProjectName }}_v{{ trimprefix .Version "v" }}_{{ .Os }}_{{ .Arch }}' builds: diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index 4a1f54963..eaba1f520 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -6,7 +6,7 @@ managementClusterName: caren-e2e images: - name: ${LOCAL_IMAGE_REGISTRY}/cluster-api-runtime-extensions-nutanix:${E2E_IMAGE_TAG} loadBehavior: mustLoad - - name: ghcr.io/nutanix-cloud-native/cluster-api-runtime-extensions-helm-chart-bundle-initializer:${E2E_IMAGE_TAG} + - name: ${LOCAL_IMAGE_REGISTRY}/cluster-api-runtime-extensions-helm-chart-bundle-initializer:${E2E_IMAGE_TAG} loadBehavior: mustLoad providers: @@ -211,7 +211,7 @@ variables: # DOCKER_HUB_PASSWORD: "" intervals: - default/wait-controllers: ["3m", "10s"] + default/wait-controllers: ["10m", "10s"] default/wait-cluster: ["10m", "10s"] default/wait-control-plane: ["10m", "10s"] default/wait-worker-nodes: ["10m", "10s"] diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index 6f38e23df..8a01ccfb6 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -116,9 +116,10 @@ var _ = Describe("Self-hosted", Serial, func() { WaitForAddonsToBeReadyInWorkloadCluster( ctx, WaitForAddonsToBeReadyInWorkloadClusterInput{ - AddonsConfig: addonsConfig, - ClusterProxy: proxy, - WorkloadCluster: workloadCluster, + AddonsConfig: addonsConfig, + ClusterProxy: proxy, + WorkloadCluster: workloadCluster, + InfrastructureProvider: lowercaseProvider, DeploymentIntervals: e2eConfig.GetIntervals( flavour, "wait-deployment",