Add support for non-OpenShift K8s to managed-gitops, testing via k3d

.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+name: Build and test the operator
+on:
+  push:
+    branches:
+      - 'main'
+
+  pull_request:
+    branches:
+      - 'main'
+
+jobs:
+
+
+  test-e2e:
+    name: Run end-to-end tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k3s-version: [ v1.27.1 ]
+    steps:
+      # - name: Public IP
+      #   id: ip
+      #   uses: haythem/[email protected]
+
+      # - name: See IP Addresses
+      #   run: |
+      #     echo ${{ steps.ip.outputs.ipv4 }}
+      #     echo ${{ steps.ip.outputs.ipv6 }}
+
+      - name: Install k3d
+        run: |
+          set -x
+          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+          sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube
+          k3d cluster create --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1
+          kubectl version
+          k3d version
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: tests-e2e/go.mod
+
+      - name: GH actions workaround - Kill XSP4 process
+        run: |
+          sudo pkill mono || true
+
+      - name: Restore go build cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/go-build
+          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
+
+      - name: Add /usr/local/bin to PATH
+        run: |
+          echo "/usr/local/bin" >> $GITHUB_PATH
+
+      - name: Run the tests
+        run: |
+          set -o pipefail
+          make download-deps
+
+          make setup-e2e-local-k8s
+
+          kubectl port-forward svc/argocd-server -n gitops-service-argocd 51212:443 &
+
+          ARGO_CD_SERVER_ADDR=127.0.0.1:51212 make start-e2e &
+
+          GITHUB_K3D=true OVERRIDE_APISERVER_ADDR_SOURCE="0.0.0.0" OVERRIDE_APISERVER_ADDR_DEST="kubernetes.default.svc"  make test-e2e     
+
+          make e2e-reset
+          pkill go
+          pkill main
+

Makefile

@@ -141,7 +141,8 @@ install-argocd-openshift: kustomize ## Using OpenShift GitOps, install Argo CD t
 	PATH=$(MAKEFILE_ROOT)/bin:$(PATH) $(MAKEFILE_ROOT)/manifests/scripts/openshift-argo-deploy/deploy.sh
 
 install-argocd-k8s: ## (Non-OpenShift): Install Argo CD to the gitops-service-argocd namespace
-	ARGO_CD_VERSION=$(ARGO_CD_VERSION) manifests/scripts/k8s-argo-deploy/deploy.sh
+	ARGO_CD_VERSION=$(ARGO_CD_VERSION) ./argocd.sh install
+	# manifests/scripts/k8s-argo-deploy/deploy.sh
 
 uninstall-argocd: ## Uninstall Argo CD from gitops-service-argocd namespace (from either OpenShift or K8s)
 	kubectl delete namespace "$(ARGO_CD_NAMESPACE)" || true
@@ -197,6 +198,8 @@ test: test-backend test-backend-shared test-cluster-agent test-appstudio-control
 
 setup-e2e-openshift: install-argocd-openshift devenv-k8s-e2e ## Setup steps for E2E tests to run with Openshift CI
 
+setup-e2e-local-k8s: install-argocd-k8s devenv-docker reset-db ## Setup steps for E2E tests to run with Local Openshift Cluster
+
 setup-e2e-local: install-argocd-openshift devenv-docker reset-db ## Setup steps for E2E tests to run with Local Openshift Cluster
 
 start-e2e: ## Start the managed gitops processes for E2E tests.

argocd.sh

@@ -4,6 +4,15 @@ ARGO_CD_VERSION="${ARGO_CD_VERSION:-stable}"
 ARGO_CD_NAMESPACE="${ARGO_CD_NAMESPACE:-gitops-service-argocd}"
 ARGO_CD_PORT="${ARGO_CD_PORT:-4000}"
 
+
+SCRIPT_PATH="$(
+  cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit
+  pwd -P
+)"
+
+export ROOT_PATH=$SCRIPT_PATH
+
+
 # This script installs ArgoCD vanilla according to https://argo-cd.readthedocs.io/en/stable/getting_started/
 
 # Checks if a binary is present on the local system
@@ -16,19 +25,36 @@ exit_if_binary_not_installed() {
     done
 }
 
+# Install Argo CD to target namespace, using kustomize.
+# - Kustomize ensures that the ClusterRoleBindings are pointing to the ServiceAccount in the proper namespace.
+build_argo_cd_manifests() {
+    KUSTOMIZE_TMP_DIR=$(mktemp -d)    
+    cp -r $ROOT_PATH/manifests/k8s-argo-cd/* $KUSTOMIZE_TMP_DIR
+
+    mv $KUSTOMIZE_TMP_DIR/kustomization.yaml $KUSTOMIZE_TMP_DIR/kustomization.yaml.old
+
+    cat $KUSTOMIZE_TMP_DIR/kustomization.yaml.old | ARGO_CD_NAMESPACE=$ARGO_CD_NAMESPACE ARGO_CD_VERSION=$ARGO_CD_VERSION envsubst > $KUSTOMIZE_TMP_DIR/kustomization.yaml
+
+}
+
 # Install 'ArgoCD Web UI' in your Kubernetes cluster
 if [ "$1" = "install" ]; then
     exit_if_binary_not_installed "kubectl"
+    exit_if_binary_not_installed "kustomize"
 
     if kubectl -n "$ARGO_CD_NAMESPACE" get pods | grep argocd-server | grep '1/1' | grep 'Running' &>/dev/null; then
         echo "ArgoCD is already running..."
         echo "Skipping ArgoCD setup."
-        exit 1
+        exit 0
     fi
 
     # Apply the argo-cd-route manifest
     kubectl create namespace "$ARGO_CD_NAMESPACE" || true
-    kubectl apply -n "$ARGO_CD_NAMESPACE" -f https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml
+
+    # Build and apply the Argo CD manifests
+    build_argo_cd_manifests
+
+    kustomize build $KUSTOMIZE_TMP_DIR | kubectl apply -f -
 
     # Get the secret
     counter=0
@@ -104,5 +130,10 @@ fi
 # Remove 'ArgoCD Web UI' from your Kubernetes cluster
 if [ "$1" = "remove" ]; then
     exit_if_binary_not_installed "kubectl"
-    kubectl delete -n "$ARGO_CD_NAMESPACE" -f https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml
+    exit_if_binary_not_installed "kustomize"
+
+    build_argo_cd_manifests
+
+    kustomize build $KUSTOMIZE_TMP_DIR | kubectl delete -f -
+    kubectl delete namespace "$ARGO_CD_NAMESPACE" || true
 fi

cluster-agent/utils/argocd_login_credentials.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	argocdclient "github.com/argoproj/argo-cd/v2/pkg/apiclient"
 	"github.com/argoproj/argo-cd/v2/util/grpc"
 	"github.com/go-logr/logr"
 	"github.com/golang/protobuf/ptypes/empty"
 	routev1 "github.com/openshift/api/route/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 
 	sharedutil "github.com/redhat-appstudio/managed-gitops/backend-shared/util"
 	logutil "github.com/redhat-appstudio/managed-gitops/backend-shared/util/log"
@@ -221,9 +222,14 @@ func (cs *CredentialService) getCredentialsFromNamespace(req credentialRequest,
 		return nil, nil, errors.New("no Argo CD admin passwords found in " + req.namespaceName)
 	}
 
+	var serverHostName string
+
 	// Retrieve the Argo CD host name from the Route
-	serverHostName := ""
-	{
+
+	argoCDServerAddrEnvVar := strings.TrimSpace(os.Getenv("ARGO_CD_SERVER_ADDR"))
+	if len(argoCDServerAddrEnvVar) > 0 {
+		serverHostName = argoCDServerAddrEnvVar
+	} else {
 
 		routeList := &routev1.RouteList{}
 
@@ -249,6 +255,7 @@ func (cs *CredentialService) getCredentialsFromNamespace(req credentialRequest,
 
 		}
 	}
+
 	if serverHostName == "" {
 		return nil, nil, errors.New("Unable to locate Route in " + req.namespaceName)
 	}

manifests/k8s-argo-cd/kustomization.yaml

@@ -0,0 +1,29 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+-  https://raw.githubusercontent.com/argoproj/argo-cd/$ARGO_CD_VERSION/manifests/install.yaml
+
+namespace: $ARGO_CD_NAMESPACE
+
+patchesStrategicMerge:
+- |-
+  apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:         
+    name: argocd-application-controller
+  subjects:         
+  - kind: ServiceAccount
+    name: argocd-application-controller
+    namespace: $ARGO_CD_NAMESPACE
+
+- |-
+  apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:         
+    name: argocd-server
+  subjects:
+  - kind: ServiceAccount
+    name: argocd-server
+    namespace: $ARGO_CD_NAMESPACE
+

tests-e2e/.gitignore

@@ -3,5 +3,5 @@ cover.out
 vendor
 bin
 coverage.out
-
+ginkgo.report
 

tests-e2e/Makefile

@@ -33,7 +33,7 @@ help: ## Display this help.
 
 .PHONY: test
 test: ## Run E2E tests.
-	ENABLE_APPPROJECT_ISOLATION="true" DEV_ONLY_ALLOW_NON_TLS_CONNECTION_TO_POSTGRESQL=true go test -v -p=1 -timeout=100m -race -count=1 -coverprofile=coverage.out ./...
+	ENABLE_APPPROJECT_ISOLATION="true" DEV_ONLY_ALLOW_NON_TLS_CONNECTION_TO_POSTGRESQL=true go test -v -p=1 -timeout=100m -race -count=1 -coverprofile=coverage.out ./core/ ./argocd/
 
 # go-get-tool will 'go install' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))

tests-e2e/argocd/argocd_gitopsengine_instance_test.go

@@ -31,6 +31,8 @@ var _ = Describe("ArgoCD instance via GitOpsEngineInstance Operations Test", fun
 
 		It("ensures that a standalone ArgoCD gets created successfully when an operation CR of resource-type GitOpsEngineInstance is created", func() {
 
+			fixture.SkipIfArgoCDOperandRequired()
+
 			dbq, err := db.NewUnsafePostgresDBQueries(true, true)
 			Expect(err).ToNot(HaveOccurred())
 			defer dbq.CloseDatabase()

tests-e2e/argocd/argocd_instance_test.go

@@ -44,6 +44,9 @@ var _ = Describe("Standalone ArgoCD instance E2E tests", func() {
 		})
 
 		testStandaloneArgoCD := func(testUpdate bool) {
+
+			fixture.SkipIfArgoCDOperandRequired()
+
 			By("creating ArgoCD resource")
 			ctx := context.Background()
 			log := log.FromContext(ctx)

tests-e2e/argocd/queryscoped_cluster_secrets_test.go

@@ -138,6 +138,11 @@ var _ = Describe("Argo CD Application tests", func() {
 
 			func(clusterScoped bool) {
 
+				if fixture.IsGitHubK3D() {
+					Skip("when running on GitHub CI via K3d, we don't have an API endpoint IP we can use, so skip")
+					return
+				}
+
 				createServiceAccountsAndSecretsForTest(clusterScoped)
 
 				for _, userName := range users {
@@ -203,6 +208,11 @@ var _ = Describe("Argo CD Application tests", func() {
 
 		DescribeTable("ensure users cannot deploy to another user's namespace, when using different Argo CD cluster secrets for each user, each with a query parameter", func(clusterScoped bool) {
 
+			if fixture.IsGitHubK3D() {
+				Skip("when running on GitHub CI via K3d, we don't have an API endpoint IP we can use, so skip")
+				return
+			}
+
 			createServiceAccountsAndSecretsForTest(clusterScoped)
 
 			Expect(users).To(HaveLen(2))

tests-e2e/core/gitopsdeployment_status_test.go

@@ -46,18 +46,6 @@ var _ = Describe("GitOpsDeployment Status Tests", func() {
 							Status: managedgitopsv1alpha1.HeathStatusCodeHealthy,
 						},
 					},
-					{
-						Group:     "route.openshift.io",
-						Version:   "v1",
-						Kind:      "Route",
-						Namespace: fixture.GitOpsServiceE2ENamespace,
-						Name:      name,
-						Status:    managedgitopsv1alpha1.SyncStatusCodeSynced,
-						Health: &managedgitopsv1alpha1.HealthStatus{
-							Status:  managedgitopsv1alpha1.HeathStatusCodeHealthy,
-							Message: "Route is healthy",
-						},
-					},
 					{
 						Group:     "",
 						Version:   "v1",

tests-e2e/core/gitopsdeployment_syncrun_test.go

@@ -22,6 +22,7 @@ import (
 )
 
 var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() {
+
 	Context("Create, update and delete GitOpsDeploymentSyncRun", func() {
 
 		var (
@@ -65,10 +66,14 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() {
 		})
 
 		AfterEach(func() {
-			err := k8sClient.Get(ctx, client.ObjectKeyFromObject(argocdCR), argocdCR)
-			Expect(err == nil || apierr.IsNotFound(err)).To(BeTrue())
 
-			removeCustomHealthCheckForDeployment(ctx, k8sClient, argocdCR)
+			if fixture.IsArgoCDOperandAvailable() {
+				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(argocdCR), argocdCR)
+				Expect(err == nil || apierr.IsNotFound(err)).To(BeTrue())
+
+				removeCustomHealthCheckForDeployment(ctx, k8sClient, argocdCR)
+			}
+
 		})
 
 		It("creating a new GitOpsDeploymentSyncRun should sync an Argo CD Application", func() {
@@ -197,6 +202,8 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() {
 		})
 
 		It("deleting the GitOpsDeploymentSyncRun should terminate a running Sync operation", func() {
+			fixture.SkipIfArgoCDOperandRequired()
+
 			gitOpsDeploymentResource = gitopsDeplFixture.BuildGitOpsDeploymentResource("test-deply-with-presync",
 				"https://github.com/managed-gitops-test-data/deployment-presync-hook", "guestbook",
 				managedgitopsv1alpha1.GitOpsDeploymentSpecType_Manual)
@@ -314,6 +321,8 @@ var _ = Describe("GitOpsDeploymentSyncRun E2E tests", func() {
 
 		It("should sync if the previous sync operation is terminated", func() {
 
+			fixture.SkipIfArgoCDOperandRequired()
+
 			gitOpsDeploymentResource = gitopsDeplFixture.BuildGitOpsDeploymentResource("test-deply-with-presync",
 				"https://github.com/managed-gitops-test-data/deployment-presync-hook", "guestbook",
 				managedgitopsv1alpha1.GitOpsDeploymentSpecType_Manual)

tests-e2e/core/gitopsdeployment_test.go

@@ -76,18 +76,6 @@ var _ = Describe("GitOpsDeployment E2E tests", func() {
 						Status: managedgitopsv1alpha1.HeathStatusCodeHealthy,
 					},
 				},
-				{
-					Group:     "route.openshift.io",
-					Version:   "v1",
-					Kind:      "Route",
-					Namespace: fixture.GitOpsServiceE2ENamespace,
-					Name:      name,
-					Status:    managedgitopsv1alpha1.SyncStatusCodeSynced,
-					Health: &managedgitopsv1alpha1.HealthStatus{
-						Status:  managedgitopsv1alpha1.HeathStatusCodeHealthy,
-						Message: "Route is healthy",
-					},
-				},
 				{
 					Group:     "",
 					Version:   "v1",

tests-e2e/core/managed_environment_status_test.go

@@ -43,8 +43,12 @@ var _ = Describe("Managed Environment Status E2E tests", func() {
 			Expect(managedEnv.Status.Conditions).To(HaveLen(1))
 			condition := managedEnv.Status.Conditions[0]
 			Expect(condition.Status).To(Equal(metav1.ConditionFalse))
-			Expect(condition.Reason).To(Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToCreateClient)))
+			Expect(condition.Reason).To(Or(
+				Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToCreateClient)),
+				Equal(string(managedgitopsv1alpha1.ConditionReasonUnableToInstallServiceAccount)),
+			))
 			Expect(condition.Message).To(ContainSubstring("no such host"))
+
 		})
 
 		It("should have a connection status condition of False when the credentials are missing a required field", func() {