feat: cilium configuration overrides for EKS provider (#1316)

**What problem does this PR solve?**:
- Stacked on
#1307
to reuse some functions and reduce merge conflicts.
- Sets Cilium default configuration for EKS to enable `eni` mode. 

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
- unit test for cilium template rendering

Tested manually for now.
- sample EKS cluster manifest
```
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  annotations:
    preflight.cluster.caren.nutanix.com/skip: all
  labels:
    cluster.x-k8s.io/provider: eks
  name: shalin-eks
spec:
  topology:
    class: eks-quick-start
    controlPlane:
      metadata:
        annotations:
          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
    variables:
    - name: clusterConfig
      value:
        addons:
          clusterAutoscaler: {}
          cni:
            provider: Cilium
          csi:
            defaultStorage:
              provider: aws-ebs
              storageClassConfig: default
            providers:
              aws-ebs:
                storageClassConfigs:
                  default: {}
            snapshotController: {}
          nfd: {}
        eks:
          region: us-west-2
    version: v1.32.9
    workers:
      machineDeployments:
      - class: default-worker
        metadata:
          annotations:
            cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "2"
            cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "2"
        name: md-0
        variables:
          overrides:
          - name: workerConfig
            value:
              eks:
                instanceType: m5.2xlarge
```

- Cilium HCP
```
apiVersion: addons.cluster.x-k8s.io/v1alpha1
kind: HelmChartProxy
...<redacted>
...
  valuesTemplate: |-
    cni:
      chainingMode: portmap
      exclusive: false
    hubble:
      enabled: true
      tls:
        auto:
          enabled: true               # enable automatic TLS certificate generation
          method: cronJob             # auto generate certificates using cronJob method
          certValidityDuration: 60    # certificates validity duration in days (default 2 months)
          schedule: "0 0 1 * *"       # schedule on the 1st day regeneration of each month
      relay:
        enabled: true
        tls:
          server:
            enabled: true
            mtls: true
        image:
          useDigest: false
        priorityClassName: system-cluster-critical
    ipam:
      mode: eni
    image:
      useDigest: false
    operator:
      image:
        useDigest: false
    certgen:
      image:
        useDigest: false
    socketLB:
      hostNamespaceOnly: true
    envoy:
      image:
        useDigest: false
    kubeProxyReplacement: true
    k8sServiceHost: "A535486E46D73CBF3C959CAE8F6831A4.gr7.us-west-2.eks.amazonaws.com"
    k8sServicePort: "443"
    enableIPv4Masquerade: false
    eni:
      enabled: true
      awsReleaseExcessIPs: true
    routingMode: native
    endpointRoutes:
      enabled: true
  version: 1.17.4
......<redacted>
...
  matchingClusters:
  - apiVersion: cluster.x-k8s.io/v1beta1
    kind: Cluster
    name: shalin-eks
    namespace: default
  observedGeneration: 3
```
- CNI and all pods running on the cluster
```
❯ kubectl get pods -A --kubeconfig shalin-eks.conf
NAMESPACE                NAME                                                              READY   STATUS              RESTARTS   AGE
default                  cluster-autoscaler-01997478-76f0-799f-b9ed-ddbff8eab94f-5ffdxsm   0/1     ContainerCreating   0          154m
kube-system              cilium-envoy-2cxcb                                                1/1     Running             0          108m
kube-system              cilium-envoy-nfp4b                                                1/1     Running             0          92m
kube-system              cilium-operator-84796b9ccf-h4lgf                                  1/1     Running             0          108m
kube-system              cilium-operator-84796b9ccf-v5hq4                                  1/1     Running             0          96m
kube-system              cilium-t4grn                                                      1/1     Running             0          92m
kube-system              cilium-zf2zm                                                      1/1     Running             0          108m
kube-system              coredns-5449774944-4hjxt                                          1/1     Running             0          155m
kube-system              coredns-5449774944-78897                                          1/1     Running             0          155m
kube-system              ebs-csi-controller-cb84bcd9-7qtqz                                 6/6     Running             0          154m
kube-system              ebs-csi-controller-cb84bcd9-dn6fg                                 6/6     Running             0          154m
kube-system              ebs-csi-node-m57pl                                                3/3     Running             0          153m
kube-system              ebs-csi-node-nnpcn                                                3/3     Running             0          92m
kube-system              hubble-relay-6b586bc6d-wd7c7                                      1/1     Running             0          108m
kube-system              snapshot-controller-6b6bf6cb95-xrm8q                              1/1     Running             0          154m
node-feature-discovery   node-feature-discovery-gc-6489bd687c-k2psp                        1/1     Running             0          154m
node-feature-discovery   node-feature-discovery-master-6fc5c44fb9-2bddp                    1/1     Running             0          154m
node-feature-discovery   node-feature-discovery-worker-tdv67                               1/1     Running             0          92m
node-feature-discovery   node-feature-discovery-worker-wwr2x                               1/1     Running             0          153m
```

- `cilium-config` configmap on EKS cluster updated to reflect eni ipam. 
**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

---------

Co-authored-by: Dimitri Koshkin <[email protected]>

Author: supershal

charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml

@@ -18,8 +18,13 @@ hubble:
     image:
       useDigest: false
     priorityClassName: system-cluster-critical
+{{- if eq .Provider "eks" }}
+ipam:
+  mode: eni
+{{- else }}
 ipam:
   mode: kubernetes
+{{- end }}
 image:
   useDigest: false
 operator:
@@ -33,7 +38,17 @@ socketLB:
 envoy:
   image:
     useDigest: false
-k8sServiceHost: auto
+k8sServiceHost: "{{ trimPrefix .ControlPlaneEndpoint.Host "https://" }}"
+k8sServicePort: "{{ .ControlPlaneEndpoint.Port }}"
 {{- if .EnableKubeProxyReplacement }}
 kubeProxyReplacement: true
 {{- end }}
+{{- if eq .Provider "eks" }}
+enableIPv4Masquerade: false
+eni:
+  enabled: true
+  awsReleaseExcessIPs: true
+routingMode: native
+endpointRoutes:
+  enabled: true
+{{- end }}

examples/capi-quick-start/eks-cluster.yaml

@@ -1,56 +1,3 @@
-apiVersion: v1
-data:
-  values.yaml: |-
-    cni:
-      exclusive: false
-    hubble:
-      enabled: true
-      tls:
-        auto:
-          enabled: true               # enable automatic TLS certificate generation
-          method: cronJob             # auto generate certificates using cronJob method
-          certValidityDuration: 60    # certificates validity duration in days (default 2 months)
-          schedule: "0 0 1 * *"       # schedule on the 1st day regeneration of each month
-      relay:
-        enabled: true
-        tls:
-          server:
-            enabled: true
-            mtls: true
-        image:
-          useDigest: false
-        priorityClassName: system-cluster-critical
-    image:
-      useDigest: false
-    operator:
-      image:
-        useDigest: false
-    certgen:
-      image:
-        useDigest: false
-    socketLB:
-      hostNamespaceOnly: true
-    envoy:
-      image:
-        useDigest: false
-    kubeProxyReplacement: true
-    k8sServiceHost: "{{ trimPrefix "https://" .Cluster.spec.controlPlaneEndpoint.host }}"
-    k8sServicePort: "{{ .Cluster.spec.controlPlaneEndpoint.port }}"
-    ipam:
-      mode: eni
-    enableIPv4Masquerade: false
-    eni:
-      enabled: true
-      awsReleaseExcessIPs: true
-    routingMode: native
-    endpointRoutes:
-      enabled: true
-kind: ConfigMap
-metadata:
-  labels:
-    cluster.x-k8s.io/provider: eks
-  name: ${CLUSTER_NAME}-cilium-cni-helm-values-template
----
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
@@ -70,10 +17,6 @@ spec:
           clusterAutoscaler: {}
           cni:
             provider: Cilium
-            values:
-              sourceRef:
-                kind: ConfigMap
-                name: ${CLUSTER_NAME}-cilium-cni-helm-values-template
           csi:
             defaultStorage:
               provider: aws-ebs

hack/addons/update-cilium-manifests.sh

@@ -25,10 +25,18 @@ envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_D
 
 cat <<EOF >"${ASSETS_DIR}/gomplate-context.yaml"
 EnableKubeProxyReplacement: false
+Provider: tmpl-capiprovider-tmpl
+ControlPlaneEndpoint:
+  Host: tmpl-controlplaneendpointhost-tmpl
+  Port: 6443
 EOF
-gomplate -f "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml" \
-  --context .="${ASSETS_DIR}/gomplate-context.yaml" \
-  >"${ASSETS_DIR}/helm-values.yaml"
+# Replace trimPrefix with strings.TrimPrefix to use the in built go function in gomplate.
+sed -e 's/trimPrefix/strings.TrimPrefix/g' \
+  -e '/k8sServiceHost:.*/,/k8sServicePort:/c\
+k8sServiceHost: auto' \
+  "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml" |
+  gomplate --context .="${ASSETS_DIR}/gomplate-context.yaml" \
+    >"${ASSETS_DIR}/helm-values.yaml"
 
 kustomize build \
   --load-restrictor LoadRestrictionsNone \

hack/examples/additional-resources/eks/cilium-configmap.yaml

@@ -5,17 +5,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../additional-resources/eks/cilium-configmap.yaml
 - ../../../bases/eks/cluster
 
 sortOptions:
   order: fifo
 
 patches:
-  # TODO: Replace with generic cilium patch and dynamically generate the correct EKS values
-  - target:
-      kind: Cluster
-    path: ../../../patches/eks/cilium-with-custom-values.yaml
   - target:
       kind: Cluster
     path: ../../../patches/skip-kube-proxy.yaml

hack/examples/overlays/clusters/eks/kustomization.yaml.tmpl

@@ -11,7 +11,3 @@
           nfd: {}
           cni:
             provider: Cilium
-            values:
-              sourceRef:
-                name: ${CLUSTER_NAME}-cilium-cni-helm-values-template
-                kind: ConfigMap

hack/examples/patches/eks/cilium-with-custom-values.yaml

@@ -266,13 +266,24 @@ func getValuesFileForChartIfNeeded(chartName, carenChartDirectory string) (strin
 		}
 
 		type input struct {
+			Provider                   string
+			ControlPlaneEndpoint       clusterv1.APIEndpoint
 			EnableKubeProxyReplacement bool
 		}
 		templateInput := input{
+			Provider: "test",
+			ControlPlaneEndpoint: clusterv1.APIEndpoint{
+				Host: "https://test.dummy.com",
+				Port: 443,
+			},
 			EnableKubeProxyReplacement: true,
 		}
 
-		err = template.Must(template.New(defaultHelmAddonFilename).ParseFiles(f)).Execute(tempFile, &templateInput)
+		funcMap := template.FuncMap{
+			"trimPrefix": strings.TrimPrefix,
+		}
+		err = template.Must(
+			template.New(defaultHelmAddonFilename).Funcs(funcMap).ParseFiles(f)).Execute(tempFile, &templateInput)
 		if err != nil {
 			return "", fmt.Errorf("failed to execute helm values template %w", err)
 		}

hack/examples/patches/eks/initialize-variables.yaml

@@ -6,11 +6,13 @@ package cilium
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"text/template"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 	apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
+	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
 )
 
 // templateValues enables kube-proxy replacement when kube-proxy is disabled.
@@ -20,18 +22,25 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
 		return "", fmt.Errorf("failed to check if kube-proxy is disabled: %w", err)
 	}
 
-	ciliumTemplate, err := template.New("").Parse(text)
+	funcMap := template.FuncMap{
+		"trimPrefix": strings.TrimPrefix,
+	}
+	ciliumTemplate, err := template.New("").Funcs(funcMap).Parse(text)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse template: %w", err)
 	}
 
 	type input struct {
+		Provider                   string
+		ControlPlaneEndpoint       clusterv1.APIEndpoint
 		EnableKubeProxyReplacement bool
 	}
 
 	// Assume when kube-proxy is disabled, we should enable Cilium's kube-proxy replacement feature.
 	templateInput := input{
 		EnableKubeProxyReplacement: kubeProxyIsDisabled,
+		Provider:                   capiutils.GetProvider(cluster),
+		ControlPlaneEndpoint:       cluster.Spec.ControlPlaneEndpoint,
 	}
 
 	var b bytes.Buffer