Migrate ingress to Envoy Gateway

.github/workflows/e2e.yaml

@@ -40,6 +40,12 @@ jobs:
       - name: Verify helm reconciliation
         run: |
           kubectl -n podinfo wait helmrelease/podinfo --for=condition=ready --timeout=5m
+      - name: Verify gateway routing
+        run: |
+          kubectl -n envoy-gateway-system port-forward \
+            $(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=envoy -o name) 8080:80 &
+          sleep 5
+          curl -sSf -H "Host: podinfo.staging" http://localhost:8080
       - name: List deployed resources
         run: flux tree ks flux-system
       - name: Debug failure

README.md

@@ -39,7 +39,7 @@ curl -s https://fluxcd.io/install.sh | sudo bash
 The Git repository contains the following top directories:
 
 - **apps** dir contains Helm releases with a custom configuration per cluster
-- **infrastructure** dir contains common infra tools such as ingress-nginx and cert-manager
+- **infrastructure** dir contains common infra tools such as Envoy Gateway and cert-manager
 - **clusters** dir contains the Flux configuration per cluster
 
 ```
@@ -73,10 +73,10 @@ The apps configuration is structured into:
 │       └── repository.yaml
 ├── production
 │   ├── kustomization.yaml
-│   └── podinfo-patch.yaml
+│   └── podinfo-values.yaml
 └── staging
     ├── kustomization.yaml
-    └── podinfo-patch.yaml
+    └── podinfo-values.yaml
 ```
 
 In **apps/base/podinfo/** dir we have a Flux `HelmRelease` with common values for both clusters:
@@ -88,6 +88,7 @@ metadata:
   name: podinfo
   namespace: podinfo
 spec:
+  interval: 50m
   releaseName: podinfo
   chart:
     spec:
@@ -96,11 +97,20 @@ spec:
         kind: HelmRepository
         name: podinfo
         namespace: flux-system
-  interval: 50m
   values:
-    ingress:
+    httpRoute:
       enabled: true
-      className: nginx
+      parentRefs:
+        - name: envoy
+          namespace: envoy-gateway-system
+          sectionName: http
+      hostnames:
+        - podinfo.local
+      rules:
+        - matches:
+            - path:
+                type: PathPrefix
+                value: /
 ```
 
 In **apps/staging/** dir we have a Kustomize patch with the staging specific values:
@@ -117,9 +127,9 @@ spec:
   test:
     enable: true
   values:
-    ingress:
-      hosts:
-        - host: podinfo.staging
+    httpRoute:
+      hostnames:
+        - podinfo.staging
 ```
 
 Note that with `version: ">=1.0.0-alpha"` we configure Flux to automatically upgrade
@@ -138,9 +148,9 @@ spec:
     spec:
       version: ">=1.0.0"
   values:
-    ingress:
-      hosts:
-        - host: podinfo.production
+    httpRoute:
+      hostnames:
+        - podinfo.production
 ```
 
 Note that with ` version: ">=1.0.0"` we configure Flux to automatically upgrade
@@ -157,10 +167,11 @@ The infrastructure is structured into:
 ./infrastructure/
 ├── configs
 │   ├── cluster-issuers.yaml
+│   ├── gateway.yaml
 │   └── kustomization.yaml
 └── controllers
     ├── cert-manager.yaml
-    ├── ingress-nginx.yaml
+    ├── envoy-gateway.yaml
     └── kustomization.yaml
 ```
 
@@ -195,6 +206,8 @@ spec:
     crds:
       enabled: true
       keep: false
+    config:
+      enableGatewayAPI: true
 ```
 
 Note that in the `OCIRepository` we configure Flux to check for new chart versions every 24 hours.
@@ -213,11 +226,14 @@ spec:
     email: fluxcdbot@users.noreply.github.com
     server: https://acme-staging-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
-      name: letsencrypt-nginx
+      name: letsencrypt
     solvers:
       - http01:
-          ingress:
-            class: nginx
+          gatewayHTTPRoute:
+            parentRefs:
+              - name: envoy
+                namespace: envoy-gateway-system
+                kind: Gateway
 ```
 
 In **clusters/production/infrastructure.yaml** we replace the Let's Encrypt server value to point to the production API:
@@ -332,21 +348,22 @@ Watch for the Helm releases being installed on staging:
 ```console
 $ watch flux get helmreleases --all-namespaces
 
-NAMESPACE    	NAME         	REVISION	SUSPENDED	READY	MESSAGE 
-cert-manager 	cert-manager 	1.19.1   	False    	True 	Helm install succeeded
-ingress-nginx	ingress-nginx	4.13.4   	False    	True 	Helm install succeeded
-podinfo      	podinfo      	6.9.2   	False    	True 	Helm install succeeded
+NAMESPACE           	NAME                	REVISION	SUSPENDED	READY	MESSAGE 
+cert-manager        	cert-manager        	1.19.1  	False    	True 	Helm install succeeded
+envoy-gateway-system	envoy-gateway       	1.8.0   	False    	True 	Helm install succeeded
+podinfo             	podinfo             	6.11.2   	False    	True 	Helm install succeeded
 ```
 
-Verify that the demo app can be accessed via ingress:
+Verify that the demo app can be accessed via the Envoy Gateway:
 
 ```console
-$ kubectl -n ingress-nginx port-forward svc/ingress-nginx-controller 8080:80 &
+$ kubectl -n envoy-gateway-system port-forward \
+    $(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=envoy -o name) 8080:80 &
 
 $ curl -H "Host: podinfo.staging" http://localhost:8080
 {
   "hostname": "podinfo-59489db7b5-lmwpn",
-  "version": "6.9.2"
+  "version": "6.11.2"
 }
 ```
 
@@ -380,7 +397,7 @@ apps                    latest@sha256:26785ee4      True    Applied revision: la
 The [Flux Operator](https://github.com/controlplaneio-fluxcd/flux-operator) offers an alternative
 to the Flux CLI bootstrap procedure. It removes the operational burden of managing Flux across fleets
 of clusters by fully automating the installation, configuration, and upgrade of the Flux controllers
-based on a declarative API called [FluxInstance](https://fluxcd.control-plane.io/operator/fluxinstance/).
+based on a declarative API called [FluxInstance](https://fluxoperator.dev/docs/crd/fluxinstance/).
 
 Install the Flux Operator CLI with Homebrew:
 
@@ -406,7 +423,7 @@ You can also provide a `FluxInstance` manifest file to the command with `flux-op
 
 > [!TIP]
 > On production systems, the Flux Operator can be installed with Helm, Terraform/OpenTofu or directly from OperatorHub.
-> For more details, please refer to the [Flux Operator documentation](https://fluxcd.control-plane.io/operator/install/).
+> For more details, please refer to the [Flux Operator documentation](https://fluxoperator.dev/docs/guides/install/).
 
 To list all the resources managed by the Flux on the cluster, use:
 
@@ -421,30 +438,32 @@ Kustomization/flux-system/flux-system
 │   │   ├── Service/podinfo/podinfo
 │   │   ├── Deployment/podinfo/podinfo
 │   │   ├── Deployment/podinfo/podinfo-redis
-│   │   └── Ingress/podinfo/podinfo
+│   │   └── HTTPRoute/podinfo/podinfo
 │   └── HelmRepository/podinfo/podinfo
 ├── Kustomization/flux-system/infra-configs
-│   └── ClusterIssuer/letsencrypt
+│   ├── ClusterIssuer/letsencrypt
+│   ├── GatewayClass/envoy
+│   └── Gateway/envoy-gateway-system/envoy
 ├── Kustomization/flux-system/infra-controllers
 │   ├── Namespace/cert-manager
-│   ├── Namespace/ingress-nginx
+│   ├── Namespace/envoy-gateway-system
 │   ├── HelmRelease/cert-manager/cert-manager
-│   ├── HelmRelease/ingress-nginx/ingress-nginx
-│   ├── HelmRepository/ingress-nginx/ingress-nginx
-│   └── OCIRepository/cert-manager/cert-manager
+│   ├── HelmRelease/envoy-gateway-system/envoy-gateway
+│   ├── OCIRepository/cert-manager/cert-manager
+│   └── OCIRepository/envoy-gateway-system/gateway-helm
 └── ArtifactGenerator/flux-system/flux-system
 ```
 
 Using Flux Operator to bootstrap Flux comes with several benefits:
 
-- The operator does not require write access to the Git repository and works with [GitHub Apps](https://fluxcd.control-plane.io/operator/flux-sync/#sync-from-a-git-repository-using-github-app-auth) and other OIDC providers.
-- Production clusters can be configured to sync their state from [Git tags](https://fluxcd.control-plane.io/operator/flux-kustomize/#cluster-sync-semver-range) instead of the main branch, allowing safe promotion of changes from staging to production.
-- The upgrade of Flux controllers and their CRDs is fully automated (can be customized via the `FluxInstance` [distribution](https://fluxcd.control-plane.io/operator/fluxinstance/#distribution-version) field).
+- The operator does not require write access to the Git repository and works with [GitHub Apps](https://fluxoperator.dev/docs/instance/sync/#sync-from-a-git-repository-using-github-app-auth) and other OIDC providers.
+- Production clusters can be configured to sync their state from [Git tags](https://fluxoperator.dev/docs/instance/customization/#cluster-sync-semver-range) instead of the main branch, allowing safe promotion of changes from staging to production.
+- The upgrade of Flux controllers and their CRDs is fully automated (can be customized via the `FluxInstance` [distribution](https://fluxoperator.dev/docs/crd/fluxinstance/#distribution-version) field).
 - The `FluxInstance` API allows configuring multi-tenancy lockdown, network policies, persistent storage, sharding, and vertical scaling of the Flux controllers.
-- The operator allows bootstrapping Flux in a [GitLess mode](https://fluxcd.control-plane.io/operator/flux-sync/#sync-from-a-container-registry), where the cluster state is stored as OCI artifacts in container registries.
-- The operator extends Flux with self-service capabilities via the [ResourceSet](https://fluxcd.control-plane.io/operator/resourcesets/) API which is designed to reduce the complexity of GitOps workflows.
+- The operator allows bootstrapping Flux in a [GitLess mode](https://fluxoperator.dev/gitless-gitops/), where the cluster state is stored as OCI artifacts in container registries.
+- The operator extends Flux with self-service capabilities via the [ResourceSet](https://fluxoperator.dev/docs/resourcesets/introduction/) API which is designed to reduce the complexity of GitOps workflows.
 
-To migrate an existing Flux installation to Flux Operator, please refer to the [bootstrap migration guide](https://fluxcd.control-plane.io/operator/flux-bootstrap-migration/).
+To migrate an existing Flux installation to Flux Operator, please refer to the [bootstrap migration guide](https://fluxoperator.dev/docs/guides/migration/).
 
 ## Testing
 

apps/base/podinfo/release.yaml

@@ -4,24 +4,37 @@ metadata:
   name: podinfo
   namespace: podinfo
 spec:
+  interval: 50m
   releaseName: podinfo
   chart:
     spec:
       chart: podinfo
       sourceRef:
         kind: HelmRepository
         name: podinfo
-  interval: 50m
   install:
-    remediation:
-      retries: 3
+    strategy:
+      name: RetryOnFailure
+  upgrade:
+    strategy:
+      name: RetryOnFailure
   # Default values
   # https://github.com/stefanprodan/podinfo/blob/master/charts/podinfo/values.yaml
   values:
     redis:
       enabled: true
       repository: public.ecr.aws/docker/library/redis
-      tag: 7.0.6
-    ingress:
+      tag: 8.6.2
+    httpRoute:
       enabled: true
-      className: nginx
+      parentRefs:
+        - name: envoy
+          namespace: envoy-gateway-system
+          sectionName: http
+      hostnames:
+        - podinfo.local
+      rules:
+        - matches:
+            - path:
+                type: PathPrefix
+                value: /

apps/production/podinfo-values.yaml

@@ -8,9 +8,6 @@ spec:
     spec:
       version: ">=1.0.0"
   values:
-    ingress:
-      hosts:
-        - host: podinfo.production
-          paths:
-            - path: /
-              pathType: ImplementationSpecific
+    httpRoute:
+      hostnames:
+        - podinfo.production

apps/staging/podinfo-values.yaml

@@ -10,9 +10,6 @@ spec:
   test:
     enable: false
   values:
-    ingress:
-      hosts:
-        - host: podinfo.staging
-          paths:
-            - path: /
-              pathType: ImplementationSpecific
+    httpRoute:
+      hostnames:
+        - podinfo.staging

infrastructure/configs/cluster-issuers.yaml

@@ -10,8 +10,11 @@ spec:
     # The server is replaced in /clusters/production/infrastructure.yaml
     server: https://acme-staging-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
-      name: letsencrypt-nginx
+      name: letsencrypt
     solvers:
       - http01:
-          ingress:
-            class: nginx
+          gatewayHTTPRoute:
+            parentRefs:
+              - name: envoy
+                namespace: envoy-gateway-system
+                kind: Gateway

infrastructure/configs/gateway.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: envoy
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: envoy
+  namespace: envoy-gateway-system
+spec:
+  gatewayClassName: envoy
+  listeners:
+    - name: http
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: All

infrastructure/configs/kustomization.yaml

@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cluster-issuers.yaml
+  - gateway.yaml

infrastructure/controllers/cert-manager.yaml

@@ -30,15 +30,15 @@ spec:
   install:
     strategy:
       name: RetryOnFailure
-      retryInterval: 2m
   upgrade:
     strategy:
       name: RetryOnFailure
-      retryInterval: 3m
   chartRef:
     kind: OCIRepository
     name: cert-manager
   values:
     crds:
       enabled: true
       keep: false
+    config:
+      enableGatewayAPI: true

infrastructure/controllers/envoy-gateway.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: envoy-gateway-system
+  labels:
+    toolkit.fluxcd.io/tenant: sre-team
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: gateway-helm
+  namespace: envoy-gateway-system
+spec:
+  interval: 24h
+  url: oci://docker.io/envoyproxy/gateway-helm
+  layerSelector:
+    mediaType: "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
+    operation: copy
+  ref:
+    semver: ">= 1.8.0-rc.0"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: envoy-gateway
+  namespace: envoy-gateway-system
+spec:
+  releaseName: envoy-gateway
+  interval: 12h
+  install:
+    strategy:
+      name: RetryOnFailure
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+  chartRef:
+    kind: OCIRepository
+    name: gateway-helm