diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ff1eab214..668e81ee2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -40,6 +40,12 @@ jobs: - name: Verify helm reconciliation run: | kubectl -n podinfo wait helmrelease/podinfo --for=condition=ready --timeout=5m + - name: Verify gateway routing + run: | + kubectl -n envoy-gateway-system port-forward \ + $(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=envoy -o name) 8080:80 & + sleep 5 + curl -sSf -H "Host: podinfo.staging" http://localhost:8080 - name: List deployed resources run: flux tree ks flux-system - name: Debug failure diff --git a/README.md b/README.md index 954a6ca91..bc6784d95 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ curl -s https://fluxcd.io/install.sh | sudo bash The Git repository contains the following top directories: - **apps** dir contains Helm releases with a custom configuration per cluster -- **infrastructure** dir contains common infra tools such as ingress-nginx and cert-manager +- **infrastructure** dir contains common infra tools such as Envoy Gateway and cert-manager - **clusters** dir contains the Flux configuration per cluster ``` @@ -73,10 +73,10 @@ The apps configuration is structured into: │   └── repository.yaml ├── production │   ├── kustomization.yaml -│   └── podinfo-patch.yaml +│   └── podinfo-values.yaml └── staging ├── kustomization.yaml - └── podinfo-patch.yaml + └── podinfo-values.yaml ``` In **apps/base/podinfo/** dir we have a Flux `HelmRelease` with common values for both clusters: @@ -88,6 +88,7 @@ metadata: name: podinfo namespace: podinfo spec: + interval: 50m releaseName: podinfo chart: spec: @@ -96,11 +97,20 @@ spec: kind: HelmRepository name: podinfo namespace: flux-system - interval: 50m values: - ingress: + httpRoute: enabled: true - className: nginx + parentRefs: + - name: envoy + namespace: envoy-gateway-system + sectionName: http + hostnames: + - podinfo.local + rules: + - matches: + - path: + type: PathPrefix + value: / ``` In **apps/staging/** dir we have a Kustomize patch with the staging specific values: @@ -117,9 +127,9 @@ spec: test: enable: true values: - ingress: - hosts: - - host: podinfo.staging + httpRoute: + hostnames: + - podinfo.staging ``` Note that with `version: ">=1.0.0-alpha"` we configure Flux to automatically upgrade @@ -138,9 +148,9 @@ spec: spec: version: ">=1.0.0" values: - ingress: - hosts: - - host: podinfo.production + httpRoute: + hostnames: + - podinfo.production ``` Note that with ` version: ">=1.0.0"` we configure Flux to automatically upgrade @@ -157,10 +167,11 @@ The infrastructure is structured into: ./infrastructure/ ├── configs │   ├── cluster-issuers.yaml +│   ├── gateway.yaml │   └── kustomization.yaml └── controllers ├── cert-manager.yaml - ├── ingress-nginx.yaml + ├── envoy-gateway.yaml └── kustomization.yaml ``` @@ -195,6 +206,8 @@ spec: crds: enabled: true keep: false + config: + enableGatewayAPI: true ``` Note that in the `OCIRepository` we configure Flux to check for new chart versions every 24 hours. @@ -213,11 +226,14 @@ spec: email: fluxcdbot@users.noreply.github.com server: https://acme-staging-v02.api.letsencrypt.org/directory privateKeySecretRef: - name: letsencrypt-nginx + name: letsencrypt solvers: - http01: - ingress: - class: nginx + gatewayHTTPRoute: + parentRefs: + - name: envoy + namespace: envoy-gateway-system + kind: Gateway ``` In **clusters/production/infrastructure.yaml** we replace the Let's Encrypt server value to point to the production API: @@ -332,21 +348,22 @@ Watch for the Helm releases being installed on staging: ```console $ watch flux get helmreleases --all-namespaces -NAMESPACE NAME REVISION SUSPENDED READY MESSAGE -cert-manager cert-manager 1.19.1 False True Helm install succeeded -ingress-nginx ingress-nginx 4.13.4 False True Helm install succeeded -podinfo podinfo 6.9.2 False True Helm install succeeded +NAMESPACE NAME REVISION SUSPENDED READY MESSAGE +cert-manager cert-manager 1.19.1 False True Helm install succeeded +envoy-gateway-system envoy-gateway 1.8.0 False True Helm install succeeded +podinfo podinfo 6.11.2 False True Helm install succeeded ``` -Verify that the demo app can be accessed via ingress: +Verify that the demo app can be accessed via the Envoy Gateway: ```console -$ kubectl -n ingress-nginx port-forward svc/ingress-nginx-controller 8080:80 & +$ kubectl -n envoy-gateway-system port-forward \ + $(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=envoy -o name) 8080:80 & $ curl -H "Host: podinfo.staging" http://localhost:8080 { "hostname": "podinfo-59489db7b5-lmwpn", - "version": "6.9.2" + "version": "6.11.2" } ``` @@ -380,7 +397,7 @@ apps latest@sha256:26785ee4 True Applied revision: la The [Flux Operator](https://github.com/controlplaneio-fluxcd/flux-operator) offers an alternative to the Flux CLI bootstrap procedure. It removes the operational burden of managing Flux across fleets of clusters by fully automating the installation, configuration, and upgrade of the Flux controllers -based on a declarative API called [FluxInstance](https://fluxcd.control-plane.io/operator/fluxinstance/). +based on a declarative API called [FluxInstance](https://fluxoperator.dev/docs/crd/fluxinstance/). Install the Flux Operator CLI with Homebrew: @@ -406,7 +423,7 @@ You can also provide a `FluxInstance` manifest file to the command with `flux-op > [!TIP] > On production systems, the Flux Operator can be installed with Helm, Terraform/OpenTofu or directly from OperatorHub. -> For more details, please refer to the [Flux Operator documentation](https://fluxcd.control-plane.io/operator/install/). +> For more details, please refer to the [Flux Operator documentation](https://fluxoperator.dev/docs/guides/install/). To list all the resources managed by the Flux on the cluster, use: @@ -421,30 +438,32 @@ Kustomization/flux-system/flux-system │ │ ├── Service/podinfo/podinfo │ │ ├── Deployment/podinfo/podinfo │ │ ├── Deployment/podinfo/podinfo-redis -│ │ └── Ingress/podinfo/podinfo +│ │ └── HTTPRoute/podinfo/podinfo │ └── HelmRepository/podinfo/podinfo ├── Kustomization/flux-system/infra-configs -│ └── ClusterIssuer/letsencrypt +│ ├── ClusterIssuer/letsencrypt +│ ├── GatewayClass/envoy +│ └── Gateway/envoy-gateway-system/envoy ├── Kustomization/flux-system/infra-controllers │ ├── Namespace/cert-manager -│ ├── Namespace/ingress-nginx +│ ├── Namespace/envoy-gateway-system │ ├── HelmRelease/cert-manager/cert-manager -│ ├── HelmRelease/ingress-nginx/ingress-nginx -│ ├── HelmRepository/ingress-nginx/ingress-nginx -│ └── OCIRepository/cert-manager/cert-manager +│ ├── HelmRelease/envoy-gateway-system/envoy-gateway +│ ├── OCIRepository/cert-manager/cert-manager +│ └── OCIRepository/envoy-gateway-system/gateway-helm └── ArtifactGenerator/flux-system/flux-system ``` Using Flux Operator to bootstrap Flux comes with several benefits: -- The operator does not require write access to the Git repository and works with [GitHub Apps](https://fluxcd.control-plane.io/operator/flux-sync/#sync-from-a-git-repository-using-github-app-auth) and other OIDC providers. -- Production clusters can be configured to sync their state from [Git tags](https://fluxcd.control-plane.io/operator/flux-kustomize/#cluster-sync-semver-range) instead of the main branch, allowing safe promotion of changes from staging to production. -- The upgrade of Flux controllers and their CRDs is fully automated (can be customized via the `FluxInstance` [distribution](https://fluxcd.control-plane.io/operator/fluxinstance/#distribution-version) field). +- The operator does not require write access to the Git repository and works with [GitHub Apps](https://fluxoperator.dev/docs/instance/sync/#sync-from-a-git-repository-using-github-app-auth) and other OIDC providers. +- Production clusters can be configured to sync their state from [Git tags](https://fluxoperator.dev/docs/instance/customization/#cluster-sync-semver-range) instead of the main branch, allowing safe promotion of changes from staging to production. +- The upgrade of Flux controllers and their CRDs is fully automated (can be customized via the `FluxInstance` [distribution](https://fluxoperator.dev/docs/crd/fluxinstance/#distribution-version) field). - The `FluxInstance` API allows configuring multi-tenancy lockdown, network policies, persistent storage, sharding, and vertical scaling of the Flux controllers. -- The operator allows bootstrapping Flux in a [GitLess mode](https://fluxcd.control-plane.io/operator/flux-sync/#sync-from-a-container-registry), where the cluster state is stored as OCI artifacts in container registries. -- The operator extends Flux with self-service capabilities via the [ResourceSet](https://fluxcd.control-plane.io/operator/resourcesets/) API which is designed to reduce the complexity of GitOps workflows. +- The operator allows bootstrapping Flux in a [GitLess mode](https://fluxoperator.dev/gitless-gitops/), where the cluster state is stored as OCI artifacts in container registries. +- The operator extends Flux with self-service capabilities via the [ResourceSet](https://fluxoperator.dev/docs/resourcesets/introduction/) API which is designed to reduce the complexity of GitOps workflows. -To migrate an existing Flux installation to Flux Operator, please refer to the [bootstrap migration guide](https://fluxcd.control-plane.io/operator/flux-bootstrap-migration/). +To migrate an existing Flux installation to Flux Operator, please refer to the [bootstrap migration guide](https://fluxoperator.dev/docs/guides/migration/). ## Testing diff --git a/apps/base/podinfo/release.yaml b/apps/base/podinfo/release.yaml index b85c8922c..c158f2e52 100644 --- a/apps/base/podinfo/release.yaml +++ b/apps/base/podinfo/release.yaml @@ -4,6 +4,7 @@ metadata: name: podinfo namespace: podinfo spec: + interval: 50m releaseName: podinfo chart: spec: @@ -11,17 +12,29 @@ spec: sourceRef: kind: HelmRepository name: podinfo - interval: 50m install: - remediation: - retries: 3 + strategy: + name: RetryOnFailure + upgrade: + strategy: + name: RetryOnFailure # Default values # https://github.com/stefanprodan/podinfo/blob/master/charts/podinfo/values.yaml values: redis: enabled: true repository: public.ecr.aws/docker/library/redis - tag: 7.0.6 - ingress: + tag: 8.6.2 + httpRoute: enabled: true - className: nginx + parentRefs: + - name: envoy + namespace: envoy-gateway-system + sectionName: http + hostnames: + - podinfo.local + rules: + - matches: + - path: + type: PathPrefix + value: / diff --git a/apps/production/podinfo-values.yaml b/apps/production/podinfo-values.yaml index dff5d6e6c..1c93bbf08 100644 --- a/apps/production/podinfo-values.yaml +++ b/apps/production/podinfo-values.yaml @@ -8,9 +8,6 @@ spec: spec: version: ">=1.0.0" values: - ingress: - hosts: - - host: podinfo.production - paths: - - path: / - pathType: ImplementationSpecific + httpRoute: + hostnames: + - podinfo.production diff --git a/apps/staging/podinfo-values.yaml b/apps/staging/podinfo-values.yaml index 86f35a6cf..83a1ec54d 100644 --- a/apps/staging/podinfo-values.yaml +++ b/apps/staging/podinfo-values.yaml @@ -10,9 +10,6 @@ spec: test: enable: false values: - ingress: - hosts: - - host: podinfo.staging - paths: - - path: / - pathType: ImplementationSpecific + httpRoute: + hostnames: + - podinfo.staging diff --git a/infrastructure/configs/cluster-issuers.yaml b/infrastructure/configs/cluster-issuers.yaml index c450efdf1..cc57c95f3 100644 --- a/infrastructure/configs/cluster-issuers.yaml +++ b/infrastructure/configs/cluster-issuers.yaml @@ -10,8 +10,11 @@ spec: # The server is replaced in /clusters/production/infrastructure.yaml server: https://acme-staging-v02.api.letsencrypt.org/directory privateKeySecretRef: - name: letsencrypt-nginx + name: letsencrypt solvers: - http01: - ingress: - class: nginx + gatewayHTTPRoute: + parentRefs: + - name: envoy + namespace: envoy-gateway-system + kind: Gateway diff --git a/infrastructure/configs/gateway.yaml b/infrastructure/configs/gateway.yaml new file mode 100644 index 000000000..039870de3 --- /dev/null +++ b/infrastructure/configs/gateway.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: envoy +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: envoy + namespace: envoy-gateway-system +spec: + gatewayClassName: envoy + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All diff --git a/infrastructure/configs/kustomization.yaml b/infrastructure/configs/kustomization.yaml index de1ebfad0..b380dbd3d 100644 --- a/infrastructure/configs/kustomization.yaml +++ b/infrastructure/configs/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cluster-issuers.yaml + - gateway.yaml diff --git a/infrastructure/controllers/cert-manager.yaml b/infrastructure/controllers/cert-manager.yaml index f31173c4c..6d71f57c2 100644 --- a/infrastructure/controllers/cert-manager.yaml +++ b/infrastructure/controllers/cert-manager.yaml @@ -30,11 +30,9 @@ spec: install: strategy: name: RetryOnFailure - retryInterval: 2m upgrade: strategy: name: RetryOnFailure - retryInterval: 3m chartRef: kind: OCIRepository name: cert-manager @@ -42,3 +40,5 @@ spec: crds: enabled: true keep: false + config: + enableGatewayAPI: true diff --git a/infrastructure/controllers/envoy-gateway.yaml b/infrastructure/controllers/envoy-gateway.yaml new file mode 100644 index 000000000..f7673aabe --- /dev/null +++ b/infrastructure/controllers/envoy-gateway.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: envoy-gateway-system + labels: + toolkit.fluxcd.io/tenant: sre-team +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: gateway-helm + namespace: envoy-gateway-system +spec: + interval: 24h + url: oci://docker.io/envoyproxy/gateway-helm + layerSelector: + mediaType: "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + operation: copy + ref: + semver: ">= 1.8.0-rc.0" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: envoy-gateway + namespace: envoy-gateway-system +spec: + releaseName: envoy-gateway + interval: 12h + install: + strategy: + name: RetryOnFailure + upgrade: + strategy: + name: RetryOnFailure + chartRef: + kind: OCIRepository + name: gateway-helm diff --git a/infrastructure/controllers/ingress-nginx.yaml b/infrastructure/controllers/ingress-nginx.yaml deleted file mode 100644 index 541b8568b..000000000 --- a/infrastructure/controllers/ingress-nginx.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - toolkit.fluxcd.io/tenant: sre-team ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: ingress-nginx - namespace: ingress-nginx -spec: - interval: 24h - url: https://kubernetes.github.io/ingress-nginx ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: ingress-nginx - namespace: ingress-nginx -spec: - dependsOn: - - name: cert-manager - namespace: cert-manager - interval: 12h - install: - strategy: - name: RetryOnFailure - retryInterval: 2m - upgrade: - strategy: - name: RetryOnFailure - retryInterval: 3m - chart: - spec: - chart: ingress-nginx - version: "*" - sourceRef: - kind: HelmRepository - name: ingress-nginx - namespace: ingress-nginx - interval: 12h - values: - controller: - admissionWebhooks: - certManager: - enabled: true - service: - type: "NodePort" diff --git a/infrastructure/controllers/kustomization.yaml b/infrastructure/controllers/kustomization.yaml index ef2de1951..d9bbf7040 100644 --- a/infrastructure/controllers/kustomization.yaml +++ b/infrastructure/controllers/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cert-manager.yaml - - ingress-nginx.yaml + - envoy-gateway.yaml