diff --git a/gateway/Makefile b/gateway/Makefile index dfb19ea86f..de040d0914 100644 --- a/gateway/Makefile +++ b/gateway/Makefile @@ -237,7 +237,7 @@ dist: clean-dist ## Build standalone gateway distribution zip @cp build.yaml build-manifest.yaml $(DIST_DIR)/ @cp -R configs/. $(DIST_DIR)/configs/ @cp -R observability $(DIST_DIR)/ - @cp gateway-controller/certificates/default-listener.crt $(DIST_DIR)/resources/certificates/ + @cp gateway-controller/certificates/default-listener.crt $(DIST_DIR)/resources/certificates/ @cp gateway-controller/listener-certs/default-listener.crt $(DIST_DIR)/resources/listener-certs/ @cp gateway-controller/listener-certs/default-listener.key $(DIST_DIR)/resources/listener-certs/ @cp -R resources/secure-backend/. $(DIST_DIR)/resources/secure-backend/ diff --git a/gateway/README.md b/gateway/README.md index b47b054cf5..da65cfaf05 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -8,7 +8,7 @@ For end-user documentation, see [docs/gateway/](../docs/gateway/). | Component | Technology | Ports | |-----------|------------|-------| -| **Gateway-Controller** | Go, Gin, oapi-codegen, bbolt, go-control-plane | 9090 (REST), 18000 (xDS) | +| **Gateway-Controller** | Go, Gin, oapi-codegen, bbolt, go-control-plane | 9090 (REST), 18443 (xDS mTLS, default), 18444 (Policy xDS mTLS, default) | | **Router** | Envoy Proxy 1.35.3 | 8080 (HTTP), 8443 (HTTPS), 9901 (Admin) | | **Policy Engine** | Go, gRPC, ext_proc, xDS, CEL | 9002 (Admin) | | **Policy Builder** | Go, Docker | Build-time only | @@ -35,8 +35,9 @@ make build-gateway-builder ### Run -Run the one-time setup (generates `api-platform.env`, the router listener TLS certificate, the AES-256 -encryption key, and the gateway-controller admin credentials), then start the stack: +Run the one-time setup (generates `api-platform.env`, the router listener TLS certificate, the xDS +mutual TLS certificates, the AES-256 encryption key, and the gateway-controller admin credentials), +then start the stack: ```bash ./scripts/setup.sh @@ -126,7 +127,8 @@ for the full set of tokens and configuration options. | Variable | Description | |----------|-------------| -| `GATEWAY_CONTROLLER_HOST` | Gateway-Controller hostname (default: `gateway-controller`). The well-known xDS ports (18000 for Router, 18001 for Policy Engine) are derived automatically. | +| `GATEWAY_CONTROLLER_HOST` | Gateway-Controller hostname (default: `gateway-controller`). The well-known xDS ports are derived automatically: 18443/18444 (Router/Policy Engine, mutual TLS) when `XDS_TLS_ENABLED=true`, else the plaintext 18000/18001. | +| `XDS_TLS_ENABLED` | Mutual TLS between gateway-runtime (Envoy + Policy Engine) and gateway-controller's xDS servers. Enabled by default in `docker-compose.yaml` (both services; `./scripts/setup.sh` provisions the certificates it uses); edit it there to fall back to plaintext xDS. | ## Component Documentation diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index df5fb32b39..55308c1752 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -50,13 +50,21 @@ ciphers = "" ecdh_curves = "X25519,P-256" [controller.server.xds_tls] -# Switches the main xDS gRPC server (serves Envoy on server.xds_port above) -# from plaintext to mutual TLS -- there is no second listener the way -# server.tls above adds one; server.xds_port itself starts speaking mTLS. -# Off by default: Envoy's xds_cluster (router/config/config-override.yaml) -# must be given a matching client cert/CA before this is turned on, or the -# connection will fail closed. +# Starts the main xDS gRPC server (serves Envoy) as mutual TLS on its own +# dedicated `port` below, in place of the plaintext listener on +# server.xds_port above -- same either/or relationship server.tls above has +# with server.api_port. Envoy's xds_cluster (router/config/config- +# override.yaml) must be given a matching client cert/CA and pointed at +# this port before this is turned on, or the connection will fail closed. +# +# Enabled by default in this repo's shipped configs/config.toml (via the +# XDS_TLS_ENABLED env var -- see scripts/setup.sh, which provisions the dev +# CA/server/client certs this needs) and off by default in the binary's own +# built-in fallback (used when no config file value is present at all, e.g. +# most unit/integration tests) -- a hand-written config that omits this +# section entirely gets the binary's off default, not this file's on. enabled = false +port = 18443 cert_file = "" key_file = "" # PEM bundle of CA certificates trusted to sign Envoy's client certificate. @@ -103,12 +111,17 @@ mutex_profile_fraction = 0 port = 18001 [controller.policy_server.tls] -# Switches the policy xDS gRPC server (serves the policy-engine on -# policy_server.port above) from plaintext to mutual TLS, on that same -# port. Off by default: policy_engine.xds.tls below must be given a -# matching client cert/CA before this is turned on, or the connection will -# fail closed. +# Starts the policy xDS gRPC server (serves the policy-engine) as mutual +# TLS on its own dedicated `port` below, in place of the plaintext listener +# on policy_server.port above -- same either/or relationship +# server.xds_tls has with server.xds_port. policy_engine.xds.tls below must +# be given a matching client cert/CA and pointed at this port before this +# is turned on, or the connection will fail closed. +# +# Same on-by-default-in-configs/config.toml / off-by-default-in-the-binary +# split as server.xds_tls above -- see its comment. enabled = false +port = 18444 # Path to TLS certificate file (required if TLS is enabled) cert_file = "./certs/server.crt" # Path to TLS private key file (required if TLS is enabled) @@ -566,7 +579,7 @@ max_reconnect_delay = "60s" [policy_engine.xds.tls] # Mutual TLS for this policy-engine's connection to gateway-controller's -# policy xDS server. Off by default; must be enabled together with +# policy xDS server. Must be enabled together with # controller.policy_server.tls above (and this cert's identity added to # controller.policy_server.tls.allowed_client_identities) -- the server # offers no server-only TLS mode for xDS, so enabling only one side leaves @@ -598,7 +611,11 @@ max_reconnect_delay = "60s" # deliberately shares one client identity across both legs; otherwise set # POLICY_ENGINE_XDS_CLIENT_* explicitly, as this repo's own docker-compose # files do. -enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "false") }}' +# +# Literal default is "true" (enabled), matching server.xds_tls/ +# policy_server.tls above -- set XDS_TLS_ENABLED=false (api-platform.env) +# to fall back to plaintext xDS across all three legs at once. +enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "true") }}' cert_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CERT_PATH" (env "XDS_CLIENT_CERT_PATH" "") }}' key_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_KEY_PATH" (env "XDS_CLIENT_KEY_PATH" "") }}' ca_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CA_PATH" (env "XDS_CLIENT_CA_PATH" "") }}' diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index e81a042d7a..b35a76d6b6 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -14,12 +14,44 @@ enabled = true [controller.server] gateway_id = '{{ env "APIP_GW_CONTROLLER_SERVER_GATEWAY_ID" "platform-gateway-id" }}' +[controller.server.xds_tls] +# mTLS on the main xDS server (serves Envoy), using the dev CA/certs +# provisioned by scripts/setup.sh into gateway-controller/xds-certs (mounted +# at /app/xds-certs, the default cert_file/key_file/client_ca_file path -- +# see defaultConfig() in pkg/config/config.go). Enabled by default -- driven +# by the same XDS_TLS_ENABLED var (api-platform.env) that gateway-runtime's +# entrypoint uses to decide whether Envoy dials this port in TLS or +# plaintext, so both sides of the connection flip together. Set +# XDS_TLS_ENABLED=false in api-platform.env to fall back to plaintext xDS. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/envoy"] + +[controller.policy_server.tls] +# Same setup as server.xds_tls above, for the policy xDS server (serves the +# policy-engine). Distinct client identity -- the policy-engine presents its +# own cert (POLICY_ENGINE_XDS_CLIENT_*_PATH in docker-compose.yaml), not +# Envoy's. Also gated by XDS_TLS_ENABLED, same as above. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/policy-engine"] + [controller.storage] type = '{{ env "APIP_GW_CONTROLLER_STORAGE_TYPE" "sqlite" }}' [controller.storage.sqlite] path = '{{ env "APIP_GW_CONTROLLER_STORAGE_SQLITE_PATH" "./data/gateway.db" }}' +[policy_engine.xds.tls] +# Client-side counterpart to controller.policy_server.tls above -- this +# policy-engine subprocess (forked by gateway-runtime's docker-entrypoint.sh +# into the same container as Envoy) presents its own POLICY_ENGINE_XDS_CLIENT_* +# identity (set in docker-compose.yaml), distinct from Envoy's XDS_CLIENT_* +# cert, since controller.policy_server.tls.allowed_client_identities only +# allows the policy-engine identity. +enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "true") }}' +cert_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CERT_PATH" (env "XDS_CLIENT_CERT_PATH" "") }}' +key_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_KEY_PATH" (env "XDS_CLIENT_KEY_PATH" "") }}' +ca_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CA_PATH" (env "XDS_CLIENT_CA_PATH" "") }}' + [policy_engine.logging] level = "info" diff --git a/gateway/distribution/README.md b/gateway/distribution/README.md index bafd745c8d..07d7646fb6 100644 --- a/gateway/distribution/README.md +++ b/gateway/distribution/README.md @@ -25,7 +25,9 @@ wso2apip-api-gateway-/ │ └── fluent-bit/ # Fluent Bit log-shipping config └── resources/ ├── certificates/ # CA certificate for upstream TLS verification - ├── listener-certs/ # HTTPS listener TLS cert/key (self-signed; shipped, rotate with --force) + ├── listener-certs/ # HTTPS listener cert/key (self-signed; shipped, rotate with --force) + │ # + xDS mutual TLS: dev CA, server, and Envoy/Policy-Engine client + │ # certs (enabled by default; ⟵ generated by setup) ├── aesgcm-keys/ # ⟵ generated by setup: AES-256 at-rest encryption key ├── secure-backend/ # Test certificates for mTLS backend testing └── gateway-controller/ @@ -49,13 +51,16 @@ bcrypting the admin password uses `htpasswd` if present, otherwise Docker. ### 1. Run the one-time setup -Run the one-time setup (generates `api-platform.env`, the router listener TLS certificate, the AES-256 -encryption key, and the gateway-controller admin credentials): +Run the one-time setup (generates `api-platform.env`, the router listener TLS certificate, the xDS +mutual TLS certificates, the AES-256 encryption key, and the gateway-controller admin credentials): - `api-platform.env` — runtime environment file (admin username + bcrypt password hash) - `resources/aesgcm-keys/default-aesgcm256-v1.bin` — AES-256 at-rest encryption key - `resources/listener-certs/default-listener.{crt,key}` — HTTPS listener certificate (a self-signed one is shipped, and kept as-is unless you pass `--force`) +- `resources/listener-certs/{ca,server,envoy-client,policy-engine-client}.{crt,key}` — xDS mutual + TLS between gateway-controller and gateway-runtime, enabled by default via `XDS_TLS_ENABLED` in + `docker-compose.yaml`; edit it there (both services) to fall back to plaintext xDS The script is **idempotent** — existing files are kept, so re-running it is safe. On a re-run it also verifies that a kept `api-platform.env` still defines the settings the gateway needs (and exits non-zero, diff --git a/gateway/distribution/docker-compose.yaml b/gateway/distribution/docker-compose.yaml index 1cb47bd091..e5c0cec16a 100644 --- a/gateway/distribution/docker-compose.yaml +++ b/gateway/distribution/docker-compose.yaml @@ -27,6 +27,12 @@ services: - "9090:9090" # REST API - "9094:9092" # Admin API - "9011:9091" # Metrics + environment: + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and + # policy_server.tls (serves the Policy Engine) -- see configs/ + # config.toml, which reads this same var. Set to "false" here (and in + # gateway-runtime's environment below) to fall back to plaintext xDS. + - XDS_TLS_ENABLED=true env_file: - path: api-platform.env required: true @@ -35,7 +41,8 @@ services: - controller-data:/app/data - ./configs/config.toml:/etc/gateway-controller/config.toml:ro - ./resources/certificates:/app/certificates - - ./resources/listener-certs:/app/listener-certs:ro + - ./resources/listener-certs:/app/listener-certs:ro # Read-only: HTTPS listener certificate (generated by scripts/setup.sh) + - ./resources/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls (generated by scripts/setup.sh) - ./resources/aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro extra_hosts: - "host.docker.internal:host-gateway" @@ -66,9 +73,22 @@ services: # Envoy admin is disabled by default in the image; enabled here for local # dev convenience since the port is already mapped to the host above. - ROUTER_ADMIN_ENABLED=true + # Enabled by default, matching gateway-controller above. Client + # cert/key paths for mutual TLS to gateway-controller's xDS servers + # (server.xds_tls / policy_server.tls) -- Envoy and the policy-engine + # present distinct client identities, so each leg gets its own + # cert/key. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro - ./configs/llm-pricing/model_prices.json:/etc/policy-engine/llm-pricing/model_prices.json:ro + - ./resources/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections networks: - gateway-network diff --git a/gateway/docker-compose.debug.yaml b/gateway/docker-compose.debug.yaml index 55bfaa1877..225f85bac1 100644 --- a/gateway/docker-compose.debug.yaml +++ b/gateway/docker-compose.debug.yaml @@ -25,10 +25,16 @@ services: ports: - "9090:9090" # REST API - "9094:9092" # Admin API - - "18000:18000" # xDS gRPC (Router) - - "18001:18001" # xDS gRPC (Policy Engine) + - "18443:18443" # xDS gRPC mTLS (Router) -- 18000 stops listening once XDS_TLS_ENABLED=true (default) + - "18444:18444" # xDS gRPC mTLS (Policy Engine) -- 18001 stops listening once XDS_TLS_ENABLED=true (default) - "9011:9091" # Metrics - "2345:2345" # dlv remote debug port + environment: + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and + # policy_server.tls (serves the Policy Engine) -- see configs/ + # config.toml, which reads this same var. Set to "false" here (and in + # gateway-runtime's environment below) to fall back to plaintext xDS. + - XDS_TLS_ENABLED=true env_file: - path: api-platform.env required: true @@ -37,7 +43,8 @@ services: - controller-data:/app/data - ./configs/config.toml:/etc/gateway-controller/config.toml:ro - ./gateway-controller/certificates:/app/certificates - - ./gateway-controller/listener-certs:/app/listener-certs:ro + - ./gateway-controller/listener-certs:/app/listener-certs:ro # Read-only: HTTPS listener certificate (generated by scripts/setup.sh) + - ./gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls (generated by scripts/setup.sh) - ./gateway-controller/aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro # AES-256 at-rest encryption key (generated by scripts/setup.sh) extra_hosts: - "host.docker.internal:host-gateway" @@ -73,8 +80,19 @@ services: # Envoy admin is disabled by default in the image; enabled here for local # dev convenience since the port is already mapped to the host above. - ROUTER_ADMIN_ENABLED=true + # Enabled by default, matching gateway-controller above -- see + # docker-compose.yaml's gateway-runtime service for the full + # explanation of these vars. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro + - ./gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections networks: - gateway-network cap_add: diff --git a/gateway/docker-compose.yaml b/gateway/docker-compose.yaml index ee4605e4d7..f971b51128 100644 --- a/gateway/docker-compose.yaml +++ b/gateway/docker-compose.yaml @@ -33,6 +33,12 @@ services: - "9094:9092" # Admin API - "9011:9091" # Metrics - "9093:9093" # REST API TLS + environment: + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and + # policy_server.tls (serves the Policy Engine) -- see configs/ + # config.toml, which reads this same var. Set to "false" here (and in + # gateway-runtime's environment below) to fall back to plaintext xDS. + - XDS_TLS_ENABLED=true env_file: - path: api-platform.env required: true @@ -41,7 +47,8 @@ services: - controller-data:/app/data - ./configs/config.toml:/etc/gateway-controller/config.toml:ro - ./gateway-controller/certificates:/app/certificates # Read-write for dynamic certificate management - - ./gateway-controller/listener-certs:/app/listener-certs:ro # Read-only for HTTPS listener certificates + - ./gateway-controller/listener-certs:/app/listener-certs:ro # Read-only: HTTPS listener certificate + - ./gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./gateway-controller/aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro # AES-256 at-rest encryption key extra_hosts: - "host.docker.internal:host-gateway" @@ -64,6 +71,24 @@ services: - "9002:9002" # Admin API - "9003:9003" # Metrics - "9004:9004" # Health + environment: + # Enabled by default, matching gateway-controller above -- set both to + # "false" together to fall back to plaintext xDS (e.g. controller and + # runtime are not co-located, or a peer build that doesn't support + # this yet). Client cert/key paths for mutual TLS to gateway- + # controller's xDS servers (server.xds_tls / policy_server.tls). + # Envoy and the policy-engine present distinct client identities, so + # each leg gets its own cert/key -- both are checked against the + # shared dev CA mounted below. See gateway-controller/listener-certs and + # gateway-runtime/docker-entrypoint.sh for how these paths are + # consumed. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt env_file: - path: api-platform.env required: true @@ -71,6 +96,7 @@ services: volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro - ./configs/llm-pricing/model_prices.json:/etc/policy-engine/llm-pricing/model_prices.json:ro + - ./gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections networks: - gateway-network diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 24ebe15ab9..f7f7757fdf 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -419,8 +419,13 @@ func main() { routerConnected := make(chan struct{}) policyEngineConnected := make(chan struct{}) - // Start xDS gRPC server with SDS support + // Start xDS gRPC server with SDS support. When server.xds_tls is enabled, + // the server listens on its own dedicated server.xds_tls.port instead of + // the plaintext server.xds_port -- same either/or relationship as the + // REST API's server.tls, so enabling TLS never leaves the previous + // plaintext xDS port silently still reachable. var xdsServerOpts []xds.ServerOption + xdsListenPort := cfg.Controller.Server.XDSPort if cfg.Controller.Server.XDSTLS.Enabled { xdsTLSConfig, err := config.BuildXDSServerTLSConfig(cfg.Controller.Server.XDSTLS) if err != nil { @@ -428,8 +433,9 @@ func main() { os.Exit(1) } xdsServerOpts = append(xdsServerOpts, xds.WithMTLS(xdsTLSConfig, cfg.Controller.Server.XDSTLS.AllowedClientIdentities)) + xdsListenPort = cfg.Controller.Server.XDSTLS.Port } - xdsServer := xds.NewServer(snapshotManager, sdsSecretManager, cfg.Controller.Server.XDSPort, log, routerConnected, xdsServerOpts...) + xdsServer := xds.NewServer(snapshotManager, sdsSecretManager, xdsListenPort, log, routerConnected, xdsServerOpts...) go func() { if err := xdsServer.Start(); err != nil { log.Error("xDS server failed", slog.Any("error", err)) @@ -450,8 +456,10 @@ func main() { cancel() } - // Initialize policy xDS server - log.Info("Initializing Policy xDS server", slog.Int("port", cfg.Controller.PolicyServer.Port)) + // Initialize policy xDS server. The actual listen port (plaintext + // policy_server.port, or policy_server.tls.port once TLS is enabled) is + // logged by policyxds.Server.Start() once it's decided below. + log.Info("Initializing Policy xDS server") // Initialize policy snapshot manager and runtime config store policySnapshotManager := policyxds.NewSnapshotManager(log) @@ -507,10 +515,14 @@ func main() { } cancel() - // Start policy xDS server in a separate goroutine + // Start policy xDS server in a separate goroutine. When policy_server.tls + // is enabled, the server listens on its own dedicated + // policy_server.tls.port instead of the plaintext policy_server.port -- + // same either/or relationship as the main xDS server's server.xds_tls. serverOpts := []policyxds.ServerOption{ policyxds.WithOnFirstConnect(policyEngineConnected), } + policyListenPort := cfg.Controller.PolicyServer.Port if cfg.Controller.PolicyServer.TLS.Enabled { policyXDSTLSConfig, err := config.BuildXDSServerTLSConfig(cfg.Controller.PolicyServer.TLS) if err != nil { @@ -518,8 +530,9 @@ func main() { os.Exit(1) } serverOpts = append(serverOpts, policyxds.WithMTLS(policyXDSTLSConfig, cfg.Controller.PolicyServer.TLS.AllowedClientIdentities)) + policyListenPort = cfg.Controller.PolicyServer.TLS.Port } - policyXDSServer := policyxds.NewServer(policySnapshotManager, apiKeySnapshotManager, lazyResourceSnapshotManager, subscriptionSnapshotManager, nil, cfg.Controller.PolicyServer.Port, log, serverOpts...) + policyXDSServer := policyxds.NewServer(policySnapshotManager, apiKeySnapshotManager, lazyResourceSnapshotManager, subscriptionSnapshotManager, nil, policyListenPort, log, serverOpts...) go func() { if err := policyXDSServer.Start(); err != nil { log.Error("Policy xDS server failed", slog.Any("error", err)) diff --git a/gateway/gateway-controller/listener-certs/ca.crt b/gateway/gateway-controller/listener-certs/ca.crt new file mode 100644 index 0000000000..406bcaeb03 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/ca.crt @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIB0TCCAXegAwIBAgIUGwxGmaPHlOaXDsviSwig8NyTHQIwCgYIKoZIzj0EAwIw +PjEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xIDAeBgNVBAMMF0FQSSBQbGF0 +Zm9ybSB4RFMgRGV2IENBMB4XDTI2MDkxMTEwMzczMloXDTM2MDkwODEwMzczMlow +PjEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xIDAeBgNVBAMMF0FQSSBQbGF0 +Zm9ybSB4RFMgRGV2IENBMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEwi619tfM +0wK7BA8QZIPpK8xwztA+2TWfBtUzgJQl7aOKcVXWFJgdjDJ9izU3o+gaxs58LKP+ +TI6wOV6oFtV1bqNTMFEwHQYDVR0OBBYEFOJ6jCg5Kf39UT4VFVlpAVUyuFlnMB8G +A1UdIwQYMBaAFOJ6jCg5Kf39UT4VFVlpAVUyuFlnMA8GA1UdEwEB/wQFMAMBAf8w +CgYIKoZIzj0EAwIDSAAwRQIhAKALvF2dX+xJVfRmCDy5ZqNfRTcVO7uVX6bVeWyh +T45AAiBemqUrU+JU7lKIQAPRe/YFMLkj767oPNZsYOLpitp/eg== +-----END CERTIFICATE----- diff --git a/gateway/gateway-controller/listener-certs/ca.key b/gateway/gateway-controller/listener-certs/ca.key new file mode 100644 index 0000000000..72de6c03b4 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/ca.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIE7C+5siU+6TqpG0oo6MmDMbkwAMHIZ7atpX5mJNE/VUoAoGCCqGSM49 +AwEHoUQDQgAEwi619tfM0wK7BA8QZIPpK8xwztA+2TWfBtUzgJQl7aOKcVXWFJgd +jDJ9izU3o+gaxs58LKP+TI6wOV6oFtV1bg== +-----END EC PRIVATE KEY----- diff --git a/gateway/gateway-controller/listener-certs/envoy-client.crt b/gateway/gateway-controller/listener-certs/envoy-client.crt new file mode 100644 index 0000000000..01981444d5 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/envoy-client.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHDCCAcKgAwIBAgIUcINYokvGDNjAZ/Uom7bQ9KH2qgQwCgYIKoZIzj0EAwIw +PjEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xIDAeBgNVBAMMF0FQSSBQbGF0 +Zm9ybSB4RFMgRGV2IENBMB4XDTI2MDkxMTEwMzczMloXDTI4MTIxNDEwMzczMlow +MzEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xFTATBgNVBAMMDGVudm95LWNs +aWVudDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKZ8OXgJugz99dDtmDbRPotk +L5sQBjJyKhzuRm8587EgBU0Zf6poZjfukpVQ2GAIY128S7TPUYGDRHZsROSidwaj +gagwgaUwCQYDVR0TBAIwADALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUH +AwIwNgYDVR0RBC8wLYYrc3BpZmZlOi8vYXBpLXBsYXRmb3JtL2dhdGV3YXktcnVu +dGltZS9lbnZveTAdBgNVHQ4EFgQUj6LS+stxH5KezkAqSUNgII/qkVkwHwYDVR0j +BBgwFoAU4nqMKDkp/f1RPhUVWWkBVTK4WWcwCgYIKoZIzj0EAwIDSAAwRQIgOQHU +F4A99uCM5O2vml+5WA/eteeb/anLb98FSBkmIDYCIQC4/dWqpSD18WEUF8fv1ns+ +Dxez5f63r0iA0YQYGa2DHA== +-----END CERTIFICATE----- diff --git a/gateway/gateway-controller/listener-certs/envoy-client.key b/gateway/gateway-controller/listener-certs/envoy-client.key new file mode 100644 index 0000000000..4125aa68eb --- /dev/null +++ b/gateway/gateway-controller/listener-certs/envoy-client.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMt9pdInAx1x55+ydMnUWoFw7SnOOvnyH4cYE0cSwJ3LoAoGCCqGSM49 +AwEHoUQDQgAEpnw5eAm6DP310O2YNtE+i2QvmxAGMnIqHO5GbznzsSAFTRl/qmhm +N+6SlVDYYAhjXbxLtM9RgYNEdmxE5KJ3Bg== +-----END EC PRIVATE KEY----- diff --git a/gateway/gateway-controller/listener-certs/policy-engine-client.crt b/gateway/gateway-controller/listener-certs/policy-engine-client.crt new file mode 100644 index 0000000000..10ec960e64 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/policy-engine-client.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdKgAwIBAgIUcINYokvGDNjAZ/Uom7bQ9KH2qgUwCgYIKoZIzj0EAwIw +PjEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xIDAeBgNVBAMMF0FQSSBQbGF0 +Zm9ybSB4RFMgRGV2IENBMB4XDTI2MDkxMTEwMzczMloXDTI4MTIxNDEwMzczMlow +OzEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xHTAbBgNVBAMMFHBvbGljeS1l +bmdpbmUtY2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAESg7NEt+BeCNF +1s1mbbad8L7BaIxffBgX1om00qQwGYBInjmqXkjHxd2mxN6UzjMGRAiYF+lzVGYZ +2JMpkG+vL6OBsDCBrTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIHgDATBgNVHSUEDDAK +BggrBgEFBQcDAjA+BgNVHREENzA1hjNzcGlmZmU6Ly9hcGktcGxhdGZvcm0vZ2F0 +ZXdheS1ydW50aW1lL3BvbGljeS1lbmdpbmUwHQYDVR0OBBYEFBOgKlVdqMTJW10u +3ESxctI/pGmQMB8GA1UdIwQYMBaAFOJ6jCg5Kf39UT4VFVlpAVUyuFlnMAoGCCqG +SM49BAMCA0gAMEUCIQCvXkrXTz5LL4XP0N9GC+urphu7/+a3nLYJaIGtvg9JmAIg +KKpx0QfsKyXw+HNrWBcviky/IZFI/7NteqlxlI/OGkI= +-----END CERTIFICATE----- diff --git a/gateway/gateway-controller/listener-certs/policy-engine-client.key b/gateway/gateway-controller/listener-certs/policy-engine-client.key new file mode 100644 index 0000000000..6a0ba00498 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/policy-engine-client.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIP0vypt2KqJrrRGNYFnhKqT7L2hrlMtOEhxKKrB/b0inoAoGCCqGSM49 +AwEHoUQDQgAESg7NEt+BeCNF1s1mbbad8L7BaIxffBgX1om00qQwGYBInjmqXkjH +xd2mxN6UzjMGRAiYF+lzVGYZ2JMpkG+vLw== +-----END EC PRIVATE KEY----- diff --git a/gateway/gateway-controller/listener-certs/server.crt b/gateway/gateway-controller/listener-certs/server.crt new file mode 100644 index 0000000000..4c28aaecd7 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/server.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICSzCCAfKgAwIBAgIUcINYokvGDNjAZ/Uom7bQ9KH2qgMwCgYIKoZIzj0EAwIw +PjEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xIDAeBgNVBAMMF0FQSSBQbGF0 +Zm9ybSB4RFMgRGV2IENBMB4XDTI2MDkxMTEwMzczMloXDTI4MTIxNDEwMzczMlow +OTEaMBgGA1UECgwRV1NPMiBBUEkgUGxhdGZvcm0xGzAZBgNVBAMMEmdhdGV3YXkt +Y29udHJvbGxlcjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM3m8/WLPQnlY50y +mpRhHEBecaeR2a0UKkDLooHJTjjAZFn2y7UKpqCbxeJama0exieggHdF9JQ7lFB+ +rWhVX1mjgdIwgc8wCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwEwYDVR0lBAwwCgYI +KwYBBQUHAwEwYAYDVR0RBFkwV4ISZ2F0ZXdheS1jb250cm9sbGVyghVpdC1nYXRl +d2F5LWNvbnRyb2xsZXKCGWl0LWdhdGV3YXktY29udHJvbGxlci14ZHOCCWxvY2Fs +aG9zdIcEfwAAATAdBgNVHQ4EFgQUQBqATuNcwExZzewRSPQdq70e3oUwHwYDVR0j +BBgwFoAU4nqMKDkp/f1RPhUVWWkBVTK4WWcwCgYIKoZIzj0EAwIDRwAwRAIgVI8E +JdT8N2pHPMjgSq7P4Gq7+KCHxwngl5kcIrZIynUCIFVU7pegZGxFxjsR/TgZdmvF +L8XSJrjPBluyoia/A1m3 +-----END CERTIFICATE----- diff --git a/gateway/gateway-controller/listener-certs/server.key b/gateway/gateway-controller/listener-certs/server.key new file mode 100644 index 0000000000..a8e943a020 --- /dev/null +++ b/gateway/gateway-controller/listener-certs/server.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEILoy07FJQVtgFbaI1fKEBrlqTId5JfP3QJgTBDH1flJ4oAoGCCqGSM49 +AwEHoUQDQgAEzebz9Ys9CeVjnTKalGEcQF5xp5HZrRQqQMuigclOOMBkWfbLtQqm +oJvF4lqZrR7GJ6CAd0X0lDuUUH6taFVfWQ== +-----END EC PRIVATE KEY----- diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 4e86f8d22b..d0bee0133d 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -312,10 +312,11 @@ type ServerConfig struct { // default. TLS ServerTLSConfig `koanf:"tls"` - // XDSTLS switches the main xDS gRPC server (serving Envoy, on XDSPort) - // from plaintext to mutual TLS. Unlike TLS above, this does not add a - // second listener -- XDSPort itself starts speaking mTLS. Off by - // default; see XDSServerTLSConfig for why xDS has no server-only mode. + // XDSTLS starts the main xDS gRPC server (serving Envoy) as mutual TLS + // on its own dedicated XDSTLS.Port, in place of the plaintext listener + // on XDSPort -- same either/or relationship TLS above has with APIPort. + // Off by default; see XDSServerTLSConfig for why xDS has no + // server-only mode. XDSTLS XDSServerTLSConfig `koanf:"xds_tls"` // ReadTimeout, ReadHeaderTimeout, WriteTimeout, and IdleTimeout bound the @@ -420,8 +421,14 @@ type PprofConfig struct { // PolicyServerConfig holds policy xDS server-related configuration type PolicyServerConfig struct { - Port int `koanf:"port"` - TLS XDSServerTLSConfig `koanf:"tls"` + // Port is the plaintext policy xDS listener port. + Port int `koanf:"port"` + + // TLS starts the policy xDS server as mutual TLS on its own dedicated + // TLS.Port, in place of the plaintext listener on Port -- same + // either/or relationship as ServerConfig.XDSTLS has with + // ServerConfig.XDSPort. Off by default. + TLS XDSServerTLSConfig `koanf:"tls"` } // PoliciesConfig holds policy-related configuration @@ -1073,6 +1080,7 @@ func defaultConfig() *Config { }, XDSTLS: XDSServerTLSConfig{ Enabled: false, + Port: 18443, CertFile: "./xds-certs/server.crt", KeyFile: "./xds-certs/server.key", ClientCAFile: "./xds-certs/ca.crt", @@ -1099,6 +1107,7 @@ func defaultConfig() *Config { Port: 18001, TLS: XDSServerTLSConfig{ Enabled: false, + Port: 18444, CertFile: "./xds-certs/server.crt", KeyFile: "./xds-certs/server.key", ClientCAFile: "./xds-certs/ca.crt", @@ -1647,6 +1656,16 @@ func (c *Config) Validate() error { return fmt.Errorf("server.xds_port must be between 1 and 65535, got: %d", c.Controller.Server.XDSPort) } + if c.Controller.PolicyServer.Port < 1 || c.Controller.PolicyServer.Port > 65535 { + return fmt.Errorf("policy_server.port must be between 1 and 65535, got: %d", c.Controller.PolicyServer.Port) + } + if c.Controller.PolicyServer.Port == c.Controller.Server.APIPort { + return fmt.Errorf("policy_server.port cannot be same as server.api_port") + } + if c.Controller.PolicyServer.Port == c.Controller.Server.XDSPort { + return fmt.Errorf("policy_server.port cannot be same as server.xds_port") + } + if strings.TrimSpace(c.Controller.Server.GatewayID) == "" { return fmt.Errorf("server.gateway_id is required and cannot be empty") } @@ -1705,6 +1724,55 @@ func (c *Config) Validate() error { return err } + // Each xDS TLS listener gets its own dedicated port (XDSServerTLSConfig.Port), + // never reusing server.xds_port/policy_server.port -- so it must be checked + // for collisions against every other configured controller port, the same + // way server.tls.port already is above. + if c.Controller.Server.XDSTLS.Enabled { + port := c.Controller.Server.XDSTLS.Port + switch port { + case c.Controller.Server.APIPort: + return fmt.Errorf("server.xds_tls.port cannot be same as server.api_port") + case c.Controller.Server.XDSPort: + return fmt.Errorf("server.xds_tls.port cannot be same as server.xds_port") + case c.Controller.PolicyServer.Port: + return fmt.Errorf("server.xds_tls.port cannot be same as policy_server.port") + } + if c.Controller.Server.TLS.Enabled && port == c.Controller.Server.TLS.Port { + return fmt.Errorf("server.xds_tls.port cannot be same as server.tls.port") + } + if c.Controller.AdminServer.Enabled && port == c.Controller.AdminServer.Port { + return fmt.Errorf("server.xds_tls.port cannot be same as admin_server.port") + } + if c.Controller.Metrics.Enabled && port == c.Controller.Metrics.Port { + return fmt.Errorf("server.xds_tls.port cannot be same as metrics.port") + } + if c.Controller.PolicyServer.TLS.Enabled && port == c.Controller.PolicyServer.TLS.Port { + return fmt.Errorf("server.xds_tls.port cannot be same as policy_server.tls.port") + } + } + + if c.Controller.PolicyServer.TLS.Enabled { + port := c.Controller.PolicyServer.TLS.Port + switch port { + case c.Controller.Server.APIPort: + return fmt.Errorf("policy_server.tls.port cannot be same as server.api_port") + case c.Controller.Server.XDSPort: + return fmt.Errorf("policy_server.tls.port cannot be same as server.xds_port") + case c.Controller.PolicyServer.Port: + return fmt.Errorf("policy_server.tls.port cannot be same as policy_server.port") + } + if c.Controller.Server.TLS.Enabled && port == c.Controller.Server.TLS.Port { + return fmt.Errorf("policy_server.tls.port cannot be same as server.tls.port") + } + if c.Controller.AdminServer.Enabled && port == c.Controller.AdminServer.Port { + return fmt.Errorf("policy_server.tls.port cannot be same as admin_server.port") + } + if c.Controller.Metrics.Enabled && port == c.Controller.Metrics.Port { + return fmt.Errorf("policy_server.tls.port cannot be same as metrics.port") + } + } + if c.Controller.AdminServer.Enabled { if c.Controller.AdminServer.Port < 1 || c.Controller.AdminServer.Port > 65535 { return fmt.Errorf("admin_server.port must be between 1 and 65535, got: %d", c.Controller.AdminServer.Port) diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index dbf2a36f1f..3badc8a2ea 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -46,6 +46,9 @@ func validConfig() *Config { IdleTimeout: 120 * time.Second, MaxHeaderBytes: 1 << 20, }, + PolicyServer: PolicyServerConfig{ + Port: 18001, + }, Storage: StorageConfig{ Type: "sqlite", SQLite: SQLiteConfig{ @@ -821,6 +824,7 @@ func TestConfig_Validate_XDSServerTLS(t *testing.T) { validXDSTLS := func() XDSServerTLSConfig { return XDSServerTLSConfig{ Enabled: true, + Port: 18443, CertFile: "./xds-certs/server.crt", KeyFile: "./xds-certs/server.key", ClientCAFile: "./xds-certs/ca.crt", @@ -850,6 +854,7 @@ func TestConfig_Validate_XDSServerTLS(t *testing.T) { t.Run("valid policy_server.tls passes", func(t *testing.T) { cfg := validConfig() tlsCfg := validXDSTLS() + tlsCfg.Port = 18444 tlsCfg.AllowedClientIdentities = []string{"spiffe://api-platform/gateway-runtime/policy-engine"} cfg.Controller.PolicyServer.TLS = tlsCfg assert.NoError(t, cfg.Validate()) @@ -858,6 +863,7 @@ func TestConfig_Validate_XDSServerTLS(t *testing.T) { t.Run("invalid policy_server.tls is rejected with a prefixed error", func(t *testing.T) { cfg := validConfig() tlsCfg := validXDSTLS() + tlsCfg.Port = 18444 tlsCfg.ClientCAFile = "" cfg.Controller.PolicyServer.TLS = tlsCfg err := cfg.Validate() @@ -869,6 +875,42 @@ func TestConfig_Validate_XDSServerTLS(t *testing.T) { cfg := validConfig() assert.NoError(t, cfg.Validate()) }) + + t.Run("server.xds_tls.port colliding with server.xds_port is rejected", func(t *testing.T) { + cfg := validConfig() + tlsCfg := validXDSTLS() + tlsCfg.Port = cfg.Controller.Server.XDSPort + cfg.Controller.Server.XDSTLS = tlsCfg + err := cfg.Validate() + assert.Error(t, err) + assert.Contains(t, err.Error(), "server.xds_tls.port cannot be same as server.xds_port") + }) + + t.Run("server.xds_tls.port colliding with policy_server.tls.port is rejected", func(t *testing.T) { + cfg := validConfig() + xdsTLSCfg := validXDSTLS() + policyTLSCfg := validXDSTLS() + xdsTLSCfg.Port = 20000 + policyTLSCfg.Port = 20000 + policyTLSCfg.AllowedClientIdentities = []string{"spiffe://api-platform/gateway-runtime/policy-engine"} + cfg.Controller.Server.XDSTLS = xdsTLSCfg + cfg.Controller.PolicyServer.TLS = policyTLSCfg + err := cfg.Validate() + assert.Error(t, err) + assert.Contains(t, err.Error(), "server.xds_tls.port cannot be same as policy_server.tls.port") + }) + + t.Run("policy_server.tls.port colliding with policy_server.port is rejected", func(t *testing.T) { + cfg := validConfig() + cfg.Controller.PolicyServer.Port = 18001 + tlsCfg := validXDSTLS() + tlsCfg.Port = 18001 + tlsCfg.AllowedClientIdentities = []string{"spiffe://api-platform/gateway-runtime/policy-engine"} + cfg.Controller.PolicyServer.TLS = tlsCfg + err := cfg.Validate() + assert.Error(t, err) + assert.Contains(t, err.Error(), "policy_server.tls.port cannot be same as policy_server.port") + }) } // TestParseServerEcdhCurves tests the ECDH curve preference parser used by diff --git a/gateway/gateway-controller/pkg/config/xds_tls.go b/gateway/gateway-controller/pkg/config/xds_tls.go index 527f6fed1d..bc949615e0 100644 --- a/gateway/gateway-controller/pkg/config/xds_tls.go +++ b/gateway/gateway-controller/pkg/config/xds_tls.go @@ -27,27 +27,37 @@ import ( ) // XDSServerTLSConfig holds mutual-TLS configuration for an xDS gRPC server: -// the main Envoy-facing ADS/SDS server on server.xds_port, and the -// policy-engine-facing server on policy_server.port. Off by default -- both -// servers keep working in plaintext either way, consistent with this -// repo's "PQC/TLS is optional-but-supported, not mandatory" posture (see -// post-quantum-cryptography.md), since not every deployment's Envoy or -// policy-engine build is configured for mTLS yet. +// the main Envoy-facing ADS/SDS server (plaintext on server.xds_port), and +// the policy-engine-facing server (plaintext on policy_server.port). Off by +// default -- both servers keep working in plaintext on their existing port +// either way, consistent with this repo's "PQC/TLS is optional-but-supported, +// not mandatory" posture (see post-quantum-cryptography.md), since not every +// deployment's Envoy or policy-engine build is configured for mTLS yet. // // Unlike ServerTLSConfig (the REST management API's TLS listener, which is // server-only TLS), this type has no server-only mode: xDS is a // control-plane channel that carries per-tenant API-key hashes, // subscription state, and full policy chains, so authenticating only the // server side is not sufficient (go-control-plane-xds-security.md -// directive 2). Whenever Enabled is true, ClientCAFile and -// AllowedClientIdentities are both required -- see ValidateXDSServerTLS. +// directive 2). Whenever Enabled is true, Port, ClientCAFile, and +// AllowedClientIdentities are all required -- see ValidateXDSServerTLS. type XDSServerTLSConfig struct { - // Enabled switches the xDS server from plaintext to mutual TLS on its - // existing port (server.xds_port or policy_server.port) -- there is no - // second listener the way ServerTLSConfig adds one for the REST API, - // since a gRPC server serves one credential type per port. + // Enabled starts the mutual-TLS listener on Port, in place of the + // plaintext listener on the server's existing port (server.xds_port or + // policy_server.port) -- same either/or relationship as + // ServerConfig.TLS.Enabled has with APIPort, so flipping this on never + // leaves the previous plaintext port silently still reachable. Off by + // default. Enabled bool `koanf:"enabled"` + // Port is this xDS server's dedicated mutual-TLS listener port -- + // separate from, and never reusing, the plaintext port + // (server.xds_port/policy_server.port) it replaces once Enabled. Must + // differ from every other configured controller port. Required when + // Enabled; disabled by default, so this port is never bound unless an + // operator explicitly opts in. + Port int `koanf:"port"` + // CertFile and KeyFile are the PEM-encoded server certificate and // private key this xDS server presents to connecting clients. Required // when Enabled. @@ -100,6 +110,9 @@ func ValidateXDSServerTLS(fieldPrefix string, cfg XDSServerTLSConfig) error { if !cfg.Enabled { return nil } + if cfg.Port < 1 || cfg.Port > 65535 { + return fmt.Errorf("%s.port must be between 1 and 65535, got: %d", fieldPrefix, cfg.Port) + } if cfg.CertFile == "" { return fmt.Errorf("%s.cert_file is required when %s.enabled", fieldPrefix, fieldPrefix) } diff --git a/gateway/gateway-controller/pkg/config/xds_tls_test.go b/gateway/gateway-controller/pkg/config/xds_tls_test.go index 11dbe1c986..094348c36b 100644 --- a/gateway/gateway-controller/pkg/config/xds_tls_test.go +++ b/gateway/gateway-controller/pkg/config/xds_tls_test.go @@ -108,6 +108,7 @@ func TestValidateXDSServerTLS(t *testing.T) { validCfg := func() XDSServerTLSConfig { return XDSServerTLSConfig{ Enabled: true, + Port: 18443, CertFile: "./certs/server.crt", KeyFile: "./certs/server.key", ClientCAFile: "./certs/ca.crt", @@ -130,6 +131,18 @@ func TestValidateXDSServerTLS(t *testing.T) { mutate: func(c *XDSServerTLSConfig) { *c = XDSServerTLSConfig{Enabled: false} }, wantErr: false, }, + { + name: "missing port", + mutate: func(c *XDSServerTLSConfig) { c.Port = 0 }, + wantErr: true, + errContains: "port must be between 1 and 65535", + }, + { + name: "out of range port", + mutate: func(c *XDSServerTLSConfig) { c.Port = 70000 }, + wantErr: true, + errContains: "port must be between 1 and 65535", + }, { name: "missing cert file", mutate: func(c *XDSServerTLSConfig) { c.CertFile = "" }, diff --git a/gateway/gateway-runtime/docker-entrypoint.sh b/gateway/gateway-runtime/docker-entrypoint.sh index 39ed921964..baac179dc8 100644 --- a/gateway/gateway-runtime/docker-entrypoint.sh +++ b/gateway/gateway-runtime/docker-entrypoint.sh @@ -95,11 +95,22 @@ done # Default configuration # GATEWAY_CONTROLLER_HOST is the primary user-facing env var to configure connectivity # to the gateway controller. The xDS ports default to well-known values: -# - ROUTER_XDS_PORT (18000): Router (Envoy) route/cluster/listener configs -# - POLICY_ENGINE_XDS_PORT (18001): Policy Engine policy chain configs +# - ROUTER_XDS_PORT (18000): Router (Envoy) route/cluster/listener configs, plaintext +# - ROUTER_XDS_TLS_PORT (18443): same, but the dedicated mTLS listener gateway- +# controller's server.xds_tls.port switches to -- only used when XDS_TLS_ENABLED=true +# below, since enabling TLS there replaces the plaintext listener on ROUTER_XDS_PORT +# rather than reusing it (matching gateway-controller's server.xds_tls.port having its +# own dedicated port, never server.xds_port, once enabled). +# - POLICY_ENGINE_XDS_PORT (18001): Policy Engine policy chain configs, plaintext +# - POLICY_ENGINE_XDS_TLS_PORT (18444): same, but the dedicated mTLS listener +# gateway-controller's policy_server.tls.port switches to -- only used when +# policy-engine's xDS TLS is enabled (POLICY_ENGINE_XDS_TLS_ENABLED, falling back to +# XDS_TLS_ENABLED, same precedence as policy_engine.xds.tls.enabled in config-template.toml) export GATEWAY_CONTROLLER_HOST="${GATEWAY_CONTROLLER_HOST:-gateway-controller}" export ROUTER_XDS_PORT="${ROUTER_XDS_PORT:-18000}" +export ROUTER_XDS_TLS_PORT="${ROUTER_XDS_TLS_PORT:-18443}" export POLICY_ENGINE_XDS_PORT="${POLICY_ENGINE_XDS_PORT:-18001}" +export POLICY_ENGINE_XDS_TLS_PORT="${POLICY_ENGINE_XDS_TLS_PORT:-18444}" export LOG_LEVEL="${LOG_LEVEL:-info}" # Performance tuning configuration @@ -144,7 +155,6 @@ export ROUTER_DRAIN_TIME_SECONDS="${ROUTER_DRAIN_TIME_SECONDS:-15}" # Derive Router (Envoy) xDS config — used by envsubst on config-override.yaml export XDS_SERVER_HOST="${GATEWAY_CONTROLLER_HOST}" -export XDS_SERVER_PORT="${ROUTER_XDS_PORT}" # Mutual TLS for the Router's (Envoy's) connection to gateway-controller's main # xDS server, off by default (plaintext) so this stays interoperable with a @@ -161,6 +171,18 @@ export XDS_CLIENT_CERT_PATH="${XDS_CLIENT_CERT_PATH:-}" export XDS_CLIENT_KEY_PATH="${XDS_CLIENT_KEY_PATH:-}" export XDS_CLIENT_CA_PATH="${XDS_CLIENT_CA_PATH:-}" +# gateway-controller's server.xds_tls.port is a dedicated port, separate from +# server.xds_port, that only starts listening once server.xds_tls.enabled=true +# -- so xds_cluster must dial ROUTER_XDS_TLS_PORT instead of ROUTER_XDS_PORT +# once TLS is turned on here, or it will try to open a plaintext connection to +# a port gateway-controller isn't listening on (server.xds_port stops serving +# the moment its TLS listener replaces it). +if [ "${XDS_TLS_ENABLED}" = "true" ]; then + export XDS_SERVER_PORT="${ROUTER_XDS_TLS_PORT}" +else + export XDS_SERVER_PORT="${ROUTER_XDS_PORT}" +fi + # TLS parameters for the same xDS connection, kept in their own vars so the # PQC hybrid group can be opted into independently of turning TLS on at all. # Classical curves only by default -- prepend "X25519MLKEM768" (FIPS 203 @@ -183,15 +205,27 @@ export XDS_CLIENT_TLS_ECDH_CURVES="${XDS_CLIENT_TLS_ECDH_CURVES:-X25519,P-256}" export XDS_CLIENT_TLS_SAN_TYPE="${XDS_CLIENT_TLS_SAN_TYPE:-DNS}" export XDS_CLIENT_TLS_SAN="${XDS_CLIENT_TLS_SAN:-${XDS_SERVER_HOST}}" -# Policy Engine xDS address -PE_XDS_SERVER="${GATEWAY_CONTROLLER_HOST}:${POLICY_ENGINE_XDS_PORT}" +# Policy Engine xDS address. gateway-controller's policy_server.tls.port is a +# dedicated port, separate from policy_server.port, that only starts listening +# once policy_server.tls.enabled=true -- so this must dial +# POLICY_ENGINE_XDS_TLS_PORT instead once the policy-engine's xDS TLS is +# enabled, or it will try to open a plaintext connection to a port +# gateway-controller isn't listening on. Same POLICY_ENGINE_XDS_TLS_ENABLED, +# falling back to XDS_TLS_ENABLED, precedence as +# policy_engine.xds.tls.enabled in config-template.toml. +POLICY_ENGINE_XDS_TLS_ENABLED_EFFECTIVE="${POLICY_ENGINE_XDS_TLS_ENABLED:-${XDS_TLS_ENABLED:-false}}" +if [ "${POLICY_ENGINE_XDS_TLS_ENABLED_EFFECTIVE}" = "true" ]; then + PE_XDS_SERVER="${GATEWAY_CONTROLLER_HOST}:${POLICY_ENGINE_XDS_TLS_PORT}" +else + PE_XDS_SERVER="${GATEWAY_CONTROLLER_HOST}:${POLICY_ENGINE_XDS_PORT}" +fi POLICY_ENGINE_SOCKET="/var/run/api-platform/policy-engine.sock" export PYTHON_EXECUTOR_SOCKET="/var/run/api-platform/python-executor.sock" log "Starting Gateway Runtime" log " Gateway Controller: ${GATEWAY_CONTROLLER_HOST}" -log " Router xDS: ${GATEWAY_CONTROLLER_HOST}:${ROUTER_XDS_PORT}" +log " Router xDS: ${GATEWAY_CONTROLLER_HOST}:${XDS_SERVER_PORT} (TLS: ${XDS_TLS_ENABLED})" log " Policy Engine xDS: ${PE_XDS_SERVER}" log " Log Level: ${LOG_LEVEL}" log " Policy Engine Socket: ${POLICY_ENGINE_SOCKET}" diff --git a/gateway/gateway-runtime/policy-engine/internal/config/traffic_log_sinks_test.go b/gateway/gateway-runtime/policy-engine/internal/config/traffic_log_sinks_test.go index a84b280e1b..f1eb1ef894 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/traffic_log_sinks_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/traffic_log_sinks_test.go @@ -332,6 +332,15 @@ func TestEffectiveShutdownTimeout(t *testing.T) { // own config, so a value that drifts from the code does not just go stale — it // documents a number the gateway will not use. func TestShippedTemplateMatchesTrafficLogDefaults(t *testing.T) { + // xds.tls.enabled defaults to true in the template but ships no cert/key/ca + // path of its own (those are inherently deployment-specific) -- supply + // placeholders so Load's validation (which only checks these are non-empty, + // never that the files exist) passes. This test is about traffic-log + // defaults, not xDS TLS, so the values themselves are unused. + t.Setenv("POLICY_ENGINE_XDS_CLIENT_CERT_PATH", "/dev/null") + t.Setenv("POLICY_ENGINE_XDS_CLIENT_KEY_PATH", "/dev/null") + t.Setenv("POLICY_ENGINE_XDS_CLIENT_CA_PATH", "/dev/null") + cfg, err := Load(filepath.Join("..", "..", "..", "..", "configs", "config-template.toml")) require.NoError(t, err, "the shipped config-template.toml must load and validate") diff --git a/gateway/it/docker-compose.test.postgres.yaml b/gateway/it/docker-compose.test.postgres.yaml index 7ea2d4286f..fad94b6939 100644 --- a/gateway/it/docker-compose.test.postgres.yaml +++ b/gateway/it/docker-compose.test.postgres.yaml @@ -85,8 +85,8 @@ services: ports: - "9090:9090" # REST API - "9092:9092" # Admin API - - "18000:18000" # xDS gRPC - - "18001:18001" + - "18443:18443" # xDS gRPC mTLS (Router) -- unused by this compose (gateway-runtime + - "18444:18444" # xDS gRPC mTLS (Policy Engine) -- dials gateway-controller-xds below), exposed for host debugging only - "9091:9091" # Metrics environment: - APIP_GW_CONTROLLER_STORAGE_TYPE=postgres @@ -106,12 +106,19 @@ services: - IT_TEMPLATE_PATH=/anything - IT_RATE_LIMIT=5 - IT_ALLOW_CREDENTIALS=true + # Enabled by default: mTLS on server.xds_tls / policy_server.tls -- see + # test-config.toml, which reads this same var. Required here even though + # nothing dials this controller's xDS server (see comment above): it shares + # test-config.toml with gateway-controller-xds below, so its own xDS/policy + # servers still start in mTLS mode and need the cert mount below. + - XDS_TLS_ENABLED=true volumes: - controller-data-tests:/app/data - ./it-aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro - ./test-config.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage depends_on: postgres: @@ -156,12 +163,17 @@ services: - IT_TEMPLATE_PATH=/anything - IT_RATE_LIMIT=5 - IT_ALLOW_CREDENTIALS=true + # Enabled by default: mTLS on server.xds_tls (serves Envoy, via gateway-runtime + # below) and policy_server.tls (serves the Policy Engine) -- see test-config.toml, + # which reads this same var. + - XDS_TLS_ENABLED=true volumes: - controller-data-tests:/app/data - ./it-aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro - ./test-config.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage depends_on: gateway-controller: @@ -200,9 +212,21 @@ services: # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage + # Enabled by default, matching gateway-controller-xds above. Client cert/key + # paths for mutual TLS to gateway-controller-xds's xDS servers -- Envoy and the + # policy-engine present distinct client identities, so each leg gets its own + # cert/key, checked against the shared dev CA mounted below. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./coverage/gateway-runtime:/coverage - ./test-config.toml:/etc/policy-engine/config.toml:ro + - ../gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections depends_on: gateway-controller-xds: condition: service_healthy diff --git a/gateway/it/docker-compose.test.sqlserver.yaml b/gateway/it/docker-compose.test.sqlserver.yaml index 12931ef129..31cc7a8f17 100644 --- a/gateway/it/docker-compose.test.sqlserver.yaml +++ b/gateway/it/docker-compose.test.sqlserver.yaml @@ -116,14 +116,17 @@ services: ports: - "9090:9090" # REST API - "9092:9092" # Admin API - - "18000:18000" # xDS gRPC - - "18001:18001" + - "18443:18443" # xDS gRPC mTLS (Router) + - "18444:18444" # xDS gRPC mTLS (Policy Engine) - "9091:9091" # Metrics environment: - APIP_GW_CONTROLLER_STORAGE_TYPE=sqlserver - APIP_GW_CONTROLLER_STORAGE_DATABASE_DSN=sqlserver://sa:${MSSQL_SA_PASSWORD:-Gateway_Strong!Pass123}@sqlserver:1433?database=gateway_test&encrypt=disable&TrustServerCertificate=true&app+name=gateway-controller - APIP_GW_CONTROLLER_LOGGING_LEVEL=debug - GOCOVERDIR=/coverage + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and policy_server.tls + # (serves the Policy Engine) -- see test-config.toml, which reads this same var. + - XDS_TLS_ENABLED=true # Control plane (mock-platform-api) for subscription event propagation - APIP_GW_CONTROLLER_CONTROLPLANE_HOST=it-mock-platform-api:9243 - APIP_GW_CONTROLLER_CONTROLPLANE_TOKEN=test-gateway-token @@ -138,6 +141,7 @@ services: - ./test-config.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage depends_on: sqlserver: @@ -180,9 +184,21 @@ services: # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage + # Enabled by default, matching gateway-controller above. Client cert/key paths + # for mutual TLS to gateway-controller's xDS servers -- Envoy and the + # policy-engine present distinct client identities, so each leg gets its own + # cert/key, checked against the shared dev CA mounted below. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./coverage/gateway-runtime:/coverage - ./test-config.toml:/etc/policy-engine/config.toml:ro + - ../gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections depends_on: gateway-controller: condition: service_healthy diff --git a/gateway/it/docker-compose.test.vhosts-multi.yaml b/gateway/it/docker-compose.test.vhosts-multi.yaml index ba4d7abac6..27e3132eba 100644 --- a/gateway/it/docker-compose.test.vhosts-multi.yaml +++ b/gateway/it/docker-compose.test.vhosts-multi.yaml @@ -33,20 +33,25 @@ services: ports: - "9090:9090" # REST API - "9092:9092" # Admin API - - "18000:18000" # xDS gRPC - - "18001:18001" + - "18443:18443" # xDS gRPC mTLS (Router) + - "18444:18444" # xDS gRPC mTLS (Policy Engine) - "9091:9091" # Metrics environment: - APIP_GW_CONTROLLER_STORAGE_TYPE=sqlite - APIP_GW_CONTROLLER_STORAGE_SQLITE_PATH=./data/gateway.db - APIP_GW_CONTROLLER_LOGGING_LEVEL=debug - GOCOVERDIR=/coverage + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and policy_server.tls + # (serves the Policy Engine) -- see test-config.vhosts-multi.toml, which reads + # this same var. + - XDS_TLS_ENABLED=true volumes: - controller-data-tests:/app/data - ./it-aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro - ./test-config.vhosts-multi.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage # healthcheck: # test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:9092/api/admin/v1/health"] @@ -82,9 +87,21 @@ services: # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage + # Enabled by default, matching gateway-controller above. Client cert/key paths + # for mutual TLS to gateway-controller's xDS servers -- Envoy and the + # policy-engine present distinct client identities, so each leg gets its own + # cert/key, checked against the shared dev CA mounted below. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./coverage/gateway-runtime:/coverage - ./test-config.vhosts-multi.toml:/etc/policy-engine/config.toml:ro + - ../gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections depends_on: # gateway-controller: # condition: service_healthy diff --git a/gateway/it/docker-compose.test.vhosts-single.yaml b/gateway/it/docker-compose.test.vhosts-single.yaml index 9ba503967b..73c578347e 100644 --- a/gateway/it/docker-compose.test.vhosts-single.yaml +++ b/gateway/it/docker-compose.test.vhosts-single.yaml @@ -33,20 +33,25 @@ services: ports: - "9090:9090" # REST API - "9092:9092" # Admin API - - "18000:18000" # xDS gRPC - - "18001:18001" + - "18443:18443" # xDS gRPC mTLS (Router) + - "18444:18444" # xDS gRPC mTLS (Policy Engine) - "9091:9091" # Metrics environment: - APIP_GW_CONTROLLER_STORAGE_TYPE=sqlite - APIP_GW_CONTROLLER_STORAGE_SQLITE_PATH=./data/gateway.db - APIP_GW_CONTROLLER_LOGGING_LEVEL=debug - GOCOVERDIR=/coverage + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and policy_server.tls + # (serves the Policy Engine) -- see test-config.vhosts-single.toml, which reads + # this same var. + - XDS_TLS_ENABLED=true volumes: - controller-data-tests:/app/data - ./it-aesgcm-keys/default-aesgcm256-v1.bin:/app/data/aesgcm-keys/default-aesgcm256-v1.bin:ro - ./test-config.vhosts-single.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage # healthcheck: # test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:9092/api/admin/v1/health"] @@ -82,9 +87,21 @@ services: # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage + # Enabled by default, matching gateway-controller above. Client cert/key paths + # for mutual TLS to gateway-controller's xDS servers -- Envoy and the + # policy-engine present distinct client identities, so each leg gets its own + # cert/key, checked against the shared dev CA mounted below. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./coverage/gateway-runtime:/coverage - ./test-config.vhosts-single.toml:/etc/policy-engine/config.toml:ro + - ../gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections depends_on: # gateway-controller: # condition: service_healthy diff --git a/gateway/it/docker-compose.test.yaml b/gateway/it/docker-compose.test.yaml index b9bd510424..a5a39ada21 100644 --- a/gateway/it/docker-compose.test.yaml +++ b/gateway/it/docker-compose.test.yaml @@ -53,14 +53,17 @@ services: ports: - "9090:9090" # REST API - "9092:9092" # Admin API - - "18000:18000" # xDS gRPC - - "18001:18001" + - "18443:18443" # xDS gRPC mTLS (Router) + - "18444:18444" # xDS gRPC mTLS (Policy Engine) - "9091:9091" # Metrics environment: - APIP_GW_CONTROLLER_STORAGE_TYPE=sqlite - APIP_GW_CONTROLLER_STORAGE_SQLITE_PATH=./data/gateway.db - APIP_GW_CONTROLLER_LOGGING_LEVEL=debug - GOCOVERDIR=/coverage + # Enabled by default: mTLS on server.xds_tls (serves Envoy) and policy_server.tls + # (serves the Policy Engine) -- see test-config.toml, which reads this same var. + - XDS_TLS_ENABLED=true # Control plane (mock-platform-api) for subscription event propagation - APIP_GW_CONTROLLER_CONTROLPLANE_HOST=it-mock-platform-api:9243 - APIP_GW_CONTROLLER_CONTROLPLANE_TOKEN=test-gateway-token @@ -75,6 +78,7 @@ services: - ./test-config.toml:/etc/gateway-controller/config.toml:ro - ../gateway-controller/certificates:/app/certificates - ../gateway-controller/listener-certs:/app/listener-certs:ro + - ../gateway-controller/listener-certs:/app/xds-certs:ro # Read-only: mTLS server certs for server.xds_tls / policy_server.tls - ./coverage/gateway-controller:/coverage depends_on: - mock-platform-api @@ -112,9 +116,21 @@ services: # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage + # Enabled by default, matching gateway-controller above. Client cert/key paths + # for mutual TLS to gateway-controller's xDS servers -- Envoy and the + # policy-engine present distinct client identities, so each leg gets its own + # cert/key, checked against the shared dev CA mounted below. + - XDS_TLS_ENABLED=true + - XDS_CLIENT_CERT_PATH=/etc/xds-certs/envoy-client.crt + - XDS_CLIENT_KEY_PATH=/etc/xds-certs/envoy-client.key + - XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt + - POLICY_ENGINE_XDS_CLIENT_CERT_PATH=/etc/xds-certs/policy-engine-client.crt + - POLICY_ENGINE_XDS_CLIENT_KEY_PATH=/etc/xds-certs/policy-engine-client.key + - POLICY_ENGINE_XDS_CLIENT_CA_PATH=/etc/xds-certs/ca.crt volumes: - ./coverage/gateway-runtime:/coverage - ./test-config.toml:/etc/policy-engine/config.toml:ro + - ../gateway-controller/listener-certs:/etc/xds-certs:ro # Read-only: mTLS client certs for Envoy + policy-engine xDS connections depends_on: # gateway-controller: # condition: service_healthy diff --git a/gateway/it/test-config.toml b/gateway/it/test-config.toml index f24e69227a..3ea3329ce0 100644 --- a/gateway/it/test-config.toml +++ b/gateway/it/test-config.toml @@ -45,6 +45,20 @@ vector_db_provider_ttl = 3600 # 1 hour in seconds # GATEWAY CONTROLLER CONFIGURATION # ============================================================================= +[controller.server.xds_tls] +# mTLS on the main xDS server (serves Envoy), using the shared dev CA/certs in +# ../gateway-controller/listener-certs (provisioned by scripts/setup.sh, mounted at +# /app/xds-certs below). Enabled by default via XDS_TLS_ENABLED (set in the compose +# file); see gateway/configs/config.toml for the full explanation of this pattern. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/envoy"] + +[controller.policy_server.tls] +# Same setup as server.xds_tls above, for the policy xDS server (serves the +# policy-engine). Distinct client identity -- the policy-engine presents its own cert. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/policy-engine"] + [controller.storage] type = '{{ env "APIP_GW_CONTROLLER_STORAGE_TYPE" "sqlite" }}' @@ -112,6 +126,17 @@ request_headers_timeout = "5s" [policy_engine.logging] level = "debug" +[policy_engine.xds.tls] +# Client-side counterpart to controller.policy_server.tls above -- this policy-engine +# subprocess presents its own client identity, distinct from Envoy's, since +# controller.policy_server.tls.allowed_client_identities only allows the +# policy-engine identity. Cert paths come from POLICY_ENGINE_XDS_CLIENT_*_PATH env +# vars (set in the compose file), same fallback pattern as gateway/configs/config.toml. +enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "true") }}' +cert_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CERT_PATH" (env "XDS_CLIENT_CERT_PATH" "") }}' +key_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_KEY_PATH" (env "XDS_CLIENT_KEY_PATH" "") }}' +ca_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CA_PATH" (env "XDS_CLIENT_CA_PATH" "") }}' + [policy_engine.admin.config_dump] # config_dump is off by default in production; enabled here for the IT suite. enabled = true diff --git a/gateway/it/test-config.vhosts-multi.toml b/gateway/it/test-config.vhosts-multi.toml index fbe649c60d..17bf1088e8 100644 --- a/gateway/it/test-config.vhosts-multi.toml +++ b/gateway/it/test-config.vhosts-multi.toml @@ -45,6 +45,20 @@ vector_db_provider_ttl = 3600 # 1 hour in seconds # GATEWAY CONTROLLER CONFIGURATION # ============================================================================= +[controller.server.xds_tls] +# mTLS on the main xDS server (serves Envoy), using the shared dev CA/certs in +# ../gateway-controller/listener-certs (provisioned by scripts/setup.sh, mounted at +# /app/xds-certs below). Enabled by default via XDS_TLS_ENABLED (set in the compose +# file); see gateway/configs/config.toml for the full explanation of this pattern. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/envoy"] + +[controller.policy_server.tls] +# Same setup as server.xds_tls above, for the policy xDS server (serves the +# policy-engine). Distinct client identity -- the policy-engine presents its own cert. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/policy-engine"] + [controller.storage] type = '{{ env "APIP_GW_CONTROLLER_STORAGE_TYPE" "sqlite" }}' @@ -97,6 +111,17 @@ default = "*-sandbox.wso2.com" [policy_engine.logging] level = "debug" +[policy_engine.xds.tls] +# Client-side counterpart to controller.policy_server.tls above -- this policy-engine +# subprocess presents its own client identity, distinct from Envoy's, since +# controller.policy_server.tls.allowed_client_identities only allows the +# policy-engine identity. Cert paths come from POLICY_ENGINE_XDS_CLIENT_*_PATH env +# vars (set in the compose file), same fallback pattern as gateway/configs/config.toml. +enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "true") }}' +cert_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CERT_PATH" (env "XDS_CLIENT_CERT_PATH" "") }}' +key_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_KEY_PATH" (env "XDS_CLIENT_KEY_PATH" "") }}' +ca_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CA_PATH" (env "XDS_CLIENT_CA_PATH" "") }}' + [policy_engine.admin.config_dump] # config_dump is off by default in production; enabled here for the IT suite. enabled = true diff --git a/gateway/it/test-config.vhosts-single.toml b/gateway/it/test-config.vhosts-single.toml index eef60c6a08..69d37ef228 100644 --- a/gateway/it/test-config.vhosts-single.toml +++ b/gateway/it/test-config.vhosts-single.toml @@ -45,6 +45,20 @@ vector_db_provider_ttl = 3600 # 1 hour in seconds # GATEWAY CONTROLLER CONFIGURATION # ============================================================================= +[controller.server.xds_tls] +# mTLS on the main xDS server (serves Envoy), using the shared dev CA/certs in +# ../gateway-controller/listener-certs (provisioned by scripts/setup.sh, mounted at +# /app/xds-certs below). Enabled by default via XDS_TLS_ENABLED (set in the compose +# file); see gateway/configs/config.toml for the full explanation of this pattern. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/envoy"] + +[controller.policy_server.tls] +# Same setup as server.xds_tls above, for the policy xDS server (serves the +# policy-engine). Distinct client identity -- the policy-engine presents its own cert. +enabled = '{{ env "XDS_TLS_ENABLED" "true" }}' +allowed_client_identities = ["spiffe://api-platform/gateway-runtime/policy-engine"] + [controller.storage] type = '{{ env "APIP_GW_CONTROLLER_STORAGE_TYPE" "sqlite" }}' @@ -95,6 +109,17 @@ default = "*-sandbox.wso2.com" [policy_engine.logging] level = "debug" +[policy_engine.xds.tls] +# Client-side counterpart to controller.policy_server.tls above -- this policy-engine +# subprocess presents its own client identity, distinct from Envoy's, since +# controller.policy_server.tls.allowed_client_identities only allows the +# policy-engine identity. Cert paths come from POLICY_ENGINE_XDS_CLIENT_*_PATH env +# vars (set in the compose file), same fallback pattern as gateway/configs/config.toml. +enabled = '{{ env "POLICY_ENGINE_XDS_TLS_ENABLED" (env "XDS_TLS_ENABLED" "true") }}' +cert_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CERT_PATH" (env "XDS_CLIENT_CERT_PATH" "") }}' +key_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_KEY_PATH" (env "XDS_CLIENT_KEY_PATH" "") }}' +ca_path = '{{ env "POLICY_ENGINE_XDS_CLIENT_CA_PATH" (env "XDS_CLIENT_CA_PATH" "") }}' + [policy_engine.admin.config_dump] # config_dump is off by default in production; enabled here for the IT suite. enabled = true diff --git a/gateway/scripts/setup.ps1 b/gateway/scripts/setup.ps1 index 041efa85f5..3f61678ff8 100644 --- a/gateway/scripts/setup.ps1 +++ b/gateway/scripts/setup.ps1 @@ -15,8 +15,11 @@ # mode: this script provisions everything the gateway needs, and the server fails # closed with a descriptive error if a required key or certificate is missing. # -# Provisions: -# - listener-certs/default-listener.{crt,key} : router HTTPS listener certificate +# Provisions (all under one listener-certs/ directory -- see $CertsDir below): +# - listener-certs/default-listener.{crt,key} : router HTTPS listener certificate +# - listener-certs/{ca,server,envoy-client,policy-engine-client}.{crt,key} : xDS mutual TLS +# between gateway-controller and gateway-runtime (Envoy + Policy Engine), enabled by +# default (XDS_TLS_ENABLED in docker-compose.yaml). # - aesgcm-keys/default-aesgcm256-v1.bin : AES-256 at-rest encryption key. The gateway's # docker compose bind-mounts this host file into the controller. # - api-platform.env : required runtime defaults for the gateway-runtime @@ -46,10 +49,17 @@ $DotEnvFile = '.env' $ProjectNamePrefix = 'wso2apip-gateway' $ProjectName = '' -# Router downstream (HTTPS ingress) listener cert/key. Referenced by -# [router.downstream_tls] in config.toml as ./listener-certs/default-listener.{crt,key} -# and mounted into the gateway-controller. The repo checkout keeps these under -# gateway-controller/listener-certs; the distribution zip stages them under resources/. +# Single directory for every TLS cert/key this script provisions: the router downstream +# (HTTPS ingress) listener cert/key (referenced by [router.downstream_tls] in config.toml +# as ./listener-certs/default-listener.{crt,key}, mounted into the gateway-controller at +# that path) and the xDS mutual TLS material (gateway-controller's server.xds_tls/ +# policy_server.tls "server" cert plus the CA and Envoy/Policy-Engine client certs, +# referenced by config.toml's default cert_file/key_file/client_ca_file paths as +# ./xds-certs/... and mounted into both gateway-controller and gateway-runtime). One host +# directory is bind-mounted at both container paths (see docker-compose.yaml) so there's a +# single place to look, even though the two Go-side defaults still expect two directory +# names inside the container. The repo checkout keeps this under +# gateway-controller/listener-certs; the distribution zip stages it under resources/. if (Test-Path -LiteralPath 'gateway-controller/listener-certs' -PathType Container) { $CertsDir = 'gateway-controller/listener-certs' } else { @@ -74,10 +84,10 @@ function Show-Usage { @' Usage: .\scripts\setup.ps1 [--force] [--certs-only] - --force regenerate the certificate and encryption key (rotates them), rewrite api-platform.env, + --force regenerate the certificates and encryption key (rotates them), rewrite api-platform.env, and re-provision the admin credentials (rotates the password) - --certs-only generate only the listener TLS certificate (skip the encryption key, api-platform.env, - and .env) + --certs-only generate only the TLS certificates (listener + xDS mTLS), skip the encryption key, + api-platform.env, and .env Admin credentials (gateway-controller REST/management API basic auth): Set ADMIN_USERNAME and/or ADMIN_PASSWORD in the environment to run non-interactively (CI). @@ -302,6 +312,91 @@ function New-ListenerCert { Write-Log " - self-signed listener certificate generated at $CertsDir/default-listener.crt" } +# Issues one client certificate under $CertsDir/ca.{crt,key}: $Name is the file basename +# (envoy-client / policy-engine-client), $Spiffe is the SPIFFE URI SAN identifying it to +# gateway-controller's allowed_client_identities check, $Tmp is a scratch directory for the CSR +# and extfile (PowerShell has no process-substitution equivalent to setup.sh's <(...), so the +# extension text goes to a real temp file instead). +function New-XdsClientCert([string]$Name, [string]$Spiffe, [string]$Tmp) { + Invoke-OpenSslQuiet { + & openssl ecparam -name prime256v1 -genkey -noout -out "$CertsDir/${Name}.key" + } "openssl failed to generate the $Name key" | Out-Null + Invoke-OpenSslQuiet { + & openssl req -new -key "$CertsDir/${Name}.key" -subj "/O=WSO2 API Platform/CN=$Name" -out "$Tmp/${Name}.csr" + } "openssl failed to generate the $Name CSR" | Out-Null + + $extFile = "$Tmp/${Name}.ext" + $utf8NoBom = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($extFile, "basicConstraints=CA:FALSE`nkeyUsage=digitalSignature`nextendedKeyUsage=clientAuth`nsubjectAltName=URI:$Spiffe`n", $utf8NoBom) + + Invoke-OpenSslQuiet { + & openssl x509 -req -in "$Tmp/${Name}.csr" -CA "$CertsDir/ca.crt" -CAkey "$CertsDir/ca.key" ` + -CAcreateserial -CAserial "$Tmp/ca.srl" -days 825 -sha256 ` + -extfile $extFile -out "$CertsDir/${Name}.crt" + } "openssl failed to sign the $Name certificate" | Out-Null +} + +function New-XdsCerts { + $haveAll = (Test-Path -LiteralPath "$CertsDir/ca.crt") -and (Test-Path -LiteralPath "$CertsDir/server.crt") ` + -and (Test-Path -LiteralPath "$CertsDir/envoy-client.crt") -and (Test-Path -LiteralPath "$CertsDir/policy-engine-client.crt") + if (-not $Force -and $haveAll) { + Set-OwnerOnlyAcl "$CertsDir/ca.key" + Set-OwnerOnlyAcl "$CertsDir/server.key" + Set-OwnerOnlyAcl "$CertsDir/envoy-client.key" + Set-OwnerOnlyAcl "$CertsDir/policy-engine-client.key" + Write-Log " - $CertsDir/*.crt already exist - keeping them" + return + } + New-Item -ItemType Directory -Force -Path $CertsDir | Out-Null + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Force -Path $tmp | Out-Null + try { + Invoke-OpenSslQuiet { + & openssl ecparam -name prime256v1 -genkey -noout -out "$CertsDir/ca.key" + } "openssl failed to generate the xDS CA key" | Out-Null + Invoke-OpenSslQuiet { + & openssl req -x509 -new -key "$CertsDir/ca.key" -sha256 -days 3650 ` + -subj "/O=WSO2 API Platform/CN=API Platform xDS Dev CA" -out "$CertsDir/ca.crt" + } "openssl failed to generate the xDS CA certificate" | Out-Null + + # Server cert presented by gateway-controller on server.xds_tls.port / policy_server.tls.port. + # SANs cover every docker-compose service/container name gateway-runtime dials this + # controller as across every compose file that shares this cert directory -- + # docker-compose.yaml/docker-compose.debug.yaml (gateway-controller), the IT suite's + # single-controller composes (it-gateway-controller), and its Postgres/EventHub replica + # sync compose (it-gateway-controller-xds, the controller gateway-runtime actually + # dials there) -- plus localhost/loopback for a controller reached directly from the + # host while debugging. + Invoke-OpenSslQuiet { + & openssl ecparam -name prime256v1 -genkey -noout -out "$CertsDir/server.key" + } "openssl failed to generate the xDS server key" | Out-Null + Invoke-OpenSslQuiet { + & openssl req -new -key "$CertsDir/server.key" -subj "/O=WSO2 API Platform/CN=gateway-controller" -out "$tmp/server.csr" + } "openssl failed to generate the xDS server CSR" | Out-Null + + $serverExtFile = "$tmp/server.ext" + $utf8NoBom = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($serverExtFile, "basicConstraints=CA:FALSE`nkeyUsage=digitalSignature,keyEncipherment`nextendedKeyUsage=serverAuth`nsubjectAltName=DNS:gateway-controller,DNS:it-gateway-controller,DNS:it-gateway-controller-xds,DNS:localhost,IP:127.0.0.1`n", $utf8NoBom) + + Invoke-OpenSslQuiet { + & openssl x509 -req -in "$tmp/server.csr" -CA "$CertsDir/ca.crt" -CAkey "$CertsDir/ca.key" ` + -CAcreateserial -CAserial "$tmp/ca.srl" -days 825 -sha256 ` + -extfile $serverExtFile -out "$CertsDir/server.crt" + } "openssl failed to sign the xDS server certificate" | Out-Null + + New-XdsClientCert 'envoy-client' 'spiffe://api-platform/gateway-runtime/envoy' $tmp + New-XdsClientCert 'policy-engine-client' 'spiffe://api-platform/gateway-runtime/policy-engine' $tmp + } finally { + Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue + } + + Set-OwnerOnlyAcl "$CertsDir/ca.key" + Set-OwnerOnlyAcl "$CertsDir/server.key" + Set-OwnerOnlyAcl "$CertsDir/envoy-client.key" + Set-OwnerOnlyAcl "$CertsDir/policy-engine-client.key" + Write-Log " - xDS mTLS dev CA + server/client certificates generated at $CertsDir" +} + function New-EncryptionKey { if (-not $Force -and (Test-Path -LiteralPath $EncKeyFile)) { Set-OwnerOnlyAcl $EncKeyFile @@ -492,6 +587,9 @@ function Set-ProjectName { Write-Log 'Provisioning listener TLS certificate ...' New-ListenerCert +Write-Log 'Provisioning xDS mutual TLS certificates ...' +New-XdsCerts + if ($CertsOnly) { exit 0 } @@ -582,6 +680,10 @@ $lines = @( 'GATEWAY_CONTROLLER_HOST=gateway-controller' 'LOG_LEVEL=info' '' + '# xDS mutual TLS between gateway-controller and gateway-runtime (Envoy + Policy Engine) is' + '# enabled by default via XDS_TLS_ENABLED in docker-compose.yaml (not here) - edit it there' + '# (both services) to fall back to plaintext xDS.' + '' "APIP_GW_CONTROLLER_AUTH_BASIC_ADMIN_USERNAME=$adminUsername" "APIP_GW_CONTROLLER_AUTH_BASIC_ADMIN_PASSWORD_HASH=$adminPasswordHash" '' diff --git a/gateway/scripts/setup.sh b/gateway/scripts/setup.sh index 28dd840b98..c0ee03a7b9 100755 --- a/gateway/scripts/setup.sh +++ b/gateway/scripts/setup.sh @@ -13,8 +13,11 @@ # mode: this script provisions everything the gateway needs, and the server fails # closed with a descriptive error if a required key or certificate is missing. # -# Provisions: -# - listener-certs/default-listener.{crt,key} : router HTTPS listener certificate +# Provisions (all under one listener-certs/ directory -- see CERTS_DIR below): +# - listener-certs/default-listener.{crt,key} : router HTTPS listener certificate +# - listener-certs/{ca,server,envoy-client,policy-engine-client}.{crt,key} : xDS mutual TLS +# between gateway-controller and gateway-runtime (Envoy + Policy Engine), enabled by +# default (XDS_TLS_ENABLED in docker-compose.yaml). # - aesgcm-keys/default-aesgcm256-v1.bin : AES-256 at-rest encryption key. The gateway's # docker compose bind-mounts this host file into the controller. # - api-platform.env : required runtime defaults for the gateway-runtime @@ -35,10 +38,17 @@ DOTENV_FILE=".env" PROJECT_NAME_PREFIX="wso2apip-gateway" PROJECT_NAME="" -# Router downstream (HTTPS ingress) listener cert/key. Referenced by -# [router.downstream_tls] in config.toml as ./listener-certs/default-listener.{crt,key} -# and mounted into the gateway-controller. The repo checkout keeps these under -# gateway-controller/listener-certs; the distribution zip stages them under resources/. +# Single directory for every TLS cert/key this script provisions: the router downstream +# (HTTPS ingress) listener cert/key (referenced by [router.downstream_tls] in config.toml +# as ./listener-certs/default-listener.{crt,key}, mounted into the gateway-controller at +# that path) and the xDS mutual TLS material (gateway-controller's server.xds_tls/ +# policy_server.tls "server" cert plus the CA and Envoy/Policy-Engine client certs, +# referenced by config.toml's default cert_file/key_file/client_ca_file paths as +# ./xds-certs/... and mounted into both gateway-controller and gateway-runtime). One host +# directory is bind-mounted at both container paths (see docker-compose.yaml) so there's a +# single place to look, even though the two Go-side defaults still expect two directory +# names inside the container. The repo checkout keeps this under gateway-controller/listener-certs; +# the distribution zip stages it under resources/. if [[ -d gateway-controller/listener-certs ]]; then CERTS_DIR="gateway-controller/listener-certs" else @@ -67,10 +77,10 @@ for arg in "$@"; do cat <<'EOF' Usage: ./scripts/setup.sh [--force] [--certs-only] - --force regenerate the certificate and encryption key (rotates them), rewrite api-platform.env, + --force regenerate the certificates and encryption key (rotates them), rewrite api-platform.env, and re-provision the admin credentials (rotates the password) - --certs-only generate only the listener TLS certificate (skip the encryption key, api-platform.env, - and .env) + --certs-only generate only the TLS certificates (listener + xDS mTLS), skip the encryption key, + api-platform.env, and .env Admin credentials (gateway-controller REST/management API basic auth): Set ADMIN_USERNAME and/or ADMIN_PASSWORD in the environment to run non-interactively (CI). @@ -158,6 +168,68 @@ gen_cert() { log " - self-signed listener certificate generated at $CERTS_DIR/default-listener.crt" } +# Issues one client certificate under $CERTS_DIR/ca.{crt,key}: $1 is the file basename +# (envoy-client / policy-engine-client), $2 is the SPIFFE URI SAN identifying it to +# gateway-controller's allowed_client_identities check, $3 is a scratch directory for the CSR. +gen_xds_client_cert() { + local name="$1" spiffe="$2" tmp="$3" + openssl ecparam -name prime256v1 -genkey -noout -out "$CERTS_DIR/$name.key" 2>/dev/null + openssl req -new -key "$CERTS_DIR/$name.key" \ + -subj "/O=WSO2 API Platform/CN=$name" -out "$tmp/$name.csr" >/dev/null 2>&1 + openssl x509 -req -in "$tmp/$name.csr" -CA "$CERTS_DIR/ca.crt" -CAkey "$CERTS_DIR/ca.key" \ + -CAcreateserial -CAserial "$tmp/ca.srl" -days 825 -sha256 \ + -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=digitalSignature\nextendedKeyUsage=clientAuth\nsubjectAltName=URI:%s\n' "$spiffe") \ + -out "$CERTS_DIR/$name.crt" >/dev/null 2>&1 +} + +gen_xds_certs() { + if [[ "$FORCE" == false && -f "$CERTS_DIR/ca.crt" && -f "$CERTS_DIR/server.crt" \ + && -f "$CERTS_DIR/envoy-client.crt" && -f "$CERTS_DIR/policy-engine-client.crt" ]]; then + chmod "$FILE_MODE" "$CERTS_DIR"/*.crt + restrict_secret_file "$CERTS_DIR/ca.key" + restrict_secret_file "$CERTS_DIR/server.key" + restrict_secret_file "$CERTS_DIR/envoy-client.key" + restrict_secret_file "$CERTS_DIR/policy-engine-client.key" + log " - $CERTS_DIR/*.crt already exist — keeping them" + return + fi + mkdir -p "$CERTS_DIR" + local tmp + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' RETURN + + openssl ecparam -name prime256v1 -genkey -noout -out "$CERTS_DIR/ca.key" 2>/dev/null + openssl req -x509 -new -key "$CERTS_DIR/ca.key" -sha256 -days 3650 \ + -subj "/O=WSO2 API Platform/CN=API Platform xDS Dev CA" \ + -out "$CERTS_DIR/ca.crt" >/dev/null 2>&1 + + # Server cert presented by gateway-controller on server.xds_tls.port / policy_server.tls.port. + # SANs cover every docker-compose service/container name gateway-runtime dials this + # controller as across every compose file that shares this cert directory -- + # docker-compose.yaml/docker-compose.debug.yaml (gateway-controller), the IT suite's + # single-controller composes (it-gateway-controller), and its Postgres/EventHub replica + # sync compose (it-gateway-controller-xds, the controller gateway-runtime actually + # dials there) -- plus localhost/loopback for a controller reached directly from the host + # while debugging. + openssl ecparam -name prime256v1 -genkey -noout -out "$CERTS_DIR/server.key" 2>/dev/null + openssl req -new -key "$CERTS_DIR/server.key" \ + -subj "/O=WSO2 API Platform/CN=gateway-controller" -out "$tmp/server.csr" >/dev/null 2>&1 + openssl x509 -req -in "$tmp/server.csr" -CA "$CERTS_DIR/ca.crt" -CAkey "$CERTS_DIR/ca.key" \ + -CAcreateserial -CAserial "$tmp/ca.srl" -days 825 -sha256 \ + -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:gateway-controller,DNS:it-gateway-controller,DNS:it-gateway-controller-xds,DNS:localhost,IP:127.0.0.1\n') \ + -out "$CERTS_DIR/server.crt" >/dev/null 2>&1 + + gen_xds_client_cert "envoy-client" "spiffe://api-platform/gateway-runtime/envoy" "$tmp" + gen_xds_client_cert "policy-engine-client" "spiffe://api-platform/gateway-runtime/policy-engine" "$tmp" + + chmod "$FILE_MODE" "$CERTS_DIR"/*.crt + restrict_secret_file "$CERTS_DIR/ca.key" + restrict_secret_file "$CERTS_DIR/server.key" + restrict_secret_file "$CERTS_DIR/envoy-client.key" + restrict_secret_file "$CERTS_DIR/policy-engine-client.key" + log " - xDS mTLS dev CA + server/client certificates generated at $CERTS_DIR" +} + gen_encryption_key() { if [[ "$FORCE" == false && -f "$ENC_KEY_FILE" ]]; then restrict_secret_file "$ENC_KEY_FILE" @@ -309,6 +381,9 @@ provision_project_name() { log "Provisioning listener TLS certificate ..." gen_cert +log "Provisioning xDS mutual TLS certificates ..." +gen_xds_certs + if [[ "$CERTS_ONLY" == true ]]; then exit 0 fi @@ -412,6 +487,10 @@ cat > "$ENV_FILE" <