From 5be33645e74fae76d7df54e4afec80353a625d8b Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 17:00:12 +0200 Subject: [PATCH 1/9] fix!: stop leaving the GKE control plane open to every address The public GKE endpoint was always enabled and master authorized networks were only configured when authorized_networks was non-empty, so the default cluster accepted any address. Master authorized networks are now always managed with Google Cloud public IP access disabled. The API is reachable only from authorized_networks, only on the private endpoint with the new enable_private_endpoint, or from any address only with the new allow_public_api_server opt-in. The plan fails until one mode is chosen, and ranges broader than /8 (IPv4) or /16 (IPv6) need the opt-in. In private-endpoint mode the kubectl_config_command output adds --internal-ip. The example takes the allowlist as a required input and uses the module source from this repository. CI runs mock-provider tests for the new rules and validates the example. BREAKING CHANGE: a configuration with an empty authorized_networks no longer plans. Set authorized_networks (an in-place update that adds master_authorized_networks_config), set enable_private_endpoint = true (an in-place update; requires enable_private_nodes), or set allow_public_api_server = true to keep the previous behaviour with no plan change. Configurations that already set authorized_networks keep working; Google Cloud public IP access is now pinned to disabled. --- .github/workflows/ci.yml | 9 ++++ README.md | 38 +++++++++++++++- examples/dev-l4-spot/README.md | 16 +++++-- examples/dev-l4-spot/main.tf | 17 ++++++- main.tf | 9 ++-- outputs.tf | 2 +- tests/api_access.tftest.hcl | 81 ++++++++++++++++++++++++++++++++++ tests/validation.tftest.hcl | 54 +++++++++++++++++++++++ variables.tf | 37 +++++++++++++++- 9 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 tests/api_access.tftest.hcl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce7be55..d2bc36d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,3 +53,12 @@ jobs: - name: Validate Terraform run: terraform validate -no-color + + - name: Run Kubernetes API access tests + run: terraform test -filter=tests/api_access.tftest.hcl -no-color + + - name: Initialize Terraform example + run: terraform -chdir=examples/dev-l4-spot init -backend=false -input=false -no-color + + - name: Validate Terraform example + run: terraform -chdir=examples/dev-l4-spot validate -no-color diff --git a/README.md b/README.md index 67c1a74..fd3e5a5 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,17 @@ Examples in `examples/` use the `infra/` submodule directly and deploy K8s resou ```bash cd examples/dev-l4-spot export TF_VAR_project_id="your-project-id" +# CIDRs allowed to reach the Kubernetes API; include this machine's egress +# address (for example the /32 of `curl -s https://checkip.amazonaws.com`). +export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]' terraform init terraform plan terraform apply ``` +`203.0.113.10/32` is a documentation placeholder. See +[Kubernetes API access](#kubernetes-api-access) for the private-endpoint mode. + After apply, configure kubectl and deploy SIE with chart `0.8.3`. The chart selects the published `v0.8.3` service images and `v0.8.3-cuda12-default` worker image for the GKE overlay. The Terraform module version is independent @@ -94,6 +100,9 @@ This creates a service account with the minimum roles needed to deploy SIE infra | `project_id` | GCP project ID | | `region` | GCP region (e.g., `us-central1`, `europe-west4`) | +You must also choose how the Kubernetes API is reached; see +[Kubernetes API access](#kubernetes-api-access). + ### Cluster | Variable | Default | Description | @@ -156,7 +165,33 @@ pod requesting N GPUs onto a node that advertises N allocatable GPUs. | `services_cidr` | `10.2.0.0/20` | Secondary CIDR range for services | | `enable_private_nodes` | `true` | No public IPs on nodes (Cloud NAT for egress) | | `master_ipv4_cidr_block` | `172.16.0.0/28` | CIDR block for the master network | -| `authorized_networks` | `[]` | CIDRs allowed to access the Kubernetes API | + +### Kubernetes API access + +The GKE control plane is never open to the whole Internet unless you ask for +it. The plan fails until you choose one mode: + +| Variable | Default | Description | +|----------|---------|-------------| +| `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. | +| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. `authorized_networks` can then list internal ranges such as a VPN. | +| `allow_public_api_server` | `false` | Explicit opt-in to accept any Internet address. With an empty `authorized_networks` the module leaves master authorized networks unmanaged. | + +Ranges broader than `/8` (IPv4) or `/16` (IPv6), including `0.0.0.0/0` and +`::/0`, are rejected unless `allow_public_api_server = true`. When master +authorized networks are managed, access from Google Cloud public IP addresses +is disabled. Every request still needs Google authentication. + +**Upgrading from 0.x.** Earlier versions left the public endpoint open to any +address when `authorized_networks` was empty. That configuration now fails the +plan with a message asking you to choose. Configurations that already set +`authorized_networks` keep working; the plan may show +`gcp_public_cidrs_access_enabled = false` if it was enabled outside Terraform. +To restrict an open cluster, set `authorized_networks`; the plan shows an +in-place update that adds `master_authorized_networks_config`. To keep the +previous behaviour explicitly, set `allow_public_api_server = true`; the plan +shows no change to the endpoint. `enable_private_endpoint` is also an in-place +update. ### Node Auto-Provisioning (NAP) @@ -294,6 +329,7 @@ See `infra/gcs_model_cache.tf` and `infra/iam.tf` for the resource definitions a This module follows GCP security best practices out of the box: +- **Restricted control plane** - the Kubernetes API accepts only `authorized_networks`, or only the private endpoint with `enable_private_endpoint`; any-address access needs `allow_public_api_server` - **Private nodes** - worker nodes have no public IPs; egress via Cloud NAT - **Shielded nodes** - Secure Boot and Integrity Monitoring on all node pools - **Workload Identity** - pods use GCP service accounts, no JSON key files diff --git a/examples/dev-l4-spot/README.md b/examples/dev-l4-spot/README.md index 20ea372..654c08c 100644 --- a/examples/dev-l4-spot/README.md +++ b/examples/dev-l4-spot/README.md @@ -6,7 +6,7 @@ Creates a minimal GKE cluster with a single L4 GPU spot node pool - ideal for de | Resource | Configuration | |----------|---------------| -| GKE cluster | Private nodes, Cloud NAT, Workload Identity | +| GKE cluster | Private nodes, Cloud NAT, Workload Identity, API endpoint restricted to `api_server_authorized_ip_ranges` | | GPU node pool | 1x NVIDIA L4 per node (g2-standard-8), spot VMs, scale 0-5 | | CPU node pool | e2-standard-4, scale 1-3 (system workloads) | | Artifact Registry | Docker repository for SIE images | @@ -16,8 +16,15 @@ Creates a minimal GKE cluster with a single L4 GPU spot node pool - ideal for de ## Usage +The Kubernetes API endpoint accepts only the CIDRs you list. Include the +address the machine running kubectl and Helm uses to reach the Internet. +`203.0.113.10/32` below is a documentation placeholder; replace it with your +own range. + ```bash export TF_VAR_project_id="your-gcp-project-id" +curl -s https://checkip.amazonaws.com # your egress address; append /32 +export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]' terraform init terraform plan @@ -43,9 +50,9 @@ helm upgrade --install sie-cluster oci://ghcr.io/superlinked/charts/sie-cluster ``` Chart `0.8.3` selects `v0.8.3` service images and the -`v0.8.3-cuda12-default` worker image. The Terraform module remains independently -versioned at `0.7.2`. The cache arguments above also configure the chart's -required payload-store bucket. +`v0.8.3-cuda12-default` worker image. The example uses the module source from +this checkout; the Terraform module is versioned independently of SIE. The +cache arguments above also configure the chart's required payload-store bucket. ## Variables @@ -56,6 +63,7 @@ required payload-store bucket. | `cluster_name` | `sie-dev` | Cluster name | | `create_artifact_registry` | `true` | Create a Docker registry for SIE images | | `deployer_service_account` | `""` | Service account email (for CI/CD; optional for interactive use) | +| `api_server_authorized_ip_ranges` | _(required)_ | CIDRs allowed to reach the Kubernetes API, such as `["203.0.113.10/32"]`; passed to the module's `authorized_networks` | ## Outputs diff --git a/examples/dev-l4-spot/main.tf b/examples/dev-l4-spot/main.tf index 16512e3..490dd0f 100644 --- a/examples/dev-l4-spot/main.tf +++ b/examples/dev-l4-spot/main.tf @@ -18,6 +18,10 @@ # # Usage: # export TF_VAR_project_id="your-project-id" +# # CIDRs allowed to reach the Kubernetes API. Include the address this +# # machine uses to reach the Internet, for example the /32 of +# # `curl -s https://checkip.amazonaws.com`. +# export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]' # terraform init # terraform plan # terraform apply @@ -79,13 +83,17 @@ variable "deployer_service_account" { default = "" } +variable "api_server_authorized_ip_ranges" { + description = "CIDR blocks allowed to reach the Kubernetes API, such as [\"203.0.113.10/32\"]. Include the egress address of the machine that runs kubectl and helm." + type = list(string) +} + # ============================================================================= # SIE GKE Infra Module # ============================================================================= module "infra" { - source = "superlinked/sie/google" - version = "0.7.2" + source = "../.." project_id = var.project_id region = var.region @@ -101,6 +109,11 @@ module "infra" { # Private cluster with NAT enable_private_nodes = true + # Kubernetes API reachable only from the listed ranges + authorized_networks = [ + for cidr in var.api_server_authorized_ip_ranges : { cidr_block = cidr, display_name = "operator" } + ] + # Node Auto-Provisioning (NAP) enable_node_auto_provisioning = true nap_max_cpu = 100 diff --git a/main.tf b/main.tf index 0b105e1..d1db817 100644 --- a/main.tf +++ b/main.tf @@ -145,14 +145,17 @@ resource "google_container_cluster" "primary" { # Private cluster configuration private_cluster_config { enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = false # Allow public access to master + enable_private_endpoint = var.enable_private_endpoint master_ipv4_cidr_block = var.enable_private_nodes ? var.master_ipv4_cidr_block : null } - # Master authorized networks + # Master authorized networks. An empty cidr_blocks list admits no external + # address; only allow_public_api_server leaves the endpoint unrestricted. dynamic "master_authorized_networks_config" { - for_each = length(var.authorized_networks) > 0 ? [1] : [] + for_each = var.allow_public_api_server && length(var.authorized_networks) == 0 ? [] : [1] content { + gcp_public_cidrs_access_enabled = false + dynamic "cidr_blocks" { for_each = var.authorized_networks content { diff --git a/outputs.tf b/outputs.tf index 03efa86..d5b3962 100644 --- a/outputs.tf +++ b/outputs.tf @@ -184,7 +184,7 @@ output "gpu_node_pools" { output "kubectl_config_command" { description = "Command to configure kubectl for this cluster" - value = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project_id}" + value = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project_id}${var.enable_private_endpoint ? " --internal-ip" : ""}" } # ============================================================================= diff --git a/tests/api_access.tftest.hcl b/tests/api_access.tftest.hcl new file mode 100644 index 0000000..8ccbcec --- /dev/null +++ b/tests/api_access.tftest.hcl @@ -0,0 +1,81 @@ +# SIE GKE Terraform - Kubernetes API access tests +# +# Run with: terraform test -filter=tests/api_access.tftest.hcl +# Uses mock providers, so no cloud credentials are needed. + +mock_provider "google" { + mock_data "google_client_openid_userinfo" { + defaults = { + email = "operator@example.com" + } + } +} + +mock_provider "google-beta" {} +mock_provider "time" {} + +variables { + project_id = "test-project" + region = "us-central1" + cluster_name = "sie-test" +} + +run "rejects_unset_api_access" { + command = plan + + expect_failures = [var.authorized_networks] +} + +run "rejects_ipv4_any_address_without_opt_in" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "0.0.0.0/0", display_name = "any" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_ipv6_any_address_without_opt_in" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "::/0", display_name = "any" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_split_any_address_without_opt_in" { + command = plan + + variables { + authorized_networks = [ + { cidr_block = "0.0.0.0/1", display_name = "low" }, + { cidr_block = "128.0.0.0/1", display_name = "high" }, + ] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_malformed_range" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "203.0.113.10", display_name = "workstation" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_private_endpoint_without_private_nodes" { + command = plan + + variables { + enable_private_nodes = false + enable_private_endpoint = true + } + + expect_failures = [var.enable_private_endpoint] +} diff --git a/tests/validation.tftest.hcl b/tests/validation.tftest.hcl index fd33aa4..7dfc3c2 100644 --- a/tests/validation.tftest.hcl +++ b/tests/validation.tftest.hcl @@ -3,6 +3,10 @@ # Run with: terraform -chdir=deploy/terraform/gcp/infra test # Requires Terraform >= 1.7.0 +variables { + authorized_networks = [{ cidr_block = "203.0.113.10/32", display_name = "test" }] +} + # ============================================================================= # Variable Validation Tests (plan-only, no infrastructure) # ============================================================================= @@ -125,6 +129,56 @@ run "validate_private_cluster_config" { } } +run "validate_api_restricted_to_authorized_networks" { + command = plan + + variables { + project_id = "test-project" + cluster_name = "sie-test" + region = "us-central1" + } + + assert { + condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == false + error_message = "An authorized-networks allowlist should keep the public endpoint enabled" + } + + assert { + condition = ( + google_container_cluster.primary.master_authorized_networks_config[0].gcp_public_cidrs_access_enabled == false + && [for block in google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks : block.cidr_block] == ["203.0.113.10/32"] + ) + error_message = "Master authorized networks should admit only the allowlisted range and no Google Cloud public addresses" + } +} + +run "validate_private_endpoint" { + command = plan + + variables { + project_id = "test-project" + cluster_name = "sie-test" + region = "us-central1" + enable_private_endpoint = true + authorized_networks = [] + } + + assert { + condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == true + error_message = "enable_private_endpoint should disable the public endpoint" + } + + assert { + condition = length(google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks) == 0 + error_message = "Master authorized networks should stay enabled with no external ranges" + } + + assert { + condition = endswith(output.kubectl_config_command, " --internal-ip") + error_message = "The kubectl command should use the private endpoint" + } +} + # ============================================================================= # Security Validation Tests # ============================================================================= diff --git a/variables.tf b/variables.tf index 0323992..9f0ec0f 100644 --- a/variables.tf +++ b/variables.tf @@ -103,12 +103,47 @@ variable "master_ipv4_cidr_block" { } variable "authorized_networks" { - description = "CIDR blocks authorized to access the cluster master" + description = "CIDR blocks authorized to reach the Kubernetes API (master authorized networks). Include every machine that runs kubectl or helm against the cluster. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Ranges broader than /8 (IPv4) or /16 (IPv6), including 0.0.0.0/0, require allow_public_api_server = true." type = list(object({ cidr_block = string display_name = string })) default = [] + + validation { + condition = alltrue([for network in var.authorized_networks : can(cidrhost(network.cidr_block, 0))]) + error_message = "Each authorized_networks[*].cidr_block must be a CIDR block such as 203.0.113.10/32." + } + + validation { + condition = var.allow_public_api_server || alltrue([ + for network in var.authorized_networks : + try(tonumber(split("/", network.cidr_block)[1]) >= (strcontains(network.cidr_block, ":") ? 16 : 8), false) + ]) + error_message = "authorized_networks contains a range broader than /8 (IPv4) or /16 (IPv6), such as 0.0.0.0/0. List the specific ranges that need API access, or set allow_public_api_server = true to accept any Internet address." + } + + validation { + condition = var.enable_private_endpoint || var.allow_public_api_server || length(var.authorized_networks) > 0 + error_message = "Choose how the Kubernetes API is reached: set authorized_networks to the CIDRs that run kubectl and helm (for example your egress address as /32), set enable_private_endpoint = true to serve the API only inside the VPC, or set allow_public_api_server = true to accept any Internet address." + } +} + +variable "enable_private_endpoint" { + description = "Serve the Kubernetes API only on the private endpoint inside the VPC and disable the public endpoint. Requires enable_private_nodes. kubectl and helm must then run from a network that reaches the VPC; authorized_networks can list those internal ranges." + type = bool + default = false + + validation { + condition = !var.enable_private_endpoint || var.enable_private_nodes + error_message = "enable_private_endpoint requires enable_private_nodes = true." + } +} + +variable "allow_public_api_server" { + description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did; ranges broader than /8 (IPv4) or /16 (IPv6) are accepted. Requests still need Google authentication." + type = bool + default = false } # ============================================================================= From 350eb656e267ccc162fc215f16e17c1d74f56565 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 17:17:52 +0200 Subject: [PATCH 2/9] fix: enforce authorized networks on the GKE private endpoint In private-endpoint mode, authorized_networks was not applied to the private endpoint, so listed internal ranges did not restrict access. Set private_endpoint_enforcement_enabled when the private endpoint is used with a non-empty list, and leave it unmanaged otherwise so the module never turns an existing enforcement off. --- README.md | 2 +- main.tf | 3 ++- tests/validation.tftest.hcl | 17 +++++++++++++++++ variables.tf | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fd3e5a5..57ca31d 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ it. The plan fails until you choose one mode: | Variable | Default | Description | |----------|---------|-------------| | `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. | -| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. `authorized_networks` can then list internal ranges such as a VPN. | +| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. A non-empty `authorized_networks` (for example a VPN range) is then enforced on the private endpoint; with an empty list any address in the VPC network can reach it. | | `allow_public_api_server` | `false` | Explicit opt-in to accept any Internet address. With an empty `authorized_networks` the module leaves master authorized networks unmanaged. | Ranges broader than `/8` (IPv4) or `/16` (IPv6), including `0.0.0.0/0` and diff --git a/main.tf b/main.tf index d1db817..2cada2c 100644 --- a/main.tf +++ b/main.tf @@ -154,7 +154,8 @@ resource "google_container_cluster" "primary" { dynamic "master_authorized_networks_config" { for_each = var.allow_public_api_server && length(var.authorized_networks) == 0 ? [] : [1] content { - gcp_public_cidrs_access_enabled = false + gcp_public_cidrs_access_enabled = false + private_endpoint_enforcement_enabled = var.enable_private_endpoint && length(var.authorized_networks) > 0 ? true : null dynamic "cidr_blocks" { for_each = var.authorized_networks diff --git a/tests/validation.tftest.hcl b/tests/validation.tftest.hcl index 7dfc3c2..305ae75 100644 --- a/tests/validation.tftest.hcl +++ b/tests/validation.tftest.hcl @@ -179,6 +179,23 @@ run "validate_private_endpoint" { } } +run "validate_private_endpoint_enforces_authorized_networks" { + command = plan + + variables { + project_id = "test-project" + cluster_name = "sie-test" + region = "us-central1" + enable_private_endpoint = true + authorized_networks = [{ cidr_block = "10.8.0.0/16", display_name = "vpn" }] + } + + assert { + condition = google_container_cluster.primary.master_authorized_networks_config[0].private_endpoint_enforcement_enabled == true + error_message = "Authorized networks should be enforced on the private endpoint when listed in private-endpoint mode" + } +} + # ============================================================================= # Security Validation Tests # ============================================================================= diff --git a/variables.tf b/variables.tf index 9f0ec0f..982d67b 100644 --- a/variables.tf +++ b/variables.tf @@ -130,7 +130,7 @@ variable "authorized_networks" { } variable "enable_private_endpoint" { - description = "Serve the Kubernetes API only on the private endpoint inside the VPC and disable the public endpoint. Requires enable_private_nodes. kubectl and helm must then run from a network that reaches the VPC; authorized_networks can list those internal ranges." + description = "Serve the Kubernetes API only on the private endpoint inside the VPC and disable the public endpoint. Requires enable_private_nodes. kubectl and helm must then run from a network that reaches the VPC. A non-empty authorized_networks is then enforced on the private endpoint, so list those internal ranges; with an empty list any address in the VPC network can reach it." type = bool default = false From 44d82932da1302dfdd5d4b692162062783495fd5 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 17:24:13 +0200 Subject: [PATCH 3/9] docs: note the GKE prerequisite for private-endpoint enforcement --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ca31d..e0f7fa0 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ it. The plan fails until you choose one mode: | Variable | Default | Description | |----------|---------|-------------| | `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. | -| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. A non-empty `authorized_networks` (for example a VPN range) is then enforced on the private endpoint; with an empty list any address in the VPC network can reach it. | +| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. A non-empty `authorized_networks` (for example a VPN range) is then enforced on the private endpoint; with an empty list any address in the VPC network can reach it. Enforcement needs a control plane at GKE `1.28.10-gke.1058000` or later with Envoy enabled; otherwise GKE rejects the update and access stays unchanged. | | `allow_public_api_server` | `false` | Explicit opt-in to accept any Internet address. With an empty `authorized_networks` the module leaves master authorized networks unmanaged. | Ranges broader than `/8` (IPv4) or `/16` (IPv6), including `0.0.0.0/0` and From 507e360fa7161d0bd09cb991a3e7376d9b165bfb Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 18:28:55 +0200 Subject: [PATCH 4/9] fix!: bound GKE authorized networks in aggregate and test the rendered cluster The mock tests only covered the validations, because the cluster_ca_certificate output indexed a master_auth block that mocks cannot produce, so no mocked plan could complete. The output now uses try(), and the CI tests assert what the module renders: the endpoint mode, the managed authorized networks with Google Cloud public access disabled, private-endpoint enforcement, the opt-in path, and the --internal-ip kubectl command. The rendered-cluster runs move out of the credentialed test file. authorized_networks now accepts IPv4 CIDR blocks only, at most 100 entries (the GKE limit), no documentation ranges, and at most one /8 of addresses in total unless allow_public_api_server is set. The new variables are non-nullable. The documentation now states that the private endpoint is reachable in-region from the VPC network and that the list restricts it only in private-endpoint mode. BREAKING CHANGE: authorized_networks rejects IPv6 entries, more than 100 entries, documentation ranges such as 203.0.113.0/24, and lists that cover more than one /8 in total unless allow_public_api_server is set. --- README.md | 49 ++++++--- examples/dev-l4-spot/README.md | 4 +- main.tf | 6 +- outputs.tf | 2 +- tests/api_access.tftest.hcl | 181 +++++++++++++++++++++++++++++++-- tests/validation.tftest.hcl | 69 +------------ variables.tf | 39 +++++-- 7 files changed, 245 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index e0f7fa0..355006f 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ terraform plan terraform apply ``` -`203.0.113.10/32` is a documentation placeholder. See +`203.0.113.10/32` is a documentation placeholder. The module rejects +documentation ranges, so replace it with your own address. See [Kubernetes API access](#kubernetes-api-access) for the private-endpoint mode. After apply, configure kubectl and deploy SIE with chart `0.8.3`. The chart @@ -173,25 +174,43 @@ it. The plan fails until you choose one mode: | Variable | Default | Description | |----------|---------|-------------| -| `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. | -| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint inside the VPC. Requires `enable_private_nodes`. A non-empty `authorized_networks` (for example a VPN range) is then enforced on the private endpoint; with an empty list any address in the VPC network can reach it. Enforcement needs a control plane at GKE `1.28.10-gke.1058000` or later with Envoy enabled; otherwise GKE rejects the update and access stays unchanged. | +| `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. The list restricts the public endpoint, and also the private endpoint in private-endpoint mode. | +| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint. Requires `enable_private_nodes`. The private endpoint is reachable from the cluster's VPC network in the cluster's region; this module does not enable access from other regions. A non-empty `authorized_networks` (for example a VPN range) is then also enforced on the private endpoint; with an empty list any address that reaches the private endpoint is admitted. Enforcement needs a control plane at GKE `1.28.10-gke.1058000` or later with Envoy enabled; otherwise GKE rejects the update and access stays unchanged. | | `allow_public_api_server` | `false` | Explicit opt-in to accept any Internet address. With an empty `authorized_networks` the module leaves master authorized networks unmanaged. | -Ranges broader than `/8` (IPv4) or `/16` (IPv6), including `0.0.0.0/0` and -`::/0`, are rejected unless `allow_public_api_server = true`. When master +Rules for `authorized_networks`: + +- Entries must be IPv4 CIDR blocks, because the module creates an IPv4 cluster. +- At most 100 entries, the GKE limit on authorized networks. +- Together the entries may cover at most 16,777,216 addresses, the size of one + `/8`. `0.0.0.0/0`, split halves such as two `/1` blocks, and several broad + ranges are rejected unless `allow_public_api_server = true`. +- Documentation ranges (`192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`) + are rejected, so an unedited placeholder fails at plan time. + +In the public-endpoint mode the list does not restrict the private endpoint, +which stays reachable from the cluster's VPC network in its region. When master authorized networks are managed, access from Google Cloud public IP addresses is disabled. Every request still needs Google authentication. -**Upgrading from 0.x.** Earlier versions left the public endpoint open to any -address when `authorized_networks` was empty. That configuration now fails the -plan with a message asking you to choose. Configurations that already set -`authorized_networks` keep working; the plan may show -`gcp_public_cidrs_access_enabled = false` if it was enabled outside Terraform. -To restrict an open cluster, set `authorized_networks`; the plan shows an -in-place update that adds `master_authorized_networks_config`. To keep the -previous behaviour explicitly, set `allow_public_api_server = true`; the plan -shows no change to the endpoint. `enable_private_endpoint` is also an in-place -update. +This module installs nothing in the cluster, so if the list stops including +your address, correct `authorized_networks` and apply again to restore access. + +**Upgrading from 0.x.** + +- Earlier versions left the public endpoint open to any address when + `authorized_networks` was empty. That configuration now fails the plan with a + message asking you to choose. +- Configurations that already set `authorized_networks` keep working if the + entries are IPv4, not documentation ranges, at most 100, and no broader than + one `/8` in total (otherwise set `allow_public_api_server = true`). The plan + may show `gcp_public_cidrs_access_enabled = false` if it was enabled outside + Terraform. +- To restrict an open cluster, set `authorized_networks`. The plan shows an + in-place update that adds `master_authorized_networks_config`. +- To keep the previous behaviour explicitly, set + `allow_public_api_server = true`. The plan shows no change to the endpoint. +- `enable_private_endpoint` is also an in-place update. ### Node Auto-Provisioning (NAP) diff --git a/examples/dev-l4-spot/README.md b/examples/dev-l4-spot/README.md index 654c08c..771d673 100644 --- a/examples/dev-l4-spot/README.md +++ b/examples/dev-l4-spot/README.md @@ -18,8 +18,8 @@ Creates a minimal GKE cluster with a single L4 GPU spot node pool - ideal for de The Kubernetes API endpoint accepts only the CIDRs you list. Include the address the machine running kubectl and Helm uses to reach the Internet. -`203.0.113.10/32` below is a documentation placeholder; replace it with your -own range. +`203.0.113.10/32` below is a documentation placeholder. The module rejects +documentation ranges, so replace it with your own address. ```bash export TF_VAR_project_id="your-gcp-project-id" diff --git a/main.tf b/main.tf index 2cada2c..5eecb02 100644 --- a/main.tf +++ b/main.tf @@ -149,8 +149,10 @@ resource "google_container_cluster" "primary" { master_ipv4_cidr_block = var.enable_private_nodes ? var.master_ipv4_cidr_block : null } - # Master authorized networks. An empty cidr_blocks list admits no external - # address; only allow_public_api_server leaves the endpoint unrestricted. + # Master authorized networks restrict the public endpoint; an empty + # cidr_blocks list admits no external address, and only + # allow_public_api_server leaves it unrestricted. They restrict the private + # endpoint only when enforcement is on, which is set in private-endpoint mode. dynamic "master_authorized_networks_config" { for_each = var.allow_public_api_server && length(var.authorized_networks) == 0 ? [] : [1] content { diff --git a/outputs.tf b/outputs.tf index d5b3962..0fd403b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -14,7 +14,7 @@ output "cluster_endpoint" { output "cluster_ca_certificate" { description = "GKE cluster CA certificate (base64 encoded)" - value = google_container_cluster.primary.master_auth[0].cluster_ca_certificate + value = try(google_container_cluster.primary.master_auth[0].cluster_ca_certificate, null) sensitive = true } diff --git a/tests/api_access.tftest.hcl b/tests/api_access.tftest.hcl index 8ccbcec..70f1960 100644 --- a/tests/api_access.tftest.hcl +++ b/tests/api_access.tftest.hcl @@ -4,6 +4,8 @@ # Uses mock providers, so no cloud credentials are needed. mock_provider "google" { + override_during = plan + mock_data "google_client_openid_userinfo" { defaults = { email = "operator@example.com" @@ -11,7 +13,10 @@ mock_provider "google" { } } -mock_provider "google-beta" {} +mock_provider "google-beta" { + override_during = plan +} + mock_provider "time" {} variables { @@ -20,6 +25,10 @@ variables { cluster_name = "sie-test" } +# ============================================================================= +# Rejected configurations +# ============================================================================= + run "rejects_unset_api_access" { command = plan @@ -36,34 +45,77 @@ run "rejects_ipv4_any_address_without_opt_in" { expect_failures = [var.authorized_networks] } -run "rejects_ipv6_any_address_without_opt_in" { +run "rejects_split_any_address_without_opt_in" { command = plan variables { - authorized_networks = [{ cidr_block = "::/0", display_name = "any" }] + authorized_networks = [ + { cidr_block = "0.0.0.0/1", display_name = "low" }, + { cidr_block = "128.0.0.0/1", display_name = "high" }, + ] } expect_failures = [var.authorized_networks] } -run "rejects_split_any_address_without_opt_in" { +run "rejects_aggregate_above_one_slash8_without_opt_in" { command = plan variables { authorized_networks = [ - { cidr_block = "0.0.0.0/1", display_name = "low" }, - { cidr_block = "128.0.0.0/1", display_name = "high" }, + { cidr_block = "11.0.0.0/8", display_name = "a" }, + { cidr_block = "12.0.0.0/8", display_name = "b" }, ] } expect_failures = [var.authorized_networks] } +run "rejects_ipv6_any_address" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "::/0", display_name = "any" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_ipv6_range_for_ipv4_cluster" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "::/16", display_name = "mapped" }] + } + + expect_failures = [var.authorized_networks] +} + run "rejects_malformed_range" { command = plan variables { - authorized_networks = [{ cidr_block = "203.0.113.10", display_name = "workstation" }] + authorized_networks = [{ cidr_block = "8.8.8.8", display_name = "workstation" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_documentation_placeholder" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "203.0.113.10/32", display_name = "placeholder" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_more_than_100_networks" { + command = plan + + variables { + authorized_networks = [for i in range(101) : { cidr_block = cidrsubnet("8.0.0.0/8", 16, i), display_name = "n${i}" }] } expect_failures = [var.authorized_networks] @@ -79,3 +131,118 @@ run "rejects_private_endpoint_without_private_nodes" { expect_failures = [var.enable_private_endpoint] } + +# ============================================================================= +# Accepted configurations +# ============================================================================= + +run "restricts_public_endpoint_to_authorized_networks" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "8.8.8.8/32", display_name = "workstation" }] + } + + assert { + condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == false + error_message = "An authorized-networks allowlist should keep the public endpoint enabled" + } + + assert { + condition = ( + length(google_container_cluster.primary.master_authorized_networks_config) == 1 + && google_container_cluster.primary.master_authorized_networks_config[0].gcp_public_cidrs_access_enabled == false + && [for block in google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks : block.cidr_block] == ["8.8.8.8/32"] + ) + error_message = "Master authorized networks should admit only the allowlisted range and no Google Cloud public addresses" + } + + assert { + condition = !endswith(output.kubectl_config_command, " --internal-ip") + error_message = "The kubectl command should use the public endpoint" + } +} + +run "accepts_networks_totalling_one_slash8" { + command = plan + + variables { + authorized_networks = [ + { cidr_block = "11.0.0.0/9", display_name = "a" }, + { cidr_block = "11.128.0.0/9", display_name = "b" }, + ] + } + + assert { + condition = length(google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks) == 2 + error_message = "Authorized networks covering exactly one /8 should be accepted" + } +} + +run "private_endpoint_disables_public_endpoint" { + command = plan + + variables { + enable_private_endpoint = true + } + + assert { + condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == true + error_message = "enable_private_endpoint should disable the public endpoint" + } + + assert { + condition = ( + length(google_container_cluster.primary.master_authorized_networks_config) == 1 + && length(google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks) == 0 + && google_container_cluster.primary.master_authorized_networks_config[0].gcp_public_cidrs_access_enabled == false + ) + error_message = "Master authorized networks should stay managed with no external ranges" + } + + assert { + condition = endswith(output.kubectl_config_command, " --internal-ip") + error_message = "The kubectl command should use the private endpoint" + } +} + +run "private_endpoint_enforces_listed_networks" { + command = plan + + variables { + enable_private_endpoint = true + authorized_networks = [{ cidr_block = "10.8.0.0/16", display_name = "vpn" }] + } + + assert { + condition = google_container_cluster.primary.master_authorized_networks_config[0].private_endpoint_enforcement_enabled == true + error_message = "Authorized networks should be enforced on the private endpoint in private-endpoint mode" + } +} + +run "opt_in_leaves_authorized_networks_unmanaged" { + command = plan + + variables { + allow_public_api_server = true + } + + assert { + condition = length(google_container_cluster.primary.master_authorized_networks_config) == 0 + error_message = "allow_public_api_server with no networks should leave master authorized networks unmanaged" + } +} + +run "opt_in_accepts_any_address_network" { + command = plan + + variables { + allow_public_api_server = true + authorized_networks = [{ cidr_block = "0.0.0.0/0", display_name = "any" }] + } + + assert { + condition = [for block in google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks : block.cidr_block] == ["0.0.0.0/0"] + error_message = "allow_public_api_server should accept 0.0.0.0/0 in authorized_networks" + } +} diff --git a/tests/validation.tftest.hcl b/tests/validation.tftest.hcl index 305ae75..4c2e766 100644 --- a/tests/validation.tftest.hcl +++ b/tests/validation.tftest.hcl @@ -4,7 +4,7 @@ # Requires Terraform >= 1.7.0 variables { - authorized_networks = [{ cidr_block = "203.0.113.10/32", display_name = "test" }] + authorized_networks = [{ cidr_block = "8.8.8.8/32", display_name = "test" }] } # ============================================================================= @@ -129,73 +129,6 @@ run "validate_private_cluster_config" { } } -run "validate_api_restricted_to_authorized_networks" { - command = plan - - variables { - project_id = "test-project" - cluster_name = "sie-test" - region = "us-central1" - } - - assert { - condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == false - error_message = "An authorized-networks allowlist should keep the public endpoint enabled" - } - - assert { - condition = ( - google_container_cluster.primary.master_authorized_networks_config[0].gcp_public_cidrs_access_enabled == false - && [for block in google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks : block.cidr_block] == ["203.0.113.10/32"] - ) - error_message = "Master authorized networks should admit only the allowlisted range and no Google Cloud public addresses" - } -} - -run "validate_private_endpoint" { - command = plan - - variables { - project_id = "test-project" - cluster_name = "sie-test" - region = "us-central1" - enable_private_endpoint = true - authorized_networks = [] - } - - assert { - condition = google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == true - error_message = "enable_private_endpoint should disable the public endpoint" - } - - assert { - condition = length(google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks) == 0 - error_message = "Master authorized networks should stay enabled with no external ranges" - } - - assert { - condition = endswith(output.kubectl_config_command, " --internal-ip") - error_message = "The kubectl command should use the private endpoint" - } -} - -run "validate_private_endpoint_enforces_authorized_networks" { - command = plan - - variables { - project_id = "test-project" - cluster_name = "sie-test" - region = "us-central1" - enable_private_endpoint = true - authorized_networks = [{ cidr_block = "10.8.0.0/16", display_name = "vpn" }] - } - - assert { - condition = google_container_cluster.primary.master_authorized_networks_config[0].private_endpoint_enforcement_enabled == true - error_message = "Authorized networks should be enforced on the private endpoint when listed in private-endpoint mode" - } -} - # ============================================================================= # Security Validation Tests # ============================================================================= diff --git a/variables.tf b/variables.tf index 982d67b..c2087a2 100644 --- a/variables.tf +++ b/variables.tf @@ -103,24 +103,41 @@ variable "master_ipv4_cidr_block" { } variable "authorized_networks" { - description = "CIDR blocks authorized to reach the Kubernetes API (master authorized networks). Include every machine that runs kubectl or helm against the cluster. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Ranges broader than /8 (IPv4) or /16 (IPv6), including 0.0.0.0/0, require allow_public_api_server = true." + description = "IPv4 CIDR blocks authorized to reach the Kubernetes API (master authorized networks), at most 100. Include every machine that runs kubectl or helm against the cluster. The list applies to the public endpoint, and also to the private endpoint when enable_private_endpoint = true. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Together the ranges may cover at most 16,777,216 addresses (one /8) unless allow_public_api_server = true. Documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) are rejected." type = list(object({ cidr_block = string display_name = string })) - default = [] + default = [] + nullable = false validation { - condition = alltrue([for network in var.authorized_networks : can(cidrhost(network.cidr_block, 0))]) - error_message = "Each authorized_networks[*].cidr_block must be a CIDR block such as 203.0.113.10/32." + condition = alltrue([for network in var.authorized_networks : can(cidrhost(network.cidr_block, 0)) && !strcontains(network.cidr_block, ":")]) + error_message = "Each authorized_networks[*].cidr_block must be an IPv4 CIDR block (an address with a prefix length, such as /32). The module creates an IPv4 cluster." } validation { - condition = var.allow_public_api_server || alltrue([ - for network in var.authorized_networks : - try(tonumber(split("/", network.cidr_block)[1]) >= (strcontains(network.cidr_block, ":") ? 16 : 8), false) + condition = length(var.authorized_networks) <= 100 + error_message = "authorized_networks accepts at most 100 entries, the GKE limit on authorized networks (https://cloud.google.com/kubernetes-engine/docs/how-to/latest/network-isolation)." + } + + validation { + condition = alltrue([ + for network in var.authorized_networks : !try( + tonumber(split("/", network.cidr_block)[1]) >= 24 + && contains(["192.0.2", "198.51.100", "203.0.113"], join(".", slice(split(".", cidrhost(network.cidr_block, 0)), 0, 3))), + false + ) ]) - error_message = "authorized_networks contains a range broader than /8 (IPv4) or /16 (IPv6), such as 0.0.0.0/0. List the specific ranges that need API access, or set allow_public_api_server = true to accept any Internet address." + error_message = "authorized_networks contains a documentation range (192.0.2.0/24, 198.51.100.0/24, or 203.0.113.0/24), such as the README placeholder. Replace it with the real egress address of the machines that need API access." + } + + validation { + condition = var.allow_public_api_server || try( + sum(concat([0], [for network in var.authorized_networks : pow(2, 32 - tonumber(split("/", network.cidr_block)[1]))])) <= pow(2, 24), + false + ) + error_message = "authorized_networks covers more than 16,777,216 addresses (one /8) in total, for example 0.0.0.0/0 or several broad ranges. List the specific ranges that need API access, or set allow_public_api_server = true to accept any Internet address." } validation { @@ -130,9 +147,10 @@ variable "authorized_networks" { } variable "enable_private_endpoint" { - description = "Serve the Kubernetes API only on the private endpoint inside the VPC and disable the public endpoint. Requires enable_private_nodes. kubectl and helm must then run from a network that reaches the VPC. A non-empty authorized_networks is then enforced on the private endpoint, so list those internal ranges; with an empty list any address in the VPC network can reach it." + description = "Serve the Kubernetes API only on the private endpoint and disable the public endpoint. Requires enable_private_nodes. The private endpoint is reachable from the cluster's VPC network in the cluster's region; this module does not enable access from other regions. kubectl and helm must run from such a network. A non-empty authorized_networks is then also enforced on the private endpoint, so list those internal ranges; with an empty list any address that reaches the private endpoint is admitted." type = bool default = false + nullable = false validation { condition = !var.enable_private_endpoint || var.enable_private_nodes @@ -141,9 +159,10 @@ variable "enable_private_endpoint" { } variable "allow_public_api_server" { - description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did; ranges broader than /8 (IPv4) or /16 (IPv6) are accepted. Requests still need Google authentication." + description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did, and authorized_networks may cover more than one /8 in total. Requests still need Google authentication." type = bool default = false + nullable = false } # ============================================================================= From 0ba1d85ee035fc8da9505a01e1e0f4bae7aa0db5 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 18:39:33 +0200 Subject: [PATCH 5/9] docs: qualify GKE public-IP propagation and the authentication statement GKE can take several hours to enforce disabled Google Cloud public IP access, so the upgrade note asks operators to verify effective access. Requests are authenticated by the Kubernetes API (including ServiceAccount tokens), not only by Google identities, so the wording now says so. --- README.md | 7 +++++-- variables.tf | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 355006f..09262e5 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,8 @@ Rules for `authorized_networks`: In the public-endpoint mode the list does not restrict the private endpoint, which stays reachable from the cluster's VPC network in its region. When master authorized networks are managed, access from Google Cloud public IP addresses -is disabled. Every request still needs Google authentication. +is disabled. Network restrictions are in addition to Kubernetes API +authentication and authorization, which every request must still pass. This module installs nothing in the cluster, so if the list stops including your address, correct `authorized_networks` and apply again to restore access. @@ -205,7 +206,9 @@ your address, correct `authorized_networks` and apply again to restore access. entries are IPv4, not documentation ranges, at most 100, and no broader than one `/8` in total (otherwise set `allow_public_api_server = true`). The plan may show `gcp_public_cidrs_access_enabled = false` if it was enabled outside - Terraform. + Terraform. GKE documents that this change can take several hours to be + enforced, so verify the effective access before treating the endpoint as + restricted. - To restrict an open cluster, set `authorized_networks`. The plan shows an in-place update that adds `master_authorized_networks_config`. - To keep the previous behaviour explicitly, set diff --git a/variables.tf b/variables.tf index c2087a2..e5c3b84 100644 --- a/variables.tf +++ b/variables.tf @@ -159,7 +159,7 @@ variable "enable_private_endpoint" { } variable "allow_public_api_server" { - description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did, and authorized_networks may cover more than one /8 in total. Requests still need Google authentication." + description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did, and authorized_networks may cover more than one /8 in total. Requests still need Kubernetes API authentication and authorization." type = bool default = false nullable = false From e2678fb35dd2ea3a6033f4081ce6a3ba8793d113 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 18:43:10 +0200 Subject: [PATCH 6/9] fix: reject GKE authorized networks that overlap documentation ranges The documentation-range guard only checked entries of /24 or narrower, so a broader range that contains a documentation range passed. Entries inside a documentation range are now always rejected, and broader entries that contain one need allow_public_api_server, so an explicit 0.0.0.0/0 under the opt-in still works. --- README.md | 5 +++-- tests/api_access.tftest.hcl | 21 +++++++++++++++++++++ variables.tf | 11 +++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 09262e5..ee70ea3 100644 --- a/README.md +++ b/README.md @@ -185,8 +185,9 @@ Rules for `authorized_networks`: - Together the entries may cover at most 16,777,216 addresses, the size of one `/8`. `0.0.0.0/0`, split halves such as two `/1` blocks, and several broad ranges are rejected unless `allow_public_api_server = true`. -- Documentation ranges (`192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`) - are rejected, so an unedited placeholder fails at plan time. +- Entries inside a documentation range (`192.0.2.0/24`, `198.51.100.0/24`, + `203.0.113.0/24`) are rejected, so an unedited placeholder fails at plan + time. Broader entries that contain one need `allow_public_api_server = true`. In the public-endpoint mode the list does not restrict the private endpoint, which stays reachable from the cluster's VPC network in its region. When master diff --git a/tests/api_access.tftest.hcl b/tests/api_access.tftest.hcl index 70f1960..b542090 100644 --- a/tests/api_access.tftest.hcl +++ b/tests/api_access.tftest.hcl @@ -111,6 +111,27 @@ run "rejects_documentation_placeholder" { expect_failures = [var.authorized_networks] } +run "rejects_documentation_placeholder_even_with_opt_in" { + command = plan + + variables { + allow_public_api_server = true + authorized_networks = [{ cidr_block = "203.0.113.10/32", display_name = "placeholder" }] + } + + expect_failures = [var.authorized_networks] +} + +run "rejects_range_containing_documentation_range" { + command = plan + + variables { + authorized_networks = [{ cidr_block = "203.0.112.0/23", display_name = "wide" }] + } + + expect_failures = [var.authorized_networks] +} + run "rejects_more_than_100_networks" { command = plan diff --git a/variables.tf b/variables.tf index e5c3b84..fa376de 100644 --- a/variables.tf +++ b/variables.tf @@ -103,7 +103,7 @@ variable "master_ipv4_cidr_block" { } variable "authorized_networks" { - description = "IPv4 CIDR blocks authorized to reach the Kubernetes API (master authorized networks), at most 100. Include every machine that runs kubectl or helm against the cluster. The list applies to the public endpoint, and also to the private endpoint when enable_private_endpoint = true. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Together the ranges may cover at most 16,777,216 addresses (one /8) unless allow_public_api_server = true. Documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) are rejected." + description = "IPv4 CIDR blocks authorized to reach the Kubernetes API (master authorized networks), at most 100. Include every machine that runs kubectl or helm against the cluster. The list applies to the public endpoint, and also to the private endpoint when enable_private_endpoint = true. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Together the ranges may cover at most 16,777,216 addresses (one /8) unless allow_public_api_server = true. Entries inside a documentation range (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) are rejected, and broader entries that contain one need allow_public_api_server = true." type = list(object({ cidr_block = string display_name = string @@ -124,12 +124,15 @@ variable "authorized_networks" { validation { condition = alltrue([ for network in var.authorized_networks : !try( - tonumber(split("/", network.cidr_block)[1]) >= 24 - && contains(["192.0.2", "198.51.100", "203.0.113"], join(".", slice(split(".", cidrhost(network.cidr_block, 0)), 0, 3))), + (tonumber(split("/", network.cidr_block)[1]) >= 24 || !var.allow_public_api_server) + && anytrue([ + for doc in ["192.0.2.0", "198.51.100.0", "203.0.113.0"] : + cidrhost("${cidrhost(network.cidr_block, 0)}/${min(tonumber(split("/", network.cidr_block)[1]), 24)}", 0) == cidrhost("${doc}/${min(tonumber(split("/", network.cidr_block)[1]), 24)}", 0) + ]), false ) ]) - error_message = "authorized_networks contains a documentation range (192.0.2.0/24, 198.51.100.0/24, or 203.0.113.0/24), such as the README placeholder. Replace it with the real egress address of the machines that need API access." + error_message = "authorized_networks overlaps a documentation range (192.0.2.0/24, 198.51.100.0/24, or 203.0.113.0/24), such as the README placeholder. Replace it with the real egress address of the machines that need API access. A broader range that contains a documentation range needs allow_public_api_server = true." } validation { From c4bf4effeace969ebb0e8b66a6f8b729fa2d511f Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 18:48:03 +0200 Subject: [PATCH 7/9] docs: note the unauthenticated health endpoints in the access wording Kubernetes serves /healthz, /livez, /readyz and /version without authentication, so the opt-in description and README now say that other requests still need Kubernetes API authentication and authorization. --- README.md | 4 +++- variables.tf | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee70ea3..8bc8cef 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,9 @@ In the public-endpoint mode the list does not restrict the private endpoint, which stays reachable from the cluster's VPC network in its region. When master authorized networks are managed, access from Google Cloud public IP addresses is disabled. Network restrictions are in addition to Kubernetes API -authentication and authorization, which every request must still pass. +authentication and authorization. Apart from the unauthenticated health and +version endpoints (such as `/healthz`, `/readyz`, and `/version`), every +request must still pass them. This module installs nothing in the cluster, so if the list stops including your address, correct `authorized_networks` and apply again to restore access. diff --git a/variables.tf b/variables.tf index fa376de..9cb8a63 100644 --- a/variables.tf +++ b/variables.tf @@ -162,7 +162,7 @@ variable "enable_private_endpoint" { } variable "allow_public_api_server" { - description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did, and authorized_networks may cover more than one /8 in total. Requests still need Kubernetes API authentication and authorization." + description = "Opt in to a public Kubernetes API endpoint that accepts any Internet address. With an empty authorized_networks the module leaves master authorized networks unmanaged, as earlier versions did, and authorized_networks may cover more than one /8 in total. Apart from the unauthenticated health and version endpoints (such as /healthz, /readyz and /version), requests still need Kubernetes API authentication and authorization." type = bool default = false nullable = false From 3f7d90ba1dfa263c440f515dd509b7917ace564a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Tue, 29 Sep 2026 19:42:23 +0200 Subject: [PATCH 8/9] fix: apply the GKE aggregate allowlist limit only to the public endpoint In private-endpoint mode authorized_networks restricts the private endpoint and holds internal ranges, so a list with all three RFC 1918 ranges was rejected unless allow_public_api_server was set, and that opt-in stayed set if private mode was later turned off. The one-/8 total now applies only while the public endpoint is enabled. The per-entry IPv4 and documentation-range checks apply in both modes. Tests cover the RFC 1918 list in private mode (accepted) and in public mode (rejected). --- README.md | 9 ++++++--- tests/api_access.tftest.hcl | 36 ++++++++++++++++++++++++++++++++++++ variables.tf | 6 +++--- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8bc8cef..d66e9e5 100644 --- a/README.md +++ b/README.md @@ -182,9 +182,12 @@ Rules for `authorized_networks`: - Entries must be IPv4 CIDR blocks, because the module creates an IPv4 cluster. - At most 100 entries, the GKE limit on authorized networks. -- Together the entries may cover at most 16,777,216 addresses, the size of one - `/8`. `0.0.0.0/0`, split halves such as two `/1` blocks, and several broad - ranges are rejected unless `allow_public_api_server = true`. +- With the public endpoint enabled, the entries together may cover at most + 16,777,216 addresses, the size of one `/8`. `0.0.0.0/0`, split halves such as + two `/1` blocks, and several broad ranges are rejected unless + `allow_public_api_server = true`. In private-endpoint mode the list holds + internal ranges (for example all three RFC 1918 ranges) and has no total + limit. - Entries inside a documentation range (`192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`) are rejected, so an unedited placeholder fails at plan time. Broader entries that contain one need `allow_public_api_server = true`. diff --git a/tests/api_access.tftest.hcl b/tests/api_access.tftest.hcl index b542090..9536ce2 100644 --- a/tests/api_access.tftest.hcl +++ b/tests/api_access.tftest.hcl @@ -227,6 +227,42 @@ run "private_endpoint_disables_public_endpoint" { } } +run "rejects_rfc1918_ranges_on_public_endpoint_without_opt_in" { + command = plan + + variables { + authorized_networks = [ + { cidr_block = "10.0.0.0/8", display_name = "rfc1918-10" }, + { cidr_block = "172.16.0.0/12", display_name = "rfc1918-172" }, + { cidr_block = "192.168.0.0/16", display_name = "rfc1918-192" }, + ] + } + + expect_failures = [var.authorized_networks] +} + +run "private_endpoint_accepts_all_rfc1918_ranges" { + command = plan + + variables { + enable_private_endpoint = true + authorized_networks = [ + { cidr_block = "10.0.0.0/8", display_name = "rfc1918-10" }, + { cidr_block = "172.16.0.0/12", display_name = "rfc1918-172" }, + { cidr_block = "192.168.0.0/16", display_name = "rfc1918-192" }, + ] + } + + assert { + condition = ( + google_container_cluster.primary.private_cluster_config[0].enable_private_endpoint == true + && length(google_container_cluster.primary.master_authorized_networks_config[0].cidr_blocks) == 3 + && google_container_cluster.primary.master_authorized_networks_config[0].private_endpoint_enforcement_enabled == true + ) + error_message = "Private-endpoint mode should accept internal ranges beyond one /8 in total without allow_public_api_server" + } +} + run "private_endpoint_enforces_listed_networks" { command = plan diff --git a/variables.tf b/variables.tf index 9cb8a63..dc8992c 100644 --- a/variables.tf +++ b/variables.tf @@ -103,7 +103,7 @@ variable "master_ipv4_cidr_block" { } variable "authorized_networks" { - description = "IPv4 CIDR blocks authorized to reach the Kubernetes API (master authorized networks), at most 100. Include every machine that runs kubectl or helm against the cluster. The list applies to the public endpoint, and also to the private endpoint when enable_private_endpoint = true. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. Together the ranges may cover at most 16,777,216 addresses (one /8) unless allow_public_api_server = true. Entries inside a documentation range (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) are rejected, and broader entries that contain one need allow_public_api_server = true." + description = "IPv4 CIDR blocks authorized to reach the Kubernetes API (master authorized networks), at most 100. Include every machine that runs kubectl or helm against the cluster. The list applies to the public endpoint, and also to the private endpoint when enable_private_endpoint = true. Leave empty only with enable_private_endpoint = true or allow_public_api_server = true. With the public endpoint enabled, the ranges together may cover at most 16,777,216 addresses (one /8) unless allow_public_api_server = true; in private-endpoint mode the list holds internal ranges and has no total limit. Entries inside a documentation range (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) are rejected, and broader entries that contain one need allow_public_api_server = true." type = list(object({ cidr_block = string display_name = string @@ -136,11 +136,11 @@ variable "authorized_networks" { } validation { - condition = var.allow_public_api_server || try( + condition = var.allow_public_api_server || var.enable_private_endpoint || try( sum(concat([0], [for network in var.authorized_networks : pow(2, 32 - tonumber(split("/", network.cidr_block)[1]))])) <= pow(2, 24), false ) - error_message = "authorized_networks covers more than 16,777,216 addresses (one /8) in total, for example 0.0.0.0/0 or several broad ranges. List the specific ranges that need API access, or set allow_public_api_server = true to accept any Internet address." + error_message = "With the public endpoint enabled, authorized_networks covers more than 16,777,216 addresses (one /8) in total, for example 0.0.0.0/0 or several broad ranges. List the specific ranges that need API access, set enable_private_endpoint = true if the list holds internal ranges for the private endpoint, or set allow_public_api_server = true to accept any Internet address." } validation { From 548c2e9055fd4ae0028924966729c85d7e4e48cc Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Wed, 30 Sep 2026 13:08:59 +0200 Subject: [PATCH 9/9] fix: restore the Registry module source in the GKE example The example is meant to be copied and used on its own, so it pins superlinked/sie/google from the Terraform Registry again instead of a local path. Before the release-please setup, each release commit bumped this pin to the version being released. release-please keeps doing that through extra-files: the version line carries an x-release-please-version marker, so the release pull request sets it to the new module version and every tag names its own release. CI still validates the example against the module code in the pull request. A workflow step rewrites the checked-out example to source "../.." and drops the marked version line before init and validate. The rewrite is never committed, and the step fails if it did not apply. A Terraform override file cannot do this, because it cannot remove the version argument, and a version constraint is rejected for a local module path. --- .github/workflows/ci.yml | 13 +++++++++++++ examples/dev-l4-spot/README.md | 6 +++--- examples/dev-l4-spot/main.tf | 3 ++- release-please-config.json | 3 +++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2bc36d..1d1e6d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,19 @@ jobs: - name: Run Kubernetes API access tests run: terraform test -filter=tests/api_access.tftest.hcl -no-color + - name: Point the example at the module source in this pull request + run: | + file=examples/dev-l4-spot/main.tf + sed -i -E \ + -e 's#^([[:space:]]*source[[:space:]]*=[[:space:]]*)"superlinked/sie/google"#\1"../.."#' \ + -e '/x-release-please-version/d' \ + "$file" + if ! grep -Eq '^[[:space:]]*source[[:space:]]*=[[:space:]]*"\.\./\.\."' "$file" \ + || grep -Eq '"superlinked/sie/google"|x-release-please-version' "$file"; then + echo "::error file=$file::could not point the example at the local module source" + exit 1 + fi + - name: Initialize Terraform example run: terraform -chdir=examples/dev-l4-spot init -backend=false -input=false -no-color diff --git a/examples/dev-l4-spot/README.md b/examples/dev-l4-spot/README.md index 771d673..25b2b2f 100644 --- a/examples/dev-l4-spot/README.md +++ b/examples/dev-l4-spot/README.md @@ -50,9 +50,9 @@ helm upgrade --install sie-cluster oci://ghcr.io/superlinked/charts/sie-cluster ``` Chart `0.8.3` selects `v0.8.3` service images and the -`v0.8.3-cuda12-default` worker image. The example uses the module source from -this checkout; the Terraform module is versioned independently of SIE. The -cache arguments above also configure the chart's required payload-store bucket. +`v0.8.3-cuda12-default` worker image. The example pins the Terraform module +release from the Registry, which is versioned independently of SIE. The cache +arguments above also configure the chart's required payload-store bucket. ## Variables diff --git a/examples/dev-l4-spot/main.tf b/examples/dev-l4-spot/main.tf index 490dd0f..b3eaa4e 100644 --- a/examples/dev-l4-spot/main.tf +++ b/examples/dev-l4-spot/main.tf @@ -93,7 +93,8 @@ variable "api_server_authorized_ip_ranges" { # ============================================================================= module "infra" { - source = "../.." + source = "superlinked/sie/google" + version = "0.7.3" # x-release-please-version project_id = var.project_id region = var.region diff --git a/release-please-config.json b/release-please-config.json index 31e0291..47ae8cf 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -8,6 +8,9 @@ "include-v-in-release-name": true, "exclude-paths": [ ".github" + ], + "extra-files": [ + "examples/dev-l4-spot/main.tf" ] } }