fix!: stop leaving the GKE control plane open to every address

.github/workflows/ci.yml

@@ -53,3 +53,25 @@ jobs:
 
       - name: Validate Terraform
         run: terraform validate -no-color
+
+      - name: Run Kubernetes API access tests
+        run: terraform test -filter=tests/api_access.tftest.hcl -no-color
+
+      - name: Point the example at the module source in this pull request
+        run: |
+          file=examples/dev-l4-spot/main.tf
+          sed -i -E \
+            -e 's#^([[:space:]]*source[[:space:]]*=[[:space:]]*)"superlinked/sie/google"#\1"../.."#' \
+            -e '/x-release-please-version/d' \
+            "$file"
+          if ! grep -Eq '^[[:space:]]*source[[:space:]]*=[[:space:]]*"\.\./\.\."' "$file" \
+            || grep -Eq '"superlinked/sie/google"|x-release-please-version' "$file"; then
+            echo "::error file=$file::could not point the example at the local module source"
+            exit 1
+          fi
+
+      - name: Initialize Terraform example
+        run: terraform -chdir=examples/dev-l4-spot init -backend=false -input=false -no-color
+
+      - name: Validate Terraform example
+        run: terraform -chdir=examples/dev-l4-spot validate -no-color

README.md

@@ -31,11 +31,18 @@ Examples in `examples/` use the `infra/` submodule directly and deploy K8s resou
 ```bash
 cd examples/dev-l4-spot
 export TF_VAR_project_id="your-project-id"
+# CIDRs allowed to reach the Kubernetes API; include this machine's egress
+# address (for example the /32 of `curl -s https://checkip.amazonaws.com`).
+export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]'
 terraform init
 terraform plan
 terraform apply
 ```
 
+`203.0.113.10/32` is a documentation placeholder. The module rejects
+documentation ranges, so replace it with your own address. See
+[Kubernetes API access](#kubernetes-api-access) for the private-endpoint mode.
+
 After apply, configure kubectl and deploy SIE with chart `0.8.3`. The chart
 selects the published `v0.8.3` service images and `v0.8.3-cuda12-default`
 worker image for the GKE overlay. The Terraform module version is independent
@@ -94,6 +101,9 @@ This creates a service account with the minimum roles needed to deploy SIE infra
 | `project_id` | GCP project ID |
 | `region` | GCP region (e.g., `us-central1`, `europe-west4`) |
 
+You must also choose how the Kubernetes API is reached; see
+[Kubernetes API access](#kubernetes-api-access).
+
 ### Cluster
 
 | Variable | Default | Description |
@@ -156,7 +166,60 @@ pod requesting N GPUs onto a node that advertises N allocatable GPUs.
 | `services_cidr` | `10.2.0.0/20` | Secondary CIDR range for services |
 | `enable_private_nodes` | `true` | No public IPs on nodes (Cloud NAT for egress) |
 | `master_ipv4_cidr_block` | `172.16.0.0/28` | CIDR block for the master network |
-| `authorized_networks` | `[]` | CIDRs allowed to access the Kubernetes API |
+
+### Kubernetes API access
+
+The GKE control plane is never open to the whole Internet unless you ask for
+it. The plan fails until you choose one mode:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `authorized_networks` | `[]` | CIDRs (with display names) allowed to reach the Kubernetes API through master authorized networks. Include every machine that runs `kubectl` or `helm` against the cluster. The list restricts the public endpoint, and also the private endpoint in private-endpoint mode. |
+| `enable_private_endpoint` | `false` | Disable the public endpoint and serve the API only on the private endpoint. Requires `enable_private_nodes`. The private endpoint is reachable from the cluster's VPC network in the cluster's region; this module does not enable access from other regions. A non-empty `authorized_networks` (for example a VPN range) is then also enforced on the private endpoint; with an empty list any address that reaches the private endpoint is admitted. Enforcement needs a control plane at GKE `1.28.10-gke.1058000` or later with Envoy enabled; otherwise GKE rejects the update and access stays unchanged. |
+| `allow_public_api_server` | `false` | Explicit opt-in to accept any Internet address. With an empty `authorized_networks` the module leaves master authorized networks unmanaged. |
+
+Rules for `authorized_networks`:
+
+- Entries must be IPv4 CIDR blocks, because the module creates an IPv4 cluster.
+- At most 100 entries, the GKE limit on authorized networks.
+- With the public endpoint enabled, the entries together may cover at most
+  16,777,216 addresses, the size of one `/8`. `0.0.0.0/0`, split halves such as
+  two `/1` blocks, and several broad ranges are rejected unless
+  `allow_public_api_server = true`. In private-endpoint mode the list holds
+  internal ranges (for example all three RFC 1918 ranges) and has no total
+  limit.
+- Entries inside a documentation range (`192.0.2.0/24`, `198.51.100.0/24`,
+  `203.0.113.0/24`) are rejected, so an unedited placeholder fails at plan
+  time. Broader entries that contain one need `allow_public_api_server = true`.
+
+In the public-endpoint mode the list does not restrict the private endpoint,
+which stays reachable from the cluster's VPC network in its region. When master
+authorized networks are managed, access from Google Cloud public IP addresses
+is disabled. Network restrictions are in addition to Kubernetes API
+authentication and authorization. Apart from the unauthenticated health and
+version endpoints (such as `/healthz`, `/readyz`, and `/version`), every
+request must still pass them.
+
+This module installs nothing in the cluster, so if the list stops including
+your address, correct `authorized_networks` and apply again to restore access.
+
+**Upgrading from 0.x.**
+
+- Earlier versions left the public endpoint open to any address when
+  `authorized_networks` was empty. That configuration now fails the plan with a
+  message asking you to choose.
+- Configurations that already set `authorized_networks` keep working if the
+  entries are IPv4, not documentation ranges, at most 100, and no broader than
+  one `/8` in total (otherwise set `allow_public_api_server = true`). The plan
+  may show `gcp_public_cidrs_access_enabled = false` if it was enabled outside
+  Terraform. GKE documents that this change can take several hours to be
+  enforced, so verify the effective access before treating the endpoint as
+  restricted.
+- To restrict an open cluster, set `authorized_networks`. The plan shows an
+  in-place update that adds `master_authorized_networks_config`.
+- To keep the previous behaviour explicitly, set
+  `allow_public_api_server = true`. The plan shows no change to the endpoint.
+- `enable_private_endpoint` is also an in-place update.
 
 ### Node Auto-Provisioning (NAP)
 
@@ -294,6 +357,7 @@ See `infra/gcs_model_cache.tf` and `infra/iam.tf` for the resource definitions a
 
 This module follows GCP security best practices out of the box:
 
+- **Restricted control plane** - the Kubernetes API accepts only `authorized_networks`, or only the private endpoint with `enable_private_endpoint`; any-address access needs `allow_public_api_server`
 - **Private nodes** - worker nodes have no public IPs; egress via Cloud NAT
 - **Shielded nodes** - Secure Boot and Integrity Monitoring on all node pools
 - **Workload Identity** - pods use GCP service accounts, no JSON key files

examples/dev-l4-spot/README.md

@@ -6,7 +6,7 @@ Creates a minimal GKE cluster with a single L4 GPU spot node pool - ideal for de
 
 | Resource | Configuration |
 |----------|---------------|
-| GKE cluster | Private nodes, Cloud NAT, Workload Identity |
+| GKE cluster | Private nodes, Cloud NAT, Workload Identity, API endpoint restricted to `api_server_authorized_ip_ranges` |
 | GPU node pool | 1x NVIDIA L4 per node (g2-standard-8), spot VMs, scale 0-5 |
 | CPU node pool | e2-standard-4, scale 1-3 (system workloads) |
 | Artifact Registry | Docker repository for SIE images |
@@ -16,8 +16,15 @@ Creates a minimal GKE cluster with a single L4 GPU spot node pool - ideal for de
 
 ## Usage
 
+The Kubernetes API endpoint accepts only the CIDRs you list. Include the
+address the machine running kubectl and Helm uses to reach the Internet.
+`203.0.113.10/32` below is a documentation placeholder. The module rejects
+documentation ranges, so replace it with your own address.
+
 ```bash
 export TF_VAR_project_id="your-gcp-project-id"
+curl -s https://checkip.amazonaws.com   # your egress address; append /32
+export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]'
 
 terraform init
 terraform plan
@@ -43,9 +50,9 @@ helm upgrade --install sie-cluster oci://ghcr.io/superlinked/charts/sie-cluster
 ```
 
 Chart `0.8.3` selects `v0.8.3` service images and the
-`v0.8.3-cuda12-default` worker image. The Terraform module remains independently
-versioned at `0.7.2`. The cache arguments above also configure the chart's
-required payload-store bucket.
+`v0.8.3-cuda12-default` worker image. The example pins the Terraform module
+release from the Registry, which is versioned independently of SIE. The cache
+arguments above also configure the chart's required payload-store bucket.
 
 ## Variables
 
@@ -56,6 +63,7 @@ required payload-store bucket.
 | `cluster_name` | `sie-dev` | Cluster name |
 | `create_artifact_registry` | `true` | Create a Docker registry for SIE images |
 | `deployer_service_account` | `""` | Service account email (for CI/CD; optional for interactive use) |
+| `api_server_authorized_ip_ranges` | _(required)_ | CIDRs allowed to reach the Kubernetes API, such as `["203.0.113.10/32"]`; passed to the module's `authorized_networks` |
 
 ## Outputs
 

examples/dev-l4-spot/main.tf

@@ -18,6 +18,10 @@
 #
 # Usage:
 #   export TF_VAR_project_id="your-project-id"
+#   # CIDRs allowed to reach the Kubernetes API. Include the address this
+#   # machine uses to reach the Internet, for example the /32 of
+#   # `curl -s https://checkip.amazonaws.com`.
+#   export TF_VAR_api_server_authorized_ip_ranges='["203.0.113.10/32"]'
 #   terraform init
 #   terraform plan
 #   terraform apply
@@ -79,13 +83,18 @@ variable "deployer_service_account" {
   default     = ""
 }
 
+variable "api_server_authorized_ip_ranges" {
+  description = "CIDR blocks allowed to reach the Kubernetes API, such as [\"203.0.113.10/32\"]. Include the egress address of the machine that runs kubectl and helm."
+  type        = list(string)
+}
+
 # =============================================================================
 # SIE GKE Infra Module
 # =============================================================================
 
 module "infra" {
   source  = "superlinked/sie/google"
-  version = "0.7.2"
+  version = "0.7.3" # x-release-please-version
 
   project_id               = var.project_id
   region                   = var.region
@@ -101,6 +110,11 @@ module "infra" {
   # Private cluster with NAT
   enable_private_nodes = true
 
+  # Kubernetes API reachable only from the listed ranges
+  authorized_networks = [
+    for cidr in var.api_server_authorized_ip_ranges : { cidr_block = cidr, display_name = "operator" }
+  ]
+
   # Node Auto-Provisioning (NAP)
   enable_node_auto_provisioning = true
   nap_max_cpu                   = 100

main.tf

@@ -145,14 +145,20 @@ resource "google_container_cluster" "primary" {
   # Private cluster configuration
   private_cluster_config {
     enable_private_nodes    = var.enable_private_nodes
-    enable_private_endpoint = false # Allow public access to master
+    enable_private_endpoint = var.enable_private_endpoint
     master_ipv4_cidr_block  = var.enable_private_nodes ? var.master_ipv4_cidr_block : null
   }
 
-  # Master authorized networks
+  # Master authorized networks restrict the public endpoint; an empty
+  # cidr_blocks list admits no external address, and only
+  # allow_public_api_server leaves it unrestricted. They restrict the private
+  # endpoint only when enforcement is on, which is set in private-endpoint mode.
   dynamic "master_authorized_networks_config" {
-    for_each = length(var.authorized_networks) > 0 ? [1] : []
+    for_each = var.allow_public_api_server && length(var.authorized_networks) == 0 ? [] : [1]
     content {
+      gcp_public_cidrs_access_enabled      = false
+      private_endpoint_enforcement_enabled = var.enable_private_endpoint && length(var.authorized_networks) > 0 ? true : null
+
       dynamic "cidr_blocks" {
         for_each = var.authorized_networks
         content {

outputs.tf

@@ -14,7 +14,7 @@ output "cluster_endpoint" {
 
 output "cluster_ca_certificate" {
   description = "GKE cluster CA certificate (base64 encoded)"
-  value       = google_container_cluster.primary.master_auth[0].cluster_ca_certificate
+  value       = try(google_container_cluster.primary.master_auth[0].cluster_ca_certificate, null)
   sensitive   = true
 }
 
@@ -184,7 +184,7 @@ output "gpu_node_pools" {
 
 output "kubectl_config_command" {
   description = "Command to configure kubectl for this cluster"
-  value       = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project_id}"
+  value       = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project_id}${var.enable_private_endpoint ? " --internal-ip" : ""}"
 }
 
 # =============================================================================

release-please-config.json

@@ -8,6 +8,9 @@
       "include-v-in-release-name": true,
       "exclude-paths": [
         ".github"
+      ],
+      "extra-files": [
+        "examples/dev-l4-spot/main.tf"
       ]
     }
   }