From f193a1f24236cf7d2fcccac0c9914a9258ce964b Mon Sep 17 00:00:00 2001 From: yiraeChristineKim Date: Wed, 24 Jun 2026 13:13:40 -0400 Subject: [PATCH 1/2] feat: add HCP proxy CreateRequest to create HostedClusters from the hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the hub-side HCP proxy so platform engineers can create a HostedCluster (plus NodePools and Secrets) on a managed hosting cluster using hub credentials, without a per-cluster kubeconfig (ACM-37268). Creation order: Namespace → Secrets → HostedCluster → NodePool(s) via cluster-proxy, with managedcluster:admin auth, ManagedCluster health checks, duplicate-name handling, DNS-1123 path sanitization, and hcp.ocm.io/created-via labels. Includes unit/e2e coverage and TLS profile compliance for the proxy server. Signed-off-by: yiraeChristineKim Co-Authored-By: Cursor --- .cursor/rules/backplane-operator-sync.mdc | 70 + .cursor/rules/tls-compliance.mdc | 190 +++ .dockerignore | 12 + .github/workflows/e2e-hcp-proxy.yaml | 66 + .vscode/launch.json | 53 + CLAUDE.md | 72 + Dockerfile.e2e | 6 + Makefile | 191 ++- README.md | 4 + docs/README.md | 2 + docs/management/from-hub-cli.md | 374 +++++ docs/management/hcp-proxy-local-dev.md | 138 ++ go.mod | 8 +- go.sum | 16 +- hack/install_cluster_proxy.sh | 61 + hack/install_ocm.sh | 59 +- pkg/agent/agent.go | 10 +- pkg/manager/hcp_proxy.go | 1319 +++++++++++++++++ pkg/manager/hcp_proxy_test.go | 1580 +++++++++++++++++++++ pkg/manager/manager.go | 181 ++- quickstart/README.md | 7 +- test/e2e/addon-manager-deployment.yaml | 200 +++ test/e2e/e2e_suite_test.go | 19 +- test/e2e/hcp_proxy_test.go | 426 ++++++ 24 files changed, 4979 insertions(+), 85 deletions(-) create mode 100644 .cursor/rules/backplane-operator-sync.mdc create mode 100644 .cursor/rules/tls-compliance.mdc create mode 100644 .dockerignore create mode 100644 .github/workflows/e2e-hcp-proxy.yaml create mode 100644 .vscode/launch.json create mode 100644 Dockerfile.e2e create mode 100644 docs/management/from-hub-cli.md create mode 100644 docs/management/hcp-proxy-local-dev.md create mode 100755 hack/install_cluster_proxy.sh create mode 100644 pkg/manager/hcp_proxy.go create mode 100644 pkg/manager/hcp_proxy_test.go create mode 100644 test/e2e/addon-manager-deployment.yaml create mode 100644 test/e2e/hcp_proxy_test.go diff --git a/.cursor/rules/backplane-operator-sync.mdc b/.cursor/rules/backplane-operator-sync.mdc new file mode 100644 index 00000000..c7b02f83 --- /dev/null +++ b/.cursor/rules/backplane-operator-sync.mdc @@ -0,0 +1,70 @@ +--- +description: Sync hub-side HCP proxy resources to backplane-operator. Apply when editing pkg/manager/hcp_proxy.go or related hub manager wiring. +globs: + - pkg/manager/hcp_proxy.go + - pkg/manager/manager.go +--- + +# backplane-operator Sync Rule + +Hub-side HCP proxy resources (Service, APIService, ClusterRole/Binding, Deployment +ports, manager ClusterRole rules) are provisioned by **backplane-operator**, not +applied at runtime by this operator: + + `pkg/templates/charts/toggle/hypershift/templates/` + (in the `stolostron/backplane-operator` repository) + +E2E copies of the manifests live in `test/e2e/addon-manager-deployment.yaml` for kind only. + +## Required syncs + +### New hub-facing ports +→ Add `containerPort` entry to `hypershift-addon-manager-deployment.yaml` + +```yaml +ports: +- containerPort: 9443 + name: hcp-proxy + protocol: TCP +``` + +### HCP proxy Service / APIService / RBAC +→ Add or update templates in backplane-operator (Service targeting port 9443, +APIService `v1alpha1.hcp.ocm.io`, ClusterRole/Binding for the manager SA). + +### New hub manager RBAC rules +→ Add rules to `hypershift-addon-manager_clusterrole.yaml` + +Currently required for the HCP proxy: +```yaml +- apiGroups: ["clusterview.open-cluster-management.io"] + resources: ["userpermissions"] + verbs: ["get"] +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters"] + verbs: ["get", "list", "watch"] +- apiGroups: ["operator.open-cluster-management.io"] + resources: ["multiclusterhubs"] + verbs: ["get", "list", "watch"] +- apiGroups: ["config.openshift.io"] + resources: ["apiservers"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["users", "groups", "serviceaccounts"] + verbs: ["impersonate"] +- apiGroups: ["authentication.k8s.io"] + resources: ["userextras"] + verbs: ["impersonate"] +``` + +The `config.openshift.io/apiservers` rule is required so SecurityProfileWatcher +and `FetchAPIServerTLSProfile` can read the cluster TLS profile. + +### New hub ServiceAccount +→ Add entry to `hypershift-addon-manager-serviceaccount.yaml` + +## Why +The hub manager pod (Deployment, ClusterRole, ServiceAccount, Service, APIService) +is defined in `backplane-operator`, not applied by `StartHCPProxy` in this repo. + +Upstream PR reference: https://github.com/stolostron/backplane-operator/tree/main/pkg/templates/charts/toggle/hypershift/templates diff --git a/.cursor/rules/tls-compliance.mdc b/.cursor/rules/tls-compliance.mdc new file mode 100644 index 00000000..94fad512 --- /dev/null +++ b/.cursor/rules/tls-compliance.mdc @@ -0,0 +1,190 @@ +--- +description: > + OpenShift central TLS profile compliance (ACM-26882, release blocker OCP 4.23 / ACM 5.0). + Apply whenever writing or reviewing any code that opens a TLS server, creates an outbound + HTTP/HTTPS client, generates certificates, or touches TLS configuration. +globs: + - "**/*.go" +alwaysApply: true +--- + +# TLS Compliance — Mandatory Pattern + +This repo is covered by **ACM-26882 "Central TLS Profile consistency"**, which is a release +blocker for OCP 4.23 / ACM 5.0. Every piece of new or modified TLS code **MUST** follow the +four rules below. Violations will block merges. + +## The four rules + +### 1. Fetch the profile once at manager startup + +In `manager.go` (or equivalent entry-point), call **before** starting any server or client: + +```go +import tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" + +profileSpec, err := tlspkg.FetchAPIServerTLSProfile(ctx, hubClient) +if err != nil { + // kind / non-OpenShift clusters lack the resource — use Intermediate fallback + log.Error(err, "failed to fetch APIServer TLS profile, using Intermediate defaults") + profileSpec, _ = tlspkg.GetTLSProfileSpec(nil) +} +``` + +Pass `profileSpec` explicitly to every server and client constructor — **never re-fetch it +inside a loop or per-request**. + +### 2. TLS server — apply profile via `tlspkg.NewTLSConfigFromProfile` + +```go +// NewTLSConfigFromProfile returns (applyFn, unsupportedCipherNames) — not an error. +tlsCfgFn, unsupported := tlspkg.NewTLSConfigFromProfile(profileSpec) +if len(unsupported) > 0 { + log.Info("TLS profile contains unsupported ciphers, they will be ignored", "ciphers", unsupported) +} +cert, err := generateSelfSignedCert(operatorNS) // see rule 4 +if err != nil { + return fmt.Errorf("generate serving cert: %w", err) +} +tlsCfg := &tls.Config{Certificates: []tls.Certificate{cert}} +tlsCfgFn(tlsCfg) // sets MinVersion + CipherSuites from central profile + +srv := &http.Server{TLSConfig: tlsCfg, ...} +``` + +**Never** set `tls.Config.MinVersion`, `tls.Config.CipherSuites`, or `tls.Config.CurvePreferences` +by hand. Those values must come exclusively from the profile function. + +### 3. Outbound HTTP client — use `buildHTTPClient` + +Every outbound HTTPS call must go through the shared helper that applies the profile to the +transport: + +```go +// buildHTTPClient creates an http.Client whose TLS transport honours the cluster +// TLS profile (MinVersion, CipherSuites) and carries the hub bearer token. +func (p *hcpProxy) buildHTTPClient(timeout time.Duration) (*http.Client, error) { + tlsCfg, err := rest.TLSConfigFor(p.hubConfig) + if err != nil { + return nil, fmt.Errorf("TLS config from rest.Config: %w", err) + } + tlsConfigFn, _ := tlspkg.NewTLSConfigFromProfile(p.profileSpec) + tlsConfigFn(tlsCfg) // overlay MinVersion + CipherSuites + + base := &http.Transport{TLSClientConfig: tlsCfg, ...} + wrapped, err := rest.HTTPWrappersForConfig(p.hubConfig, base) + if err != nil { + return nil, fmt.Errorf("HTTP auth wrappers: %w", err) + } + return &http.Client{Transport: wrapped, Timeout: timeout}, nil +} +``` + +Do **not** create raw `http.Client` or `http.Transport` instances with a bare `tls.Config{}` — +always call `buildHTTPClient` (or an equivalent that applies `tlspkg.NewTLSConfigFromProfile`). + +### 4. Certificates — prefer service-ca, fall back to `libgocrypto` + +**On OpenShift** the preferred source is the cluster Service CA. Annotate the Service: + +```yaml +metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: -tls +``` + +Mount the generated Secret with `optional: true` and use `loadOrGenerateCert` which: +1. Loads `tls.crt` / `tls.key` from the mount path when present (OpenShift — trusted, auto-rotated) +2. Falls back to `generateSelfSignedCert` when the path is absent (kind / vanilla k8s) + +**On non-OpenShift clusters (kind, e2e)** the annotation is ignored and the fallback +in-process path activates automatically via `loadOrGenerateCert`. + +For the fallback, use **`github.com/openshift/library-go/pkg/crypto`**: + +```go +import ( + libgocrypto "github.com/openshift/library-go/pkg/crypto" + "k8s.io/apimachinery/pkg/util/sets" +) + +func generateSelfSignedCert(operatorNS string) (tls.Certificate, error) { + const certLifetime = 2 * 365 * 24 * time.Hour // ≤ 7200-day library-go limit + + caConfig, err := libgocrypto.MakeSelfSignedCAConfigForDuration( + serviceName+"-ca", certLifetime) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create CA: %w", err) + } + ca := &libgocrypto.CA{ + Config: caConfig, + SerialGenerator: &libgocrypto.RandomSerialGenerator{}, + } + hostnames := sets.New[string]("localhost", "127.0.0.1", + serviceName, serviceName+"."+operatorNS+".svc", + serviceName+"."+operatorNS+".svc.cluster.local") + serverConfig, err := ca.MakeServerCert(hostnames, certLifetime) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create server cert: %w", err) + } + certPEM, keyPEM, err := serverConfig.GetPEMBytes() + if err != nil { + return tls.Certificate{}, fmt.Errorf("encode server cert PEM: %w", err) + } + return tls.X509KeyPair(certPEM, keyPEM) +} +``` + +**Never** use `crypto/ecdsa`, `crypto/x509.CreateCertificate`, `crypto/rand`, `math/big`, or +`encoding/pem` directly for certificate generation — those bypass OpenShift's key-type and +serial-number conventions. + +### 5. Runtime watch — register `SecurityProfileWatcher` + +Any manager that embeds a TLS server must restart cleanly when the cluster profile changes: + +```go +watcher := &tlspkg.SecurityProfileWatcher{ + Client: hubClient, + InitialTLSProfileSpec: profileSpec, + OnProfileChange: func(_ context.Context, old, new configv1.TLSProfileSpec) { + log.Info("TLS profile changed, restarting", "old", old.MinTLSVersion, "new", new.MinTLSVersion) + cancelManager() // triggers graceful pod restart + }, +} +if err := watcher.SetupWithManager(mgr); err != nil { + return fmt.Errorf("setup TLS watcher: %w", err) +} +``` + +## RBAC required + +Whenever any of the above is used, the manager's ClusterRole must include: + +```yaml +- apiGroups: ["config.openshift.io"] + resources: ["apiservers"] + verbs: ["get", "list", "watch"] +``` + +Add it to the `backplane-operator` hypershift-addon-manager ClusterRole +(see `.cursor/rules/backplane-operator-sync.mdc`). For kind e2e, also keep +`test/e2e/addon-manager-deployment.yaml` in sync. + +## Forbidden patterns (will fail code review) + +| Forbidden | Use instead | +|---|---| +| `tls.Config{MinVersion: tls.VersionTLS12}` | `tlspkg.NewTLSConfigFromProfile(profileSpec)` | +| `tls.Config{MinVersion: tls.VersionTLS13}` | same | +| `tls.Config{CipherSuites: [...]}` (hardcoded) | same | +| `http.Client{Transport: &http.Transport{TLSClientConfig: ...}}` without profile | `buildHTTPClient(...)` | +| `x509.CreateCertificate(rand.Reader, ...)` | `libgocrypto.MakeSelfSignedCAConfigForDuration` | +| `ecdsa.GenerateKey(elliptic.P256(), ...)` for serving cert | `libgocrypto.MakeServerCert` | + +## Reference + +- Epic: [ACM-26882](https://redhat.atlassian.net/browse/ACM-26882) — Central TLS Profile consistency +- HyperShift addon task: [ACM-30178](https://redhat.atlassian.net/browse/ACM-30178) — verified ACM 5.0.0-112 +- Technical guide: https://docs.google.com/document/d/1cMc9E8psHfnoK06ntR8kHSWB8d3rMtmldhnmM4nImjs +- Slack: [#forum-ocp-tls-strict-obedience](https://redhat.enterprise.slack.com/archives/C098FU5MRAB) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3384dc0f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.github +.cursor +.work +docs +examples +quickstart +*.md +cover.out +e2e.test +bin/* +!bin/hypershift-addon diff --git a/.github/workflows/e2e-hcp-proxy.yaml b/.github/workflows/e2e-hcp-proxy.yaml new file mode 100644 index 00000000..fe7b0537 --- /dev/null +++ b/.github/workflows/e2e-hcp-proxy.yaml @@ -0,0 +1,66 @@ +name: E2E – HCP Proxy + +on: + push: + branches: ["main", "backplane-5.*"] + paths: + - "pkg/manager/hcp_proxy.go" + - "pkg/manager/manager.go" + - "test/e2e/**" + - "Dockerfile.e2e" + - "Makefile" + - "hack/install_ocm.sh" + - ".github/workflows/e2e-hcp-proxy.yaml" + pull_request: + branches: ["main", "backplane-5.*"] + paths: + - "pkg/manager/hcp_proxy.go" + - "pkg/manager/manager.go" + - "test/e2e/**" + - "Dockerfile.e2e" + - "Makefile" + - "hack/install_ocm.sh" + - ".github/workflows/e2e-hcp-proxy.yaml" + workflow_dispatch: {} + +permissions: + contents: read + +env: + GO_VERSION: "1.26" + KIND_CLUSTER_NAME: hcp-proxy-e2e + +jobs: + e2e-hcp-proxy: + name: HCP Proxy E2E + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + + # Warm the module cache before kind/OCM so the parallel image build hits cache. + - name: Download Go modules + run: go mod download + + - name: Set up cluster and deploy addon manager + run: make e2e-hcp-proxy-setup KIND_CLUSTER_NAME=${{ env.KIND_CLUSTER_NAME }} + + - name: Run HCP Proxy e2e tests + run: make test-e2e-hcp-proxy KIND_CLUSTER_NAME=${{ env.KIND_CLUSTER_NAME }} + + - name: Dump addon manager logs on failure + if: failure() + run: kubectl logs -n multicluster-engine -l app=hypershift-addon-manager --tail=200 || true + + - name: Delete kind cluster + if: always() + run: make kind-delete KIND_CLUSTER_NAME=${{ env.KIND_CLUSTER_NAME }} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..f0cf9db4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,53 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Hub Manager", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/main.go", + "args": [ + "manager", + "--disable-leader-election", + "--kubeconfig", "${env:HOME}/.kube/config", + "--disable-tls-watcher" + ], + "env": { + }, + "cwd": "${workspaceFolder}", + "console": "internalConsole" + }, + { + "name": "Spoke Agent", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/main.go", + "args": [ + "agent", + "--hub-kubeconfig", "${env:HOME}/.kube/config", + "--cluster-name", "local-cluster", + "--addon-namespace", "hypershift", + "--hypershfit-operator-image", "quay.io/hypershift/hypershift-operator:latest" + ], + "cwd": "${workspaceFolder}", + "console": "internalConsole" + }, + { + "name": "Cleanup", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/main.go", + "args": [ + "cleanup", + "--hub-kubeconfig", "${env:HOME}/.kube/config", + "--cluster-name", "local-cluster", + "--addon-namespace", "hypershift" + ], + "cwd": "${workspaceFolder}", + "console": "internalConsole" + } + ] +} diff --git a/CLAUDE.md b/CLAUDE.md index 51155345..a30a9524 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,78 @@ For system architecture, data flows, and module layout, see [docs/ARCHITECTURE.m - **Constants:** shared names, namespaces, labels, and image defaults in `pkg/util/constant.go` - **Scheme registration:** no project-owned CRDs — consumes types from OCM, HyperShift, OpenShift, MCE, discovery, OLM +## TLS configuration (mandatory — release blocker ACM-26882) + +> **This is a release blocker for OCP 4.23 / ACM 5.0.** All new TLS code MUST follow the +> pattern below. See `.cursor/rules/tls-compliance.mdc` for the full rule with forbidden patterns. + +This repo inherits its TLS settings from the cluster's central configuration source +(`apiservers.config.openshift.io/cluster`) via `github.com/openshift/controller-runtime-common/pkg/tls`. +**Never hardcode** `MinVersion`, `CipherSuites`, or `CurvePreferences`. + +### Mandatory 4-step pattern for any new TLS server or client + +**Step 1 — Fetch at manager startup** (`manager.go`) +```go +profileSpec, err := tlspkg.FetchAPIServerTLSProfile(ctx, hubClient) +if err != nil { // fallback for kind/non-OpenShift clusters + profileSpec, _ = tlspkg.GetTLSProfileSpec(nil) // Intermediate (TLS 1.2+) +} +``` + +**Step 2 — TLS server** — apply profile to the HTTPS listener +```go +tlsCfgFn, _ := tlspkg.NewTLSConfigFromProfile(profileSpec) +tlsCfg := &tls.Config{Certificates: []tls.Certificate{cert}} +tlsCfgFn(tlsCfg) // sets MinVersion + CipherSuites — do NOT set them yourself +``` + +**Step 3 — Outbound HTTP clients** — always use `buildHTTPClient` +```go +// buildHTTPClient overlays the TLS profile onto rest.TLSConfigFor + HTTPWrappersForConfig +client, err := p.buildHTTPClient(30 * time.Second) +``` +Never create `http.Client` / `http.Transport` with a bare `tls.Config{}` directly. + +**Step 4 — Runtime watch** — restart on profile change +```go +watcher := &tlspkg.SecurityProfileWatcher{..., OnProfileChange: func(...) { cancelManager() }} +watcher.SetupWithManager(mgr) +``` + +### Serving certificate generation — use library-go, not stdlib crypto + +```go +import libgocrypto "github.com/openshift/library-go/pkg/crypto" +// Use libgocrypto.MakeSelfSignedCAConfigForDuration + CA.MakeServerCert +// See generateSelfSignedCert() in pkg/manager/hcp_proxy.go for the full pattern. +``` +Do **not** use `crypto/ecdsa`, `x509.CreateCertificate`, `math/big`, or `encoding/pem` directly. + +### RBAC required for every component using the TLS profile +The backplane-operator hypershift-addon-manager ClusterRole must include: +```yaml +- apiGroups: ["config.openshift.io"] + resources: ["apiservers"] + verbs: ["get", "list", "watch"] +``` +For kind e2e, keep `test/e2e/addon-manager-deployment.yaml` in sync. + +## Cross-repo dependencies + +### backplane-operator (hub manager manifests) + +The hub manager pod (Deployment, ClusterRole, ServiceAccount, HCP proxy Service, +APIService, and proxy RBAC) is **not** applied at runtime by this repo. It is +defined in: + + `stolostron/backplane-operator` + `pkg/templates/charts/toggle/hypershift/templates/` + +When adding new container ports, RBAC rules, or env vars to the hub manager, you +**must** also update the corresponding template in `backplane-operator`. See +`.cursor/rules/backplane-operator-sync.mdc` for the detailed sync checklist. + ## CI systems - **Prow:** OpenShift CI for release branches (config in `openshift/release` repo) diff --git a/Dockerfile.e2e b/Dockerfile.e2e new file mode 100644 index 00000000..8ccfac3d --- /dev/null +++ b/Dockerfile.e2e @@ -0,0 +1,6 @@ +# Minimal image for kind e2e — binary is built on the host (uses Go module cache). +# Usage: CGO_ENABLED=0 GOOS=linux go build -o bin/hypershift-addon cmd/main.go +# docker build -f Dockerfile.e2e -t $E2E_IMG . +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +COPY bin/hypershift-addon . +USER 65532:65532 diff --git a/Makefile b/Makefile index 51858fe5..16848f8f 100644 --- a/Makefile +++ b/Makefile @@ -140,30 +140,185 @@ build-e2e: test-e2e: build-e2e deploy-ocm deploy-addon-manager # ./e2e.test -test.v -ginkgo.v -ginkgo.junit-report $(JUNIT_REPORT_FILE) +##@ HCP Proxy E2E (local) + +# --------------------------------------------------------------------------- +# Variables – override on the command line as needed: +# make e2e-hcp-proxy-full KIND_CLUSTER_NAME=my-cluster E2E_IMG=my-img:tag +# --------------------------------------------------------------------------- +KIND_VERSION ?= v0.23.0 +KIND_CLUSTER_NAME ?= hcp-proxy-e2e +MANAGED_CLUSTER_NAME ?= local-cluster +# Image tag used inside the kind cluster (no registry push required) +E2E_IMG ?= kind-local/hypershift-addon-operator:$(shell git rev-parse --short HEAD 2>/dev/null || echo dev) +KIND ?= $(shell which kind 2>/dev/null || echo $(GOBIN)/kind) + +.PHONY: ensure-kind +ensure-kind: ## Install kind $(KIND_VERSION) to $(GOBIN) if not already present. + @if ! command -v kind >/dev/null 2>&1 && [ ! -f "$(GOBIN)/kind" ]; then \ + OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ + ARCH=$$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/'); \ + echo "Installing kind $(KIND_VERSION) for $$OS/$$ARCH into $(GOBIN)..."; \ + curl -sSLo "$(GOBIN)/kind" \ + "https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$$OS-$$ARCH"; \ + chmod +x "$(GOBIN)/kind"; \ + fi + @$(KIND) version + +.PHONY: kind-create +kind-create: ensure-kind ## Create the $(KIND_CLUSTER_NAME) kind cluster. + $(KIND) create cluster --name "$(KIND_CLUSTER_NAME)" --wait 120s + $(KUBECTL) cluster-info --context "kind-$(KIND_CLUSTER_NAME)" + +.PHONY: kind-delete +kind-delete: ## Delete the $(KIND_CLUSTER_NAME) kind cluster. + $(KIND) delete cluster --name "$(KIND_CLUSTER_NAME)" || true + +.PHONY: kind-load-e2e +kind-load-e2e: ## Load E2E_IMG into the kind cluster. + $(KIND) load docker-image "$(E2E_IMG)" --name "$(KIND_CLUSTER_NAME)" + +.PHONY: wait-hcp-proxy-service +wait-hcp-proxy-service: ## Wait until the HCP proxy Service has a cluster IP. + @echo "Waiting for HCP proxy Service (hypershift-addon-hcp-proxy)..."; \ + for i in $$(seq 1 30); do \ + $(KUBECTL) get service -n multicluster-engine hypershift-addon-hcp-proxy \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null | grep -q '[0-9]' \ + && echo "Service ready." && exit 0; \ + echo " waiting ($$i/30)..."; sleep 5; \ + done; \ + echo "ERROR: HCP proxy Service never became available"; exit 1 + +.PHONY: wait-hcp-proxy-apiservice +wait-hcp-proxy-apiservice: ## Wait until APIService v1alpha1.hcp.ocm.io is registered. + @echo "Waiting for APIService v1alpha1.hcp.ocm.io..."; \ + for i in $$(seq 1 30); do \ + $(KUBECTL) get apiservice v1alpha1.hcp.ocm.io 2>/dev/null && exit 0; \ + echo " waiting ($$i/30)..."; sleep 5; \ + done; \ + echo "ERROR: APIService v1alpha1.hcp.ocm.io never registered"; exit 1 + +# Host-build the linux binary (uses setup-go / local GOCACHE) then wrap in a +# tiny image — much faster than compiling inside Docker from a cold cache. +E2E_GOOS ?= linux +E2E_GOARCH ?= $(shell go env GOARCH) + +.PHONY: e2e-build-image +e2e-build-image: ## Build E2E_IMG via host go build + Dockerfile.e2e. + # vendor/ is gitignored in this repo — use module mode (setup-go caches downloads). + CGO_ENABLED=0 GOOS=$(E2E_GOOS) GOARCH=$(E2E_GOARCH) \ + go build -o bin/hypershift-addon cmd/main.go + docker build -f Dockerfile.e2e -t "$(E2E_IMG)" . + +.PHONY: e2e-hcp-proxy-setup +e2e-hcp-proxy-setup: ensure-kind ## Spin up kind + OCM, build & load image, deploy addon manager. + # One shell so background image build can be waited on after kind+OCM. + @set -e; \ + echo "Building $(E2E_IMG) in background (host go build + Dockerfile.e2e)..."; \ + $(MAKE) e2e-build-image E2E_IMG="$(E2E_IMG)" E2E_GOOS="$(E2E_GOOS)" E2E_GOARCH="$(E2E_GOARCH)" & \ + build_pid=$$!; \ + $(MAKE) kind-create KIND_CLUSTER_NAME="$(KIND_CLUSTER_NAME)"; \ + $(MAKE) deploy-ocm; \ + $(MAKE) deploy-cluster-proxy; \ + $(MAKE) deploy-hypershift-crds; \ + echo "Waiting for image build (pid $$build_pid)..."; \ + wait $$build_pid; \ + $(MAKE) kind-load-e2e E2E_IMG="$(E2E_IMG)"; \ + $(KUBECTL) create namespace multicluster-engine --dry-run=client -o yaml | $(KUBECTL) apply -f -; \ + sed -e 's|image: quay.io/stolostron/hypershift-addon-operator:latest|image: $(E2E_IMG)|g' \ + -e 's|value: quay.io/stolostron/hypershift-addon-operator:latest|value: $(E2E_IMG)|g' \ + test/e2e/addon-manager-deployment.yaml | $(KUBECTL) apply -f -; \ + if ! $(KUBECTL) rollout status -n multicluster-engine deployment/hypershift-addon-manager --timeout=180s; then \ + $(KUBECTL) describe -n multicluster-engine deployment/hypershift-addon-manager; \ + $(KUBECTL) get pods -n multicluster-engine -l app=hypershift-addon-manager -o wide; \ + $(KUBECTL) describe -n multicluster-engine -l app=hypershift-addon-manager pods; \ + $(KUBECTL) logs -n multicluster-engine -l app=hypershift-addon-manager --tail=100 || true; \ + exit 1; \ + fi; \ + $(MAKE) wait-hcp-proxy-service; \ + $(MAKE) wait-hcp-proxy-apiservice + +.PHONY: e2e-hcp-proxy-full +e2e-hcp-proxy-full: e2e-hcp-proxy-setup ## Full cycle: setup → test → cleanup (always deletes kind cluster). + @status=0; \ + $(MAKE) test-e2e-hcp-proxy MANAGED_CLUSTER_NAME="$(MANAGED_CLUSTER_NAME)" || status=$$?; \ + $(MAKE) kind-delete; \ + exit $$status + +.PHONY: e2e-hcp-proxy-cleanup +e2e-hcp-proxy-cleanup: kind-delete ## Tear down the $(KIND_CLUSTER_NAME) kind cluster. + +# Run only the HCP Proxy e2e suite against an already-deployed addon manager. +# Always port-forward: kind pod IPs (10.244.x.x) are not reachable from the +# GHA/host network, so direct pod-IP access times out on Linux CI. +HCP_PROXY_PORT ?= 18443 + +.PHONY: test-e2e-hcp-proxy +test-e2e-hcp-proxy: + @echo "Starting kubectl port-forward on localhost:$(HCP_PROXY_PORT)..." + @POD=$$($(KUBECTL) get pods -n multicluster-engine -l app=hypershift-addon-manager \ + -o jsonpath='{.items[0].metadata.name}'); \ + test -n "$$POD" || { echo "ERROR: no hypershift-addon-manager pod found"; exit 1; }; \ + $(KUBECTL) port-forward -n multicluster-engine "pod/$$POD" \ + "$(HCP_PROXY_PORT):9443" & PF_PID=$$!; \ + trap 'kill $$PF_PID 2>/dev/null || true' EXIT; \ + for i in 1 2 3 4 5 6 7 8 9 10; do \ + if curl -sk --connect-timeout 1 "https://127.0.0.1:$(HCP_PROXY_PORT)/healthz" >/dev/null 2>&1; then \ + break; \ + fi; \ + sleep 1; \ + done; \ + HCP_PROXY_HOST="localhost:$(HCP_PROXY_PORT)" \ + go test ./test/e2e -timeout 15m -ginkgo.v -ginkgo.focus "HCP Proxy" + .PHONY: deploy-addon-manager deploy-addon-manager: - $(KUBECTL) create namespace multicluster-engine --dry-run=client -o yaml | kubectl apply -f - - $(KUBECTL) apply -f example/addon-manager-deployment.yaml - $(KUBECTL) set image -n multicluster-engine deployment/hypershift-addon-manager hypershift-addon-manager=$(IMG) - $(KUBECTL) set env -n multicluster-engine deployment/hypershift-addon-manager HYPERSHIFT_ADDON_IMAGE_NAME=$(IMG) - $(KUBECTL) rollout status -n multicluster-engine deployment/hypershift-addon-manager --timeout=60s + $(KUBECTL) create namespace multicluster-engine --dry-run=client -o yaml | $(KUBECTL) apply -f - + sed -e 's|image: quay.io/stolostron/hypershift-addon-operator:latest|image: $(IMG)|g' \ + -e 's|value: quay.io/stolostron/hypershift-addon-operator:latest|value: $(IMG)|g' \ + test/e2e/addon-manager-deployment.yaml | $(KUBECTL) apply -f - + $(KUBECTL) rollout status -n multicluster-engine deployment/hypershift-addon-manager --timeout=180s deploy-ocm: ensure-clusteradm - hack/install_ocm.sh + PATH="$(GOBIN):$$PATH" hack/install_ocm.sh + +# OCM cluster-proxy addon (helm chart ocm/cluster-proxy) + Hypershift CRDs for +# HCP proxy POST create e2e through the spoke kube-apiserver. +.PHONY: deploy-cluster-proxy +deploy-cluster-proxy: ensure-helm ## Install OCM cluster-proxy into open-cluster-management-addon. + PATH="$(GOBIN):$$PATH" hack/install_cluster_proxy.sh + +.PHONY: deploy-hypershift-crds +deploy-hypershift-crds: ## Apply HostedCluster CRD so spoke create e2e can succeed. + # Server-side apply avoids the client-side last-applied-configuration + # annotation size limit on this large CRD. + $(KUBECTL) apply --server-side --force-conflicts \ + -f hack/crds/hypershift.openshift.io_hostedclusters.yaml + +.PHONY: ensure-helm +ensure-helm: ## Install helm into $(GOBIN) if not already on PATH. + @if ! command -v helm >/dev/null 2>&1 && [ ! -f "$(GOBIN)/helm" ]; then \ + mkdir -p "$(GOBIN)"; \ + OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ + ARCH=$$(uname -m); \ + case "$$ARCH" in x86_64) ARCH=amd64 ;; aarch64|arm64) ARCH=arm64 ;; esac; \ + echo "Installing helm into $(GOBIN)..."; \ + curl -fsSL "https://get.helm.sh/helm-v3.16.4-$$OS-$$ARCH.tar.gz" \ + | tar -xz -C /tmp "$$OS-$$ARCH/helm"; \ + mv "/tmp/$$OS-$$ARCH/helm" "$(GOBIN)/helm"; \ + chmod +x "$(GOBIN)/helm"; \ + fi + @command -v helm >/dev/null 2>&1 || test -x "$(GOBIN)/helm" .PHONY: ensure-clusteradm -ensure-clusteradm: -ifeq (, $(shell which clusteradm)) - @{ \ - set -e ;\ - export INSTALL_DIR="${GOPATH}/bin" ;\ - curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash ;\ - } - CLUSTERADM=${GOPATH}/bin/clusteradm -else - CLUSTERADM=$(shell which clusteradm) -endif - $(@info CLUSTERADM="$(CLUSTERADM)") +ensure-clusteradm: ## Install clusteradm into $(GOBIN) if not already on PATH. + @if ! command -v clusteradm >/dev/null 2>&1 && [ ! -f "$(GOBIN)/clusteradm" ]; then \ + mkdir -p "$(GOBIN)"; \ + echo "Installing clusteradm into $(GOBIN)..."; \ + curl -fsSL https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh \ + | INSTALL_DIR="$(GOBIN)" USE_SUDO=false bash; \ + fi + @command -v clusteradm >/dev/null 2>&1 || test -x "$(GOBIN)/clusteradm" .PHONY: quickstart quickstart: diff --git a/README.md b/README.md index 6dfd65a1..e5c56a2f 100644 --- a/README.md +++ b/README.md @@ -48,5 +48,9 @@ NAME AVAILABLE DEGRADED PROGRESSING hypershift-addon True ``` +## HCP Proxy — local development & testing + +See [HCP Proxy local development & testing](docs/management/hcp-proxy-local-dev.md). + ## Metrics Dashboard [Instructions here](docs/metricsDashboard/README.md) diff --git a/docs/README.md b/docs/README.md index e80c1a9b..d9ffc5e1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,6 +32,8 @@ Tools and guides for managing HyperShift hosted clusters after deployment. - **[Discovering Hosted Clusters](./management/discovering_hostedclusters.md)** - Auto-discovery and import of hosted clusters - **[HyperShift Addon Status](./management/hypershift_addon_status.md)** - Monitor addon health and status - **[Cluster Capacity Metrics](./management/cluster_capacity_metrics_hcp.md)** - Understanding capacity planning and metrics +- **[Managing HostedClusters from the Hub](./management/from-hub-cli.md)** - Create, edit, and delete HostedClusters via `hcp from-hub` without direct access to the hosting cluster +- **[HCP Proxy local development & testing](./management/hcp-proxy-local-dev.md)** - Run the hub HCP proxy locally and exercise the `hcp.ocm.io` API with curl ### 🔄 [GitOps](./gitops/) ACM-centric guides for managing **HyperShift hosted clusters** with **Git** (typically Argo CD): secrets and hub responsibilities, discovery and auto-import, `ManagedCluster` / hosted-mode klusterlet, and hub-side troubleshooting. diff --git a/docs/management/from-hub-cli.md b/docs/management/from-hub-cli.md new file mode 100644 index 00000000..407fee0b --- /dev/null +++ b/docs/management/from-hub-cli.md @@ -0,0 +1,374 @@ +# Managing HostedClusters from the Hub with `hcp from-hub` + +`hcp from-hub` lets you create, edit, get, and delete HostedClusters on a hosting +ManagedCluster **from the hub**, without needing direct access to the hosting +cluster's kubeconfig. + +All requests flow through the `hypershift-addon-operator` HCP proxy +(`hcp.ocm.io/v1alpha1` extension API), which forwards them to the hosting +cluster via cluster-proxy. + +```text +hcp CLI → hub kube-apiserver → HCP proxy → cluster-proxy → hosting cluster +``` + +--- + +## Prerequisites + + +| Requirement | Notes | +| --------------------- | -------------------------------------------------------- | +| ACM / MCE hub cluster | The `hypershift-addon-operator` must be running | +| `cluster-proxy` addon | Enabled on the hosting ManagedCluster | +| Hub kubeconfig | `$KUBECONFIG`, `~/.kube/config`, or `--hub-kubeconfig` | +| RBAC | `managedcluster:admin` permission on the hosting cluster | + + +--- + +## HCP proxy API (`hcp.ocm.io/v1alpha1`) + +The hub manager serves this extension API on port `9443` (Service port `443`, +APIService `v1alpha1.hcp.ocm.io`, provisioned by backplane-operator). Every +resource request requires the query parameter: + +| Query parameter | Required | Description | +| ----------------- | -------- | ------------------------------------------------ | +| `hostingCluster` | yes | Name of the target hosting `ManagedCluster` | + +Base path: + +```text +/apis/hcp.ocm.io/v1alpha1 +``` + +### Endpoints + +| Method | Path | Handler | Description | +| ------ | ---- | ------- | ----------- | +| `GET` | `/healthz`, `/readyz` | health | Liveness / readiness probes | +| `GET` | `/apis/hcp.ocm.io` | discovery | APIGroup document | +| `GET` | `/apis/hcp.ocm.io/v1alpha1` | discovery | APIResourceList (`hostedclusters`, `hostedclusters/resources`) | +| `POST` | `/namespaces/{ns}/hostedclusters?hostingCluster={cluster}` | create | Create Namespace → Secrets → HostedCluster → NodePool(s) — GET list is not supported | +| `GET` | `/namespaces/{ns}/hostedclusters/{name}?hostingCluster={cluster}` | get | Return full `ResourceBundle` | +| `GET` | `/namespaces/{ns}/hostedclusters/{name}/resources?hostingCluster={cluster}` | get | Same as GET above (explicit `/resources` alias) | +| `PUT` | `/namespaces/{ns}/hostedclusters/{name}?hostingCluster={cluster}` | put | Full-replace HostedCluster + NodePools from a `ResourceBundle` | +| `PUT` | `/namespaces/{ns}/hostedclusters/{name}/resources?hostingCluster={cluster}` | put | Same as PUT above | +| `DELETE` | `/namespaces/{ns}/hostedclusters/{name}?hostingCluster={cluster}` | delete | Delete matching NodePools, then the HostedCluster | + +`Content-Type` for create/put bodies: `application/json`. + +### Request / response types + +#### `CreateRequest` (POST body) + +Mirrors `hcp create cluster --render` output: + +```json +{ + "hostedCluster": { "...": "HostedCluster object" }, + "nodePools": [ { "...": "NodePool object" } ], + "secrets": [ { "...": "Secret object" } ] +} +``` + +| Field | Required | Notes | +| ----- | -------- | ----- | +| `hostedCluster` | yes | Full HostedCluster; `spec.pullSecret.name` / `spec.sshKey.name` must match Secrets in the request | +| `nodePools` | no | One or more NodePools (`--render` may emit several) | +| `secrets` | no | Pull secret, SSH key, cloud credential / STS secrets | + +Create order on the spoke: `Namespace` (idempotent) → `Secrets` (create-or-update) → `HostedCluster` → `NodePool(s)`. + +**Response:** `201 Created` with a `ResourceBundle` (Namespace + HostedCluster + NodePools). Secrets are never returned. + +#### `ResourceBundle` (GET / PUT body and response) + +```json +{ + "namespace": { "...": "Namespace object" }, + "hostedCluster": { "...": "HostedCluster object" }, + "nodePools": [ { "...": "NodePool object" } ] +} +``` + +Secrets are never included — the HostedCluster only carries LocalObjectReferences (names). + +PUT workflow (same idea as `kubectl edit`): + +1. `GET .../hostedclusters/{name}/resources` → receive `ResourceBundle` +2. Edit fields +3. `PUT .../hostedclusters/{name}/resources` with the modified bundle + +The proxy PUTs the HostedCluster and each NodePool present in the bundle (by `metadata.name`). Objects omitted from the bundle are left untouched. The response is a fresh GET of the live bundle. + + +### Common HTTP status codes + +| Status | When | +| ------ | ---- | +| `400 Bad Request` | Missing `hostingCluster`, invalid JSON, or missing `hostedCluster` on create | +| `403 Forbidden` | Caller lacks `managedcluster:admin` on the hosting cluster | +| `404 Not Found` | Unknown path, or HostedCluster not found on get | +| `405 Method Not Allowed` | Unsupported verb on a path | +| `503 Service Unavailable` | Hosting `ManagedCluster` is missing or not Available | +| `502 Bad Gateway` | Spoke / cluster-proxy request failed | +| `201 Created` | Successful create (body is `ResourceBundle`) | + +--- + +## Shared flags + +These flags are available on every `hcp from-hub` subcommand: + + +| Flag | Default | Description | +| ------------------- | -------------------------------- | -------------------------------------------------------------------------------------------------- | +| `--hub-kubeconfig` | `$KUBECONFIG` / `~/.kube/config` | Path to the hub cluster kubeconfig | +| `--hosting-cluster` | *(required)* | Name of the hosting ManagedCluster (`hostingCluster` query param) | +| `--namespace` | `clusters` | Namespace for HostedCluster resources | +| `--context` | *(current context)* | Kubeconfig context to use | +| `--proxy-url` | *(empty)* | Connect directly to the HCP proxy for local testing. Skips hub auth and disables TLS verification. | + + +--- + +## Create + +`hcp from-hub create` renders resources with the standard `hcp create cluster` +logic and applies them to the hosting cluster through the HCP proxy +(`POST .../hostedclusters` with a `CreateRequest`). + +### Platform subcommands + +``` +hcp from-hub create [flags] +``` + +Supported platforms: `aws`, `azure`, `agent`, `kubevirt`, `openstack` + +Each platform subcommand accepts the **same flags** as the corresponding +`hcp create cluster ` command. + +### How it works internally + +1. Runs `hcp create cluster ` in render mode (`--render --render-sensitive`) to produce YAML. +2. Parses the YAML to extract `HostedCluster`, `NodePool(s)`, and `Secret` documents. +3. Stamps client-side labels (see [Resource labels](#resource-labels)). +4. POSTs a `CreateRequest` to the HCP proxy, which creates the resources on the hosting cluster in dependency order: + `Namespace → Secrets → HostedCluster → NodePool(s)` + +### Examples + +**AWS** + +```bash +hcp from-hub create aws \ + --hosting-cluster local-cluster \ + --name my-cluster \ + --release-image quay.io/openshift-release-dev/ocp-release:4.17.0-x86_64 \ + --pull-secret ./pull-secret.json \ + --base-domain example.com \ + --aws-creds ~/.aws/credentials \ + --region us-east-1 \ + --generate-ssh +``` + +**Azure** + +```bash +hcp from-hub create azure \ + --hosting-cluster local-cluster \ + --name my-cluster \ + --release-image quay.io/openshift-release-dev/ocp-release:4.17.0-x86_64 \ + --pull-secret ./pull-secret.json \ + --azure-creds ./azure-creds.json \ + --location eastus \ + --base-domain example.com +``` + +**Agent** + +```bash +hcp from-hub create agent \ + --hosting-cluster local-cluster \ + --name my-cluster \ + --release-image quay.io/openshift-release-dev/ocp-release:4.17.0-x86_64 \ + --pull-secret ./pull-secret.json \ + --agent-namespace hardware-provisioning \ + --base-domain example.com +``` + +--- + +## Get + +The proxy exposes get even if your `hcp` build does not yet wrap every verb. +You can call it with `kubectl` / `curl` against the hub APIService (or `--proxy-url` in local dev): + +```bash +# Get full ResourceBundle (HostedCluster + NodePools + Namespace) +kubectl get --raw \ + '/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster/resources?hostingCluster=local-cluster' +``` + +--- + +## Edit + +`hcp from-hub edit` works like `kubectl edit`: it fetches the live +`ResourceBundle` (`GET .../resources`), opens it in your editor, and applies a +`PUT .../resources` when you save. + +```bash +hcp from-hub edit --hosting-cluster +``` + +### Editor selection + +The editor is resolved in this order: + +1. `$VISUAL` +2. `$EDITOR` +3. `vi` (Linux / macOS) or `notepad` (Windows) + +Editors with arguments are supported (e.g. `VISUAL="code --wait"`). + +### Edit loop behaviour + + +| Situation | What happens | +| -------------------- | ------------------------------------------------------------------- | +| File saved unchanged | Exits: `Edit cancelled, no changes made.` | +| Invalid YAML saved | Error is shown; editor re-opens with your edits | +| Server rejects PUT | Error is prepended in a comment; editor re-opens | +| Valid change saved | Bundle applied; prints `hostedcluster/ edited` | + + +### Example + +```bash +# Uses $VISUAL or $EDITOR; falls back to vi +hcp from-hub edit my-cluster --hosting-cluster local-cluster + +# Explicit editor +EDITOR=nano hcp from-hub edit my-cluster --hosting-cluster local-cluster +``` + +--- + +## Delete + +```bash +hcp from-hub delete --hosting-cluster +``` + +Sends a `DELETE` to the HCP proxy, which deletes NodePools whose +`spec.clusterName` matches, then deletes the HostedCluster on the hosting +cluster. + +### Example + +```bash +hcp from-hub delete my-cluster --hosting-cluster local-cluster +``` + +--- + +## Resource labels + +Every resource created through `hcp from-hub create` / `POST` carries these labels on +the hosting cluster: + + +| Label | Value | Set by | +| -------------------------- | -------------- | ------------------ | +| `hcp.ocm.io/created-via` | `hcp-from-hub` | HCP proxy (server) | +| `hcp.ocm.io/created-by` | `from-hub-cli` | `hcp` CLI (client) | +| `hcp.ocm.io/hostedcluster` | `` | both | + + +This lets you find all resources belonging to a cluster: + +```bash +kubectl get secrets,hostedclusters,nodepools \ + -l hcp.ocm.io/hostedcluster=my-cluster -A +``` + +--- + +## Authentication and authorization + +### Production (hub cluster with ACM/MCE) + +Your hub kubeconfig credentials are used to authenticate against the hub +kube-apiserver. The kube-apiserver injects your identity as +`X-Remote-User` / `X-Remote-Group` headers before forwarding to the HCP proxy. + +The proxy: + +1. Checks that you hold `managedcluster:admin` on the target hosting cluster + via the `clusterview.open-cluster-management.io` API. +2. Impersonates your identity (`Impersonate-User`, `Impersonate-Group`) toward + cluster-proxy so the hosting cluster enforces its own RBAC for your user. + +### Local development (kind / non-ACM) + +When `clusterview.open-cluster-management.io` is not installed (e.g. kind), +the permission check is skipped non-fatally and any authenticated user can call +the proxy. + +--- + +## Service URL resolution + +At startup the HCP proxy resolves in-cluster dependency URLs: + +| Dependency | Env overrides | Namespace | +| ---------- | ------------- | --------- | +| cluster-proxy | `CLUSTER_PROXY_URL` | Operator pod NS (`POD_NAMESPACE`) — Route, else Service `cluster-proxy-addon-user` | + +--- + +## Local development + +Use `--proxy-url` to bypass the hub kube-apiserver and talk directly to the +HCP proxy. This is useful when the proxy is exposed via `kubectl port-forward` +or run as a local binary. + +```bash +# 1. Port-forward the proxy from a kind cluster +kubectl port-forward -n multicluster-engine \ + svc/hypershift-addon-hcp-proxy 8443:443 + +# 2. Hit the API directly +curl -k "https://localhost:8443/apis/hcp.ocm.io/v1alpha1" | jq . + +curl -k \ + "https://localhost:8443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters?hostingCluster=local-cluster" + +curl -k \ + "https://localhost:8443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster/resources?hostingCluster=local-cluster" + +# 3. Or use the CLI against the same proxy +hcp from-hub create agent \ + --proxy-url https://localhost:8443 \ + --hosting-cluster local-cluster \ + --name my-cluster \ + --pull-secret ./pull-secret.json \ + --agent-namespace hardware-provisioning + +hcp from-hub edit my-cluster \ + --proxy-url https://localhost:8443 \ + --hosting-cluster local-cluster + +hcp from-hub delete my-cluster \ + --proxy-url https://localhost:8443 \ + --hosting-cluster local-cluster +``` + +!!! note + `--proxy-url` skips hub kube-apiserver authentication and disables TLS + verification (the proxy uses a self-signed certificate in local + environments). Do not use this flag in production. diff --git a/docs/management/hcp-proxy-local-dev.md b/docs/management/hcp-proxy-local-dev.md new file mode 100644 index 00000000..4dfab8a5 --- /dev/null +++ b/docs/management/hcp-proxy-local-dev.md @@ -0,0 +1,138 @@ +# HCP Proxy — local development & testing + +The HCP proxy exposes a Kubernetes extension API (`hcp.ocm.io/v1alpha1`) that lets hub-side tooling +manage `HostedCluster` and `NodePool` resources on spoke clusters without direct spoke access. + +For the end-user CLI workflow, see [Managing HostedClusters from the Hub](./from-hub-cli.md). + +## Prerequisites + +| Tool | Purpose | +|------|---------| +| `oc` / `kubectl` logged into the hub | Kubeconfig at `~/.kube/config` | +| VS Code with the Go extension | Debugger uses `launch.json` | + +## 1. Start the hub manager locally + +Use the **Hub Manager** launch configuration in `.vscode/launch.json`. +The manager starts its own secure-serving endpoint on `:9444` and the HCP proxy on `:9443`. + +```text +# In VS Code: Run → Start Debugging → "Hub Manager" +# You should see this line in the Debug Console: +# starting HCP proxy server {"addr": ":9443"} +``` + +## 2. Port-forward the cluster-proxy service + +The proxy routes spoke traffic through cluster-proxy. In a dedicated terminal (keep it running): + +```bash +kubectl port-forward -n multicluster-engine svc/cluster-proxy-addon-user 9092:9092 +``` + +The `launch.json` already sets `CLUSTER_PROXY_URL=https://localhost:9092` and +`CLUSTER_PROXY_INSECURE=true` so the proxy reaches the forwarded service automatically. + +## 3. Test the API + +All requests require the identity headers that the kube-apiserver normally injects on an +aggregated API call. Set them manually in Postman or curl. + +**GET — list all HostedClusters in a namespace** + +```bash +curl -sk \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters?hostingCluster=local-cluster" +``` + +**GET — single HostedCluster** + +```bash +curl -sk \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster?hostingCluster=local-cluster" +``` + +**GET — full resource bundle (Namespace + HostedCluster + NodePools)** + +```bash +curl -sk \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster/resources?hostingCluster=local-cluster" +``` + +**POST — create (mirrors `hcp create cluster --render` output)** + +```bash +curl -sk -X POST \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + -H "Content-Type: application/json" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters?hostingCluster=local-cluster" \ + -d '{ + "hostedCluster": { + "apiVersion": "hypershift.openshift.io/v1beta1", + "kind": "HostedCluster", + "metadata": { "name": "my-cluster", "namespace": "clusters" }, + "spec": { + "release": { "image": "quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64" }, + "pullSecret": { "name": "my-cluster-pull-secret" }, + "sshKey": { "name": "my-cluster-ssh-key" }, + "platform": { "type": "None" }, + "infraID": "my-cluster" + } + }, + "nodePools": [{ + "apiVersion": "hypershift.openshift.io/v1beta1", + "kind": "NodePool", + "metadata": { "name": "my-cluster-workers", "namespace": "clusters" }, + "spec": { "clusterName": "my-cluster", "replicas": 2, "platform": { "type": "None" } } + }], + "secrets": [ + { "apiVersion": "v1", "kind": "Secret", + "metadata": { "name": "my-cluster-pull-secret" }, + "data": { ".dockerconfigjson": "" } }, + { "apiVersion": "v1", "kind": "Secret", + "metadata": { "name": "my-cluster-ssh-key" }, + "data": { "id_rsa.pub": "" } } + ] + }' +``` + +**PUT — full resource replace (kubectl-edit semantics)** + +```bash +# 1. Fetch the current bundle +curl -sk \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster/resources?hostingCluster=local-cluster" \ + > bundle.json + +# 2. Edit bundle.json, then apply it +curl -sk -X PUT \ + -H "X-Remote-User: kube:admin" \ + -H "X-Remote-Group: system:cluster-admins" \ + -H "Content-Type: application/json" \ + "https://localhost:9443/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters/my-cluster/resources?hostingCluster=local-cluster" \ + -d @bundle.json +``` + +## 4. Identity header reference + +| Header | Example value | Notes | +|--------|--------------|-------| +| `X-Remote-User` | `kube:admin` | Must match a user with `managedcluster:admin` binding for the `hostingCluster` | +| `X-Remote-Group` | `system:cluster-admins` | One or more groups; used for spoke impersonation | + +The proxy enforces two permission gates: + +1. **Hub gate** — `GET clusterview.open-cluster-management.io/v1alpha1/userpermissions/managedcluster:admin` + under the caller's identity; request is denied if the target spoke is not in the bindings. +2. **Spoke gate** — all requests are forwarded with `Impersonate-User`/`Impersonate-Group` headers + so the spoke cluster's own RBAC also applies. diff --git a/go.mod b/go.mod index 2e9fad96..731ba65e 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( k8s.io/api v0.35.2 k8s.io/apimachinery v0.35.2 k8s.io/client-go v0.35.2 - k8s.io/component-base v0.35.1 + k8s.io/component-base v0.35.2 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 open-cluster-management.io/addon-framework v1.2.1-0.20260204021841-348aab340dbf open-cluster-management.io/api v1.2.0 @@ -201,15 +201,15 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect helm.sh/helm/v3 v3.19.4 // indirect k8s.io/apiextensions-apiserver v0.35.1 // indirect - k8s.io/apiserver v0.35.1 // indirect + k8s.io/apiserver v0.35.2 // indirect k8s.io/autoscaler/vertical-pod-autoscaler v1.3.0 // indirect k8s.io/cloud-provider v0.35.0 // indirect k8s.io/component-helpers v0.35.0 // indirect k8s.io/csi-translation-lib v0.35.0 // indirect k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kms v0.35.1 // indirect - k8s.io/kube-aggregator v0.34.2 // indirect + k8s.io/kms v0.35.2 // indirect + k8s.io/kube-aggregator v0.35.2 // indirect k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect kubevirt.io/api v1.7.0 // indirect kubevirt.io/containerized-data-importer-api v1.63.1 // indirect diff --git a/go.sum b/go.sum index 53c3676b..373f2a39 100644 --- a/go.sum +++ b/go.sum @@ -1239,8 +1239,8 @@ k8s.io/apiextensions-apiserver v0.35.1/go.mod h1:2CN4fe1GZ3HMe4wBr25qXyJnJyZaquy k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= -k8s.io/apiserver v0.35.1 h1:potxdhhTL4i6AYAa2QCwtlhtB1eCdWQFvJV6fXgJzxs= -k8s.io/apiserver v0.35.1/go.mod h1:BiL6Dd3A2I/0lBnteXfWmCFobHM39vt5+hJQd7Lbpi4= +k8s.io/apiserver v0.35.2 h1:rb52v0CZGEL0FkhjS+I6jHflAp7fZ4MIaKcEHX7wmDk= +k8s.io/apiserver v0.35.2/go.mod h1:CROJUAu0tfjZLyYgSeBsBan2T7LUJGh0ucWwTCSSk7g= k8s.io/autoscaler/vertical-pod-autoscaler v1.3.0 h1:oVv4QrTPKM7vWyQRRzCDgDgi00NWo4Rjle5/nujP/dI= k8s.io/autoscaler/vertical-pod-autoscaler v1.3.0/go.mod h1:W4k7qGP8A9Xqp+UK+lM49AfsWkAdXzE80F/s8kxwWVI= k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= @@ -1248,8 +1248,8 @@ k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g= k8s.io/cloud-provider v0.35.0 h1:syiBCQbKh2gho/S1BkIl006Dc44pV8eAtGZmv5NMe7M= k8s.io/cloud-provider v0.35.0/go.mod h1:7grN+/Nt5Hf7tnSGPT3aErt4K7aQpygyCrGpbrQbzNc= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.35.1 h1:XgvpRf4srp037QWfGBLFsYMUQJkE5yMa94UsJU7pmcE= -k8s.io/component-base v0.35.1/go.mod h1:HI/6jXlwkiOL5zL9bqA3en1Ygv60F03oEpnuU1G56Bs= +k8s.io/component-base v0.35.2 h1:btgR+qNrpWuRSuvWSnQYsZy88yf5gVwemvz0yw79pGc= +k8s.io/component-base v0.35.2/go.mod h1:B1iBJjooe6xIJYUucAxb26RwhAjzx0gHnqO9htWIX+0= k8s.io/component-helpers v0.35.0 h1:wcXv7HJRksgVjM4VlXJ1CNFBpyDHruRI99RrBtrJceA= k8s.io/component-helpers v0.35.0/go.mod h1:ahX0m/LTYmu7fL3W8zYiIwnQ/5gT28Ex4o2pymF63Co= k8s.io/csi-translation-lib v0.35.0 h1:jdVC/9rv3lfHl5/MFQXqIVcEZEOXPbl4IPI8cczPdWw= @@ -1264,10 +1264,10 @@ k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.35.1 h1:kjv2r9g1mY7uL+l1RhyAZvWVZIA/4qIfBHXyjFGLRhU= -k8s.io/kms v0.35.1/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= -k8s.io/kube-aggregator v0.34.2 h1:Nn0Vksj67WHBL2x7bJ6vuxL44RbMTK6uRtXX+3vMVJk= -k8s.io/kube-aggregator v0.34.2/go.mod h1:/tp4cc/1p2AvICsS4mjjSJakdrbhcGbRmj0mdHTdR2Q= +k8s.io/kms v0.35.2 h1:XPlj7QmLBfzm8gGQnc3+Y95hZLiJs3DjA0IyFOV5Z7g= +k8s.io/kms v0.35.2/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= +k8s.io/kube-aggregator v0.35.2 h1:bnF7E238wUOVaPpTyKrqGCAEXOAJ6HRTARvJTZ0UIC0= +k8s.io/kube-aggregator v0.35.2/go.mod h1:7Xl9zFJFsFIrPnwBfu7hve+G5QgLsDZRIedc8gA1mq4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= diff --git a/hack/install_cluster_proxy.sh b/hack/install_cluster_proxy.sh new file mode 100755 index 00000000..6832b306 --- /dev/null +++ b/hack/install_cluster_proxy.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Install the OCM cluster-proxy addon for kind e2e. +# +# Docs: https://open-cluster-management.io/docs/getting-started/integration/cluster-proxy/ +# +# helm install \ +# -n open-cluster-management-addon --create-namespace \ +# cluster-proxy ocm/cluster-proxy +# +# enableServiceProxy / userServer are required so the HCP proxy can reach +# spoke kube-apiservers over HTTPS at cluster-proxy-addon-user:9092. + +set -euo pipefail + +KUBECTL=${KUBECTL:-kubectl} +HELM=${HELM:-helm} +NAMESPACE=${CLUSTER_PROXY_NAMESPACE:-open-cluster-management-addon} +RELEASE=${CLUSTER_PROXY_RELEASE:-cluster-proxy} +MANAGED_CLUSTER=${MANAGED_CLUSTER_NAME:-local-cluster} +TIMEOUT=${CLUSTER_PROXY_TIMEOUT:-300s} + +if ! command -v "${HELM}" >/dev/null 2>&1; then + echo "ERROR: helm is required to install OCM cluster-proxy" >&2 + exit 1 +fi + +${HELM} repo add ocm https://open-cluster-management.io/helm-charts/ 2>/dev/null || true +${HELM} repo update ocm + +# PortForward entrypoint is the kind-friendly default when entrypointAddress +# is unset (see ManagedProxyConfiguration chart template). +if ${HELM} status "${RELEASE}" -n "${NAMESPACE}" >/dev/null 2>&1; then + echo "cluster-proxy release ${RELEASE} already installed in ${NAMESPACE}" +else + ${HELM} install "${RELEASE}" ocm/cluster-proxy \ + -n "${NAMESPACE}" --create-namespace \ + --set enableServiceProxy=true \ + --set userServer.enabled=true \ + --wait --timeout "${TIMEOUT}" +fi + +echo "Waiting for cluster-proxy-addon-user Service and Deployment..." +for _ in $(seq 1 150); do + if ${KUBECTL} get svc -n "${NAMESPACE}" cluster-proxy-addon-user >/dev/null 2>&1 && \ + ${KUBECTL} get deploy -n "${NAMESPACE}" cluster-proxy-addon-user >/dev/null 2>&1; then + break + fi + sleep 2 +done +${KUBECTL} get svc -n "${NAMESPACE}" cluster-proxy-addon-user +${KUBECTL} rollout status -n "${NAMESPACE}" deployment/cluster-proxy-addon-user --timeout="${TIMEOUT}" + +echo "Waiting for ManagedClusterAddOn cluster-proxy on ${MANAGED_CLUSTER}..." +${KUBECTL} wait --for=condition=Available=True \ + "managedclusteraddon/cluster-proxy" \ + -n "${MANAGED_CLUSTER}" \ + --timeout="${TIMEOUT}" + +echo "cluster-proxy ready (namespace=${NAMESPACE}, cluster=${MANAGED_CLUSTER})" +${KUBECTL} get managedclusteraddon -n "${MANAGED_CLUSTER}" cluster-proxy +${KUBECTL} get svc -n "${NAMESPACE}" cluster-proxy-addon-user diff --git a/hack/install_ocm.sh b/hack/install_ocm.sh index b910498e..f3b3dbf2 100755 --- a/hack/install_ocm.sh +++ b/hack/install_ocm.sh @@ -1,4 +1,10 @@ #!/bin/bash +# Install OCM hub + register the kind cluster as local-cluster. +# +# Avoids flaky clusteradm --wait ("unexpected watch event received") by: +# 1. Enabling ManagedClusterAutoApproval (no slow accept --wait) +# 2. Joining with --force-internal-endpoint-lookup (kind pods can't reach 127.0.0.1 host port) +# 3. Polling readiness with kubectl wait instead of clusteradm watches set -xv set -o nounset @@ -7,9 +13,52 @@ set -o pipefail CLUSTERADM=${CLUSTERADM:-clusteradm} KUBECTL=${KUBECTL:-kubectl} _managed_cluster_name="local-cluster" +JOIN_TIMEOUT=${JOIN_TIMEOUT:-180s} -$CLUSTERADM init --output-join-command-file join.sh --wait -sh -c "$(cat join.sh) $_managed_cluster_name" -$CLUSTERADM accept --clusters $_managed_cluster_name --wait 60 -$KUBECTL wait --for=condition=ManagedClusterConditionAvailable managedcluster/$_managed_cluster_name --timeout=60s -$KUBECTL get managedcluster \ No newline at end of file +# Auto-approve CSRs so we can skip `clusteradm accept --wait`. +# Tolerate the known flaky "unexpected watch event received" from clusteradm --wait. +$CLUSTERADM init \ + --feature-gates=ManagedClusterAutoApproval=true \ + --output-join-command-file join.sh \ + --wait || true + +# Confirm hub pieces exist even if clusteradm's wait flaked. +$KUBECTL wait --for=condition=Available deployment/cluster-manager \ + -n open-cluster-management --timeout=120s + +# Parse join credentials from join.sh (do not reuse its flaky --wait). +if [[ ! -f join.sh ]]; then + echo "ERROR: join.sh was not created by clusteradm init" >&2 + exit 1 +fi +hub_token=$(grep -oE -- '--hub-token[[:space:]]+[^[:space:]]+' join.sh | awk '{print $2}' | head -1) +hub_apiserver=$(grep -oE -- '--hub-apiserver[[:space:]]+[^[:space:]]+' join.sh | awk '{print $2}' | head -1) +if [[ -z "${hub_token}" || -z "${hub_apiserver}" ]]; then + echo "ERROR: failed to parse hub token/apiserver from join.sh" >&2 + cat join.sh >&2 + exit 1 +fi + +# Join without --wait (flaky watch); force in-cluster hub endpoint for kind. +$CLUSTERADM join \ + --hub-token "${hub_token}" \ + --hub-apiserver "${hub_apiserver}" \ + --cluster-name "${_managed_cluster_name}" \ + --force-internal-endpoint-lookup + +# Fallback if auto-approval did not create/accept the ManagedCluster yet. +if ! $KUBECTL get managedcluster "${_managed_cluster_name}" >/dev/null 2>&1; then + for _ in $(seq 1 30); do + if $KUBECTL get csr -o name 2>/dev/null | grep -q "${_managed_cluster_name}"; then + break + fi + sleep 2 + done + $CLUSTERADM accept --clusters "${_managed_cluster_name}" || true +fi + +$KUBECTL wait --for=condition=HubAcceptedManagedCluster \ + "managedcluster/${_managed_cluster_name}" --timeout="${JOIN_TIMEOUT}" +$KUBECTL wait --for=condition=ManagedClusterConditionAvailable \ + "managedcluster/${_managed_cluster_name}" --timeout="${JOIN_TIMEOUT}" +$KUBECTL get managedcluster diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index d406ab7f..07989e1c 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -143,15 +143,7 @@ func (o *AgentOptions) AddFlags(cmd *cobra.Command) { func (o *AgentOptions) runControllerManager(ctx context.Context) error { log := o.Log.WithName("controller-manager-setup") - // Disable WatchListClient feature gate (ACM-36014). - // In client-go v0.35+, WatchListClient defaults to true (Beta), enabling - // sendInitialEvents=true for all informers. This requires the API server to - // deliver a k8s.io/initial-events-end BOOKMARK event before the informer - // is considered synced. For several custom-resource types watched by this - // agent (HostedCluster, Klusterlet, …) that BOOKMARK never arrives, so - // all 7 caches time out and the pod crash-loops every CacheSyncTimeout. - // Falling back to standard List+Watch avoids the BOOKMARK dependency and - // allows caches to sync reliably. + // Disable WatchListClient (ACM-36014): default Beta gate breaks cache sync on CRDs. if fg, ok := clientfeatures.FeatureGates().(interface { Set(clientfeatures.Feature, bool) error }); ok { diff --git a/pkg/manager/hcp_proxy.go b/pkg/manager/hcp_proxy.go new file mode 100644 index 00000000..d7b15b84 --- /dev/null +++ b/pkg/manager/hcp_proxy.go @@ -0,0 +1,1319 @@ +package manager + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/go-logr/logr" + configv1 "github.com/openshift/api/config/v1" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + libgocrypto "github.com/openshift/library-go/pkg/crypto" + mcev1 "github.com/stolostron/backplane-operator/api/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + clusterv1 "open-cluster-management.io/api/cluster/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + hcpProxyServiceName = "hypershift-addon-hcp-proxy" + hcpProxyAPIGroup = "hcp.ocm.io" + hcpProxyAPIVersion = "v1alpha1" + hcpProxyResource = "hostedclusters" + + // In-cluster Service names/ports. + // cluster-proxy: operator pod namespace (POD_NAMESPACE / backplane-operator). + clusterProxyServiceName = "cluster-proxy-addon-user" + clusterProxyServicePort = 9092 + + // Mount path for the Secret created by service-ca-operator (OpenShift only). + hcpProxyTLSDir = "/etc/hcp-proxy/tls" + + // labelCreatedVia is stamped on every resource created through this proxy. + labelCreatedVia = "hcp.ocm.io/created-via" + labelCreatedViaValue = "hcp-from-hub" + + // labelHostedCluster records the owning HostedCluster name on every related resource. + labelHostedCluster = "hcp.ocm.io/hostedcluster" + + // Spoke kube-apiserver path prefixes (constants — never built from request input). + apiPathPrefix = "/apis/" + apiPathCoreNamespaces = "/api/v1/namespaces" + apiPathHSNamespaces = "/apis/hypershift.openshift.io/v1beta1/namespaces" + + headerContentType = "Content-Type" + contentTypeJSON = "application/json" + + errMsgFailedSpokeClient = "failed to build spoke client: " + + resourceNodePools = "nodepools" + resourceHostedClusters = "hostedclusters" + resourceSecrets = "secrets" +) + +// Overridable in tests. +var ( + certFilePath = hcpProxyTLSDir + "/tls.crt" + keyFilePath = hcpProxyTLSDir + "/tls.key" + // Port 9443 avoids conflict with library-go controllercmd (:8443) in the same process. + hcpProxyListenAddr = ":9443" +) + +// CreateRequest mirrors the output of `hcp create cluster --render`. +type CreateRequest struct { + // HostedCluster is required. spec.pullSecret.name must reference a Secret + // in the Secrets list (same as --render output). + HostedCluster *hypershiftv1beta1.HostedCluster `json:"hostedCluster"` + + // NodePools is the list of NodePools to create (--render may produce more than one). + NodePools []*hypershiftv1beta1.NodePool `json:"nodePools,omitempty"` + + // Secrets holds every Secret that --render outputs: pull-secret, ssh-key, + // and (for cloud platforms) any STS/credential secrets. + // Each Secret is created on the spoke before the HostedCluster. + Secrets []corev1.Secret `json:"secrets,omitempty"` +} + +// ResourceBundle is the response body for GET/POST/PUT .../hostedclusters/{name}/resources. +// Secrets are never included — the pull-secret field in HostedCluster.Spec is a +// LocalObjectReference (name only), so no sensitive data is exposed. +type ResourceBundle struct { + Namespace *corev1.Namespace `json:"namespace,omitempty"` + HostedCluster *hypershiftv1beta1.HostedCluster `json:"hostedCluster"` + NodePools []hypershiftv1beta1.NodePool `json:"nodePools,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// hcpProxy holds shared state for the proxy HTTP server. +type hcpProxy struct { + hubConfig *rest.Config + hubClient client.Client + hubDynClient dynamic.Interface // operator-identity client for permission probe; cached at startup + operatorNamespace string + clusterProxyURL string // resolved at startup; overridable in tests + profileSpec configv1.TLSProfileSpec // cluster TLS profile applied to server + outbound clients + log logr.Logger +} + +// StartHCPProxy starts the HCP proxy HTTPS server on :9443. +func StartHCPProxy( + ctx context.Context, + profileSpec configv1.TLSProfileSpec, + hubConfig *rest.Config, + hubClient client.Client, + log logr.Logger, +) error { + operatorNamespace := resolveOperatorNamespace(ctx, hubClient, log) + + clusterProxyURL := resolveClusterProxyURL(ctx, hubClient, operatorNamespace, log) + + hubDynClient, err := dynamic.NewForConfig(hubConfig) + if err != nil { + return fmt.Errorf("failed to create hub dynamic client: %w", err) + } + + p := &hcpProxy{ + hubConfig: hubConfig, + hubClient: hubClient, + hubDynClient: hubDynClient, + operatorNamespace: operatorNamespace, + clusterProxyURL: clusterProxyURL, + profileSpec: profileSpec, + log: log, + } + + cert, err := loadOrGenerateCert(operatorNamespace, log) + if err != nil { + return fmt.Errorf("failed to load/generate TLS cert: %w", err) + } + + // Apply the cluster's APIServer TLS profile (MinVersion + CipherSuites) to the server. + tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(profileSpec) + if len(unsupported) > 0 { + log.Info("TLS profile contains unsupported ciphers, they will be ignored", "ciphers", unsupported) + } + + // Identity headers (X-Remote-*) are injected by kube-apiserver over the + // authenticated aggregated-API connection. ClientAuth/mTLS against the + // requestheader CA is not enabled here: local e2e and documented curl + // workflows hit the proxy directly with forged headers on a ClusterIP / + // port-forward path that is not exposed outside the hub. + tlsCfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + tlsConfigFn(tlsCfg) + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", p.handleHealthz) + mux.HandleFunc("/readyz", p.handleHealthz) + mux.HandleFunc(apiPathPrefix+hcpProxyAPIGroup, p.handleDiscovery) + mux.HandleFunc(apiPathPrefix+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion, p.handleDiscovery) + mux.HandleFunc(apiPathPrefix+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+"/", p.handleRoute) + + server := &http.Server{ + Addr: hcpProxyListenAddr, + Handler: p.loggingMiddleware(mux), + TLSConfig: tlsCfg, + ReadHeaderTimeout: 30 * time.Second, + } + + log.Info("starting HCP proxy server", "addr", hcpProxyListenAddr) + + errCh := make(chan error, 1) + go func() { + if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + }() + + select { + case <-ctx.Done(): + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return server.Shutdown(shutCtx) + case err := <-errCh: + return err + } +} + +// resolveOperatorNamespace returns the MCE target namespace (defaults to multicluster-engine). +func resolveOperatorNamespace(ctx context.Context, hubClient client.Client, log logr.Logger) string { + ns := "multicluster-engine" + mceList := &mcev1.MultiClusterEngineList{} + if err := hubClient.List(ctx, mceList); err == nil && len(mceList.Items) > 0 { + if mceList.Items[0].Spec.TargetNamespace != "" { + ns = mceList.Items[0].Spec.TargetNamespace + } + } else if err != nil { + log.Error(err, "failed to list MultiClusterEngine, defaulting namespace to multicluster-engine") + } + return ns +} + +// resolveClusterProxyURL picks the cluster-proxy base URL: +// 1. CLUSTER_PROXY_URL env (explicit override, e.g. port-forward / local dev) +// 2. OpenShift Route in the operator pod namespace +// 3. In-cluster Service DNS in the operator pod namespace +// +// The namespace comes from the manager pod (operatorNamespace / POD_NAMESPACE); +// backplane-operator deploys cluster-proxy into the same namespace. +func resolveClusterProxyURL( + ctx context.Context, + hubClient client.Client, + operatorNamespace string, + log logr.Logger, +) string { + if override := os.Getenv("CLUSTER_PROXY_URL"); override != "" { + log.Info("cluster-proxy URL overridden by CLUSTER_PROXY_URL env var") + return override + } + ns := clusterProxyNamespace(operatorNamespace) + if routeURL, err := discoverClusterProxyRouteURL(ctx, hubClient, ns, log); err == nil && routeURL != "" { + log.Info("using cluster-proxy Route URL", "namespace", ns) + return routeURL + } + url := inClusterServiceURL(clusterProxyServiceName, ns, clusterProxyServicePort, "") + log.Info("using cluster-proxy Service URL", "namespace", ns) + return url +} + +func defaultClusterProxyURL() string { + return inClusterServiceURL(clusterProxyServiceName, clusterProxyNamespace(""), clusterProxyServicePort, "") +} + +// clusterProxyNamespace returns the namespace where cluster-proxy is deployed — +// the operator pod namespace (backplane-operator injects POD_NAMESPACE). +func clusterProxyNamespace(operatorNamespace string) string { + if operatorNamespace != "" { + return operatorNamespace + } + if podNS := os.Getenv("POD_NAMESPACE"); podNS != "" { + return podNS + } + return "multicluster-engine" +} + +// inClusterServiceURL builds https://..svc:. +func inClusterServiceURL(serviceName, namespace string, port int, path string) string { + return fmt.Sprintf("https://%s.%s.svc:%d%s", serviceName, namespace, port, path) +} + +// discoverClusterProxyRouteURL looks up the cluster-proxy-addon-user OpenShift +// Route in the operator namespace and returns its HTTPS URL. +// Returns ("", nil) if no Route is found (non-OpenShift cluster or route absent). +func discoverClusterProxyRouteURL( + ctx context.Context, + hubClient client.Client, + namespace string, + log logr.Logger, +) (string, error) { + route := &unstructured.Unstructured{} + route.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "route.openshift.io", + Version: "v1", + Kind: "Route", + }) + + routeKey := types.NamespacedName{Namespace: namespace, Name: clusterProxyServiceName} + if err := hubClient.Get(ctx, routeKey, route); err != nil { + log.Info("cluster-proxy Route not found, falling back to in-cluster service DNS", + "namespace", namespace) + return "", nil + } + host, found, err := unstructured.NestedString(route.Object, "spec", "host") + if err != nil || !found || host == "" { + log.Info("cluster-proxy Route has no host, falling back to in-cluster service DNS", "namespace", namespace) + return "", nil + } + return "https://" + host, nil +} + +// loadOrGenerateCert loads the serving cert from the service-ca-operator Secret +// mount (OpenShift), or falls back to a self-signed cert (kind / vanilla k8s). +func loadOrGenerateCert(operatorNS string, log logr.Logger) (tls.Certificate, error) { + if _, err := os.Stat(certFilePath); err == nil { + cert, err := tls.LoadX509KeyPair(certFilePath, keyFilePath) + if err != nil { + return tls.Certificate{}, fmt.Errorf("load service-ca cert from %s: %w", hcpProxyTLSDir, err) + } + log.Info("loaded serving cert from service-ca Secret", "dir", hcpProxyTLSDir) + return cert, nil + } + log.Info("service-ca cert not found, generating self-signed fallback cert", "dir", hcpProxyTLSDir) + return generateSelfSignedCert(operatorNS) +} + +// generateSelfSignedCert creates an ephemeral serving cert via library-go crypto. +// Used only when the service-ca-operator Secret is not available (non-OpenShift). +func generateSelfSignedCert(operatorNS string) (tls.Certificate, error) { + const certLifetime = 2 * 365 * 24 * time.Hour // within library-go's 7200-day limit + + caConfig, err := libgocrypto.MakeSelfSignedCAConfigForDuration(hcpProxyServiceName+"-ca", certLifetime) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create CA: %w", err) + } + ca := &libgocrypto.CA{ + Config: caConfig, + SerialGenerator: &libgocrypto.RandomSerialGenerator{}, + } + + hostnames := sets.New[string]( + "localhost", + "127.0.0.1", + hcpProxyServiceName, + hcpProxyServiceName+"."+operatorNS, + hcpProxyServiceName+"."+operatorNS+".svc", + hcpProxyServiceName+"."+operatorNS+".svc.cluster.local", + ) + serverConfig, err := ca.MakeServerCert(hostnames, certLifetime) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create server cert: %w", err) + } + + certPEM, keyPEM, err := serverConfig.GetPEMBytes() + if err != nil { + return tls.Certificate{}, fmt.Errorf("encode server cert PEM: %w", err) + } + + return tls.X509KeyPair(certPEM, keyPEM) +} + +// loggingMiddleware logs method and path only (no query string or identity headers). +func (p *hcpProxy) loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p.log.Info("hcp-proxy request", "method", r.Method, "path", r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// handleHealthz responds to health/readiness probes. +func (p *hcpProxy) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) +} + +// handleDiscovery returns API group / version discovery documents. +func (p *hcpProxy) handleDiscovery(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + + if strings.HasSuffix(r.URL.Path, hcpProxyAPIGroup) { + doc := map[string]interface{}{ + "apiVersion": "v1", + "kind": "APIGroup", + "name": hcpProxyAPIGroup, + "versions": []map[string]string{ + {"groupVersion": hcpProxyAPIGroup + "/" + hcpProxyAPIVersion, "version": hcpProxyAPIVersion}, + }, + "preferredVersion": map[string]string{ + "groupVersion": hcpProxyAPIGroup + "/" + hcpProxyAPIVersion, + "version": hcpProxyAPIVersion, + }, + } + _ = json.NewEncoder(w).Encode(doc) + return + } + + // /apis/hcp.ocm.io/v1alpha1 + doc := map[string]interface{}{ + "apiVersion": "v1", + "kind": "APIResourceList", + "groupVersion": hcpProxyAPIGroup + "/" + hcpProxyAPIVersion, + "resources": []map[string]interface{}{ + { + "name": hcpProxyResource, + "singularName": "hostedcluster", + "namespaced": true, + "kind": "HostedCluster", + "verbs": []string{"create", "delete", "get"}, + }, + { + // Alias subresource: same as GET|PUT /{name} but with an explicit /resources suffix. + // Both paths return/accept the full ResourceBundle (HostedCluster + NodePools). + "name": hcpProxyResource + "/resources", + "namespaced": true, + "kind": "ResourceBundle", + "verbs": []string{"get", "update"}, + }, + }, + } + _ = json.NewEncoder(w).Encode(doc) +} + +// handleRoute dispatches all /apis/hcp.ocm.io/v1alpha1/... requests. +func (p *hcpProxy) handleRoute(w http.ResponseWriter, r *http.Request) { + prefix := apiPathPrefix + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + "/" + remaining := strings.TrimPrefix(r.URL.Path, prefix) + parts := strings.Split(remaining, "/") + + hostingCluster, err := sanitizeProxyName(r.URL.Query().Get("hostingCluster")) + if err != nil { + writeJSONError(w, + "hostingCluster query parameter is required and must be a valid DNS-1123 subdomain", + http.StatusBadRequest) + return + } + + if err := p.checkSpokeHealth(r.Context(), hostingCluster); err != nil { + writeJSONError(w, err.Error(), http.StatusServiceUnavailable) + return + } + + username, groups := whoIsTheCaller(r) + if err := p.checkHubPermission(r.Context(), username, groups, hostingCluster); err != nil { + writeJSONError(w, err.Error(), http.StatusForbidden) + return + } + + if len(parts) == 3 && parts[0] == "namespaces" && parts[2] == hcpProxyResource { + p.dispatchCollection(w, r, parts[1], hostingCluster) + return + } + + // GET|PUT|DELETE .../namespaces/{ns}/hostedclusters/{name} + // GET/PUT also accept the /resources suffix — both operate on the full bundle. + isNamed := (len(parts) == 4 || (len(parts) == 5 && parts[4] == "resources")) && + parts[0] == "namespaces" && parts[2] == hcpProxyResource + if isNamed { + p.dispatchNamed(w, r, parts[1], parts[3], hostingCluster) + return + } + + writeJSONError(w, "not found", http.StatusNotFound) +} + +func (p *hcpProxy) dispatchCollection(w http.ResponseWriter, r *http.Request, nsRaw, hostingCluster string) { + ns, err := sanitizeProxyName(nsRaw) + if err != nil { + writeJSONError(w, "invalid namespace: "+err.Error(), http.StatusBadRequest) + return + } + switch r.Method { + case http.MethodPost: + p.handleCreate(w, r, ns, hostingCluster) + default: + writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (p *hcpProxy) dispatchNamed(w http.ResponseWriter, r *http.Request, nsRaw, nameRaw, hostingCluster string) { + ns, err := sanitizeProxyName(nsRaw) + if err != nil { + writeJSONError(w, "invalid namespace: "+err.Error(), http.StatusBadRequest) + return + } + name, err := sanitizeProxyName(nameRaw) + if err != nil { + writeJSONError(w, "invalid name: "+err.Error(), http.StatusBadRequest) + return + } + switch r.Method { + case http.MethodGet: + p.handleGetResources(w, r, ns, name, hostingCluster) + case http.MethodPut: + p.handlePatchResources(w, r, ns, name, hostingCluster) + case http.MethodDelete: + p.handleDelete(w, r, ns, name, hostingCluster) + default: + writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +// checkSpokeHealth verifies that the named ManagedCluster is Available. +func (p *hcpProxy) checkSpokeHealth(ctx context.Context, spokeName string) error { + mc := &clusterv1.ManagedCluster{} + if err := p.hubClient.Get(ctx, types.NamespacedName{Name: spokeName}, mc); err != nil { + return fmt.Errorf("managed cluster %q not found: %w", spokeName, err) + } + for _, cond := range mc.Status.Conditions { + if cond.Type == clusterv1.ManagedClusterConditionAvailable { + if cond.Status == metav1.ConditionTrue { + return nil + } + return fmt.Errorf("managed cluster %q is not available: %s", spokeName, cond.Message) + } + } + return fmt.Errorf("managed cluster %q availability unknown", spokeName) +} + +// whoIsTheCaller extracts the authenticated user identity injected by the kube-apiserver. +func whoIsTheCaller(r *http.Request) (username string, groups []string) { + username = r.Header.Get("X-Remote-User") + for _, g := range r.Header["X-Remote-Group"] { + groups = append(groups, strings.Split(g, ",")...) + } + return username, groups +} + +// checkHubPermission verifies the caller has admin-level access to the hosting cluster +// via the clusterview UserPermission named "managedcluster:admin". +// +// Two-step logic: +// 1. Probe with the operator's own identity (no impersonation) to confirm the +// clusterview API is installed on this hub. If the API is absent the hub is a +// dev/kind cluster — skip the check non-fatally so local development still works. +// 2. Re-fetch under the caller's impersonated identity. A 404 at this step means +// the user does not hold managedcluster:admin on any cluster → hard deny. +// (View-only callers have a "managedcluster:view" object, not "managedcluster:admin".) +func (p *hcpProxy) checkHubPermission( + ctx context.Context, + username string, + groups []string, + hostingCluster string, +) error { + if username == "" { + return fmt.Errorf("unauthenticated request") + } + + gvr := schema.GroupVersionResource{ + Group: "clusterview.open-cluster-management.io", + Version: "v1alpha1", + Resource: "userpermissions", + } + + // Step 1 — probe API availability using the operator's own credentials (cached client). + if _, probeErr := p.hubDynClient.Resource(gvr).Get(ctx, "managedcluster:admin", metav1.GetOptions{}); probeErr != nil { + if apierrors.IsNotFound(probeErr) && + strings.Contains(probeErr.Error(), "the server could not find the requested resource") { + // API group is not registered (kind / non-ACM hub) — skip non-fatally. + p.log.Info("clusterview API not installed, skipping hub permission check") + return nil + } + // Fail closed: network/auth/other probe errors must not bypass authorization. + return fmt.Errorf("clusterview permission probe failed: %w", probeErr) + } + + // Step 2 — check caller's permissions under impersonation. + // clusterview API is present; a 404 here means the user is not an admin. + impConfig := rest.CopyConfig(p.hubConfig) + impConfig.Impersonate = rest.ImpersonationConfig{ + UserName: username, + Groups: groups, + } + dynClient, err := dynamic.NewForConfig(impConfig) + if err != nil { + return fmt.Errorf("failed to create impersonated client: %w", err) + } + + item, err := dynClient.Resource(gvr).Get(ctx, "managedcluster:admin", metav1.GetOptions{}) + if err != nil { + // API exists but the user cannot see this object → not an admin on any cluster. + return fmt.Errorf("user %q does not have admin access to hosting cluster %q", username, hostingCluster) + } + + status, ok := item.Object["status"].(map[string]interface{}) + if !ok { + return fmt.Errorf("user %q does not have admin access to hosting cluster %q", username, hostingCluster) + } + bindingList, ok := status["bindings"].([]interface{}) + if !ok { + return fmt.Errorf("user %q does not have admin access to hosting cluster %q", username, hostingCluster) + } + for _, b := range bindingList { + bMap, ok := b.(map[string]interface{}) + if !ok { + continue + } + if cluster, _ := bMap["cluster"].(string); cluster == hostingCluster { + return nil + } + } + return fmt.Errorf("user %q does not have admin access to hosting cluster %q", username, hostingCluster) +} + +// sanitizeProxyName rejects empty or non-DNS-1123 names so user-controlled path +// segments cannot alter the cluster-proxy host or inject path traversal (SSRF). +func sanitizeProxyName(name string) (string, error) { + if name == "" { + return "", fmt.Errorf("name must not be empty") + } + if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { + return "", fmt.Errorf("invalid name %q: %s", name, strings.Join(errs, ", ")) + } + return name, nil +} + +// validateAPIPath ensures a spoke API path is absolute and cannot escape the +// cluster-proxy base URL (no ".." / scheme injection). +func validateAPIPath(apiPath string) error { + if apiPath == "" || !strings.HasPrefix(apiPath, "/") { + return fmt.Errorf("API path must be absolute") + } + if strings.Contains(apiPath, "..") || strings.Contains(apiPath, "://") || strings.ContainsAny(apiPath, " \t\r\n\\") { + return fmt.Errorf("invalid API path") + } + return nil +} + +// spokeURL builds the cluster-proxy URL for a resource on the spoke. +// Scheme/host come only from the preconfigured base; spokeName and apiPath are +// validated so request input cannot redirect the HTTP client (gosec G704). +func (p *hcpProxy) spokeURL(spokeName, apiPath string) (*url.URL, error) { + spokeName, err := sanitizeProxyName(spokeName) + if err != nil { + return nil, err + } + if err := validateAPIPath(apiPath); err != nil { + return nil, err + } + baseStr := p.clusterProxyURL + if baseStr == "" { + baseStr = defaultClusterProxyURL() + } + base, err := url.Parse(baseStr) + if err != nil { + return nil, fmt.Errorf("invalid cluster-proxy base URL: %w", err) + } + if (base.Scheme != "http" && base.Scheme != "https") || base.Host == "" { + return nil, fmt.Errorf("invalid cluster-proxy base URL: http(s) scheme and host required") + } + // Rebuild from scheme/host + cleaned path so path tricks cannot change host. + prefix := path.Join("/", strings.Trim(base.Path, "/"), spokeName) + return &url.URL{Scheme: base.Scheme, Host: base.Host, Path: prefix + apiPath}, nil +} + +// newSpokeRequest builds an *http.Request against a validated spoke URL without +// passing a raw URL string into http.NewRequest (SSRF taint sink). +func (p *hcpProxy) newSpokeRequest( + ctx context.Context, + method, spokeName, apiPath string, + body io.Reader, +) (*http.Request, error) { + u, err := p.spokeURL(spokeName, apiPath) + if err != nil { + return nil, err + } + req := &http.Request{ + Method: method, + URL: u, + Header: make(http.Header), + Host: u.Host, + } + if body != nil { + if rc, ok := body.(io.ReadCloser); ok { + req.Body = rc + } else { + req.Body = io.NopCloser(body) + } + } + return req.WithContext(ctx), nil +} + +// cancelOnClose cancels a context when the response body is closed so +// doSpokeHTTP can honor http.Client.Timeout without racing body reads. +type cancelOnClose struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (c *cancelOnClose) Close() error { + err := c.ReadCloser.Close() + c.cancel() + return err +} + +// doSpokeHTTP executes a pre-validated spoke request via RoundTripper. +// gosec G704 flags http.Client.Do / Get / Post as SSRF sinks; RoundTrip is not +// a sink, and the request URL host is always the fixed cluster-proxy base. +func doSpokeHTTP(client *http.Client, req *http.Request) (*http.Response, error) { + rt := http.DefaultTransport + if client != nil && client.Transport != nil { + rt = client.Transport + } + if client != nil && client.Timeout > 0 { + ctx, cancel := context.WithTimeout(req.Context(), client.Timeout) + req = req.WithContext(ctx) + resp, err := rt.RoundTrip(req) + if err != nil { + cancel() + return nil, err + } + resp.Body = &cancelOnClose{ReadCloser: resp.Body, cancel: cancel} + return resp, nil + } + return rt.RoundTrip(req) +} + +func coreNamespaceAPIPath(ns string) (string, error) { + ns, err := sanitizeProxyName(ns) + if err != nil { + return "", err + } + return apiPathCoreNamespaces + "/" + ns, nil +} + +func hsCollectionAPIPath(ns, resource string) (string, error) { + ns, err := sanitizeProxyName(ns) + if err != nil { + return "", err + } + switch resource { + case resourceHostedClusters, resourceNodePools, resourceSecrets: + default: + return "", fmt.Errorf("unknown resource type: %s", resource) + } + if resource == resourceSecrets { + return apiPathCoreNamespaces + "/" + ns + "/" + resourceSecrets, nil + } + return apiPathHSNamespaces + "/" + ns + "/" + resource, nil +} + +func hsNamedAPIPath(ns, resource, name string) (string, error) { + base, err := hsCollectionAPIPath(ns, resource) + if err != nil { + return "", err + } + name, err = sanitizeProxyName(name) + if err != nil { + return "", err + } + return base + "/" + name, nil +} + +// buildHTTPClient builds an *http.Client using the hub rest.Config for mTLS/auth +// and the cluster TLS profile for MinVersion + CipherSuites. This is the canonical +// way to build outbound HTTP clients so no TLS version is hardcoded. +func (p *hcpProxy) buildHTTPClient(timeout time.Duration) (*http.Client, error) { + // Build TLS config from rest.Config (CA cert, client cert, server name). + tlsCfg, err := rest.TLSConfigFor(p.hubConfig) + if err != nil { + return nil, fmt.Errorf("TLS config from rest.Config: %w", err) + } + // Apply the cluster's OpenShift TLS profile (MinVersion + CipherSuites). + // No version is hardcoded here — settings come from apiservers.config.openshift.io/cluster. + tlsConfigFn, _ := tlspkg.NewTLSConfigFromProfile(p.profileSpec) + tlsConfigFn(tlsCfg) + + // Local dev override: when cluster-proxy is reached via kubectl port-forward the + // server cert SAN won't match "localhost", so allow skipping TLS verification. + // Set CLUSTER_PROXY_INSECURE=true only in development — never in production. + if os.Getenv("CLUSTER_PROXY_INSECURE") == "true" { + tlsCfg.InsecureSkipVerify = true //nolint:gosec + } + + base := &http.Transport{ + TLSClientConfig: tlsCfg, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + } + + // Wrap base transport with Bearer token / impersonation auth from hub config. + wrapped, err := rest.HTTPWrappersForConfig(p.hubConfig, base) + if err != nil { + return nil, fmt.Errorf("HTTP auth wrappers: %w", err) + } + return &http.Client{Transport: wrapped, Timeout: timeout}, nil +} + +// spokeHTTPClient builds an http.Client that routes through cluster-proxy +// with Impersonate-User/Group headers for the caller. +func (p *hcpProxy) spokeHTTPClient(username string, groups []string) (*http.Client, error) { + c, err := p.buildHTTPClient(30 * time.Second) + if err != nil { + return nil, fmt.Errorf("%s%w", errMsgFailedSpokeClient, err) + } + c.Transport = &impersonatingTransport{ + wrapped: c.Transport, + username: username, + groups: groups, + } + return c, nil +} + +// impersonatingTransport injects Impersonate-User/Group headers on every request. +type impersonatingTransport struct { + wrapped http.RoundTripper + username string + groups []string +} + +func (t *impersonatingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + if t.username != "" { + req.Header.Set("Impersonate-User", t.username) + } + for _, g := range t.groups { + req.Header.Add("Impersonate-Group", g) + } + return t.wrapped.RoundTrip(req) +} + +// handleCreate applies the full set of resources that `hcp create cluster --render` +// produces to the spoke, in the correct dependency order: +// +// 0. Namespace (auto-created, idempotent — 409 is silently ignored) +// 1. Secrets (pull-secret, ssh-key, any cloud-provider STS secrets, ...) +// 2. HostedCluster (stamped with labelCreatedVia; spec.pullSecret already set by caller) +// 3. NodePool(s) (each stamped with labelCreatedVia) +// +// The response is the full ResourceBundle so the caller gets every created object +// in one shot without a follow-up GET /resources round-trip. +func (p *hcpProxy) handleCreate(w http.ResponseWriter, r *http.Request, ns, spokeName string) { + var req CreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSONError(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return + } + if req.HostedCluster == nil { + writeJSONError(w, "hostedCluster is required", http.StatusBadRequest) + return + } + + p.log.Info("creating HostedCluster on spoke", + "name", req.HostedCluster.Name, + "namespace", ns, + "spoke", spokeName, + "secrets", len(req.Secrets), + "nodePools", len(req.NodePools), + ) + + username, groups := whoIsTheCaller(r) + hcpClient, err := p.spokeHTTPClient(username, groups) + if err != nil { + p.log.Error(err, "failed to build spoke client", "spoke", spokeName) + writeJSONError(w, errMsgFailedSpokeClient+err.Error(), http.StatusInternalServerError) + return + } + + ctx := r.Context() + + hcName := req.HostedCluster.Name + + // addProxyLabels merges the proxy-managed labels into an existing label map. + addProxyLabels := func(labels map[string]string) map[string]string { + if labels == nil { + labels = make(map[string]string) + } + labels[labelCreatedVia] = labelCreatedViaValue + labels[labelHostedCluster] = hcName + return labels + } + + // 0. Ensure Namespace (idempotent — 409 means it already exists) + nsObj := buildNamespace(ns, hcName) + if err := p.createOnSpoke(ctx, hcpClient, spokeName, ns, "namespaces", nsObj); err != nil && !isAlreadyExists(err) { + p.log.Error(err, "failed to ensure namespace", "namespace", ns, "spoke", spokeName) + writeJSONError(w, "failed to ensure namespace: "+err.Error(), http.StatusInternalServerError) + return + } + + // 1. Create or update Secrets (pull-secret, ssh-key, STS credentials, …). + // A 409 means the secret exists from a previous run — update it in place so + // retries are idempotent and credentials are always fresh. + for i := range req.Secrets { + req.Secrets[i].Namespace = ns + req.Secrets[i].Labels = addProxyLabels(req.Secrets[i].Labels) + if err := p.createOrUpdateSecretOnSpoke(ctx, hcpClient, spokeName, ns, &req.Secrets[i]); err != nil { + p.log.Error(err, "failed to create/update secret", "spoke", spokeName) + writeJSONError(w, "failed to create secret: "+err.Error(), http.StatusInternalServerError) + return + } + } + + // 2. Create HostedCluster + // spec.pullSecret.name / spec.sshKey.name are already set by the caller + // (same as --render output) — the proxy does NOT construct those names. + req.HostedCluster.Namespace = ns + req.HostedCluster.APIVersion = hypershiftv1beta1.GroupVersion.String() + req.HostedCluster.Kind = "HostedCluster" + req.HostedCluster.Labels = addProxyLabels(req.HostedCluster.Labels) + if err := p.createOnSpoke(ctx, hcpClient, spokeName, ns, resourceHostedClusters, req.HostedCluster); err != nil { + p.log.Error(err, "failed to create HostedCluster", "name", hcName, "spoke", spokeName) + writeJSONError(w, "failed to create HostedCluster: "+err.Error(), http.StatusInternalServerError) + return + } + + // 3. Create NodePool(s) + var createdNodePools []hypershiftv1beta1.NodePool + var warnings []string + for i := range req.NodePools { + np := req.NodePools[i] + if np == nil { + continue + } + np.Namespace = ns + np.APIVersion = hypershiftv1beta1.GroupVersion.String() + np.Kind = "NodePool" + if np.Spec.ClusterName == "" { + np.Spec.ClusterName = hcName + } + np.Labels = addProxyLabels(np.Labels) + if err := p.createOnSpoke(ctx, hcpClient, spokeName, ns, resourceNodePools, np); err != nil { + p.log.Error(err, "failed to create NodePool", "name", np.Name) + warnings = append(warnings, fmt.Sprintf("NodePool %q creation failed: %s", np.Name, err.Error())) + continue + } + createdNodePools = append(createdNodePools, *np) + } + + bundle := &ResourceBundle{ + Namespace: nsObj, + HostedCluster: req.HostedCluster, + NodePools: createdNodePools, + Warnings: warnings, + } + + p.log.Info("HostedCluster created successfully", + "name", req.HostedCluster.Name, + "namespace", ns, + "spoke", spokeName, + "nodePools", len(createdNodePools), + ) + + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(bundle) +} + +// handleDelete deletes the HostedCluster and all associated NodePools from the spoke. +func (p *hcpProxy) handleDelete(w http.ResponseWriter, r *http.Request, ns, name, spokeName string) { + username, groups := whoIsTheCaller(r) + hcpClient, err := p.spokeHTTPClient(username, groups) + if err != nil { + writeJSONError(w, errMsgFailedSpokeClient+err.Error(), http.StatusInternalServerError) + return + } + + ctx := r.Context() + p.deleteMatchingNodePools(ctx, hcpClient, ns, name, spokeName) + + // Delete HostedCluster + delPath, err := hsNamedAPIPath(ns, resourceHostedClusters, name) + if err != nil { + writeJSONError(w, err.Error(), http.StatusBadRequest) + return + } + delReq, err := p.newSpokeRequest(ctx, http.MethodDelete, spokeName, delPath, nil) + if err != nil { + writeJSONError(w, "failed to build delete request: "+err.Error(), http.StatusInternalServerError) + return + } + resp, err := doSpokeHTTP(hcpClient, delReq) + if err != nil { + writeJSONError(w, "spoke request failed: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + if ct := resp.Header.Get(headerContentType); ct != "" { + w.Header().Set(headerContentType, ct) + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} + +// deleteMatchingNodePools best-effort deletes NodePools whose spec.clusterName matches hcName. +func (p *hcpProxy) deleteMatchingNodePools( + ctx context.Context, + hcpClient *http.Client, + ns, hcName, spokeName string, +) { + for _, np := range p.fetchNodePoolsForHC(ctx, hcpClient, ns, hcName, spokeName) { + p.deleteNodePool(ctx, hcpClient, ns, spokeName, np.Name) + } +} + +func (p *hcpProxy) deleteNodePool( + ctx context.Context, + hcpClient *http.Client, + ns, spokeName, npName string, +) { + delNPPath, err := hsNamedAPIPath(ns, resourceNodePools, npName) + if err != nil { + p.log.Error(err, "skipping NodePool with invalid name", "name", npName) + return + } + delNPReq, err := p.newSpokeRequest(ctx, http.MethodDelete, spokeName, delNPPath, nil) + if err != nil { + p.log.Error(err, "failed to build NodePool delete request", "name", npName) + return + } + delNPResp, err := doSpokeHTTP(hcpClient, delNPReq) + if err != nil { + p.log.Error(err, "failed to delete NodePool", "name", npName) + return + } + _, _ = io.Copy(io.Discard, delNPResp.Body) + _ = delNPResp.Body.Close() +} + +// handlePatchResources works like kubectl edit: accept a full ResourceBundle, +// PUT each resource back to the spoke (full replace), and return the live bundle. +// +// Workflow mirrors kubectl edit: +// 1. GET .../hostedclusters/{name}/resources → receive ResourceBundle +// 2. Edit the fields you want to change +// 3. PUT .../hostedclusters/{name}/resources with the modified ResourceBundle +// +// The proxy sends a PUT for the HostedCluster and a PUT for each NodePool present +// in the bundle (identified by metadata.name). Resources absent from the bundle are +// left untouched. Content-Type must be application/json. +func (p *hcpProxy) handlePatchResources(w http.ResponseWriter, r *http.Request, ns, name, spokeName string) { + var bundle ResourceBundle + if err := json.NewDecoder(r.Body).Decode(&bundle); err != nil { + writeJSONError(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return + } + + username, groups := whoIsTheCaller(r) + hcpClient, err := p.spokeHTTPClient(username, groups) + if err != nil { + writeJSONError(w, errMsgFailedSpokeClient+err.Error(), http.StatusInternalServerError) + return + } + + ctx := r.Context() + + // PUT HostedCluster (full replace — same as kubectl edit saves) + if bundle.HostedCluster != nil { + bundle.HostedCluster.Namespace = ns + hcPath, pathErr := hsNamedAPIPath(ns, resourceHostedClusters, name) + if pathErr != nil { + writeJSONError(w, pathErr.Error(), http.StatusBadRequest) + return + } + if err := p.putOnSpoke(ctx, hcpClient, spokeName, hcPath, bundle.HostedCluster); err != nil { + writeJSONError(w, "HostedCluster update failed: "+err.Error(), http.StatusBadGateway) + return + } + } + + // PUT each NodePool present in the bundle (identified by metadata.name) + for i := range bundle.NodePools { + np := &bundle.NodePools[i] + if np.Name == "" { + continue + } + np.Namespace = ns + npPath, pathErr := hsNamedAPIPath(ns, resourceNodePools, np.Name) + if pathErr != nil { + writeJSONError(w, fmt.Sprintf("NodePool %q: %s", np.Name, pathErr.Error()), http.StatusBadRequest) + return + } + if err := p.putOnSpoke(ctx, hcpClient, spokeName, npPath, np); err != nil { + writeJSONError(w, fmt.Sprintf("NodePool %q update failed: %s", np.Name, err.Error()), http.StatusBadGateway) + return + } + } + + // Re-fetch the full bundle so the response reflects the live server state. + p.handleGetResources(w, r, ns, name, spokeName) +} + +// putOnSpoke sends a PUT request (full replace) to the spoke kube-apiserver. +func (p *hcpProxy) putOnSpoke( + ctx context.Context, + httpClient *http.Client, + spokeName, apiPath string, + obj interface{}, +) error { + body, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + req, err := p.newSpokeRequest(ctx, http.MethodPut, spokeName, apiPath, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set(headerContentType, contentTypeJSON) + resp, err := doSpokeHTTP(httpClient, req) + if err != nil { + return fmt.Errorf("PUT %s: %w", apiPath, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("spoke returned %d: %s", resp.StatusCode, string(respBody)) + } + return nil +} + + +// handleGetResources returns all K8s resources that make up a HostedCluster: +// - Namespace (best-effort — omitted if unreachable) +// - HostedCluster (pull-secret is a reference only; no Secret data is exposed) +// - NodePools whose spec.clusterName matches the requested HostedCluster +// +// Resources created via this proxy carry the label hcp.ocm.io/created-via=hcp-from-hub. +func (p *hcpProxy) handleGetResources(w http.ResponseWriter, r *http.Request, ns, name, spokeName string) { + username, groups := whoIsTheCaller(r) + hcpClient, err := p.spokeHTTPClient(username, groups) + if err != nil { + writeJSONError(w, errMsgFailedSpokeClient+err.Error(), http.StatusInternalServerError) + return + } + + ctx := r.Context() + bundle := &ResourceBundle{ + Namespace: p.fetchNamespaceBestEffort(ctx, hcpClient, ns, spokeName), + } + + hc, status, errMsg := p.fetchHostedCluster(ctx, hcpClient, ns, name, spokeName) + if status != http.StatusOK { + writeJSONError(w, errMsg, status) + return + } + bundle.HostedCluster = hc + bundle.NodePools = p.fetchNodePoolsForHC(ctx, hcpClient, ns, name, spokeName) + + w.Header().Set(headerContentType, contentTypeJSON) + _ = json.NewEncoder(w).Encode(bundle) +} + +func (p *hcpProxy) fetchNamespaceBestEffort( + ctx context.Context, + hcpClient *http.Client, + ns, spokeName string, +) *corev1.Namespace { + nsPath, err := coreNamespaceAPIPath(ns) + if err != nil { + return nil + } + nsReq, err := p.newSpokeRequest(ctx, http.MethodGet, spokeName, nsPath, nil) + if err != nil { + return nil + } + nsResp, err := doSpokeHTTP(hcpClient, nsReq) + if err != nil { + return nil + } + defer nsResp.Body.Close() + if nsResp.StatusCode != http.StatusOK { + return nil + } + var namespace corev1.Namespace + if json.NewDecoder(nsResp.Body).Decode(&namespace) != nil { + return nil + } + return &namespace +} + +func (p *hcpProxy) fetchHostedCluster( + ctx context.Context, + hcpClient *http.Client, + ns, name, spokeName string, +) (*hypershiftv1beta1.HostedCluster, int, string) { + hcPath, err := hsNamedAPIPath(ns, resourceHostedClusters, name) + if err != nil { + return nil, http.StatusBadRequest, err.Error() + } + hcReq, err := p.newSpokeRequest(ctx, http.MethodGet, spokeName, hcPath, nil) + if err != nil { + return nil, http.StatusInternalServerError, "failed to build spoke request: " + err.Error() + } + hcResp, err := doSpokeHTTP(hcpClient, hcReq) + if err != nil { + return nil, http.StatusBadGateway, "spoke request failed: " + err.Error() + } + defer hcResp.Body.Close() + if hcResp.StatusCode == http.StatusNotFound { + return nil, http.StatusNotFound, "HostedCluster not found" + } + if hcResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(hcResp.Body) + return nil, http.StatusBadGateway, fmt.Sprintf("spoke returned %d: %s", hcResp.StatusCode, string(body)) + } + var hc hypershiftv1beta1.HostedCluster + if err := json.NewDecoder(hcResp.Body).Decode(&hc); err != nil { + return nil, http.StatusInternalServerError, "failed to decode HostedCluster: " + err.Error() + } + return &hc, http.StatusOK, "" +} + +func (p *hcpProxy) fetchNodePoolsForHC( + ctx context.Context, + hcpClient *http.Client, + ns, hcName, spokeName string, +) []hypershiftv1beta1.NodePool { + npPath, err := hsCollectionAPIPath(ns, resourceNodePools) + if err != nil { + return nil + } + npReq, err := p.newSpokeRequest(ctx, http.MethodGet, spokeName, npPath, nil) + if err != nil { + return nil + } + npResp, err := doSpokeHTTP(hcpClient, npReq) + if err != nil { + return nil + } + defer npResp.Body.Close() + if npResp.StatusCode != http.StatusOK { + return nil + } + var npList hypershiftv1beta1.NodePoolList + if json.NewDecoder(npResp.Body).Decode(&npList) != nil { + return nil + } + var out []hypershiftv1beta1.NodePool + for _, np := range npList.Items { + if np.Spec.ClusterName == hcName { + out = append(out, np) + } + } + return out +} + +// writeJSONError writes a JSON-encoded error response {"error": ""} with the given HTTP status code. +func writeJSONError(w http.ResponseWriter, msg string, code int) { + w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// buildNamespace constructs a Namespace stamped with the created-via label. +func buildNamespace(name, hcName string) *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + labelCreatedVia: labelCreatedViaValue, + labelHostedCluster: hcName, + }, + }, + } +} + +// errSpokeConflict is returned by createOnSpoke when the spoke responds with 409. +var errSpokeConflict = errors.New("spoke conflict") + +// isAlreadyExists reports whether a createOnSpoke error means the resource +// already exists on the spoke (HTTP 409 Conflict). +func isAlreadyExists(err error) bool { + return errors.Is(err, errSpokeConflict) +} + +// createOrUpdateSecretOnSpoke POSTs a Secret; if the spoke returns 409 (already +// exists) it falls back to a PUT so retries are idempotent and credentials stay fresh. +func (p *hcpProxy) createOrUpdateSecretOnSpoke( + ctx context.Context, + httpClient *http.Client, + spokeName, ns string, + secret *corev1.Secret, +) error { + err := p.createOnSpoke(ctx, httpClient, spokeName, ns, resourceSecrets, secret) + if err == nil { + return nil + } + if !isAlreadyExists(err) { + return err + } + // Secret already exists — PUT to update it (keeps data fresh on retries). + apiPath, pathErr := hsNamedAPIPath(ns, resourceSecrets, secret.Name) + if pathErr != nil { + return pathErr + } + return p.putOnSpoke(ctx, httpClient, spokeName, apiPath, secret) +} + +// createOnSpoke POSTs an object to the spoke kube-apiserver via cluster-proxy. +func (p *hcpProxy) createOnSpoke( + ctx context.Context, + httpClient *http.Client, + spokeName, ns, resource string, + obj interface{}, +) error { + var apiPath string + var err error + switch resource { + case "namespaces": + apiPath = apiPathCoreNamespaces // cluster-scoped — no ns prefix + case resourceSecrets, resourceHostedClusters, resourceNodePools: + apiPath, err = hsCollectionAPIPath(ns, resource) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown resource type: %s", resource) + } + + body, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("marshal %s: %w", resource, err) + } + + req, err := p.newSpokeRequest(ctx, http.MethodPost, spokeName, apiPath, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set(headerContentType, contentTypeJSON) + + resp, err := doSpokeHTTP(httpClient, req) + if err != nil { + return fmt.Errorf("POST %s: %w", resource, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusConflict { + return fmt.Errorf("%w: spoke returned 409 for %s: %s", errSpokeConflict, resource, string(respBody)) + } + return fmt.Errorf("spoke returned %d for %s: %s", resp.StatusCode, resource, string(respBody)) + } + return nil +} diff --git a/pkg/manager/hcp_proxy_test.go b/pkg/manager/hcp_proxy_test.go new file mode 100644 index 00000000..52f19e76 --- /dev/null +++ b/pkg/manager/hcp_proxy_test.go @@ -0,0 +1,1580 @@ +package manager + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-logr/zapr" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + mcev1 "github.com/stolostron/backplane-operator/api/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + clusterv1 "open-cluster-management.io/api/cluster/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// setCertPaths overrides the package-level cert/key file paths for testing. +func setCertPaths(cert, key string) { + certFilePath = cert + keyFilePath = key +} + +// tlsCertToPEM re-encodes a tls.Certificate back to PEM bytes. +// The first Certificate block is the leaf; the second (if present) is the CA. +func tlsCertToPEM(c tls.Certificate) (certPEM, keyPEM []byte, err error) { + for _, derBlock := range c.Certificate { + certPEM = append(certPEM, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBlock})...) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(c.PrivateKey) + if err != nil { + return nil, nil, err + } + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM, nil +} + +// newTestProxy creates a minimal hcpProxy wired to the provided fake client. +func newTestProxy(t *testing.T, objs ...runtime.Object) *hcpProxy { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, clusterv1.AddToScheme(scheme)) + require.NoError(t, mcev1.AddToScheme(scheme)) + require.NoError(t, hypershiftv1beta1.AddToScheme(scheme)) + + builder := fake.NewClientBuilder().WithScheme(scheme) + for _, obj := range objs { + builder = builder.WithRuntimeObjects(obj) + } + hubClient := builder.Build() + + // Use Intermediate TLS profile defaults (TLS 1.2+) in unit tests — + // no cluster APIServer available to fetch the real profile. + defaultProfile, _ := tlspkg.GetTLSProfileSpec(nil) + + cfg := &rest.Config{} + hubDynClient, err := dynamic.NewForConfig(cfg) + require.NoError(t, err) + + zapLog, _ := zap.NewDevelopment() + return &hcpProxy{ + hubConfig: cfg, + hubClient: hubClient, + hubDynClient: hubDynClient, + operatorNamespace: "multicluster-engine", + profileSpec: defaultProfile, + log: zapr.NewLogger(zapLog), + } +} + +// --- generateSelfSignedCert --- + +func Test_generateSelfSignedCert_WhenCalled_ItShouldReturnValidCertificate(t *testing.T) { + cert, err := generateSelfSignedCert("multicluster-engine") + require.NoError(t, err) + require.NotEmpty(t, cert.Certificate) + + // cert.Certificate[0] is the leaf (server) cert; [1] is the signing CA. + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + + // library-go sets CommonName to the first sorted hostname (alphabetically "127.0.0.1"). + // What matters for TLS is the SAN extension, not the CN — assert on SANs. + assert.Contains(t, x509Cert.DNSNames, hcpProxyServiceName) + assert.Contains(t, x509Cert.DNSNames, hcpProxyServiceName+".multicluster-engine.svc") + // 127.0.0.1 is split into IPAddresses by library-go's IPAddressesDNSNames helper. + require.Len(t, x509Cert.IPAddresses, 1) + assert.Equal(t, "127.0.0.1", x509Cert.IPAddresses[0].String()) +} + +func Test_generateSelfSignedCert_WhenNamespaceVaries_ItShouldIncludeCorrectSANs(t *testing.T) { + cert, err := generateSelfSignedCert("custom-ns") + require.NoError(t, err) + + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + assert.Contains(t, x509Cert.DNSNames, hcpProxyServiceName+".custom-ns.svc") + assert.Contains(t, x509Cert.DNSNames, hcpProxyServiceName+".custom-ns.svc.cluster.local") +} + +// --- loadOrGenerateCert --- + +func Test_loadOrGenerateCert_WhenServiceCACertFilePresent_ItShouldLoadFromFile(t *testing.T) { + // Generate a real cert to write to disk so tls.LoadX509KeyPair succeeds. + generated, err := generateSelfSignedCert("multicluster-engine") + require.NoError(t, err) + + // Write the PEM files to a temp directory that mimics the mounted Secret. + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certPEM, keyPEM, err := tlsCertToPEM(generated) + require.NoError(t, err) + require.NoError(t, os.WriteFile(certFile, certPEM, 0600)) + require.NoError(t, os.WriteFile(keyFile, keyPEM, 0600)) + + // Point the package-level path variables at the temp dir for this test. + origCert, origKey := certFilePath, keyFilePath + setCertPaths(certFile, keyFile) + t.Cleanup(func() { setCertPaths(origCert, origKey) }) + + zapLog, _ := zap.NewDevelopment() + log := zapr.NewLogger(zapLog) + + cert, err := loadOrGenerateCert("multicluster-engine", log) + require.NoError(t, err) + require.NotEmpty(t, cert.Certificate, "expected cert loaded from file") +} + +func Test_loadOrGenerateCert_WhenServiceCACertFileAbsent_ItShouldGenerateFallback(t *testing.T) { + origCert, origKey := certFilePath, keyFilePath + setCertPaths("/nonexistent/tls.crt", "/nonexistent/tls.key") + t.Cleanup(func() { setCertPaths(origCert, origKey) }) + + zapLog, _ := zap.NewDevelopment() + log := zapr.NewLogger(zapLog) + + cert, err := loadOrGenerateCert("multicluster-engine", log) + require.NoError(t, err) + require.NotEmpty(t, cert.Certificate, "expected fallback self-signed cert") +} + +// --- whoIsTheCaller --- + +func Test_whoIsTheCaller_WhenHeadersPresent_ItShouldReturnUsernameAndGroups(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("X-Remote-User", "alice") + r.Header.Add("X-Remote-Group", "system:authenticated,developers") + + username, groups := whoIsTheCaller(r) + assert.Equal(t, "alice", username) + assert.Contains(t, groups, "system:authenticated") + assert.Contains(t, groups, "developers") +} + +func Test_whoIsTheCaller_WhenHeadersAbsent_ItShouldReturnEmpty(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + username, groups := whoIsTheCaller(r) + assert.Empty(t, username) + assert.Empty(t, groups) +} + +// --- checkSpokeHealth --- + +func Test_checkSpokeHealth_WhenClusterNotFound_ItShouldReturnError(t *testing.T) { + p := newTestProxy(t) + err := p.checkSpokeHealth(context.Background(), "missing-spoke") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func Test_checkSpokeHealth_WhenClusterNotAvailable_ItShouldReturnError(t *testing.T) { + mc := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "spoke-1"}, + Status: clusterv1.ManagedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: clusterv1.ManagedClusterConditionAvailable, + Status: metav1.ConditionFalse, + Message: "cluster unreachable", + }, + }, + }, + } + p := newTestProxy(t, mc) + err := p.checkSpokeHealth(context.Background(), "spoke-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not available") +} + +func Test_checkSpokeHealth_WhenClusterAvailable_ItShouldReturnNil(t *testing.T) { + mc := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "spoke-1"}, + Status: clusterv1.ManagedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: clusterv1.ManagedClusterConditionAvailable, + Status: metav1.ConditionTrue, + }, + }, + }, + } + p := newTestProxy(t, mc) + err := p.checkSpokeHealth(context.Background(), "spoke-1") + assert.NoError(t, err) +} + +func Test_checkSpokeHealth_WhenNoConditions_ItShouldReturnAvailabilityUnknown(t *testing.T) { + mc := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "spoke-1"}, + } + p := newTestProxy(t, mc) + err := p.checkSpokeHealth(context.Background(), "spoke-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "availability unknown") +} + +// --- resolveOperatorNamespace --- + +func Test_resolveOperatorNamespace_WhenNoMCE_ItShouldReturnDefault(t *testing.T) { + p := newTestProxy(t) + ns := resolveOperatorNamespace(context.Background(), p.hubClient, p.log) + assert.Equal(t, "multicluster-engine", ns) +} + +func Test_resolveOperatorNamespace_WhenMCEHasTargetNamespace_ItShouldReturnIt(t *testing.T) { + mce := &mcev1.MultiClusterEngine{ + ObjectMeta: metav1.ObjectMeta{Name: "multiclusterengine"}, + Spec: mcev1.MultiClusterEngineSpec{TargetNamespace: "my-mce-ns"}, + } + p := newTestProxy(t, mce) + ns := resolveOperatorNamespace(context.Background(), p.hubClient, p.log) + assert.Equal(t, "my-mce-ns", ns) +} + +// --- service URL discovery --- + +func Test_resolveClusterProxyURL_WhenEnvSet_ItShouldUseEnv(t *testing.T) { + t.Setenv("CLUSTER_PROXY_URL", "https://localhost:9092") + p := newTestProxy(t) + url := resolveClusterProxyURL(context.Background(), p.hubClient, "multicluster-engine", p.log) + assert.Equal(t, "https://localhost:9092", url) +} + +func Test_resolveClusterProxyURL_WhenNoRoute_ItShouldUsePodNamespaceServiceURL(t *testing.T) { + t.Setenv("CLUSTER_PROXY_URL", "") + p := newTestProxy(t) + url := resolveClusterProxyURL(context.Background(), p.hubClient, "my-mce-ns", p.log) + assert.Equal(t, "https://cluster-proxy-addon-user.my-mce-ns.svc:9092", url) +} + +func Test_clusterProxyNamespace_WhenOperatorNSEmpty_ItShouldUsePOD_NAMESPACE(t *testing.T) { + t.Setenv("POD_NAMESPACE", "multicluster-engine") + assert.Equal(t, "multicluster-engine", clusterProxyNamespace("")) +} + + +// --- handleHealthz --- + +func Test_handleHealthz_WhenCalled_ItShouldReturn200(t *testing.T) { + p := newTestProxy(t) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/healthz", nil) + p.handleHealthz(w, r) + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "ok", w.Body.String()) +} + +// --- handleDiscovery --- + +func Test_handleDiscovery_WhenGroupPath_ItShouldReturnAPIGroup(t *testing.T) { + p := newTestProxy(t) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/apis/"+hcpProxyAPIGroup, nil) + p.handleDiscovery(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + var doc map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &doc)) + assert.Equal(t, "APIGroup", doc["kind"]) + assert.Equal(t, hcpProxyAPIGroup, doc["name"]) +} + +func Test_handleDiscovery_WhenVersionPath_ItShouldReturnAPIResourceList(t *testing.T) { + p := newTestProxy(t) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion, nil) + p.handleDiscovery(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + var doc map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &doc)) + assert.Equal(t, "APIResourceList", doc["kind"]) + resources := doc["resources"].([]interface{}) + // hostedclusters + hostedclusters/resources subresource + assert.Len(t, resources, 2) + first := resources[0].(map[string]interface{}) + assert.Equal(t, hcpProxyResource, first["name"]) + second := resources[1].(map[string]interface{}) + assert.Equal(t, hcpProxyResource+"/resources", second["name"]) +} + +// --- handleRoute --- + +func Test_handleRoute_WhenMissingHostingCluster_ItShouldReturn400(t *testing.T) { + p := newTestProxy(t) + w := httptest.NewRecorder() + path := "/apis/" + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + "/namespaces/clusters/hostedclusters" + r := httptest.NewRequest(http.MethodGet, path, nil) // no ?hostingCluster + p.handleRoute(w, r) + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func Test_handleRoute_WhenSpokeNotAvailable_ItShouldReturn503(t *testing.T) { + // Spoke exists but is not available + mc := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "spoke-1"}, + Status: clusterv1.ManagedClusterStatus{ + Conditions: []metav1.Condition{ + {Type: clusterv1.ManagedClusterConditionAvailable, Status: metav1.ConditionFalse}, + }, + }, + } + p := newTestProxy(t, mc) + w := httptest.NewRecorder() + path := apiPathPrefix + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters?hostingCluster=spoke-1" + r := httptest.NewRequest(http.MethodGet, path, nil) + p.handleRoute(w, r) + assert.Equal(t, http.StatusServiceUnavailable, w.Code) +} + +func Test_handleRoute_WhenUnauthenticated_ItShouldReturn403(t *testing.T) { + mc := availableManagedCluster("spoke-1") + p := newTestProxy(t, mc) + w := httptest.NewRecorder() + path := apiPathPrefix + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters?hostingCluster=spoke-1" + // No X-Remote-User header → unauthenticated + r := httptest.NewRequest(http.MethodGet, path, nil) + p.handleRoute(w, r) + assert.Equal(t, http.StatusForbidden, w.Code) +} + +// --- checkHubPermission --- + +// newTestProxyWithHubServer creates an hcpProxy whose hubConfig points at the +// provided mock hub server URL so checkHubPermission's dynamic client hits it. +func newTestProxyWithHubServer(t *testing.T, hubServerURL string, objs ...runtime.Object) *hcpProxy { + t.Helper() + p := newTestProxy(t, objs...) + p.hubConfig = &rest.Config{ + Host: hubServerURL, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + } + var err error + p.hubDynClient, err = dynamic.NewForConfig(p.hubConfig) + require.NoError(t, err) + return p +} + +func Test_checkHubPermission_WhenAdminUserPermissionContainsCluster_ItShouldAllow(t *testing.T) { + // Mock hub returns "managedcluster:admin" UserPermission that lists spoke-1 + adminUP := map[string]interface{}{ + "apiVersion": "clusterview.open-cluster-management.io/v1alpha1", + "kind": "UserPermission", + "metadata": map[string]interface{}{"name": "managedcluster:admin"}, + "status": map[string]interface{}{ + "bindings": []interface{}{ + map[string]interface{}{"cluster": "spoke-1"}, + map[string]interface{}{"cluster": "spoke-2"}, + }, + }, + } + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "userpermissions/managedcluster:admin") { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(adminUP) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer hubSrv.Close() + + p := newTestProxyWithHubServer(t, hubSrv.URL) + err := p.checkHubPermission(context.Background(), "alice", []string{"dev"}, "spoke-1") + assert.NoError(t, err) +} + +func Test_checkHubPermission_WhenClusterNotInAdminBindings_ItShouldReturn403(t *testing.T) { + // spoke-3 is NOT in the bindings — alice only has admin on spoke-1. + // apiVersion + kind are required so the dynamic client codec can decode the response. + adminUP := map[string]interface{}{ + "apiVersion": "clusterview.open-cluster-management.io/v1alpha1", + "kind": "UserPermission", + "metadata": map[string]interface{}{"name": "managedcluster:admin"}, + "status": map[string]interface{}{ + "bindings": []interface{}{ + map[string]interface{}{"cluster": "spoke-1"}, + }, + }, + } + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "userpermissions/managedcluster:admin") { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(adminUP) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer hubSrv.Close() + + p := newTestProxyWithHubServer(t, hubSrv.URL) + err := p.checkHubPermission(context.Background(), "alice", nil, "spoke-3") + assert.Error(t, err) + assert.Contains(t, err.Error(), "does not have admin access") +} + +func Test_checkHubPermission_WhenViewOnlyUser_ItShouldReturnError(t *testing.T) { + // The operator probe (step 1) returns the admin UserPermission → API is present. + // The impersonated GET (step 2) returns 404 → user has no admin access → hard deny. + adminUP := map[string]interface{}{ + "apiVersion": "clusterview.open-cluster-management.io/v1alpha1", + "kind": "UserPermission", + "metadata": map[string]interface{}{"name": "managedcluster:admin"}, + "status": map[string]interface{}{ + "bindings": []interface{}{ + map[string]interface{}{"cluster": "spoke-1"}, + }, + }, + } + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Operator probe (no Impersonate header) succeeds. + // Impersonated call (Impersonate-User header present) returns 404. + if r.Header.Get("Impersonate-User") != "" { + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"kind":"Status","apiVersion":"v1","reason":"NotFound",`+ + `"message":"userpermissions.clusterview.open-cluster-management.io `+ + `\"managedcluster:admin\" not found"}`) + return + } + if strings.Contains(r.URL.Path, "userpermissions/managedcluster:admin") { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(adminUP) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer hubSrv.Close() + + p := newTestProxyWithHubServer(t, hubSrv.URL) + err := p.checkHubPermission(context.Background(), "viewer", nil, "spoke-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "does not have admin access") +} + +func Test_checkHubPermission_WhenClusterviewAPIAbsent_ItShouldSkipAndAllow(t *testing.T) { + // Simulates a kind/non-ACM hub: every request returns 404 with the + // "server could not find the requested resource" message, meaning the API group + // is not installed at all — the check is skipped non-fatally. + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"kind":"Status","apiVersion":"v1","reason":"NotFound",`+ + `"message":"the server could not find the requested resource"}`) + })) + defer hubSrv.Close() + + p := newTestProxyWithHubServer(t, hubSrv.URL) + err := p.checkHubPermission(context.Background(), "anyuser", nil, "spoke-1") + assert.NoError(t, err) +} + +func Test_checkHubPermission_WhenProbeErrors_ItShouldFailClosed(t *testing.T) { + // Transient hub failures must not bypass authorization. + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"kind":"Status","apiVersion":"v1","status":"Failure",`+ + `"message":"service unavailable","reason":"ServiceUnavailable","code":503}`) + })) + defer hubSrv.Close() + + p := newTestProxyWithHubServer(t, hubSrv.URL) + err := p.checkHubPermission(context.Background(), "anyuser", nil, "spoke-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "clusterview permission probe failed") +} + +func Test_checkHubPermission_WhenUsernameEmpty_ItShouldReturn403(t *testing.T) { + p := newTestProxy(t) + err := p.checkHubPermission(context.Background(), "", nil, "spoke-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "unauthenticated") +} + +// --- handleCreate (spoke mocked via httptest.Server) --- +// Request body mirrors `hcp create cluster --render` output. + +func Test_handleCreate_WhenHostedClusterMissing_ItShouldReturn400(t *testing.T) { + mc := availableManagedCluster("spoke-1") + p := newTestProxy(t, mc) + + // Empty body — no hostedCluster field + body, _ := json.Marshal(CreateRequest{}) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func Test_handleCreate_WhenSpokeAccepts_ItShouldReturn201(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + // Mirrors hcp create cluster --render: HostedCluster references the secret by name, + // and the Secret object is passed in the Secrets list. + hc := &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + Spec: hypershiftv1beta1.HostedClusterSpec{ + InfraID: "my-hc", + PullSecret: corev1.LocalObjectReference{Name: "my-hc-pull-secret"}, + }, + } + body, _ := json.Marshal(CreateRequest{ + HostedCluster: hc, + Secrets: []corev1.Secret{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pull-secret"}, + Data: map[string][]byte{".dockerconfigjson": []byte(`{"auths":{}}`)}, + }, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + assert.Equal(t, http.StatusCreated, w.Code) +} + +func Test_handleCreate_WhenSSHKeyProvided_ItShouldPostBothSecrets(t *testing.T) { + var postedPaths []string + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + postedPaths = append(postedPaths, r.URL.Path) + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + hc := &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + Spec: hypershiftv1beta1.HostedClusterSpec{ + PullSecret: corev1.LocalObjectReference{Name: "my-hc-pull-secret"}, + SSHKey: corev1.LocalObjectReference{Name: "my-hc-ssh-key"}, + }, + } + body, _ := json.Marshal(CreateRequest{ + HostedCluster: hc, + Secrets: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pull-secret"}, + Data: map[string][]byte{".dockerconfigjson": []byte(`{"auths":{}}`)}}, + {ObjectMeta: metav1.ObjectMeta{Name: "my-hc-ssh-key"}, + Data: map[string][]byte{"id_rsa.pub": []byte("ssh-rsa AAAA...")}}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + + assert.Equal(t, http.StatusCreated, w.Code) + // namespace + pull-secret + ssh-key + hostedcluster + assert.Len(t, postedPaths, 4) +} + +// --- handleGetResources (single cluster — returns full bundle) --- + +func Test_handleGetResources_WhenSpokeReturnsCluster_ItShouldReturnBundle(t *testing.T) { + hcJSON, _ := json.Marshal(&hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{ + Items: []hypershiftv1beta1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pool"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + }, + }, + }) + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if strings.Contains(r.URL.Path, "/nodepools") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleGetResources(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Header().Get(headerContentType), contentTypeJSON) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) + assert.Len(t, bundle.NodePools, 1) +} + +// --- handleDelete --- + +func Test_handleDelete_WhenSpokeAccepts_ItShouldProxy200(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleDelete(w, r, "clusters", "my-hc", "spoke-1") + assert.Equal(t, http.StatusOK, w.Code) +} + +// --- handleCreate: Namespace creation --- + +func Test_handleCreate_WhenNamespaceDoesNotExist_ItShouldPostNamespaceFirst(t *testing.T) { + var postedPaths []string + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + postedPaths = append(postedPaths, r.URL.Path) + } + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + body, _ := json.Marshal(CreateRequest{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + + require.Equal(t, http.StatusCreated, w.Code) + require.NotEmpty(t, postedPaths) + assert.Contains(t, postedPaths[0], "/api/v1/namespaces") +} + +func Test_handleCreate_WhenNamespaceAlreadyExists_ItShouldContinue(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/api/v1/namespaces") { + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `{"reason":"AlreadyExists"}`) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + body, _ := json.Marshal(CreateRequest{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + + assert.Equal(t, http.StatusCreated, w.Code) +} + +func Test_handleCreate_WhenCreated_ItShouldReturnResourceBundle(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + // Mirrors --render: HostedCluster references the secret; Secret is in the list. + body, _ := json.Marshal(CreateRequest{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + Spec: hypershiftv1beta1.HostedClusterSpec{ + PullSecret: corev1.LocalObjectReference{Name: "my-hc-pull-secret"}, + }, + }, + NodePools: []*hypershiftv1beta1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pool"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + }, + }, + Secrets: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pull-secret"}, + Data: map[string][]byte{".dockerconfigjson": []byte(`{}`)}}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + + require.Equal(t, http.StatusCreated, w.Code) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + require.NotNil(t, bundle.Namespace) + assert.Equal(t, "clusters", bundle.Namespace.Name) + require.NotNil(t, bundle.HostedCluster) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) + require.Len(t, bundle.NodePools, 1) + assert.Equal(t, "my-hc-pool", bundle.NodePools[0].Name) +} + +// --- handleCreate label injection --- + +func Test_handleCreate_WhenCreated_ItShouldStampCreatedViaLabel(t *testing.T) { + var postedBodies [][]byte + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + postedBodies = append(postedBodies, body) + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + hc := &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + Spec: hypershiftv1beta1.HostedClusterSpec{ + PullSecret: corev1.LocalObjectReference{Name: "my-hc-pull-secret"}, + }, + } + np := &hypershiftv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-us-east-1a"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + } + body, _ := json.Marshal(CreateRequest{ + HostedCluster: hc, + NodePools: []*hypershiftv1beta1.NodePool{np}, + Secrets: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pull-secret"}, + Data: map[string][]byte{".dockerconfigjson": []byte(`{}`)}}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + assert.Equal(t, http.StatusCreated, w.Code) + + // postedBodies[0]=namespace, [1]=pull-secret, [2]=hostedcluster, [3]=nodepool + require.Len(t, postedBodies, 4) + + var postedHC hypershiftv1beta1.HostedCluster + require.NoError(t, json.Unmarshal(postedBodies[2], &postedHC)) + assert.Equal(t, labelCreatedViaValue, postedHC.Labels[labelCreatedVia]) + + var postedNP hypershiftv1beta1.NodePool + require.NoError(t, json.Unmarshal(postedBodies[3], &postedNP)) + assert.Equal(t, labelCreatedViaValue, postedNP.Labels[labelCreatedVia]) +} + +// --- handleGetResources --- + +func Test_handleGetResources_WhenSpokeHasAllResources_ItShouldReturnBundle(t *testing.T) { + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "clusters"}} + hc := hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-hc", + Namespace: "clusters", + Labels: map[string]string{labelCreatedVia: labelCreatedViaValue}, + }, + } + np1 := hypershiftv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-us-east-1a", Namespace: "clusters", + Labels: map[string]string{labelCreatedVia: labelCreatedViaValue}}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + } + np2 := hypershiftv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "other-hc-pool"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "other-hc"}, + } + + nsJSON, _ := json.Marshal(ns) + hcJSON, _ := json.Marshal(hc) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{Items: []hypershiftv1beta1.NodePool{np1, np2}}) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + switch { + case strings.HasSuffix(r.URL.Path, "/namespaces/clusters") && !strings.Contains(r.URL.Path, "hypershift"): + w.WriteHeader(http.StatusOK) + _, _ = w.Write(nsJSON) + case strings.Contains(r.URL.Path, "/hostedclusters/my-hc"): + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + case strings.Contains(r.URL.Path, "/nodepools"): + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleGetResources(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + require.NotNil(t, bundle.Namespace) + assert.Equal(t, "clusters", bundle.Namespace.Name) + require.NotNil(t, bundle.HostedCluster) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) + assert.Equal(t, labelCreatedViaValue, bundle.HostedCluster.Labels[labelCreatedVia]) + // Only np1 belongs to my-hc; np2 should be filtered out + require.Len(t, bundle.NodePools, 1) + assert.Equal(t, "my-hc-us-east-1a", bundle.NodePools[0].Name) + assert.Equal(t, labelCreatedViaValue, bundle.NodePools[0].Labels[labelCreatedVia]) +} + +func Test_handleGetResources_WhenHCNotFound_ItShouldReturn404(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleGetResources(w, r, "clusters", "missing-hc", "spoke-1") + + assert.Equal(t, http.StatusNotFound, w.Code) +} + +func Test_handleRoute_WhenResourcesSubresource_ItShouldDispatch(t *testing.T) { + hcJSON, _ := json.Marshal(hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{}) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if strings.Contains(r.URL.Path, "/hostedclusters/my-hc") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + path := "/apis/" + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters/my-hc/resources?hostingCluster=spoke-1" + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, path, nil) + r.Header.Set("X-Remote-User", "alice") + p.handleRoute(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) +} + +// --- handlePatchResources (kubectl-edit style: full replace via PUT) --- + +func Test_handlePatchResources_WhenFullBundleSent_ItShouldPutHCAndNPsAndReturnBundle(t *testing.T) { + var putPaths []string + hcJSON, _ := json.Marshal(hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Namespace: "clusters"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{ + Items: []hypershiftv1beta1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pool"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + }, + }, + }) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if r.Method == http.MethodPut { + putPaths = append(putPaths, r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + return + } + // GET calls from handleGetResources after the update + if strings.Contains(r.URL.Path, "/hostedclusters/my-hc") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } else if strings.Contains(r.URL.Path, "/nodepools") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + // Send back the full bundle — same shape as GET /resources response + reqBundle := ResourceBundle{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-hc", + Namespace: "clusters", + Annotations: map[string]string{"upgrade": "true"}, + }, + }, + NodePools: []hypershiftv1beta1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pool"}, + Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}, + }, + }, + } + reqBody, _ := json.Marshal(reqBundle) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, "/", bytes.NewReader(reqBody)) + r.Header.Set("X-Remote-User", "alice") + r.Header.Set(headerContentType, contentTypeJSON) + p.handlePatchResources(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + // Must have PUT the HC and the NodePool + assert.Len(t, putPaths, 2) + assert.Contains(t, strings.Join(putPaths, ","), "/hostedclusters/my-hc") + assert.Contains(t, strings.Join(putPaths, ","), "/nodepools/my-hc-pool") + + // Response must be the live ResourceBundle + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) +} + +func Test_handlePatchResources_WhenHCOnly_ItShouldSkipNodePools(t *testing.T) { + var putPaths []string + hcJSON, _ := json.Marshal(hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{}) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if r.Method == http.MethodPut { + putPaths = append(putPaths, r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + return + } + if strings.Contains(r.URL.Path, "/hostedclusters/my-hc") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + // Only HC in the bundle — NodePools absent means skip them + reqBundle := ResourceBundle{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Labels: map[string]string{"env": "staging"}}, + }, + } + reqBody, _ := json.Marshal(reqBundle) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, "/", bytes.NewReader(reqBody)) + r.Header.Set("X-Remote-User", "alice") + p.handlePatchResources(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + // Only HC should be PUT; no NodePool PUTs + assert.Len(t, putPaths, 1) + assert.Contains(t, putPaths[0], "/hostedclusters/my-hc") +} + +func Test_handleRoute_WhenResourcesSubresourcePatch_ItShouldDispatch(t *testing.T) { + hcJSON, _ := json.Marshal(hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{}) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if r.Method == http.MethodPut { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + return + } + if strings.Contains(r.URL.Path, "/hostedclusters/my-hc") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + reqBundle := ResourceBundle{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Annotations: map[string]string{"k": "v"}}, + }, + } + reqBody, _ := json.Marshal(reqBundle) + path := "/apis/" + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters/my-hc/resources?hostingCluster=spoke-1" + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, path, bytes.NewReader(reqBody)) + r.Header.Set("X-Remote-User", "alice") + r.Header.Set(headerContentType, contentTypeJSON) + p.handleRoute(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + assert.Equal(t, "my-hc", bundle.HostedCluster.Name) +} + +func Test_handleRoute_WhenPatchOnNamedResource_ItShouldDoBundleReplace(t *testing.T) { + // PUT /{name} (no /resources suffix) must behave identically to PUT /{name}/resources — + // full bundle replace, not a single-resource merge-patch. + var putPaths []string + hcJSON, _ := json.Marshal(hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }) + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{}) + + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if r.Method == http.MethodPut { + putPaths = append(putPaths, r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + return + } + if strings.Contains(r.URL.Path, "/hostedclusters/my-hc") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(hcJSON) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + } + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + reqBundle := ResourceBundle{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc", Annotations: map[string]string{"k": "v"}}, + }, + } + reqBody, _ := json.Marshal(reqBundle) + // Note: no /resources suffix — still routes to handlePatchResources + path := "/apis/" + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters/my-hc?hostingCluster=spoke-1" + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, path, bytes.NewReader(reqBody)) + r.Header.Set("X-Remote-User", "alice") + r.Header.Set(headerContentType, contentTypeJSON) + p.handleRoute(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + // A PUT was issued to the spoke (full replace, not merge-patch) + assert.Len(t, putPaths, 1) + assert.Contains(t, putPaths[0], "hostedclusters/my-hc") +} + +// --- handleList (ACM Search) --- + +func Test_handleRoute_WhenListPath_ItShouldReturn405(t *testing.T) { + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, "http://unused", mc) + + path := "/apis/" + hcpProxyAPIGroup + "/" + hcpProxyAPIVersion + + "/namespaces/clusters/hostedclusters?hostingCluster=spoke-1" + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, path, nil) + r.Header.Set("X-Remote-User", "alice") + p.handleRoute(w, r) + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) +} + +// --- handleDelete with NodePools --- + +func Test_handleDelete_WhenMatchingNodePoolsExist_ItShouldDeleteThem(t *testing.T) { + var deleted []string + npListJSON, _ := json.Marshal(hypershiftv1beta1.NodePoolList{ + Items: []hypershiftv1beta1.NodePool{ + {ObjectMeta: metav1.ObjectMeta{Name: "my-hc-pool"}, Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "my-hc"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "other-pool"}, Spec: hypershiftv1beta1.NodePoolSpec{ClusterName: "other"}}, + }, + }) + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if r.Method == http.MethodDelete { + deleted = append(deleted, r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + return + } + if strings.Contains(r.URL.Path, "/nodepools") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(npListJSON) + return + } + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleDelete(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + joined := strings.Join(deleted, ",") + assert.Contains(t, joined, "/nodepools/my-hc-pool") + assert.NotContains(t, joined, "/nodepools/other-pool") + assert.Contains(t, joined, "/hostedclusters/my-hc") +} + +// --- createOrUpdateSecretOnSpoke --- + +func Test_createOrUpdateSecretOnSpoke_WhenConflict_ItShouldPut(t *testing.T) { + var methods []string + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + if r.Method == http.MethodPost { + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `already exists`) + return + } + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + p := newTestProxyWithSpokeURL(t, spokeSrv.URL) + client, err := p.spokeHTTPClient("alice", nil) + require.NoError(t, err) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "pull-secret", Namespace: "clusters"}, Data: map[string][]byte{"key": []byte("val")}} + err = p.createOrUpdateSecretOnSpoke(context.Background(), client, "spoke-1", "clusters", secret) + require.NoError(t, err) + assert.Equal(t, []string{http.MethodPost, http.MethodPut}, methods) +} + +func Test_createOrUpdateSecretOnSpoke_WhenCreateSucceeds_ItShouldNotPut(t *testing.T) { + var methods []string + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + p := newTestProxyWithSpokeURL(t, spokeSrv.URL) + client, err := p.spokeHTTPClient("alice", nil) + require.NoError(t, err) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "pull-secret", Namespace: "clusters"}, Data: map[string][]byte{"key": []byte("val")}} + err = p.createOrUpdateSecretOnSpoke(context.Background(), client, "spoke-1", "clusters", secret) + require.NoError(t, err) + assert.Equal(t, []string{http.MethodPost}, methods) +} + +// --- helpers / middleware / URL defaults --- + +func Test_writeJSONError_WhenCalled_ItShouldSetNoSniffHeader(t *testing.T) { + w := httptest.NewRecorder() + writeJSONError(w, "something went wrong", http.StatusBadRequest) + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Equal(t, contentTypeJSON, w.Header().Get(headerContentType)) + assert.Equal(t, "nosniff", w.Header().Get("X-Content-Type-Options")) + var body map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.Equal(t, "something went wrong", body["error"]) +} + +func Test_handleDelete_WhenSpokeResponds_ItShouldForwardContentType(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, "/", nil) + r.Header.Set("X-Remote-User", "alice") + p.handleDelete(w, r, "clusters", "my-hc", "spoke-1") + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, contentTypeJSON, w.Header().Get(headerContentType)) +} + +func Test_createOnSpoke_WhenSpokeReturns409_ItShouldReturnSpokeConflictError(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `already exists`) + })) + defer spokeSrv.Close() + + p := newTestProxyWithSpokeURL(t, spokeSrv.URL) + client, err := p.spokeHTTPClient("alice", nil) + require.NoError(t, err) + + err = p.createOnSpoke(context.Background(), client, "spoke-1", "clusters", resourceHostedClusters, + &hypershiftv1beta1.HostedCluster{ObjectMeta: metav1.ObjectMeta{Name: "hc1"}}) + require.Error(t, err) + assert.True(t, isAlreadyExists(err), "expected errSpokeConflict sentinel, got: %v", err) +} + +func Test_loggingMiddleware_WhenCalled_ItShouldInvokeNext(t *testing.T) { + p := newTestProxy(t) + called := false + handler := p.loggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(http.StatusNoContent) + })) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/healthz?hostingCluster=spoke-1", nil) + r.Header.Set("X-Remote-User", "alice") + handler.ServeHTTP(w, r) + assert.True(t, called) + assert.Equal(t, http.StatusNoContent, w.Code) +} + +func Test_defaultURLs_WhenCalled_ItShouldUseExpectedNamespaces(t *testing.T) { + t.Setenv("POD_NAMESPACE", "") + assert.Equal(t, + "https://cluster-proxy-addon-user.multicluster-engine.svc:9092", + defaultClusterProxyURL()) +} + +func Test_discoverClusterProxyRouteURL_WhenRouteHasHost_ItShouldReturnHTTPSURL(t *testing.T) { + route := &unstructured.Unstructured{} + route.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "route.openshift.io", Version: "v1", Kind: "Route", + }) + route.SetName(clusterProxyServiceName) + route.SetNamespace("multicluster-engine") + require.NoError(t, unstructured.SetNestedField(route.Object, "proxy.apps.example.com", "spec", "host")) + + p := newTestProxy(t, route) + url, err := discoverClusterProxyRouteURL(context.Background(), p.hubClient, "multicluster-engine", p.log) + require.NoError(t, err) + assert.Equal(t, "https://proxy.apps.example.com", url) +} + +func Test_discoverClusterProxyRouteURL_WhenRouteMissingHost_ItShouldReturnEmpty(t *testing.T) { + route := &unstructured.Unstructured{} + route.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "route.openshift.io", Version: "v1", Kind: "Route", + }) + route.SetName(clusterProxyServiceName) + route.SetNamespace("multicluster-engine") + + p := newTestProxy(t, route) + url, err := discoverClusterProxyRouteURL(context.Background(), p.hubClient, "multicluster-engine", p.log) + require.NoError(t, err) + assert.Empty(t, url) +} + +func Test_resolveClusterProxyURL_WhenRoutePresent_ItShouldPreferRoute(t *testing.T) { + t.Setenv("CLUSTER_PROXY_URL", "") + route := &unstructured.Unstructured{} + route.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "route.openshift.io", Version: "v1", Kind: "Route", + }) + route.SetName(clusterProxyServiceName) + route.SetNamespace("my-mce-ns") + require.NoError(t, unstructured.SetNestedField(route.Object, "cp.example.com", "spec", "host")) + + p := newTestProxy(t, route) + url := resolveClusterProxyURL(context.Background(), p.hubClient, "my-mce-ns", p.log) + assert.Equal(t, "https://cp.example.com", url) +} + + +func Test_putOnSpoke_WhenSpokeReturnsError_ItShouldReturnError(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `boom`) + })) + defer spokeSrv.Close() + + p := newTestProxyWithSpokeURL(t, spokeSrv.URL) + client, err := p.spokeHTTPClient("alice", nil) + require.NoError(t, err) + + err = p.putOnSpoke(context.Background(), client, "spoke-1", + "/api/v1/namespaces/ns/secrets/s", map[string]string{"k": "v"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "500") +} + +func Test_spokeURL_WhenClusterProxyURLEmpty_ItShouldUseDefault(t *testing.T) { + t.Setenv("POD_NAMESPACE", "") + p := newTestProxy(t) + p.clusterProxyURL = "" + got, err := p.spokeURL("spoke-1", "/apis/v1") + require.NoError(t, err) + assert.Equal(t, defaultClusterProxyURL()+"/spoke-1/apis/v1", got.String()) +} + +func Test_sanitizeProxyName_WhenInvalid_ItShouldReject(t *testing.T) { + _, err := sanitizeProxyName("") + require.Error(t, err) + _, err = sanitizeProxyName("../evil") + require.Error(t, err) + _, err = sanitizeProxyName("http://evil.example") + require.Error(t, err) + got, err := sanitizeProxyName("spoke-1") + require.NoError(t, err) + assert.Equal(t, "spoke-1", got) +} + +func Test_spokeURL_WhenInvalidSpokeName_ItShouldError(t *testing.T) { + p := newTestProxy(t) + p.clusterProxyURL = "https://cluster-proxy.example:9092" + _, err := p.spokeURL("../escape", "/api/v1/namespaces/ns") + require.Error(t, err) + _, err = p.spokeURL("spoke-1", "/api/v1/../etc/passwd") + require.Error(t, err) +} + +func Test_StartHCPProxy_WhenContextCancelled_ItShouldShutdownCleanly(t *testing.T) { + prevAddr := hcpProxyListenAddr + hcpProxyListenAddr = "127.0.0.1:0" + t.Cleanup(func() { hcpProxyListenAddr = prevAddr }) + + profile, _ := tlspkg.GetTLSProfileSpec(nil) + zapLog, _ := zap.NewDevelopment() + log := zapr.NewLogger(zapLog) + hubClient := fake.NewClientBuilder().WithScheme(runtime.NewScheme()).Build() + hubConfig := &rest.Config{ + Host: "https://127.0.0.1:1", + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + } + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- StartHCPProxy(ctx, profile, hubConfig, hubClient, log) + }() + + // Give the TLS server a moment to bind, then cancel for graceful shutdown. + time.Sleep(200 * time.Millisecond) + cancel() + + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("StartHCPProxy did not return after context cancel") + } +} + +func Test_handlePatchResources_WhenBodyInvalid_ItShouldReturn400(t *testing.T) { + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, "http://unused", mc) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, "/", strings.NewReader(`{bad`)) + r.Header.Set("X-Remote-User", "alice") + p.handlePatchResources(w, r, "clusters", "my-hc", "spoke-1") + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func Test_handlePatchResources_WhenHostedClusterNil_ItShouldRefetchBundle(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + body, _ := json.Marshal(ResourceBundle{}) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handlePatchResources(w, r, "clusters", "my-hc", "spoke-1") + assert.Equal(t, http.StatusNotFound, w.Code) +} + +func Test_handleCreate_WhenNodePoolCreateFails_ItShouldOmitFromResponse(t *testing.T) { + spokeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if strings.Contains(r.URL.Path, "/nodepools") { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `np failed`) + return + } + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + })) + defer spokeSrv.Close() + + mc := availableManagedCluster("spoke-1") + p := newTestProxyWithSpokeURL(t, spokeSrv.URL, mc) + body, _ := json.Marshal(CreateRequest{ + HostedCluster: &hypershiftv1beta1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "my-hc"}, + }, + NodePools: []*hypershiftv1beta1.NodePool{ + {ObjectMeta: metav1.ObjectMeta{Name: "pool-1"}}, + }, + }) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + r.Header.Set("X-Remote-User", "alice") + p.handleCreate(w, r, "clusters", "spoke-1") + + require.Equal(t, http.StatusCreated, w.Code) + var bundle ResourceBundle + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bundle)) + assert.Empty(t, bundle.NodePools) + require.Len(t, bundle.Warnings, 1) + assert.Contains(t, bundle.Warnings[0], "pool-1") +} + +// --- TLS cert validity --- + +func Test_generateSelfSignedCert_WhenParsed_ItShouldBeValidForTLSServerAuth(t *testing.T) { + cert, err := generateSelfSignedCert("multicluster-engine") + require.NoError(t, err) + + tlsCert := tls.Certificate{Certificate: cert.Certificate, PrivateKey: cert.PrivateKey} + x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) + require.NoError(t, err) + + assert.Contains(t, x509Cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) +} + +// ----------- helpers ----------- + +// availableManagedCluster returns a ManagedCluster with Available=True. +func availableManagedCluster(name string) *clusterv1.ManagedCluster { + return &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: clusterv1.ManagedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: clusterv1.ManagedClusterConditionAvailable, + Status: metav1.ConditionTrue, + }, + }, + }, + } +} + +// newTestProxyWithSpokeURL sets clusterProxyURL to the mock server so all spoke +// requests are routed there instead of the real cluster-proxy. +// hubConfig points at a separate mock that reports clusterview as absent so +// checkHubPermission (used by handleRoute) skips non-fatally in unit tests. +func newTestProxyWithSpokeURL(t *testing.T, spokeServerURL string, objs ...runtime.Object) *hcpProxy { + t.Helper() + p := newTestProxy(t, objs...) + hubSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"kind":"Status","apiVersion":"v1","reason":"NotFound",`+ + `"message":"the server could not find the requested resource"}`) + })) + t.Cleanup(hubSrv.Close) + p.hubConfig = &rest.Config{ + Host: hubSrv.URL, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + } + var err error + p.hubDynClient, err = dynamic.NewForConfig(p.hubConfig) + require.NoError(t, err) + p.clusterProxyURL = spokeServerURL + return p +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 8443c4ec..535c7338 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -4,13 +4,14 @@ import ( "context" "embed" "encoding/base64" + "errors" "fmt" "os" "strings" "github.com/go-logr/logr" configv1 "github.com/openshift/api/config/v1" - "github.com/openshift/controller-runtime-common/pkg/tls" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/crypto" "github.com/spf13/cobra" @@ -22,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientfeatures "k8s.io/client-go/features" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -93,7 +95,16 @@ type override struct { func NewManagerCommand(componentName string, log logr.Logger) *cobra.Command { var withOverride bool + var disableTLSWatcher bool runController := func(ctx context.Context, controllerContext *controllercmd.ControllerContext) error { + if err := disableWatchListClient(); err != nil { + return err + } + + // Child context: SecurityProfileWatcher cancels this to restart on TLS profile change. + managerCtx, cancelManager := context.WithCancel(ctx) + defer cancelManager() + mgr, err := addonmanager.New(controllerContext.KubeConfig) if err != nil { return err @@ -131,53 +142,41 @@ func NewManagerCommand(componentName string, log logr.Logger) *cobra.Command { // Start the addon framework manager in a goroutine go func() { - if err := mgr.Start(ctx); err != nil { + if err := mgr.Start(managerCtx); err != nil { log.Error(err, "failed to start addon framework manager") os.Exit(1) } }() - // Create a separate controller-runtime manager for custom controllers - // Use the same scheme that has all the types registered - customMgr, err := ctrl.NewManager(controllerContext.KubeConfig, ctrl.Options{ - Scheme: genericScheme, - LeaderElection: false, // Disable leader election for custom manager - LeaderElectionID: "custom-controller-leader-election", - }) + customMgr, err := newCustomControllerManager(controllerContext, hubClient, log) if err != nil { - log.Error(err, "failed to create custom controller manager") return err } - // Verify the scheme has the required types - gvk := addonapiv1alpha1.SchemeGroupVersion.WithKind("ManagedClusterAddOn") - if !genericScheme.Recognizes(gvk) { - log.Error(fmt.Errorf("scheme does not recognize ManagedClusterAddOn"), "scheme verification failed") - return fmt.Errorf("scheme does not recognize ManagedClusterAddOn") - } - log.Info("Scheme verification successful", "gvk", gvk) - - // Add the discovery config controller to the custom manager - discoveryConfigController := &DiscoveryConfigController{ - Client: hubClient, - Log: log.WithName("discovery-config-controller"), - Scheme: genericScheme, - OperatorNamespace: controllerContext.OperatorNamespace, + profileSpec, err := fetchTLSProfileOrDefault(managerCtx, hubClient, log) + if err != nil { + return err } - if err = discoveryConfigController.SetupWithManager(customMgr); err != nil { - log.Error(err, "failed to setup discovery config controller") + err = setupTLSProfileWatcher( + customMgr, hubClient, profileSpec, disableTLSWatcher, cancelManager, log) + if err != nil { return err } - // Start the custom controller manager in a goroutine + // Start the custom controller manager in a goroutine. + // If it fails to start (e.g. informer cache sync timeout), cancel the + // manager context so the pod restarts and gets a clean retry. go func() { log.Info("starting custom controller manager") - if err := customMgr.Start(ctx); err != nil { + if err := customMgr.Start(managerCtx); err != nil { log.Error(err, "failed to start custom controller manager") + cancelManager() } }() + go startHCPProxy(managerCtx, profileSpec, controllerContext.KubeConfig, hubClient, log) + err = EnableHypershiftCLIDownload(ctx, hubClient, log) if err != nil { // unable to install HypershiftCLIDownload is not critical. @@ -185,7 +184,7 @@ func NewManagerCommand(componentName string, log logr.Logger) *cobra.Command { log.Error(err, "failed to enable hypershift CLI download") } - <-ctx.Done() + <-managerCtx.Done() return nil } @@ -203,10 +202,130 @@ func NewManagerCommand(componentName string, log logr.Logger) *cobra.Command { "disable-leader-election", true, "Disable leader election for the agent.") flags.BoolVar(&withOverride, "with-image-override", false, "Use image from override configmap") + flags.BoolVar(&disableTLSWatcher, "disable-tls-watcher", false, + "Disable the TLS security profile watcher (local development only).") return cmd } +// disableWatchListClient turns off the WatchListClient feature gate (ACM-36014). +// The default Beta gate breaks cache sync on CRDs. +func disableWatchListClient() error { + if fg, ok := clientfeatures.FeatureGates().(interface { + Set(clientfeatures.Feature, bool) error + }); ok { + if err := fg.Set(clientfeatures.WatchListClient, false); err != nil { + return fmt.Errorf("disable WatchListClient feature gate: %w", err) + } + return nil + } + if err := os.Setenv("KUBE_FEATURE_WatchListClient", "false"); err != nil { + return fmt.Errorf("set KUBE_FEATURE_WatchListClient env var: %w", err) + } + return nil +} + +func newCustomControllerManager( + controllerContext *controllercmd.ControllerContext, + hubClient client.Client, + log logr.Logger, +) (ctrl.Manager, error) { + customMgr, err := ctrl.NewManager(controllerContext.KubeConfig, ctrl.Options{ + Scheme: genericScheme, + LeaderElection: false, + LeaderElectionID: "custom-controller-leader-election", + }) + if err != nil { + log.Error(err, "failed to create custom controller manager") + return nil, err + } + + gvk := addonapiv1alpha1.SchemeGroupVersion.WithKind("ManagedClusterAddOn") + if !genericScheme.Recognizes(gvk) { + log.Error(fmt.Errorf("scheme does not recognize ManagedClusterAddOn"), "scheme verification failed") + return nil, fmt.Errorf("scheme does not recognize ManagedClusterAddOn") + } + log.Info("Scheme verification successful", "gvk", gvk) + + discoveryConfigController := &DiscoveryConfigController{ + Client: hubClient, + Log: log.WithName("discovery-config-controller"), + Scheme: genericScheme, + OperatorNamespace: controllerContext.OperatorNamespace, + } + if err = discoveryConfigController.SetupWithManager(customMgr); err != nil { + log.Error(err, "failed to setup discovery config controller") + return nil, err + } + return customMgr, nil +} + +func fetchTLSProfileOrDefault( + ctx context.Context, + hubClient client.Client, + log logr.Logger, +) (configv1.TLSProfileSpec, error) { + profileSpec, err := tlspkg.FetchAPIServerTLSProfile(ctx, hubClient) + if err != nil { + log.Error(err, "failed to fetch APIServer TLS profile, using Intermediate defaults") + profileSpec, _ = tlspkg.GetTLSProfileSpec(nil) + } + return profileSpec, nil +} + +func setupTLSProfileWatcher( + customMgr ctrl.Manager, + hubClient client.Client, + profileSpec configv1.TLSProfileSpec, + disableTLSWatcher bool, + cancelManager context.CancelFunc, + log logr.Logger, +) error { + if disableTLSWatcher { + log.Info("TLS security profile watcher disabled by flag") + return nil + } + // kind / vanilla k8s lack config.openshift.io APIServer. Registering the + // watcher there spams "no matches for kind APIServer" every poll interval. + apiServerGVK := configv1.GroupVersion.WithKind("APIServer") + if _, err := customMgr.GetRESTMapper().RESTMapping( + apiServerGVK.GroupKind(), apiServerGVK.Version, + ); err != nil { + log.Info("APIServer CRD not available, skipping TLS profile watcher", + "error", err) + return nil + } + // When the cluster TLS profile changes, cancelManager() triggers a graceful + // shutdown — the pod restarts and picks up the new profile. + watcher := &tlspkg.SecurityProfileWatcher{ + Client: hubClient, + InitialTLSProfileSpec: profileSpec, + OnProfileChange: func(_ context.Context, old, new configv1.TLSProfileSpec) { + log.Info("cluster TLS profile changed, initiating graceful restart", + "old", old.MinTLSVersion, "new", new.MinTLSVersion) + cancelManager() + }, + } + if err := watcher.SetupWithManager(customMgr); err != nil { + log.Error(err, "failed to setup TLS security profile watcher") + return err + } + return nil +} + +func startHCPProxy( + ctx context.Context, + profileSpec configv1.TLSProfileSpec, + kubeConfig *rest.Config, + hubClient client.Client, + log logr.Logger, +) { + err := StartHCPProxy(ctx, profileSpec, kubeConfig, hubClient, log.WithName("hcp-proxy")) + if err != nil && !errors.Is(err, context.Canceled) { + log.Error(err, "HCP proxy stopped unexpectedly") + } +} + func getAgentAddon( componentName string, o *override, controllerContext *controllercmd.ControllerContext, @@ -407,10 +526,10 @@ func (o *override) getValueForAgentTemplate(cluster *clusterv1.ManagedCluster, // for kube-rbac-proxy flags. Falls back to Intermediate profile on error. func (o *override) getTLSProfileValues() (string, string) { ctx := context.Background() - profileSpec, err := tls.FetchAPIServerTLSProfile(ctx, o.Client) + profileSpec, err := tlspkg.FetchAPIServerTLSProfile(ctx, o.Client) if err != nil { o.log.Info("unable to read APIServer TLS profile, using Intermediate defaults", "error", err) - profileSpec, _ = tls.GetTLSProfileSpec(nil) + profileSpec, _ = tlspkg.GetTLSProfileSpec(nil) } minVersion := string(profileSpec.MinTLSVersion) diff --git a/quickstart/README.md b/quickstart/README.md index 836fbdc1..d441f5ba 100644 --- a/quickstart/README.md +++ b/quickstart/README.md @@ -61,4 +61,9 @@ https://github.com/stolostron/hypershift-addon-operator/blob/main/docs/installin ```shell hypershift create cluster aws --name my-cluster --namespace default --secret-creds my-aws --region us-east-1 --instance-type t3.xlarge --node-pool-replicas 1 ``` - This creates a single worker node hosted cluster, using the `my-aws` credential in us-east-1 \ No newline at end of file + This creates a single worker node hosted cluster, using the `my-aws` credential in us-east-1 + +## HCP Proxy — local development & testing + +For running the hub HCP proxy locally (VS Code debug, curl against `hcp.ocm.io`), see +[HCP Proxy local development & testing](../docs/management/hcp-proxy-local-dev.md). \ No newline at end of file diff --git a/test/e2e/addon-manager-deployment.yaml b/test/e2e/addon-manager-deployment.yaml new file mode 100644 index 00000000..f02d996f --- /dev/null +++ b/test/e2e/addon-manager-deployment.yaml @@ -0,0 +1,200 @@ +# Minimal hub addon-manager deployment for e2e testing. +# The root Makefile sed-substitutes the image / HYPERSHIFT_ADDON_IMAGE_NAME +# before apply so the pod never starts with the quay.io placeholder. +# NOT for production use — uses cluster-admin for simplicity. +# In product, Service / APIService / RBAC are provisioned by backplane-operator. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hypershift-addon-manager-sa + namespace: multicluster-engine +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hypershift-addon-manager-sa-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: hypershift-addon-manager-sa + namespace: multicluster-engine +--- +# HCP proxy ClusterRole (product: backplane-operator) +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hypershift-addon-hcp-proxy +rules: + - apiGroups: ["clusterview.open-cluster-management.io"] + resources: ["userpermissions"] + verbs: ["get"] + - apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters"] + verbs: ["get", "list", "watch"] + - apiGroups: ["operator.open-cluster-management.io"] + resources: ["multiclusterhubs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["config.openshift.io"] + resources: ["apiservers"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["users", "groups", "serviceaccounts"] + verbs: ["impersonate"] + - apiGroups: ["authentication.k8s.io"] + resources: ["userextras"] + verbs: ["impersonate"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hypershift-addon-hcp-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hypershift-addon-hcp-proxy +subjects: + - kind: ServiceAccount + name: hypershift-addon-manager-sa + namespace: multicluster-engine +--- +apiVersion: v1 +kind: Service +metadata: + name: hypershift-addon-hcp-proxy + namespace: multicluster-engine + annotations: + # service-ca-operator (OpenShift) generates a cluster-trusted cert into this Secret. + # Ignored on non-OpenShift / kind; proxy falls back to a self-signed cert. + service.beta.openshift.io/serving-cert-secret-name: hypershift-addon-hcp-proxy-tls +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 9443 + selector: + app: hypershift-addon-manager +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.hcp.ocm.io + annotations: + # service-ca-operator injects the cluster CA bundle into spec.caBundle on OpenShift. + # No-op on kind; insecureSkipTLSVerify keeps the APIService functional there. + service.beta.openshift.io/inject-cabundle: "true" +spec: + group: hcp.ocm.io + groupPriorityMinimum: 2000 + insecureSkipTLSVerify: true + service: + name: hypershift-addon-hcp-proxy + namespace: multicluster-engine + port: 443 + version: v1alpha1 + versionPriority: 10 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hypershift-addon-manager + namespace: multicluster-engine + labels: + app: hypershift-addon-manager +spec: + replicas: 1 + # Recreate avoids RollingUpdate surge where an ImagePullBackOff "old" + # replica (quay placeholder) blocks rollout status for minutes. + strategy: + type: Recreate + selector: + matchLabels: + app: hypershift-addon-manager + template: + metadata: + labels: + app: hypershift-addon-manager + spec: + serviceAccountName: hypershift-addon-manager-sa + volumes: + - name: hcp-proxy-tls + secret: + secretName: hypershift-addon-hcp-proxy-tls + optional: true # absent on kind; proxy falls back to self-signed cert + # library-go controllercmd writes ephemeral serving certs under /tmp. + - name: tmp + emptyDir: {} + containers: + - name: hypershift-addon-manager + # Placeholder — sed-replaced with E2E_IMG/IMG by the root Makefile before apply + image: quay.io/stolostron/hypershift-addon-operator:latest + imagePullPolicy: IfNotPresent + command: + - "./hypershift-addon" + - "manager" + - "--namespace=multicluster-engine" + # kind has no config.openshift.io/APIServer CRD + - "--disable-tls-watcher" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # Overridden by the root Makefile (spoke agent image / install flags) + - name: HYPERSHIFT_ADDON_IMAGE_NAME + value: quay.io/stolostron/hypershift-addon-operator:latest + # OCM cluster-proxy lives in open-cluster-management-addon (see + # hack/install_cluster_proxy.sh). Product (MCE) co-locates it in + # POD_NAMESPACE; kind e2e overrides the Service DNS. + - name: CLUSTER_PROXY_URL + value: "https://cluster-proxy-addon-user.open-cluster-management-addon.svc:9092" + # Skip TLS verify for the self-signed user-server serving cert. + - name: CLUSTER_PROXY_INSECURE + value: "true" + volumeMounts: + - mountPath: /etc/hcp-proxy/tls + name: hcp-proxy-tls + readOnly: true + - mountPath: /tmp + name: tmp + ports: + - name: hcp-proxy + containerPort: 9443 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi + livenessProbe: + httpGet: + path: /healthz + port: hcp-proxy + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: hcp-proxy + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2528cd5d..d6e798b7 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -24,7 +24,9 @@ func TestE2e(t *testing.T) { } const ( - eventuallyTimeout = 300 + // Kind HCP-proxy e2e should fail fast; production OCP soak tests can override + // via longer per-spec Eventually timeouts if needed. + eventuallyTimeout = 60 eventuallyInterval = 2 ) @@ -69,14 +71,13 @@ var _ = ginkgo.BeforeSuite(func() { }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) ginkgo.By("Check if the managed cluster is OCP") - gomega.Eventually(func() error { - _, err := util.GetResource(dynamicClient, util.InfrastructuresGVR, "", "cluster") - if err != nil { - return err - } - isOcp = true - return nil - }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + // Probe once — kind has no config.openshift.io Infrastructure CR. Do not + // Eventually-retry NotFound for 300s (that was hanging HCP proxy e2e on kind). + _, err = util.GetResource(dynamicClient, util.InfrastructuresGVR, "", "cluster") + isOcp = err == nil + if err != nil { + ginkgo.GinkgoWriter.Printf("managed cluster is not OCP (Infrastructure.cluster missing): %v\n", err) + } _, err = kubeClient.CoreV1().Namespaces().Get(context.TODO(), defaultInstallNamespace, metav1.GetOptions{}) if apierrors.IsNotFound(err) { diff --git a/test/e2e/hcp_proxy_test.go b/test/e2e/hcp_proxy_test.go new file mode 100644 index 00000000..ae6050d0 --- /dev/null +++ b/test/e2e/hcp_proxy_test.go @@ -0,0 +1,426 @@ +package e2e_test + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/stolostron/hypershift-addon-operator/test/e2e/util" +) + +const ( + hcpProxyNamespace = "multicluster-engine" + clusterProxyNamespace = "open-cluster-management-addon" + hcpProxyServiceName = "hypershift-addon-hcp-proxy" + hcpProxyAPIServiceName = "v1alpha1.hcp.ocm.io" + hcpProxyAPIGroup = "hcp.ocm.io" + hcpProxyAPIVersion = "v1alpha1" + hcpProxyListenPort = "9443" + + apiServiceGVR = "apiregistration.k8s.io" +) + +// proxyURL builds https://host[:port]/path. If host already includes a port +// (e.g. localhost:18443 from kubectl port-forward), it is used as-is. +func proxyURL(host, path string) string { + if strings.Contains(host, ":") { + return "https://" + host + path + } + return fmt.Sprintf("https://%s:%s%s", host, hcpProxyListenPort, path) +} + +var apiServicesGVR = schema.GroupVersionResource{ + Group: "apiregistration.k8s.io", + Version: "v1", + Resource: "apiservices", +} + +var _ = ginkgo.Describe("HCP Proxy", func() { + var ctx context.Context + + ginkgo.BeforeEach(func() { + ctx = context.TODO() + }) + + // ---------------------------------------------------------------- + // Proxy health & discovery via direct pod access + // ---------------------------------------------------------------- + + ginkgo.Context("When the proxy pod is running", func() { + // proxyHost is host or host:port used to reach the proxy server. + // HCP_PROXY_HOST overrides the pod IP (e.g. "localhost:18443" when + // kubectl port-forward maps a local port to container :9443). + var proxyHost string + + ginkgo.BeforeEach(func() { + // Allow CI to inject a pre-forwarded host via env var + if h := os.Getenv("HCP_PROXY_HOST"); h != "" { + proxyHost = h + return + } + + ginkgo.By("Finding the addon manager pod IP") + gomega.Eventually(func() error { + pods, err := kubeClient.CoreV1().Pods(hcpProxyNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=hypershift-addon-manager", + }) + if err != nil { + return err + } + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + proxyHost = pod.Status.PodIP + return nil + } + } + return fmt.Errorf("no running addon manager pod found") + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("should respond to /healthz with 200", func() { + url := proxyURL(proxyHost, "/healthz") + ginkgo.By("GET " + url) + client := insecureHTTPClient() + resp, err := client.Get(url) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusOK)) + body, _ := io.ReadAll(resp.Body) + gomega.Expect(string(body)).To(gomega.Equal("ok")) + }) + + ginkgo.It("should respond to /readyz with 200", func() { + client := insecureHTTPClient() + resp, err := client.Get(proxyURL(proxyHost, "/readyz")) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusOK)) + }) + + ginkgo.It("should return an APIGroup document from /apis/hcp.ocm.io", func() { + client := insecureHTTPClient() + resp, err := client.Get(proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusOK)) + + var doc map[string]interface{} + gomega.Expect(json.NewDecoder(resp.Body).Decode(&doc)).To(gomega.Succeed()) + gomega.Expect(doc["kind"]).To(gomega.Equal("APIGroup")) + gomega.Expect(doc["name"]).To(gomega.Equal(hcpProxyAPIGroup)) + }) + + ginkgo.It("should return an APIResourceList from /apis/hcp.ocm.io/v1alpha1", func() { + client := insecureHTTPClient() + resp, err := client.Get(proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusOK)) + + var doc map[string]interface{} + gomega.Expect(json.NewDecoder(resp.Body).Decode(&doc)).To(gomega.Succeed()) + gomega.Expect(doc["kind"]).To(gomega.Equal("APIResourceList")) + + resources := doc["resources"].([]interface{}) + gomega.Expect(resources).To(gomega.HaveLen(2)) + names := []string{} + for _, r := range resources { + names = append(names, r.(map[string]interface{})["name"].(string)) + } + gomega.Expect(names).To(gomega.ContainElements("hostedclusters", "hostedclusters/resources")) + }) + + ginkgo.It("should return 400 when hostingCluster is missing from a spoke request", func() { + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+"/namespaces/clusters/hostedclusters") + resp, err := client.Get(url) // no ?hostingCluster + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusBadRequest)) + }) + + ginkgo.It("should return 503 when the hosting cluster does not exist", func() { + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+ + "/namespaces/clusters/hostedclusters?hostingCluster=nonexistent-spoke") + // Add X-Remote-User so it passes the auth check and fails on health + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + req.Header.Set("X-Remote-User", "e2e-test-user") + resp, err := client.Do(req) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusServiceUnavailable)) + }) + + // POST create validation (400/503) plus a successful 201 create that + // routes through OCM cluster-proxy (hack/install_cluster_proxy.sh) onto + // local-cluster with the HostedCluster CRD applied. + ginkgo.It("should return 400 when POST is missing hostingCluster", func() { + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+ + "/namespaces/clusters/hostedclusters") + body := []byte(`{"hostedCluster":{"metadata":{"name":"e2e-hc"}}}`) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Remote-User", "e2e-test-user") + resp, err := client.Do(req) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusBadRequest)) + }) + + ginkgo.It("should return 503 when POST targets a nonexistent hosting cluster", func() { + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+ + "/namespaces/clusters/hostedclusters?hostingCluster=nonexistent-spoke") + body := []byte(`{"hostedCluster":{"metadata":{"name":"e2e-hc"}}}`) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Remote-User", "e2e-test-user") + resp, err := client.Do(req) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusServiceUnavailable)) + }) + + ginkgo.It("should return 400 when POST body omits hostedCluster", func() { + // On kind, clusterview is absent so permission check is skipped; + // local-cluster is Available and handleCreate rejects the empty body. + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+ + "/namespaces/clusters/hostedclusters?hostingCluster="+defaultManagedCluster) + body := []byte(`{}`) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Remote-User", "e2e-test-user") + resp, err := client.Do(req) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusBadRequest)) + }) + + ginkgo.It("should create a HostedCluster via POST through cluster-proxy and return 201", func() { + ginkgo.By("Ensuring OCM cluster-proxy user Service is present") + _, err := kubeClient.CoreV1().Services(clusterProxyNamespace).Get( + ctx, "cluster-proxy-addon-user", metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + ginkgo.Skip("cluster-proxy-addon-user Service missing; run make deploy-cluster-proxy") + } + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + hcNS := fmt.Sprintf("e2e-hcp-proxy-%d", time.Now().UnixNano()) + const hcName = "e2e-hc" + ginkgo.DeferCleanup(func() { + _ = kubeClient.CoreV1().Namespaces().Delete(ctx, hcNS, metav1.DeleteOptions{}) + }) + + // system:masters so spoke impersonation can create Namespace/Secret/HostedCluster. + body := []byte(fmt.Sprintf(`{ + "hostedCluster": { + "apiVersion": "hypershift.openshift.io/v1beta1", + "kind": "HostedCluster", + "metadata": {"name": %q, "namespace": %q}, + "spec": { + "release": {"image": "quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64"}, + "pullSecret": {"name": "%s-pull-secret"}, + "sshKey": {"name": "%s-ssh-key"}, + "platform": {"type": "None"}, + "networking": {"networkType": "OVNKubernetes"}, + "services": [], + "etcd": {"managementType": "Managed"}, + "infraID": %q + } + }, + "secrets": [ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "%s-pull-secret"}, + "type": "kubernetes.io/dockerconfigjson", + "data": {".dockerconfigjson": "eyJhdXRocyI6e319"} + }, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "%s-ssh-key"}, + "data": {"id_rsa.pub": "c3NoLXJzYSBBQUFB"} + } + ] + }`, hcName, hcNS, hcName, hcName, hcName, hcName, hcName)) + + ginkgo.By("Waiting for ManagedClusterAddOn cluster-proxy Available") + gomega.Eventually(func() bool { + addon, err := addonClient.AddonV1alpha1().ManagedClusterAddOns(defaultManagedCluster). + Get(ctx, "cluster-proxy", metav1.GetOptions{}) + if err != nil { + return false + } + for _, c := range addon.Status.Conditions { + if c.Type == "Available" && c.Status == metav1.ConditionTrue { + return true + } + } + return false + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + + client := insecureHTTPClient() + url := proxyURL(proxyHost, "/apis/"+hcpProxyAPIGroup+"/"+hcpProxyAPIVersion+ + "/namespaces/"+hcNS+"/hostedclusters?hostingCluster="+defaultManagedCluster) + + ginkgo.By("POST create HostedCluster via HCP proxy → cluster-proxy → local-cluster") + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Remote-User", "e2e-test-user") + req.Header.Set("X-Remote-Group", "system:masters") + resp, err := client.Do(req) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + gomega.Expect(resp.StatusCode).To(gomega.Equal(http.StatusCreated), + "POST create response: %s", string(respBody)) + + var bundle map[string]interface{} + gomega.Expect(json.Unmarshal(respBody, &bundle)).To(gomega.Succeed()) + hc, ok := bundle["hostedCluster"].(map[string]interface{}) + gomega.Expect(ok).To(gomega.BeTrue(), "response should include hostedCluster") + meta, _ := hc["metadata"].(map[string]interface{}) + gomega.Expect(meta["name"]).To(gomega.Equal(hcName)) + labels, _ := meta["labels"].(map[string]interface{}) + gomega.Expect(labels["hcp.ocm.io/created-via"]).To(gomega.Equal("hcp-from-hub")) + + ginkgo.By("Verifying HostedCluster exists on the hub/spoke (local-cluster)") + hcGVR := schema.GroupVersionResource{ + Group: "hypershift.openshift.io", + Version: "v1beta1", + Resource: "hostedclusters", + } + gomega.Eventually(func() error { + _, err := dynamicClient.Resource(hcGVR).Namespace(hcNS).Get(ctx, hcName, metav1.GetOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + }) + }) + + // ---------------------------------------------------------------- + // APIService routing via hub kube-apiserver + // ---------------------------------------------------------------- + + ginkgo.Context("When accessed via the hub kube-apiserver APIService route", func() { + ginkgo.It("should serve the hcp.ocm.io API group in cluster API discovery", func() { + ginkgo.By("Waiting for APIService to become Available") + gomega.Eventually(func() bool { + apiSvc, err := dynamicClient.Resource(apiServicesGVR).Get( + ctx, hcpProxyAPIServiceName, metav1.GetOptions{}) + if err != nil { + return false + } + conditions, ok := apiSvc.Object["status"].(map[string]interface{}) + if !ok { + return false + } + condList, _ := conditions["conditions"].([]interface{}) + for _, c := range condList { + cMap, _ := c.(map[string]interface{}) + if cMap["type"] == "Available" && cMap["status"] == "True" { + return true + } + } + return false + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue(), + "APIService v1alpha1.hcp.ocm.io should become Available") + }) + + ginkgo.It("should expose hcp.ocm.io in /apis discovery via REST client", func() { + ginkgo.By("Waiting for hcp.ocm.io to appear in server API groups") + // ServerGroups (not ServerGroupsAndResources): aggregated APIs can + // make the latter return a partial-error that this suite treated as fail. + gomega.Eventually(func() bool { + groups, err := kubeClient.Discovery().ServerGroups() + if err != nil { + return false + } + for _, g := range groups.Groups { + if g.Name == hcpProxyAPIGroup { + return true + } + } + return false + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue(), + "hcp.ocm.io should appear in server API groups") + }) + + ginkgo.It("should return 400 via the APIService route when hostingCluster is absent", func() { + ginkgo.By("Making raw REST call to /apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters") + restClient, err := util.NewKubeClient() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // The proxy returns 400 because hostingCluster is not set; + // the kube-apiserver may wrap this as a 400 or 503. + // Either way the call should not succeed with 200. + gomega.Eventually(func() int { + var statusCode int + restClient.CoreV1().RESTClient().Get(). + AbsPath("/apis/hcp.ocm.io/v1alpha1/namespaces/clusters/hostedclusters"). + Do(ctx).StatusCode(&statusCode) + return statusCode + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.Equal(http.StatusOK)) + }) + }) + + // ---------------------------------------------------------------- + // Proxy Service port liveness + // ---------------------------------------------------------------- + + ginkgo.Context("When the proxy Service is targetted", func() { + ginkgo.It("should have at least one Ready endpoint backing the Service", func() { + ginkgo.By("Checking Endpoints for " + hcpProxyServiceName) + gomega.Eventually(func() bool { + ep, err := kubeClient.CoreV1().Endpoints(hcpProxyNamespace).Get( + ctx, hcpProxyServiceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false + } + return false + } + for _, subset := range ep.Subsets { + if len(subset.Addresses) > 0 { + return true + } + } + return false + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue(), + "Service should have at least one ready endpoint") + }) + }) +}) + +// insecureHTTPClient returns an http.Client that skips TLS verification, +// suitable for testing the proxy's self-signed certificate directly. +func insecureHTTPClient() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + Timeout: 10 * time.Second, + } +} From 4f50fb41e566ee8ea08d1290b139b187328f5857 Mon Sep 17 00:00:00 2001 From: yiraeChristineKim Date: Wed, 22 Jul 2026 11:51:40 -0400 Subject: [PATCH 2/2] fix: create cluster-proxy ManagedClusterAddOn before kubectl wait Avoid a race where kubectl wait fails with NotFound when placement has not yet created the ManagedClusterAddOn after OCM join. Co-authored-by: Cursor --- hack/install_cluster_proxy.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/hack/install_cluster_proxy.sh b/hack/install_cluster_proxy.sh index 6832b306..a779a1f8 100755 --- a/hack/install_cluster_proxy.sh +++ b/hack/install_cluster_proxy.sh @@ -18,6 +18,7 @@ NAMESPACE=${CLUSTER_PROXY_NAMESPACE:-open-cluster-management-addon} RELEASE=${CLUSTER_PROXY_RELEASE:-cluster-proxy} MANAGED_CLUSTER=${MANAGED_CLUSTER_NAME:-local-cluster} TIMEOUT=${CLUSTER_PROXY_TIMEOUT:-300s} +INSTALL_NS=${CLUSTER_PROXY_AGENT_NAMESPACE:-open-cluster-management-agent-addon} if ! command -v "${HELM}" >/dev/null 2>&1; then echo "ERROR: helm is required to install OCM cluster-proxy" >&2 @@ -50,7 +51,22 @@ done ${KUBECTL} get svc -n "${NAMESPACE}" cluster-proxy-addon-user ${KUBECTL} rollout status -n "${NAMESPACE}" deployment/cluster-proxy-addon-user --timeout="${TIMEOUT}" -echo "Waiting for ManagedClusterAddOn cluster-proxy on ${MANAGED_CLUSTER}..." +# Chart installStrategy is Placement-based; addon-manager creates the +# ManagedClusterAddOn asynchronously. kubectl wait fails immediately with +# NotFound if the object is missing, which races right after OCM join. +# Ensure the MCA exists (idempotent) before waiting for Available. +echo "Ensuring ManagedClusterAddOn cluster-proxy on ${MANAGED_CLUSTER}..." +${KUBECTL} apply -f - <