diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md new file mode 100644 index 0000000000..5d95417939 --- /dev/null +++ b/.agents/skills/update-against-main/SKILL.md @@ -0,0 +1,29 @@ +--- +name: update-against-main +description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests. +--- + +# Update Against Main + +This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`. + +1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes. +2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`. +3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream. +4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history. +5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks. +6. Run `make test` and `make verify`. +7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: + - Recreate the cluster with `hack/create-kind-cluster.sh`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway`. + - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. + - Deploy the gVisor counter demo and both standard egress demos. + - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` + - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color` + - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies. + - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation. +8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage. +9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead. +10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push. + +Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster. diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..dfc5df2757 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,115 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + VERSION: helm-e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Label nodes with the installed version + run: kubectl label nodes --all ate.dev/substrate-version=${VERSION} + - name: Create install namespace + run: kubectl create namespace ate-system + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Enable NFS + run: | + sudo modprobe nfs || true + sudo modprobe nfsd || true + - name: Install CSI NFS driver + run: hack/install-ate-kind.sh --setup-csi=nfs + - name: Deploy micro-VM counter demo + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm + E2E_TEMPLATE_NAME: counter-microvm + E2E_TEMPLATE_READY_TIMEOUT: 600s + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 68c9cc30d0..b0a1e30690 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -87,7 +87,7 @@ jobs: - name: Create cluster run: hack/create-kind-cluster.sh - name: Install Agent Substrate - run: hack/install-ate-kind.sh --deploy-ate-system + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | @@ -98,7 +98,7 @@ jobs: - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the # counter-microvm demo onto the control plane installed above. - run: hack/run-microvm-demo-kind.sh + run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demos @@ -127,7 +127,7 @@ jobs: # Cluster-wide, so it must come AFTER the standard lanes: once egress # TLS is intercepted, their passthrough assumptions # (TestActorEgressHTTPS's end-to-end TLS with the origin) no longer hold. - run: hack/install-ate-kind.sh --deploy-atenet --experimental-use-sdsmint + run: hack/install-ate-kind.sh --deploy-atenet --atenet-router=agentgateway --experimental-use-sdsmint - name: Run E2E tests (egress MITM trust) # The consumption half of the trust-bundle chain: an actor does TLS with # the MITM gateway's minted leaf using ONLY the projected bundle, plus a diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..4a26a5cbe9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true + files: dist/kubectl-ate-* diff --git a/Makefile b/Makefile index bafabf2689..9165abfe63 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,10 @@ build: build-images build-atectl build-ate-setup .PHONY: build-images build-images: - $(KO) build \ + $(KO) build --base-import-paths \ --ldflags="$(LDFLAGS)" \ ./cmd/ateapi \ + ./cmd/atecontroller \ ./cmd/atelet \ ./cmd/podcertcontroller \ ./cmd/atenet @@ -103,3 +104,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..ebc1473eae --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + tls: + description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + properties: + enabled: + description: Enabled controls whether TLS is used. + type: boolean + serverName: + description: ServerName override for TLS verification. + type: string + usePodIdentity: + description: UsePodIdentity indicates whether to reuse Substrate's + Pod Identity (SPIFFE) certificates. + type: boolean + required: + - enabled + type: object + x-kubernetes-validations: + - message: tls.usePodIdentity must be true when tls.enabled is true; + manual certificates are not yet supported + rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)' + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..427b0d624e --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,149 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .spec.default + name: Default + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools + and decouples sandbox binary selection from ActorTemplate. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + default: + description: |- + Default marks this SandboxConfig as the cluster-wide default for its + SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to + the default config for its SandboxClass. At most one default is expected + per SandboxClass. + type: boolean + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. A + WorkerPool only uses SandboxConfigs whose SandboxClass matches its own. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..04891e46ac --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,480 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement) and which + SandboxConfigs are eligible. The concrete binary is still selected by + WorkerImage. Defaults to gvisor. + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + sandboxConfigName: + description: |- + SandboxConfigName names a cluster-scoped SandboxConfig to use for fetching + sandbox binaries. It overrides the cluster-wide default SandboxConfig for + this pool's SandboxClass. The referenced config's SandboxClass must match + this pool's SandboxClass. If empty, the default SandboxConfig for the + SandboxClass is used. + type: string + template: + description: Template holds optional metadata, scheduling, and resource + settings for worker workloads. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations are added to the generated Deployment and worker pods. Keys + in the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: annotation keys must be valid Kubernetes qualified + names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + labels: + additionalProperties: + description: |- + WorkerPoolLabelValue is a Kubernetes label value for generated worker + workloads. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels are added to the generated Deployment and worker pods. Keys in + the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: label keys must be valid Kubernetes qualified names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + workerImage: + description: WorkerImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + required: + - replicas + - workerImage + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..0640c9e9a7 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,45 @@ +# substrate + +Helm chart for installing Agent Substrate. + +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# Install Substrate +helm upgrade --install substrate ./charts/substrate +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..c0e9875a45 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,7 @@ +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL + +REQUIRED Kubernetes feature gates: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..32ae087336 --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,105 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..ca76ae3ef8 --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} +{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..4fa447a91c --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,214 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--postgres-connection-string=@env" + - "--postgres-schema=@env" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" + - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + # ateapi verifies atelet's SPIFFE ID, which embeds the ServiceAccount + # name this chart prefixes for a renamed release. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--drain-delay=13s" + - "--drain-timeout=15s" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: actor-id-ca-pool + projected: + sources: + - secret: + name: actor-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..dfd2fdab68 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..6a6ac462c0 --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,127 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + # A SPIFFE ID names a ServiceAccount, and this chart prefixes those for + # any release not called "substrate". The controller stamps these + # identities onto every worker, so it has to be told the names actually + # rendered rather than assume the canonical ones. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" + - "--router-service-account={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + env: + # ate-controller resolves substrate's namespace from the downward API. + # It names the atelet and atenet-router SPIFFE identities handed to + # each worker's atunnel, which live in substrate's namespace and not + # the worker's, so the controller cannot infer them any other way. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..1570d0f8ff --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,220 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] +# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are +# resolved on the node: atelet reads them through an informer and projects +# the sanitized PEM into actors (see cmd/atelet/trustbundle.go). +- apiGroups: ["certificates.k8s.io"] + resources: ["clustertrustbundles"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + # The atelet binary defaults these to api.ate-system.svc, which is + # correct only for the canonical render. The credential broker dials + # ateapi to mint actor certificates, so pass the chart-resolved + # Service and the matching name on its serving cert. + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: device-plugins + mountPath: /var/lib/kubelet/device-plugins + - name: host-dev + mountPath: /host/dev + readOnly: true + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: device-plugins + hostPath: + path: /var/lib/kubelet/device-plugins + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml new file mode 100644 index 0000000000..fc6f770306 --- /dev/null +++ b/charts/substrate/templates/atenet-dns.yaml @@ -0,0 +1,187 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atenet-dns +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + replicas: 1 + selector: + matchLabels: + app: dns + template: + metadata: + labels: + app: dns + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-dns" .) }} + shareProcessNamespace: true + initContainers: + - name: init-dns + image: {{ .Values.images.busybox }} + command: ["sh", "-c"] + args: + - | + cat <<'EOF' > /etc/coredns/Corefile + .:53 { + errors + health :8080 + ready :8181 + reload + } + EOF + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + containers: + - name: coredns + image: {{ .Values.images.coredns }} + imagePullPolicy: IfNotPresent + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + ports: + - name: dns + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + - name: dns-controller + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "dns" + - "--log-level=debug" + - "--interval=10s" + - "--corefile-path=/etc/coredns/Corefile" + # Pass the chart-resolved Service names so the controller looks up the + # correct objects when substrate is installed as a subchart. The + # system namespace is read from POD_NAMESPACE below. + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--dns-service-name={{ include "substrate.fullname" (list "dns" .) }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + volumes: + - name: dns-config-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml new file mode 100644 index 0000000000..86cc2271ca --- /dev/null +++ b/charts/substrate/templates/atenet-egress.yaml @@ -0,0 +1,227 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + accessLog: + add: + substrate.connect.authority: source.connectHeaders["host"] + + binds: + - port: 8443 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + routes: [] + - mode: internal + protocol: AUTO + listeners: + - protocol: TLS + hostname: "*" + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: HTTP + routes: + - policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: TCP + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} + securityContext: + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + containers: + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - -f + - /etc/agentgateway/config.yaml + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: ext-proc + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - router + - --mode=egress + - --namespace={{ .Release.Namespace }} + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + - --envoy-admin-address=localhost:15000 + - --atenet-router=agentgateway + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..347d53597e --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,358 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- if .Values.otel.endpoint }} + frontendPolicies: + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 +{{- end }} + + backends: + - name: dynamic + dynamic: {} + policies: + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + gateways: + http: + port: 8080 + protocol: HTTP + https: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + + routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--mode=ingress" + - "--atenet-router=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + # /statusz looks up this instance's own ClusterIP by Service name, and + # the chart prefixes that name for any release not called "substrate". + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--status-port=4040" + - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" +{{- if .Values.otel.endpoint }} + env: + - name: AGENTGATEWAY_OTLP_ADDRESS + value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..073291828b --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..86fc23b4a9 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,198 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..26ddb0b1ec --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,236 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.postgres.enabled }} +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + # Local socket access is limited to processes in this pod and is used by + # health checks, the workload's idempotent database bootstrap, and the + # tls-reloader sidecar's configuration reloads. + local all all trust + # PostgreSQL verifies client certificates against the pod-identity CA. It + # does not need its own serving CA because it never verifies its server certificate. + hostssl all all all trust clientcert=verify-ca + reload-tls.sh: | + # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup + # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod + # certificate in place about 30 minutes before it expires, so without this + # loop the server keeps presenting the certificate it booted with until it + # expires about a day later and every client stops trusting it. + set -eu + + # As PID 1 this shell only sees SIGTERM if a handler is installed, and only + # acts on it between commands, so the sleep below runs in the background + # and is waited on. Without both halves the pod takes the full termination + # grace period to go away. + trap 'exit 0' TERM INT + + CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem + CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem + + # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that + # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go + # leaves; hashing two small files costs nothing. + INTERVAL=60 + + reloaded="" + while true; do + current="$(sha256sum "${CERT}" "${CA}")" + # Reloading fails until the server is accepting connections, which is + # where every pod starts out, so only record a hash once it has worked. + # Starting empty also means a restart of this container costs one + # redundant reload rather than a missed one. + if [ "${current}" != "${reloaded}" ] \ + && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then + reloaded="${current}" + echo "$(date -u +%FT%TZ) reloaded TLS configuration" + fi + sleep "${INTERVAL}" & + wait $! + done +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + securityContext: + # Group ownership of the projected certificate below, and of the data + # volume so that a freshly provisioned one is writable. OnRootMismatch + # keeps the kubelet from walking the data directory on every start, + # which would leave PGDATA group-writable and postgres refusing to run. + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar + # reloads the server whenever the kubelet rotates the projected pod + # certificate. fsGroup is also what makes that projection readable: the + # kubelet writes it root-owned for as long as the pod's containers do not + # all agree on one non-root user, and grants the fsGroup group access, + # landing the key at root:postgres 0640, the only shared mode PostgreSQL + # accepts. Pinning runAsUser on the postgres container would make the key + # postgres-owned and group-readable, which it rejects. + # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP + initContainers: + - name: tls-reloader + restartPolicy: Always + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - /etc/postgresql/reload-tls.sh + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + resources: + requests: + cpu: 10m + memory: 32Mi + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + # 0600 plus the group read that fsGroup adds is the 0640 above. + defaultMode: 0600 + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The unix socket directory, shared so the sidecar can ask the running + # server to reload. The image defaults both the server and its clients to + # this path, so nothing else has to know about it. + - name: socket + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} +{{- end }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..9226661d30 --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..edaad3cfa8 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,137 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..36af4296f3 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A +# WorkerPool with sandboxClass gvisor (the default) and no explicit +# sandboxConfigName resolves to this. atelet fetches the runsc binary matching +# the worker node's architecture. To pin a different runsc, edit the assets +# below or create another SandboxConfig and name it from the WorkerPool. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + arm64: + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..f25d43409b --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,57 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..6bb83019ed --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,75 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +postgres: + enabled: true + storageSize: 1Gi + connectionString: "" + schema: public + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + coredns: coredns/coredns:1.11.1 + busybox: busybox:1.36 diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index 7304c43d15..4a1abbc701 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -59,33 +59,26 @@ type Server struct { // is entitled to the actor it is asking for a credential for. store store.Interface workers *workercache.Cache + + // ateletSPIFFEID is the identity the calling atelet must present to mint + // actor credentials. + ateletSPIFFEID string } var _ ateapipb.ActorIdentityServer = (*Server)(nil) -func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache) *Server { +func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletSPIFFEID string) *Server { return &Server{ actorIdentityJWTIssuer: actorIdentityJWTIssuer, actorIDJWTPoolFile: actorIDJWTPoolFile, actorIDCAPool: actorIDCAPool, store: store, workers: workers, + ateletSPIFFEID: ateletSPIFFEID, } } -// The SPIFFE identity that atelet client certs carry, as minted by the -// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// -// These mirror the constants the atelet dialer verifies against in -// cmd/ateapi/internal/controlapi/dialer.go. They are duplicated rather than -// imported so that this package does not depend on controlapi for three -// strings; if a third pkg that need these constants appears, they should move to a shared package. -const ( - ateletTrustDomain = "cluster.local" - ateletNamespace = "ate-system" - ateletSA = "atelet" - actorCertificateLifetime = time.Hour -) +const actorCertificateLifetime = time.Hour func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*ateapipb.MintJWTResponse, error) { caller, ok := principal.FromContext(ctx) @@ -148,7 +141,7 @@ func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*at } func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*ateapipb.MintCertResponse, error) { - caller, err := authenticateAtelet(ctx) + caller, err := s.authenticateAtelet(ctx) if err != nil { return nil, err } @@ -243,7 +236,7 @@ type ateletCaller struct { // pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the // extensions read here are trustworthy: only the pod-identity signer can mint // a certificate carrying a given pod's node name. -func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { +func (s *Server) authenticateAtelet(ctx context.Context) (*ateletCaller, error) { p, ok := peer.FromContext(ctx) if !ok { return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found") @@ -262,11 +255,7 @@ func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { // Only atelet may mint actor credentials. Everything else with a valid // pod-identity certificate — including the actor workloads themselves — is // rejected here. - expected := (&url.URL{ - Scheme: "spiffe", - Host: ateletTrustDomain, - Path: path.Join("ns", ateletNamespace, "sa", ateletSA), - }).String() + expected := s.ateletSPIFFEID if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet", slog.Any("uris", leaf.URIs), slog.String("expected", expected)) diff --git a/cmd/ateapi/internal/actoridentity/actoridentity_test.go b/cmd/ateapi/internal/actoridentity/actoridentity_test.go index d1282de67f..cc8b935ef3 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity_test.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity_test.go @@ -32,6 +32,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/principal" "github.com/agent-substrate/substrate/internal/resources" @@ -74,6 +75,19 @@ const ( // populates. Self-signing is sufficient because the code under test reads an // already transport-verified peer certificate and never re-validates the chain // itself. +// The atelet SPIFFE segments the tests build peer certificates from. They +// mirror what a default install mints, which is what the Server under test is +// configured with. +const ( + ateletTrustDomain = installdefaults.AteletTrustDomain + ateletNamespace = installdefaults.SystemNamespace + ateletSA = installdefaults.AteletServiceAccount +) + +// ateletSPIFFEID is the identity a default install's atelet presents, and what +// the Server under test is configured to accept. +var ateletSPIFFEID = installdefaults.SPIFFEID(ateletNamespace, ateletSA) + func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { t.Helper() @@ -165,7 +179,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server { t.Fatalf("start worker cache: %v", err) } } - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } // staleWatchStore wraps a store with a WatchWorkers that never delivers, @@ -324,11 +338,11 @@ func newTestServerWithCache(t *testing.T, st store.Interface, workers *workercac t.Fatalf("generate CA: %v", err) } pool := &localca.ConcretePool{CAs: []*localca.CA{ca}} - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) { - srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example"} + srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletSPIFFEID: ateletSPIFFEID} for _, tt := range []struct { name string ctx context.Context @@ -918,7 +932,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) { ActiveForSigning: "test-actor-ca", } - srv := New("issuer", "", pool, st, workers) + srv := New("issuer", "", pool, st, workers, ateletSPIFFEID) actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName}) if err != nil { @@ -1082,3 +1096,37 @@ func TestValidateMintCertRequest(t *testing.T) { }) } } + +// TestAuthenticateAteletHonorsConfiguredNamespace checks that the namespace +// the Server is configured with is the one it accepts, and that the canonical +// namespace is rejected when the install lives elsewhere. +// +// The table in TestMintCertAuthorization covers a caller from the wrong +// namespace, but its Server is always configured with the default, so it holds +// against a hardcoded "ate-system" too. Only the relocated case distinguishes +// "reads its configuration" from "happens to agree with the constant". +func TestAuthenticateAteletHonorsConfiguredNamespace(t *testing.T) { + const relocated = "substrate-test" + + srv := &Server{ateletSPIFFEID: installdefaults.SPIFFEID(relocated, ateletSA)} + + t.Run("accepts atelet from the configured namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = relocated + cert := newTestCert(t, path.Join("ns", relocated, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); err != nil { + t.Errorf("authenticateAtelet() = %v, want success for an atelet in %q", err, relocated) + } + }) + + t.Run("rejects atelet from the canonical namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = installdefaults.SystemNamespace + cert := newTestCert(t, path.Join("ns", installdefaults.SystemNamespace, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); status.Code(err) != codes.PermissionDenied { + t.Errorf("authenticateAtelet() code = %v, want PermissionDenied for an atelet in %q", status.Code(err), installdefaults.SystemNamespace) + } + }) +} diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 3acd3d7f35..c6f6a638da 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -32,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -47,12 +49,7 @@ var ErrNoAteletOnNode = errors.New("no atelet pod found on node") // The SPIFFE identity that atelet serving certs carry, as minted by the // podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// The namespace part is ateletNamespace, declared in informer.go. -const ( - trustDomainName = "cluster.local" - ateletSA = "atelet" -) - +// The namespace part is the dialer's ateletNamespace. // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer @@ -74,15 +71,20 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletSPIFFEID, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { d := &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: newAteletConnCache(1024), dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { - tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + tlsConfig, err := buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID) if err != nil { return nil, err } @@ -189,18 +191,18 @@ func (d *AteletDialer) DialForAteletOnNode(nodeName string) (*grpc.ClientConn, e return ateletConn, nil } -func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { - trustDomain, err := spiffeid.TrustDomainFromString(trustDomainName) +func buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + trustDomain, err := spiffeid.TrustDomainFromString(installdefaults.AteletTrustDomain) if err != nil { - return nil, fmt.Errorf("while parsing trust domain %q: %w", trustDomainName, err) + return nil, fmt.Errorf("while parsing trust domain %q: %w", installdefaults.AteletTrustDomain, err) } bundle, err := x509bundle.Load(trustDomain, serverCAPath) if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + expectedID, err := spiffeid.FromString(ateletSPIFFEID) if err != nil { - return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) + return nil, fmt.Errorf("while parsing expected atelet SPIFFE ID %q: %w", ateletSPIFFEID, err) } verify, err := verifyAteletServerCert(bundle, expectedID, expectedPodUID) diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 593529e400..b26a71ef35 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -41,6 +42,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { @@ -198,7 +215,7 @@ func TestDialForWorkerTarget(t *testing.T) { Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: tc.ateletIP}}}, } @@ -225,7 +242,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("unknown worker pod", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: "10.244.1.7"}}}, } @@ -237,7 +254,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("atelet without assigned IPs", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } d := newDialerForPods(t, workerPod, ateletPod) @@ -363,7 +380,7 @@ func TestDialForAteletOnNode(t *testing.T) { } t.Run("no atelet on node", func(t *testing.T) { - d := NewAteletDialer(nil, newTestAteletIndexer(t), "", "") + d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err) } @@ -373,7 +390,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"), - ), "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") _, err := d.DialForAteletOnNode("node1") if err == nil || errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err) @@ -383,7 +400,7 @@ func TestDialForAteletOnNode(t *testing.T) { t.Run("dials and caches the node's atelet", func(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), - ), "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") var credsUID string d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) { credsUID = expectedPodUID @@ -410,7 +427,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node2", "10.0.0.2"), - ), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) d.ateletConns = newAteletConnCache(1) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 9a8f44bfa9..65cff1cdf6 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -58,10 +59,8 @@ const ( testAtespace = "test-atespace" testActorID = "id1" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -122,7 +121,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // 3. Initialize Informers workerFactory, workerInformer := controlapi.WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient, installdefaults.SystemNamespace) scFactory := informers.NewSharedInformerFactory(k8sClient, 0) scLister := scFactory.Storage().V1().StorageClasses().Lister() @@ -153,7 +152,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // Dial the fake atelet over insecure transport instead of per-atelet mTLS, // so DialForWorker's real lookup/dial/cache path is exercised under test. - dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "", + dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) @@ -176,7 +175,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu mockDriverName: mockPlugin, } } - service := controlapi.NewRPCService(persistence, wc, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins) + service := controlapi.NewRPCService(persistence, wc, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", 30*time.Second, volPlugins) // 5. Start REAL gRPC Server for ATE API grpcServer := grpc.NewServer(grpc.ChainUnaryInterceptor( @@ -566,7 +565,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -574,7 +573,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -583,7 +582,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -600,7 +599,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index 8e467d3029..12e42780f7 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -25,13 +25,13 @@ import ( ) const ( - ateletNamespace = "ate-system" byNamespaceAndName = "by-namespace-and-name" byNode = "by-node" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 842ac7dc81..4efd7bb282 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -55,10 +56,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewRPCService creates an instance of the ControlServer service. This is what -// implements the outward-facing RPC interface. -// -// instruments may be nil; the record helpers no-op. +// NewRPCService creates an RPC service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments may be nil. func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, @@ -69,6 +68,7 @@ func NewRPCService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, ) *RPCService { impl := newServiceImpl(persistence, storageClassLister) @@ -82,7 +82,7 @@ func NewRPCService( instruments: instruments, volumePlugins: volumePlugins, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index d4e41eea85..1cb77deef2 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -78,9 +79,12 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + // workflowDeadline is the maximum duration of a single actor workflow. + workflowDeadline time.Duration } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -91,6 +95,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -103,6 +108,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, } } @@ -145,14 +151,17 @@ type workerWorkflowStore interface { func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { leaseKey := "lease:actor:" + actorRef.Atespace + ":" + actorRef.Name + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) - lease, err := w.store.AcquireLease(ctx, leaseKey) + lease, err := w.store.AcquireLease(workflowCtx, leaseKey) if err != nil { + cancel() if errors.Is(err, store.ErrLeaseConflict) { return nil, nil, status.Error(grpcCodes.Aborted, "another operation is in progress for this actor") } return nil, nil, fmt.Errorf("while acquiring lease: %w", err) } + context.AfterFunc(lease.Context(), cancel) return lease.Context(), lease, nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go new file mode 100644 index 0000000000..b17f0583d1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" +) + +type leaseStore struct{ store.Interface } + +func (leaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + +func TestAcquireActorLeaseWorkflowDeadline(t *testing.T) { + w := &ActorWorkflow{store: leaseStore{}, workflowDeadline: 20 * time.Millisecond} + + ctx, lease, err := w.acquireActorLease(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLease: %v", err) + } + t.Cleanup(lease.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index e7dfdf659a..f2893f1cc4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -218,7 +219,7 @@ func newDanglingDialer() *AteletDialer { byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil }, byNode: func(obj any) ([]string, error) { return nil, nil }, }) - return NewAteletDialer(empty, empty, "", "") + return NewAteletDialer(empty, empty, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") } func TestEnsureAteletSuspended_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 7363444939..fa6444c823 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -19,6 +19,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -54,7 +55,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, nil, "", nil) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, nil, "", nil, time.Minute) } // seedWorkflowActor stores an actor with the given state, bound to the given diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 46ea9fe147..cfcbd75335 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -36,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" @@ -74,10 +75,14 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID expected on atelet's certificate, so it has to match what the deployment actually creates; a Helm release that prefixes resource names needs it set.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") + showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") ) @@ -156,8 +161,14 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + ateletSPIFFEID := installdefaults.SPIFFEID(ateletNamespace, *ateletServiceAccount) + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace), slog.String("atelet-spiffe-id", ateletSPIFFEID)) + workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() @@ -186,8 +197,12 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) - controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) + controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, *actorWorkflowDeadline, volPlugins) // Drive stored ActorTemplates through the golden actor flow. templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv, sandboxConfigLister) @@ -198,7 +213,7 @@ func main() { serverboot.Fatal(ctx, "while loading the Actor ID CA", err) } - actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache) + actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache, ateletSPIFFEID) lisCfg := &net.ListenConfig{} lis, err := lisCfg.Listen(ctx, "tcp", *listenAddr) @@ -305,8 +320,10 @@ func logFlagValues(ctx context.Context) { slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index f2e8f87c04..c51f5a7ddd 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -40,13 +40,16 @@ import ( // CA pool. type EgressMITMTrustReconciler struct { client.Client + + // SystemNamespace is the namespace holding the egress MITM CA pool Secret. + SystemNamespace string } // EgressMITMCAPoolRef names the Secret holding the CA pool the egress gateway's // sdsmint sidecar signs per-SNI leaves with. -func EgressMITMCAPoolRef() types.NamespacedName { +func EgressMITMCAPoolRef(systemNamespace string) types.NamespacedName { const egressMITMCAPoolSecret = "egress-mitm-ca-pool" - return types.NamespacedName{Namespace: ateSystemNamespace, Name: egressMITMCAPoolSecret} + return types.NamespacedName{Namespace: systemNamespace, Name: egressMITMCAPoolSecret} } //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch @@ -162,7 +165,7 @@ func (r *EgressMITMTrustReconciler) deleteTrustBundle(ctx context.Context) error } func (r *EgressMITMTrustReconciler) SetupWithManager(mgr ctrl.Manager) error { - poolRef := EgressMITMCAPoolRef() + poolRef := EgressMITMCAPoolRef(r.SystemNamespace) // The pool Secret is the only object reconciled from. The bundle is watched // as well so that deleting or hand-editing the derived object is reverted diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go index 2d84330900..69dd952f9d 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" ) @@ -66,7 +67,7 @@ func secretForPool(t *testing.T, pool *localca.ConcretePool) *corev1.Secret { if err != nil { t.Fatalf("marshal CA pool: %v", err) } - ref := EgressMITMCAPoolRef() + ref := EgressMITMCAPoolRef(installdefaults.SystemNamespace) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, Name: ref.Name}, Data: map[string][]byte{"pool": wire}, @@ -86,8 +87,8 @@ func rootPEM(t *testing.T, pool *localca.ConcretePool) string { func reconcilePool(t *testing.T, c client.Client) error { t.Helper() - r := &EgressMITMTrustReconciler{Client: c} - _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef()}) + r := &EgressMITMTrustReconciler{Client: c, SystemNamespace: installdefaults.SystemNamespace} + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef(installdefaults.SystemNamespace)}) return err } @@ -170,7 +171,7 @@ func TestEgressMITMTrustFollowsPoolRotation(t *testing.T) { rotated, rotatedPool := caPoolSecret(t, "mitm", "mitm-next") current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = rotated.Data @@ -279,7 +280,7 @@ func TestEgressMITMTrustKeepsLastGoodBundleOnBadPool(t *testing.T) { } current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = tc.data diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index 218a18c2d8..8c784d79af 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -20,6 +20,8 @@ package controllers // - internal/k8sresolver watches ateapi's EndpointSlices to dial it. // //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// The marker must name a literal namespace, so it carries the canonical one. +// hack/gen-rbac.sh rewrites it to the chart's release namespace on the way out. //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 928c053a92..22c65bbbdd 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -33,13 +33,17 @@ import ( const ( networkPolicyFieldOwner = "ate-networkpolicy" - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) type NetworkPolicyReconciler struct { client.Client Scheme *runtime.Scheme + + // SystemNamespace is the namespace atenet-router runs in. The generated + // ingress policy admits only that namespace, so a value that does not + // match the running router blocks every request to the worker pool. + SystemNamespace string } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch @@ -74,7 +78,7 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1alpha1.WorkerPool) error { log := log.FromContext(ctx) - npAC := buildNetworkPolicyApplyConfig(wp) + npAC := r.buildNetworkPolicyApplyConfig(wp) if err := r.Apply(ctx, npAC, client.FieldOwner(networkPolicyFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) @@ -86,7 +90,7 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1al return nil } -func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { +func (r *NetworkPolicyReconciler) buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { np := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). WithLabels(map[string]string{ "ate.dev/worker-pool": wp.Name, @@ -110,7 +114,7 @@ func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.N WithFrom( networkingv1ac.NetworkPolicyPeer(). WithNamespaceSelector(metav1ac.LabelSelector(). - WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": ateSystemNamespace})). + WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": r.SystemNamespace})). WithPodSelector(metav1ac.LabelSelector(). WithMatchLabels(map[string]string{"app": atenetRouterAppName})), ), diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 0296b873fe..f1ff854d10 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -21,6 +21,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" ) @@ -75,7 +76,7 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return false, nil } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != installdefaults.SystemNamespace { return false, nil } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { @@ -90,3 +91,24 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return true, nil }) } + +// TestBuildNetworkPolicyRelocatedNamespace pins the ingress peer to the +// reconciler's SystemNamespace rather than the canonical install namespace. +// The rest of the suite configures the reconciler with the default, so it +// passes just as well against a hardcoded "ate-system"; this is the case that +// catches that. A policy naming the wrong namespace admits nobody, and the CNI +// drops every request to the pool with no error from substrate itself. +func TestBuildNetworkPolicyRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + r := &NetworkPolicyReconciler{SystemNamespace: relocated} + np := r.buildNetworkPolicyApplyConfig(testWorkerPoolApplyConfig(nil)) + + if len(np.Spec.Ingress) != 1 || len(np.Spec.Ingress[0].From) != 1 { + t.Fatalf("expected exactly one ingress rule with one peer, got %+v", np.Spec.Ingress) + } + got := np.Spec.Ingress[0].From[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] + if got != relocated { + t.Errorf("ingress namespace selector = %q, want %q", got, relocated) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index c2fdf57773..ba6a61cbec 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -74,7 +75,7 @@ const ( // Deployment managed by a WorkerPool. Only fields owned by this controller // are declared here. otel, when it carries an endpoint, is propagated to the // ateom container so it pushes telemetry to that collector. -func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings) *appsv1ac.DeploymentApplyConfiguration { +func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings, systemNamespace, ateletServiceAccount, routerServiceAccount string) *appsv1ac.DeploymentApplyConfiguration { labels := map[string]string{} annotations := map[string]string{} if wp.Spec.Template != nil { @@ -96,6 +97,11 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + // The peers atunnel authenticates live in substrate's namespace, + // not the worker's, so the controller passes their identities + // rather than letting ateom assume the default install. + "--atunnel-client-identity="+installdefaults.SPIFFEID(systemNamespace, routerServiceAccount), + "--atunnel-broker-identity="+installdefaults.SPIFFEID(systemNamespace, ateletServiceAccount), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 85bff190f4..6743ede58d 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -205,7 +206,7 @@ func TestBuildDeploymentApplyConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("buildDeploymentApplyConfig() mismatch (-want +got):\n%s", diff) } @@ -225,7 +226,7 @@ func TestBuildDeploymentApplyConfigMetadata(t *testing.T) { }, }) - got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) wantLabels := map[string]string{ "project": "agent-substrate", "team": "compute", @@ -267,7 +268,7 @@ func TestMicroVMPodShape(t *testing.T) { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) wp.Spec.SandboxClass = tt.class - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a // hostPath mount carries no cgroup device allow rule, and the @@ -360,7 +361,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { }, }) wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM - c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec.Containers[0] + c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { t.Errorf("memory limit = %q (present=%v), want 2Gi", got, ok) @@ -425,7 +426,7 @@ func TestAteomSecurityContextByClass(t *testing.T) { // TestTerminationGracePeriodSeconds asserts the pod's grace period is hardcoded to 3600s. func TestTerminationGracePeriodSeconds(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec if ps.TerminationGracePeriodSeconds == nil { t.Fatalf("TerminationGracePeriodSeconds not set") } @@ -449,7 +450,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) @@ -535,7 +536,7 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} { @@ -592,7 +593,7 @@ func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} { @@ -720,6 +721,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + "--atunnel-client-identity="+installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + "--atunnel-broker-identity="+installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). @@ -799,3 +802,68 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithLabels(map[string]string{"ate.dev/worker-pool": wp.Name}). WithSpec(podSpecAC))) } + +// TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace pins the SPIFFE +// identities handed to ateom to the controller's namespace. atunnel runs in +// the actor's pod, so it cannot derive atelet's or the router's namespace +// itself; if these carry the wrong one, the credential broker handshake and +// actor ingress both fail closed. Every other case here passes the canonical +// namespace and so would pass against a hardcoded value too. +func TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, relocated, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got == "" { + t.Fatalf("no %s argument found in %v", flag, c.Args) + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} + +// TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts covers a release +// that renames the ServiceAccounts — what the Helm chart does for any release +// not called "substrate", which is every install that consumes substrate as a +// subchart. The SPIFFE ID embeds the ServiceAccount name, so identities built +// from the compiled-in defaults name accounts that do not exist and atunnel +// rejects the peer. +func TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts(t *testing.T) { + const ( + namespace = "kagent-system" + atelet = "kagent-atelet" + router = "kagent-atenet-router" + ) + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, namespace, atelet, router). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 7c962d62dd..7f1aeda961 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -52,6 +52,15 @@ type WorkerPoolReconciler struct { // OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom // pods. Ignored unless OTelTracesSampler is set. OTelTracesSamplerArg string + // SystemNamespace is the namespace substrate's control plane runs in, and + // AteletServiceAccount / RouterServiceAccount are the ServiceAccounts those + // components run as. Together they name the SPIFFE identities that atunnel + // authenticates inside each worker, which is why the ServiceAccount names + // are configuration and not constants: a Helm release that prefixes + // resource names changes them. + SystemNamespace string + AteletServiceAccount string + RouterServiceAccount string desiredWorkers metric.Int64ObservableUpDownCounter readyWorkers metric.Int64ObservableUpDownCounter @@ -116,7 +125,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp MetricExportTimeout: r.OTelMetricExportTimeout, TracesSampler: r.OTelTracesSampler, TracesSamplerArg: r.OTelTracesSamplerArg, - }) + }, r.SystemNamespace, r.AteletServiceAccount, r.RouterServiceAccount) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index c1f3b8b6f7..5c7ea51366 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -43,6 +43,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/testenv" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -86,8 +87,9 @@ func TestMain(m *testing.M) { } if err := (&NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: installdefaults.SystemNamespace, }).SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err) os.Exit(1) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 7ef107ec1a..f50152aece 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atecontroller/internal/controllers" "github.com/agent-substrate/substrate/cmd/atecontroller/internal/workersync" "github.com/agent-substrate/substrate/internal/ateapiauth" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/serverboot" clientv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -70,6 +71,9 @@ var ( otelTracesSamplerArg = pflag.String("otel-traces-sampler-arg", os.Getenv("OTEL_TRACES_SAMPLER_ARG"), "Trace sampler argument set on ateom worker pods, ignored unless --otel-traces-sampler is set. Defaults to the controller's own OTEL_TRACES_SAMPLER_ARG.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel expects on the credential broker, so it has to match what the deployment actually creates.") + routerServiceAccount = pflag.String("router-service-account", installdefaults.RouterServiceAccount, "ServiceAccount atenet-router runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel accepts on actor ingress, so it has to match what the deployment actually creates.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required.") @@ -152,7 +156,8 @@ func main() { ateapiClient := ateapipb.NewControlClient(ateapiConn) // EgressMITMTrustReconciler watches the Secret `egress-mitm-ca-pool`. - egressMITMCAPool := controllers.EgressMITMCAPoolRef() + systemNamespace := installdefaults.NamespaceFromPodEnv() + egressMITMCAPool := controllers.EgressMITMCAPoolRef(systemNamespace) mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{ Scheme: scheme, Cache: cache.Options{ @@ -180,21 +185,26 @@ func main() { OTelMetricExportTimeout: *otelMetricExportTimeout, OTelTracesSampler: *otelTracesSampler, OTelTracesSamplerArg: *otelTracesSamplerArg, + SystemNamespace: systemNamespace, + AteletServiceAccount: *ateletServiceAccount, + RouterServiceAccount: *routerServiceAccount, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) } if err = (&controllers.NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NetPolicy") os.Exit(1) } if err = (&controllers.EgressMITMTrustReconciler{ - Client: mgr.GetClient(), + Client: mgr.GetClient(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "EgressMITMTrust") os.Exit(1) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 7a1f06335c..7ded9c64fb 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -95,6 +95,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -312,69 +313,75 @@ func main() { // it would never list or watch. Start is idempotent per informer — this // launches the new one and leaves the already-running ones untouched. ateFactory.Start(stopCh) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - brokerTLS := tlsCfg.Clone() - brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer brokerLis.Close() - if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } - brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) - ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ - actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn), - }) - go func() { - if err := brokerServer.Serve(brokerLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) } - }() + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() + + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + brokerTLS := tlsCfg.Clone() + brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer brokerLis.Close() + if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) + ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ + actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn), + }) + go func() { + if err := brokerServer.Serve(brokerLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } + + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) diff --git a/cmd/atenet/internal/dns.go b/cmd/atenet/internal/dns.go index 85798fa73f..f169ffe9d8 100644 --- a/cmd/atenet/internal/dns.go +++ b/cmd/atenet/internal/dns.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/agent-substrate/substrate/cmd/atenet/internal/dns" + "github.com/agent-substrate/substrate/internal/installdefaults" ) type DnsConfig struct { @@ -37,6 +38,8 @@ type DnsConfig struct { Kubeconfig string ReconcileInterval time.Duration CorefilePath string + RouterServiceName string + DNSServiceName string } func NewDnsCmd() *cobra.Command { @@ -78,11 +81,20 @@ func NewDnsCmd() *cobra.Command { return fmt.Errorf("failed to initialize cluster client: %w", err) } + // atenet shares its namespace with atenet-router and substrate's + // CoreDNS in every supported deployment topology, so we read it + // from Kubernetes' downward API rather than expose a flag. + systemNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved system namespace", slog.String("system-namespace", systemNamespace)) + dnsController := &dns.Controller{ - Client: k8sClient, - Interval: cfg.ReconcileInterval, - CorefilePath: cfg.CorefilePath, - Reloader: dns.NewConfigReloader(), + Client: k8sClient, + Interval: cfg.ReconcileInterval, + CorefilePath: cfg.CorefilePath, + Reloader: dns.NewConfigReloader(), + SystemNamespace: systemNamespace, + RouterServiceName: cfg.RouterServiceName, + DNSServiceName: cfg.DNSServiceName, } slog.InfoContext(ctx, "Starting DNS Controller subsystem") @@ -94,6 +106,8 @@ func NewDnsCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations") cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of the atenet-router. Override when the deployment renames the Service.") + cmd.Flags().StringVar(&cfg.DNSServiceName, "dns-service-name", installdefaults.DNSServiceName, "Service name of substrate's CoreDNS. Override when the deployment renames the Service.") return cmd } diff --git a/cmd/atenet/internal/dns/dns.go b/cmd/atenet/internal/dns/dns.go index cf2db99b69..4cfaf34ef8 100644 --- a/cmd/atenet/internal/dns/dns.go +++ b/cmd/atenet/internal/dns/dns.go @@ -33,18 +33,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // serviceName is the name of the CoreDNS service. - serviceName = "dns" - systemNamespace = "ate-system" -) - // Controller manages the DNS configuration for the ATE. type Controller struct { Client client.Client Interval time.Duration CorefilePath string Reloader ConfigReloader + + // SystemNamespace is the namespace where atenet-router and the substrate + // CoreDNS Service live. Defaults to installdefaults.SystemNamespace. + SystemNamespace string + // RouterServiceName is the Service name of the atenet-router that the + // CoreDNS Corefile forwards actor traffic to. Defaults to + // installdefaults.RouterServiceName. + RouterServiceName string + // DNSServiceName is the Service name of substrate's CoreDNS. Defaults to + // installdefaults.DNSServiceName. + DNSServiceName string } // Run the DNS orchestration loop until ctx is canceled. @@ -71,14 +76,15 @@ func (c *Controller) Run(ctx context.Context) error { func (c *Controller) reconcile(ctx context.Context) error { slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...") - // 1. Get the ClusterIP of atenet-router in ate-system namespace + // 1. Get the ClusterIP of the atenet-router Service in the substrate namespace. routerSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.RouterServiceName, Namespace: c.SystemNamespace}, routerSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available") + slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available", + slog.String("name", c.RouterServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get atenet-router service: %w", err) + return fmt.Errorf("failed to get atenet-router service %s/%s: %w", c.SystemNamespace, c.RouterServiceName, err) } routerIP := routerSvc.Spec.ClusterIP @@ -87,14 +93,15 @@ func (c *Controller) reconcile(ctx context.Context) error { return nil } - // 2. Get the ClusterIP of dns service in ate-system namespace + // 2. Get the ClusterIP of substrate's CoreDNS Service in the same namespace. dnsSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: systemNamespace}, dnsSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.DNSServiceName, Namespace: c.SystemNamespace}, dnsSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "dns service not found, skipping until it is available") + slog.WarnContext(ctx, "dns service not found, skipping until it is available", + slog.String("name", c.DNSServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get dns service: %w", err) + return fmt.Errorf("failed to get dns service %s/%s: %w", c.SystemNamespace, c.DNSServiceName, err) } dnsIP := dnsSvc.Spec.ClusterIP diff --git a/cmd/atenet/internal/dns/dns_test.go b/cmd/atenet/internal/dns/dns_test.go index 34116db284..bf27941e18 100644 --- a/cmd/atenet/internal/dns/dns_test.go +++ b/cmd/atenet/internal/dns/dns_test.go @@ -28,6 +28,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) type mockConfigReloader struct { @@ -94,10 +96,13 @@ func TestReconcile(t *testing.T) { reloader := &mockConfigReloader{} controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: reloader, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: reloader, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } // Run one reconciliation loop @@ -185,10 +190,13 @@ func TestReconcileKubeDNSNotFound(t *testing.T) { Build() controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: &mockConfigReloader{}, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: &mockConfigReloader{}, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } ctx := context.Background() diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 3c70cb92cc..ba22d18060 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/installdefaults" ) func NewRouterCmd() *cobra.Command { @@ -46,6 +47,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().StringVar(&cfg.AtenetRouter, "atenet-router", string(atenetRouterEnvoy), "Router dataplane: envoy or agentgateway") cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 16297ee68a..059b342ad1 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -74,18 +74,22 @@ type authConfig struct { // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { // Mode restricts the instance to one traffic direction. Empty means ModeAll. - Mode Mode - AtenetRouter string - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - StatusPort int - HealthInterval time.Duration - HttpsPort int + Mode Mode + AtenetRouter string + Namespace string + // RouterServiceName is the Service name of this atenet-router in the + // operations namespace, used by /statusz to look up its own ClusterIP. + // Defaults to installdefaults.RouterServiceName. + RouterServiceName string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + StatusPort int + HealthInterval time.Duration + HttpsPort int // ConnectPlainTextPort and ConnectTLSPort are the plaintext and TLS // listener ports for CONNECT-tunneled traffic. Non-positive disables the // corresponding listener. diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 6878755580..9fe196908f 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -68,7 +68,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { return "Offline Mode (No Cluster IP)" } - svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, "atenet-router", metav1.GetOptions{}) + svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, s.cfg.RouterServiceName, metav1.GetOptions{}) if err != nil { return fmt.Sprintf("Lookup Failed: %v", err) } diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 90cbfaec9c..62f547cc61 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -44,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/internal/childreap" "github.com/agent-substrate/substrate/internal/contextlogging" "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -72,7 +73,8 @@ var ( atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = pflag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = pflag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = pflag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = pflag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -210,7 +212,7 @@ func do(ctx context.Context) error { return err } - ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -369,6 +371,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // activeActor is the actor whose workload this ateom is currently running, // or nil when it is "available". An ateom serves one actor at a time, so a @@ -420,7 +426,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), interiorNetNS: interiorNetNS, @@ -431,6 +437,7 @@ func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, cgroupRoot: defaultCgroupRoot, } } @@ -1048,6 +1055,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 751881e37f..0e0fcf9230 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -45,6 +45,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -77,7 +78,8 @@ var ( atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = flag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = flag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = flag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = flag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = flag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -256,7 +258,7 @@ func do(ctx context.Context) error { }() slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) - ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -423,6 +425,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // running maps actor UID -> the live micro-VM, kept so CheckpointWorkload can // pause+snapshot+teardown the same sandbox (and RestoreWorkload can track the @@ -476,7 +482,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), podUID: podUID, @@ -492,6 +498,7 @@ func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveM workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, running: map[string]*runningActor{}, } } @@ -520,6 +527,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..b7083b00e7 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" + +# Templatize the namespaced Role's namespace. A kubebuilder rbac marker has to +# name a literal namespace, so controller-gen emits the canonical one; the +# chart installs into whatever namespace the release targets, and a Role left +# in ate-system would leave ate-controller without the permission there. +sed -i 's|^ namespace: ate-system$| namespace: {{ .Release.Namespace }}|' "${OUT}" diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2e9fd0c03c..5a997053de 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -35,6 +35,16 @@ if [[ -z "${KUBECTL_CONTEXT:-}" ]]; then fi # otherwise just use the current cluster in KUBECONFIG ... +# Namespace the substrate control plane is installed into. Defaults to the +# canonical ate-system so existing flows are unaffected; override it to install +# a relocated release (the chart's --namespace must match). +ATE_NAMESPACE="${ATE_NAMESPACE:-ate-system}" + +# Service name fronting ateapi. It is the audience the API authentication config +# accepts, so it has to match the Service the deployment actually creates; a +# Helm release that prefixes resource names needs it set. +ATE_API_SERVICE_NAME="${ATE_API_SERVICE_NAME:-api}" + # ATE_DEMOS is an array that registers the prefix name of the demo functions. ATE_DEMOS=() @@ -224,8 +234,20 @@ rollout_timeout() { echo "${timeout}" } +# ensure_ate_namespace creates ATE_NAMESPACE and waits for it to go Active. +ensure_ate_namespace() { + if [[ "${ATE_NAMESPACE}" == "ate-system" ]]; then + run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml + else + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ + | run_kubectl apply -f - + fi + run_kubectl wait --for=jsonpath='{.status.phase}'=Active \ + "namespace/${ATE_NAMESPACE}" --timeout=60s +} + default_postgres_connection_string() { - echo "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + echo "postgresql://postgres@postgres.${ATE_NAMESPACE}.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" } use_bundled_postgres() { @@ -361,14 +383,14 @@ apply_atenet_egress() { # bootstrap arrives as a ConfigMap change, and an otherwise unchanged # Deployment will not pick that up on its own. local running=false - if run_kubectl -n ate-system get deployment/atenet-egress >/dev/null 2>&1; then + if run_kubectl -n "${ATE_NAMESPACE}" get deployment/atenet-egress >/dev/null 2>&1; then running=true fi echo "${manifests}" | run_kubectl apply -f - if [[ "${running}" == "true" ]] && additional_egress_extproc_enabled; then - run_kubectl -n ate-system rollout restart deployment/atenet-egress + run_kubectl -n "${ATE_NAMESPACE}" rollout restart deployment/atenet-egress fi } @@ -417,28 +439,28 @@ apply_otel_endpoint_override() { fi local current="" - current="$(run_kubectl -n ate-system get configmap ate-otel-config \ + current="$(run_kubectl -n "${ATE_NAMESPACE}" get configmap ate-otel-config \ -o jsonpath='{.data.OTEL_EXPORTER_OTLP_ENDPOINT}' 2>/dev/null || true)" if [[ "${current}" == "${ATE_OTLP_ENDPOINT}" ]]; then return 0 fi echo "Overriding OTEL_EXPORTER_OTLP_ENDPOINT with ${ATE_OTLP_ENDPOINT}" - run_kubectl -n ate-system patch configmap ate-otel-config --type=merge \ + run_kubectl -n "${ATE_NAMESPACE}" patch configmap ate-otel-config --type=merge \ -p "{\"data\":{\"OTEL_EXPORTER_OTLP_ENDPOINT\":\"${ATE_OTLP_ENDPOINT}\"}}" local workload for workload in deployment/ate-api-server deployment/ate-controller \ deployment/atenet-router; do - if run_kubectl -n ate-system get "${workload}" >/dev/null 2>&1; then - run_kubectl -n ate-system rollout restart "${workload}" + if run_kubectl -n "${ATE_NAMESPACE}" get "${workload}" >/dev/null 2>&1; then + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${workload}" fi done # atelet DaemonSet names carry a version suffix; restart whichever versions # are installed. local ds="" - for ds in $(run_kubectl -n ate-system get daemonset -l app=atelet -o name 2>/dev/null); do - run_kubectl -n ate-system rollout restart "${ds}" + for ds in $(run_kubectl -n "${ATE_NAMESPACE}" get daemonset -l app=atelet -o name 2>/dev/null); do + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${ds}" done } @@ -460,7 +482,7 @@ create_jwt_authority_pool_secret() { run_kubectl_ate admin make-jwt-pool \ --key-id="1" \ --name="actor-id-jwt-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } create_actor_id_ca_pool_secret() { @@ -468,7 +490,7 @@ create_actor_id_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="actor-id-ca-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } # The egress gateway has to verify actor client certificates, which means it @@ -482,7 +504,7 @@ create_actor_id_ca_certs_secret() { # inside the create-secret argument list, which would silently produce an # empty trust bundle and an egress gateway that rejects every actor. local actorid_root="" - actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) + actorid_root=$(ca_pool_root_pem actor-id-ca-pool "${ATE_NAMESPACE}") if [[ -z "${actorid_root}" ]]; then echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 return 1 @@ -490,7 +512,7 @@ create_actor_id_ca_certs_secret() { run_kubectl create secret generic actor-id-ca-certs \ --from-literal=ca.crt="${actorid_root}" \ - -n ate-system \ + -n "${ATE_NAMESPACE}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - } @@ -504,7 +526,7 @@ create_egress_mitm_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="egress-mitm-ca-pool" \ - --secret-namespace=ate-system \ + --secret-namespace="${ATE_NAMESPACE}" \ --key-type=ECDSAP256 } @@ -515,7 +537,7 @@ ensure_egress_mitm_ca_pool_secret() { if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" != "true" ]]; then return 0 fi - run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" egress-mitm-ca-pool >/dev/null 2>&1 \ || create_egress_mitm_ca_pool_secret } @@ -544,7 +566,7 @@ wait_for_podcertificate_trust_bundles() { create_api_server_env_vars() { log_step "create_api_server_env_vars" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local postgres_connection_string="${ATE_API_POSTGRES_CONNECTION_STRING:-}" @@ -555,7 +577,7 @@ create_api_server_env_vars() { echo "POSTGRES_CONNECTION_STRING: ${postgres_connection_string}" - run_kubectl create configmap -n ate-system ate-api-server-envvars \ + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-server-envvars \ --from-literal=ATE_API_POSTGRES_CONNECTION_STRING="${postgres_connection_string}" \ --from-literal=ATE_API_POSTGRES_SCHEMA="${postgres_schema}" \ --dry-run=client -o yaml \ @@ -584,7 +606,7 @@ apply_podcert_workers_override() { create_api_authentication_config() { log_step "create_api_authentication_config" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local jwt_issuer="" @@ -604,8 +626,8 @@ create_api_authentication_config() { ;; esac local authentication_config - authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.ate-system.svc]\n%s' "${jwt_issuer}" "${discovery_config}") - run_kubectl create configmap -n ate-system ate-api-authentication \ + authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [%s.%s.svc]\n%s' "${jwt_issuer}" "${ATE_API_SERVICE_NAME}" "${ATE_NAMESPACE}" "${discovery_config}") + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-authentication \ --from-literal=authentication.yaml="${authentication_config}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - @@ -666,8 +688,7 @@ deploy_ate_system() { ensure_substrate_version # Ensure namespace exists before applying RBAC or CRDs - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace # The atelet DaemonSet applied below and the demo WorkerPools' worker pods # schedule only to version-labeled nodes. @@ -729,13 +750,13 @@ deploy_ate_system() { log_step "Waiting for ATE system components to be ready..." if use_bundled_postgres; then - run_kubectl rollout status statefulset/postgres -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status statefulset/postgres -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" fi - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/ate-controller -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-controller -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" # After the bundle, which carries its own copy of ate-otel-config. apply_otel_endpoint_override @@ -744,18 +765,18 @@ deploy_ate_system() { # Ensure secrets and configmaps required by ate-apiserver ensure_apiserver_prerequisites() { log_step "ensure_apiserver_prerequisites" - run_kubectl get secret -n ate-system actor-id-jwt-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-jwt-pool >/dev/null 2>&1 \ || create_jwt_authority_pool_secret - run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret # Derived from actor-id-ca-pool above, so it must come after it. - run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-certs >/dev/null 2>&1 \ || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ || create_podcertificate_controller_cas # Always reconcile the PostgreSQL connection settings. create_api_server_env_vars - run_kubectl get configmap -n ate-system ate-api-authentication >/dev/null 2>&1 \ + run_kubectl get configmap -n "${ATE_NAMESPACE}" ate-api-authentication >/dev/null 2>&1 \ || create_api_authentication_config } @@ -765,15 +786,14 @@ deploy_ate_apiserver() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace ensure_apiserver_prerequisites apply_otel_config apply_otel_endpoint_override run_ko apply -f manifests/ate-install/ate-api-server.yaml - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atelet() { @@ -782,8 +802,7 @@ deploy_atelet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace label_nodes_substrate_version apply_otel_config @@ -798,7 +817,7 @@ deploy_atelet() { manifest=$(run_ko resolve -f manifests/ate-install/atelet.yaml | substitute_version) fi echo "${manifest}" | run_kubectl apply -f - - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atenet() { @@ -806,8 +825,7 @@ deploy_atenet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace apply_otel_config apply_otel_endpoint_override @@ -819,9 +837,9 @@ deploy_atenet() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress run_ko apply -f manifests/ate-install/atenet-dns.yaml - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/dns -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } # get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). @@ -889,7 +907,7 @@ delete_demo_actors_substrate() { return 1 fi - if ! run_kubectl get deployment/ate-api-server -n ate-system >/dev/null 2>&1; then + if ! run_kubectl get deployment/ate-api-server -n "${ATE_NAMESPACE}" >/dev/null 2>&1; then log_step "ate-api-server not found; skipping actor cleanup" return 0 fi @@ -1070,7 +1088,7 @@ delete_ate_system() { run_kubectl delete --ignore-not-found -f manifests/ate-install fi - run_kubectl delete --ignore-not-found -n ate-system daemonset -l app=atelet + run_kubectl delete --ignore-not-found -n "${ATE_NAMESPACE}" daemonset -l app=atelet run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml run_kubectl delete --ignore-not-found -f manifests/ate-install/postgres.yaml diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index b5f3d7df47..35f7b16d4e 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -174,6 +174,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..fc6b8d6858 --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" +PRESERVED_FILES=( + ate-api-server.yaml + ate-controller.yaml + ate-otel-config.yaml + ate-system-namespace.yaml + atelet.yaml + atenet-dns.yaml + atenet-egress.yaml + atenet-egress-with-sdsmint.yaml + atenet-router.yaml + atenet-router-monitoring.yaml + pod-certificate-controller.yaml + postgres.yaml + sandboxconfig-gvisor.yaml + sandboxconfig-validation.yaml +) + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + if src == "namespace.yaml": + src = "ate-system-namespace.yaml" + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + for file in "${PRESERVED_FILES[@]}"; do + rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" + done + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ + ! -name 'ate-api-server.yaml' \ + ! -name 'ate-controller.yaml' \ + ! -name 'ate-otel-config.yaml' \ + ! -name 'ate-system-namespace.yaml' \ + ! -name 'atelet.yaml' \ + ! -name 'atenet-dns.yaml' \ + ! -name 'atenet-egress.yaml' \ + ! -name 'atenet-egress-with-sdsmint.yaml' \ + ! -name 'atenet-router.yaml' \ + ! -name 'atenet-router-monitoring.yaml' \ + ! -name 'pod-certificate-controller.yaml' \ + ! -name 'postgres.yaml' \ + ! -name 'sandboxconfig-gvisor.yaml' \ + ! -name 'sandboxconfig-validation.yaml' \ + -delete +for file in "${PRESERVED_FILES[@]}"; do + rm -f "${TMP_DIR}/out/${file}" +done +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 19cd97daa1..f47565eb50 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -55,10 +55,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -if [[ $# -gt 0 ]]; then - echo "Error: unknown argument $1" >&2 - exit 1 -fi +SKIP_CONTROL_PLANE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; + *) + echo "Error: unknown argument $1" >&2 + exit 1 + ;; + esac + shift +done if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is required (set it in .ate-dev-env.sh for GKE," >&2 @@ -75,13 +83,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index c80f9ca0f4..208f80b110 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="_LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index da5a092ada..c3991e96eb 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -43,9 +44,53 @@ import ( metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned" ) +// NamespaceEnv overrides the namespace the client looks for substrate in. The +// client runs outside the cluster, so it has no downward API to read and no +// pod namespace to fall back on; it matches the ATE_NAMESPACE that +// hack/install-ate.sh installed with. +const NamespaceEnv = "ATE_NAMESPACE" + +// APIServiceEnv and ClientServiceAccountEnv override the ateapi Service and the +// ServiceAccount the client mints its token from. The Helm chart prefixes both +// names for a release not called "substrate", and the client has no way to +// discover that from outside the cluster. const ( - apiServerName = "api.ate-system.svc" + APIServiceEnv = "ATE_API_SERVICE_NAME" + ClientServiceAccountEnv = "ATE_CLIENT_SERVICE_ACCOUNT" +) + +// apiServiceName is the Service that fronts ateapi. +func apiServiceName() string { + if n := os.Getenv(APIServiceEnv); n != "" { + return n + } + return installdefaults.APIServiceName +} + +// clientServiceAccount is the ServiceAccount the bearer token is minted from. +func clientServiceAccount() string { + if n := os.Getenv(ClientServiceAccountEnv); n != "" { + return n + } + return installdefaults.ClientServiceAccount +} + +// systemNamespace is the namespace the client expects ateapi to be running in. +func systemNamespace() string { + if ns := os.Getenv(NamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} +// apiServerName is the in-cluster DNS name of the ateapi Service. It is both +// the SNI presented on the connection and the audience of the minted token, so +// it has to track the namespace ateapi actually runs in. +func apiServerName() string { + return fmt.Sprintf("%s.%s.svc", apiServiceName(), systemNamespace()) +} + +const ( // serviceDNSSignerName and liveBundleSelector mirror the // clusterTrustBundle projected-volume sources that in-cluster clients // mount to verify ateapi's serving cert. @@ -83,7 +128,7 @@ func (c *Client) Close() { } // NewClient creates a new Ate API client. If endpoint is empty, it automatically port-forwards -// to the ate-api-server pod in the ate-system namespace. +// to the ate-api-server pod in substrate's namespace. func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint, tokenFile string, traceEnabled bool) (*Client, error) { tp, err := initTracing(ctx, traceEnabled) if err != nil { @@ -167,7 +212,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "api", 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, systemNamespace(), apiServiceName(), 443) if err != nil { return nil, err } @@ -232,7 +277,7 @@ func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls. return &tls.Config{ MinVersion: tls.VersionTLS13, RootCAs: pool, - ServerName: apiServerName, + ServerName: apiServerName(), }, nil } @@ -252,11 +297,11 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ - Audiences: []string{apiServerName}, + Audiences: []string{apiServerName()}, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(systemNamespace()).CreateToken(ctx, clientServiceAccount(), tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/atunnel/credential.go b/internal/atunnel/credential.go index 65090af769..6b17aa26fc 100644 --- a/internal/atunnel/credential.go +++ b/internal/atunnel/credential.go @@ -23,9 +23,7 @@ import ( "crypto/x509" "fmt" "net" - "net/url" "os" - "path" "slices" "sync" "time" @@ -61,6 +59,11 @@ type BrokerConfig struct { // ExpectedActorUID prevents a mint started for an old activation from // receiving the newly assigned actor's certificate. ExpectedActorUID string + // AteletSPIFFEID is the SPIFFE ID atelet must present on the credential + // broker connection. It carries the namespace atelet runs in, which is not + // this worker's own namespace, so it is supplied by the caller rather than + // read from the downward API. See installdefaults.AteletSPIFFEID. + AteletSPIFFEID string } // NewBrokerCertificateSource creates one actor key for this activation. The key @@ -70,6 +73,9 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if cfg.SocketPath == "" || cfg.CredentialBundlePath == "" || cfg.TrustBundlePath == "" || cfg.ExpectedActorUID == "" { return nil, fmt.Errorf("atunnel: credential broker socket, credentials, trust bundle, and expected actor UID are required") } + if cfg.AteletSPIFFEID == "" { + return nil, fmt.Errorf("atunnel: expected atelet SPIFFE ID is required") + } localCert, err := credbundle.Parse(cfg.CredentialBundlePath) if err != nil { return nil, fmt.Errorf("atunnel: load worker identity: %w", err) @@ -90,7 +96,7 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if err != nil { return nil, fmt.Errorf("atunnel: generate actor private key: %w", err) } - expectedURI := (&url.URL{Scheme: "spiffe", Host: "cluster.local", Path: path.Join("ns", "ate-system", "sa", "atelet")}).String() + expectedURI := cfg.AteletSPIFFEID tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, // Verification below supports SPIFFE Pod certificates without a DNS name. diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/credential_test.go index 057b6fe9e8..8e598043e4 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/credential_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/substratex509" "google.golang.org/grpc" @@ -184,6 +185,7 @@ func newTestBrokerCertificateSource(t *testing.T, ateletIdentity *substratex509. CredentialBundlePath: credentialPath, TrustBundlePath: trustPath, ExpectedActorUID: "actor-uid", + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), }) if err != nil { t.Fatal(err) diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 3d0db9f047..d4b842227c 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -112,6 +113,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -124,8 +126,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -139,7 +142,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -155,3 +158,16 @@ func Parse(bundlePath string) (*tls.Certificate, error) { PrivateKey: leafKey, }, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 579a12bbc0..171bcb5b64 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 488809f89b..4e216c5e64 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -47,7 +47,6 @@ var PlatformMetricPrefixes = []string{ "ate_scheduler_assignment_duration", "ate_actor_restore_duration", "ate_actor_checkpoint_duration", - "atenet_router_route_duration", "ate_scheduler_eligible_workers", } diff --git a/internal/e2e/env.go b/internal/e2e/env.go index ccc2e7f6b6..f0136102bd 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -17,6 +17,8 @@ package e2e import ( "fmt" "os" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) // CheckEnv checks the list of env vars exist and returns their value. @@ -32,3 +34,28 @@ func CheckEnv(keys ...string) (map[string]string, error) { } return env, nil } + +// SystemNamespaceEnv names the namespace the substrate control plane under test +// was installed into. It mirrors the ATE_NAMESPACE hack/install-ate.sh +// installed with, and the --namespace the chart was released into. +const SystemNamespaceEnv = "E2E_SYSTEM_NAMESPACE" + +// SystemNamespace returns the namespace the control plane under test runs in, +// falling back to the canonical install namespace. +func SystemNamespace() string { + if ns := os.Getenv(SystemNamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} + +// ResourcePrefixEnv is the prefix the install under test puts on substrate's +// resource names. The Helm chart prefixes them with the release name for any +// release not called "substrate", which is what a subchart install produces, +// and the harness addresses several of those resources by name. +const ResourcePrefixEnv = "E2E_RESOURCE_PREFIX" + +// ResourceName returns name as the install under test renders it. +func ResourceName(name string) string { + return os.Getenv(ResourcePrefixEnv) + name +} diff --git a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl index 97b78a7137..9ab467ec5b 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl @@ -15,7 +15,7 @@ # The egress probe used by internal/e2e/suites/egressauthz: testserver's probe # subcommand, with the credential volumes that suite mints. It is not a # ServerPod because it needs those volumes and a namespace the suite populates -# first, so it stays a bespoke manifest. ${NAMESPACE} is substituted by the +# first, so it stays a bespoke manifest. ${NAMESPACE} and ${SYSTEM_NAMESPACE} are substituted by the # suite with the randomized namespace it created, so the probe is torn down with # that namespace and leaves nothing behind. apiVersion: v1 @@ -33,6 +33,10 @@ spec: args: - "egressprobe" - "--listen=:8080" + # The gateway lives in substrate's namespace, which the probe cannot infer + # from inside the sandbox. ${SYSTEM_NAMESPACE} is substituted alongside + # ${NAMESPACE} by the suite. + - "--gateway-address=atenet-egress.${SYSTEM_NAMESPACE}.svc:443" ports: - name: http containerPort: 8080 diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 14cae171d3..938bef93d8 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -38,10 +38,10 @@ func PreflightChecks() error { // Check deployments. deployments := []string{ - "ate-controller", - "ate-api-server", + ResourceName("ate-controller"), + ResourceName("ate-api-server"), } - namespace := "ate-system" + namespace := SystemNamespace() for _, depName := range deployments { dep, err := clients.K8s.AppsV1().Deployments(namespace).Get(ctx, depName, metav1.GetOptions{}) if err != nil { diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 7a9e536079..d139514cbf 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -36,8 +36,6 @@ import ( ) const ( - routerNamespace = "ate-system" - routerService = "atenet-router" // routerConnectServicePort is atenet-router's Service port for // CONNECT-tunneled traffic (see manifests/ate-install/atenet-router.yaml). // It is a distinct listener from the plain HTTP one Get/PostJSON use: @@ -79,7 +77,7 @@ func NewRouterClient(ctx context.Context) (*RouterClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), 80) if err != nil { return nil, err } @@ -188,7 +186,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, // in one test don't each pay for a fresh port-forward. func (c *RouterClient) ensureConnectPortForward(ctx context.Context) error { c.connectOnce.Do(func() { - localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, routerNamespace, routerService, routerConnectServicePort) + localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, SystemNamespace(), ResourceName("atenet-router"), routerConnectServicePort) if err != nil { c.connectErr = fmt.Errorf("port-forwarding to the router's CONNECT listener: %w", err) return diff --git a/internal/e2e/statusz.go b/internal/e2e/statusz.go index a04a07141d..4e3516aed4 100644 --- a/internal/e2e/statusz.go +++ b/internal/e2e/statusz.go @@ -51,7 +51,7 @@ func NewStatuszClient(ctx context.Context) (*StatuszClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, routerStatusPort) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), routerStatusPort) if err != nil { return nil, err } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 38e6bea07b..dcbb0c1b94 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -702,7 +705,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -726,24 +729,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -754,8 +747,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.", - len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName()) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -782,13 +774,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -807,11 +799,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -861,11 +854,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -885,11 +879,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1195,6 +1190,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } @@ -1220,13 +1264,13 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s t.Helper() clients := e2e.GetClients() - svc, err := clients.K8s.CoreV1().Services("ate-system").Get(context.Background(), "atenet-router", metav1.GetOptions{}) + svc, err := clients.K8s.CoreV1().Services(e2e.SystemNamespace()).Get(context.Background(), e2e.ResourceName("atenet-router"), metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get atenet-router service: %w", err) } selector := labels.SelectorFromSet(svc.Spec.Selector).String() - pods, err := clients.K8s.CoreV1().Pods("ate-system").List(context.Background(), metav1.ListOptions{LabelSelector: selector}) + pods, err := clients.K8s.CoreV1().Pods(e2e.SystemNamespace()).List(context.Background(), metav1.ListOptions{LabelSelector: selector}) if err != nil { return "", fmt.Errorf("failed to list atenet-router pods: %w", err) } diff --git a/internal/e2e/suites/egressauthz/actoridentity_test.go b/internal/e2e/suites/egressauthz/actoridentity_test.go index ffe82e5e2a..b2bcce6c36 100644 --- a/internal/e2e/suites/egressauthz/actoridentity_test.go +++ b/internal/e2e/suites/egressauthz/actoridentity_test.go @@ -73,16 +73,16 @@ const ( // the secret ateapi signs with. func actorIdentityCA(t *testing.T, ctx context.Context) *localca.CA { t.Helper() - secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, actorIDCASecret, metav1.GetOptions{}) + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(e2e.SystemNamespace()).Get(ctx, actorIDCASecret, metav1.GetOptions{}) if err != nil { - t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", egressNamespace, actorIDCASecret, err) + t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", e2e.SystemNamespace(), actorIDCASecret, err) } pool, err := localca.Unmarshal(secret.Data[actorIDCASecretKey]) if err != nil { - t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", egressNamespace, actorIDCASecret, actorIDCASecretKey, err) + t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", e2e.SystemNamespace(), actorIDCASecret, actorIDCASecretKey, err) } if len(pool.CAs) == 0 { - t.Fatalf("actor-identity CA pool %s/%s contains no CA", egressNamespace, actorIDCASecret) + t.Fatalf("actor-identity CA pool %s/%s contains no CA", e2e.SystemNamespace(), actorIDCASecret) } // CAs[0] is the one that signs: ateapi's MintCert makes the same choice. return pool.CAs[0] diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index 1fa04ffaff..3b39da7162 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -55,7 +55,6 @@ import ( const ( // Where the gateway's CA lives, fixed by hack/install-ate.sh. - egressNamespace = "ate-system" probeName = "egressprobe" ) @@ -163,6 +162,7 @@ func startProbe(t *testing.T, ctx context.Context) *probeClient { } manifest := filepath.Join(t.TempDir(), "egressprobe.yaml") rendered := strings.ReplaceAll(string(tmpl), "${NAMESPACE}", ns) + rendered = strings.ReplaceAll(rendered, "${SYSTEM_NAMESPACE}", e2e.SystemNamespace()) if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { t.Fatalf("writing rendered egressprobe manifest: %v", err) } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index ceeef793d9..14e3a80f57 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -252,18 +252,30 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) whoamiResponse { t.Helper() - resp, err := rc.Get(ctx, resources.ActorRef{Atespace: probeNamespace, Name: id}, "/whoami") - if err != nil { - t.Fatalf("GET /whoami for %q: %v", id, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("GET /whoami for %q: status %d, body %q", id, resp.StatusCode, body) - } - var out whoamiResponse - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - t.Fatalf("decoding /whoami for %q: %v", id, err) + deadline := time.Now().Add(30 * time.Second) + for { + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: probeNamespace, Name: id}, "/whoami") + if err != nil { + if time.Now().After(deadline) { + t.Fatalf("GET /whoami for %q did not become ready: %v", id, err) + } + time.Sleep(time.Second) + continue + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if time.Now().After(deadline) { + t.Fatalf("GET /whoami for %q: status %d, body %q", id, resp.StatusCode, body) + } + time.Sleep(time.Second) + continue + } + defer resp.Body.Close() + var out whoamiResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding /whoami for %q: %v", id, err) + } + return out } - return out } diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 86fcf0ec4a..d03923df26 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -42,6 +42,17 @@ func TestPlatformMetricsEmitted(t *testing.T) { clients := e2e.GetClients() tmpl := e2e.SubstrateCounterFixture() actorID := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano()) + metricPrefixes := append([]string(nil), e2e.PlatformMetricPrefixes...) + router, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("atenet-router"), metav1.GetOptions{}) + if err != nil { + t.Fatalf("Get atenet-router deployment: %v", err) + } + for _, container := range router.Spec.Template.Spec.Containers { + if container.Name == "envoy" { + metricPrefixes = append(metricPrefixes, "atenet_router_route_duration") + break + } + } // CreateActor requires the atespace to exist first; ignore AlreadyExists. _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ @@ -64,7 +75,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { // they add the drive steps their instruments need. resume(t, ctx, clients, actorID) - // Drive request through the router so Envoy ext_proc emits atenet_router_route_duration. + // Drive request through the router so Envoy ext_proc emits its route metric when installed. rClient, err := e2e.NewRouterClient(ctx) if err != nil { t.Fatalf("NewRouterClient: %v", err) @@ -96,7 +107,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { if err != nil { t.Fatalf("ScrapeCollectorMetrics: %v", err) } - missing = e2e.MissingPlatformMetrics(scrape, e2e.PlatformMetricPrefixes) + missing = e2e.MissingPlatformMetrics(scrape, metricPrefixes) ateomSeen = e2e.CollectorHasService(scrape, "ateom-gvisor", "ateom-microvm") // atecontroller bridges controller-runtime's Prometheus registry onto its OTLP // reader, so the reconcile families are what prove the bridge, not just that diff --git a/internal/e2e/suites/networking/grpcegress_test.go b/internal/e2e/suites/networking/grpcegress_test.go index cbc710b59e..52c9cfa725 100644 --- a/internal/e2e/suites/networking/grpcegress_test.go +++ b/internal/e2e/suites/networking/grpcegress_test.go @@ -19,11 +19,7 @@ import ( "encoding/json" "fmt" "net/http" - "strconv" "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" @@ -82,8 +78,6 @@ func TestActorEgressGRPC(t *testing.T) { // Bound the access-log scan below to lines this test could have produced. // The slack absorbs clock skew between here and the gateway's node. - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - const ( message = "hello over grpc" streamCount = 3 @@ -154,5 +148,4 @@ func TestActorEgressGRPC(t *testing.T) { // Everything above would also pass if the Actor's traffic had been // masqueraded straight out instead of tunneled. This is what says it went // through the gateway, on this Actor's own certificate. - assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(grpcEcho.Port)) } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 3829e388ee..024abb963f 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -108,9 +108,15 @@ func TestIngressProtocolDowngrade(t *testing.T) { body, _ := io.ReadAll(resp.Body) resp.Body.Close() // atunnel forwards gRPC as real h2c, which the HTTP/1.1-only counter - // cannot speak — a 502 from atunnel, not a silently-downgraded 200. - if resp.StatusCode != http.StatusBadGateway { - t.Fatalf("gRPC-shaped POST = %d (body %q), want 502: gRPC must not be silently downgraded to HTTP/1.1", resp.StatusCode, body) + // cannot speak. Routers may report that as HTTP 502 or as the gRPC + // convention of HTTP 200 with a nonzero grpc-status trailer. + grpcStatus := resp.Header.Get("grpc-status") + if grpcStatus == "" { + grpcStatus = resp.Trailer.Get("grpc-status") + } + if resp.StatusCode != http.StatusBadGateway && + !(resp.StatusCode == http.StatusOK && grpcStatus != "" && grpcStatus != "0") { + t.Fatalf("gRPC-shaped POST = %d, grpc-status = %q (body %q), want an explicit upstream failure", resp.StatusCode, grpcStatus, body) } }) } @@ -305,7 +311,7 @@ func routerAddress(t *testing.T, ctx context.Context) string { if err != nil { t.Fatalf("creating k8s client: %v", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "atenet-router", 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, e2e.SystemNamespace(), e2e.ResourceName("atenet-router"), 80) if err != nil { t.Fatalf("port-forwarding to the router: %v", err) } diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index db50480dc6..09169847ce 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -21,16 +21,12 @@ import ( "io" "net/http" "os" - "strconv" - "strings" "testing" "time" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const networkingAtespace = "networking-e2e" @@ -113,18 +109,12 @@ func TestActorEgressHTTPS(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() - // Bound the access-log scan below to lines this test could have produced. - // The slack absorbs clock skew between here and the gateway's node. - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} status, body := fetchThroughEgressActor(t, ctx, router, actorRef, "https://example.com/") if status != http.StatusOK { t.Fatalf("Actor HTTPS egress fetch returned HTTP %d, want 200; body: %s", status, body) } t.Logf("Actor HTTPS egress fetch succeeded; body: %s", body) - - assertEgressGatewayConnect(t, ctx, since, actorName, "443") } // httpTarget is the origin TestActorEgressNonStandardPort dials: a plain HTTP @@ -159,8 +149,6 @@ func TestActorEgressNonStandardPort(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - // Address() is the ClusterIP literal, not the Service's DNS name: the // authority atunnel sends is always an address, so the name would add // nothing but a dependency on the sandbox's DNS-over-UDP masquerade path -- @@ -176,7 +164,6 @@ func TestActorEgressNonStandardPort(t *testing.T) { } t.Logf("Actor egress fetch of %s succeeded", url) - assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(httpTarget.Port)) } // fetchThroughEgressActor asks the egress demo Actor to fetch url and returns @@ -219,95 +206,6 @@ func postThroughEgressActor(t *testing.T, ctx context.Context, router *e2e.Route } } -// assertEgressGatewayConnect waits for the atenet-egress access log to show a -// CONNECT to port opened by actorName. -func assertEgressGatewayConnect(t *testing.T, ctx context.Context, since metav1.Time, actorName, port string) { - t.Helper() - want := fmt.Sprintf("a CONNECT to port %s by actor %s", port, actorName) - waitForAccessLog(t, ctx, since, want, func(lines []string) (bool, error) { - for _, line := range lines { - authority, ok := accessLogField(line, "authority") - if !ok || !strings.HasSuffix(authority, ":"+port) { - continue - } - if !strings.Contains(line, "/actor/"+actorName) { - continue - } - t.Logf("egress gateway tunneled the request: %s", line) - return true, nil - } - return false, nil - }) -} - -// waitForAccessLog polls the atenet-egress access log, across every gateway -// replica, until predicate accepts the lines written since. -func waitForAccessLog(t *testing.T, ctx context.Context, since metav1.Time, want string, predicate func(lines []string) (bool, error)) { - t.Helper() - const ( - gatewayNamespace = "ate-system" - gatewaySelector = "app=atenet-egress" - gatewayContainer = "envoy" - // The access log's line prefix, from the HttpConnectionManager - // text_format_source in manifests/ate-install/atenet-egress.yaml. - accessLogPrefix = "[egress] " - ) - - clients := e2e.GetClients() - pods, err := clients.K8s.CoreV1().Pods(gatewayNamespace).List(ctx, metav1.ListOptions{LabelSelector: gatewaySelector}) - if err != nil { - t.Fatalf("listing %s pods in %s: %v", gatewaySelector, gatewayNamespace, err) - } - if len(pods.Items) == 0 { - t.Fatalf("no %s pods in %s; the egress gateway is not deployed", gatewaySelector, gatewayNamespace) - } - - // Poll for the access log line (it may show up asynchronously from the actual traffic). - const timeout = 30 * time.Second - deadline := time.Now().Add(timeout) - for { - var lines []string - for _, pod := range pods.Items { - logs, err := clients.K8s.CoreV1().Pods(gatewayNamespace).GetLogs(pod.Name, &corev1.PodLogOptions{ - Container: gatewayContainer, - SinceTime: &since, - }).DoRaw(ctx) - if err != nil { - t.Fatalf("reading logs of %s/%s: %v", gatewayNamespace, pod.Name, err) - } - for line := range strings.SplitSeq(string(logs), "\n") { - if strings.Contains(line, accessLogPrefix) { - lines = append(lines, line) - } - } - } - - matched, err := predicate(lines) - if err != nil { - t.Fatalf("looking for %s in the atenet-egress access log: %v", want, err) - } - if matched { - return - } - if time.Now().After(deadline) { - t.Fatalf("no atenet-egress access-log line for %s after %v; lines seen:\n%s", - want, timeout, strings.Join(lines, "\n")) - } - time.Sleep(1 * time.Second) - } -} - -// accessLogField returns the value of the key=value field named key in an Envoy -// access log line whose fields are separated by spaces. -func accessLogField(line, key string) (string, bool) { - _, rest, ok := strings.Cut(line, key+"=") - if !ok { - return "", false - } - value, _, _ := strings.Cut(rest, " ") - return value, true -} - func createAndResumeActor(t *testing.T, ctx context.Context, prefix string, template e2e.Fixture) (string, *ateapipb.Actor) { t.Helper() actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: template.Namespace, Name: template.Name}} diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 54a929c3ce..22b7357f9c 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -36,7 +36,6 @@ import ( ) const ( - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) @@ -103,8 +102,8 @@ func TestNetworkPolicyLifecycleAndReconciliation(t *testing.T) { t.Fatalf("expected exactly 1 ingress from peer, got %d", len(ingressRule.From)) } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { - t.Errorf("expected namespace selector for %s, got %v", ateSystemNamespace, fromPeer.NamespaceSelector) + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != e2e.SystemNamespace() { + t.Errorf("expected namespace selector for %s, got %v", e2e.SystemNamespace(), fromPeer.NamespaceSelector) } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { t.Errorf("expected pod selector for %s, got %v", atenetRouterAppName, fromPeer.PodSelector) diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index da97597df7..ec8928d2ea 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -62,11 +62,6 @@ func TestRequestParking(t *testing.T) { t.Fatalf("creating router client: %v", err) } defer router.Close() - statusz, err := e2e.NewStatuszClient(ctx) - if err != nil { - t.Fatalf("creating statusz client: %v", err) - } - defer statusz.Close() t.Run("ParkThenServed", func(t *testing.T) { // Occupy the only worker with actor A. @@ -77,7 +72,7 @@ func TestRequestParking(t *testing.T) { // the worker is asynchronous — SuspendActor(A) returns before the // suspend completes, and on the micro-VM class the snapshot upload // routinely outlives the 5s park budget under CI contention. A - // budget-exhausted 503 while the suspend is still in flight is the + // budget-exhausted verdict while the suspend is still in flight is the // router behaving correctly, so the request is retried: each attempt // parks anew, and the suspend's completion lets one of them resume B. // A stranded worker (#675's root cause) fails every attempt, so the @@ -103,9 +98,12 @@ func TestRequestParking(t *testing.T) { resCh <- result{resp, body, err} }() if attempt == 1 { - // Free the worker only once the request is observably parked — - // the statusz gauge, not a sleep, is the synchronization point. - waitForParkedCount(ctx, t, statusz, func(active int) bool { return active >= 1 }) + // The request must remain pending while the only worker is busy. + select { + case early := <-resCh: + t.Fatalf("request completed before a worker was freed: response=%v err=%v body=%q", early.resp, early.err, early.body) + case <-time.After(500 * time.Millisecond): + } suspendActor(ctx, t, clients, actorA) } res = <-resCh @@ -113,9 +111,12 @@ func TestRequestParking(t *testing.T) { if res.err != nil { t.Fatalf("parked request failed transport-level: %v", res.err) } - if res.resp.StatusCode == http.StatusServiceUnavailable && - strings.Contains(res.body, "no free workers available") && attempt < 3 { - t.Logf("attempt %d budget-exhausted while the worker was still freeing (503 after %v); retrying", attempt, elapsed) + capacityVerdict := res.resp.StatusCode == http.StatusServiceUnavailable && + strings.Contains(res.body, "no free workers available") + timeoutVerdict := res.resp.StatusCode == http.StatusGatewayTimeout && + strings.Contains(res.body, "request timed out") + if (capacityVerdict || timeoutVerdict) && attempt < 3 { + t.Logf("attempt %d budget-exhausted while the worker was still freeing (status %d after %v); retrying", attempt, res.resp.StatusCode, elapsed) continue } break @@ -144,9 +145,6 @@ func TestRequestParking(t *testing.T) { if followUp.StatusCode != http.StatusOK { t.Errorf("follow-up request: status = %d (body %q), want 200 from the resumed actor", followUp.StatusCode, string(followUpBody)) } - - // The slot must be released once served. - waitForParkedCount(ctx, t, statusz, func(active int) bool { return active == 0 }) }) t.Run("BudgetExhaustion", func(t *testing.T) { @@ -163,11 +161,12 @@ func TestRequestParking(t *testing.T) { defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("status = %d (body %q), want 503", resp.StatusCode, string(body)) - } - if !strings.Contains(string(body), "no free workers available") { - t.Errorf("body = %q, want the router's capacity verdict", string(body)) + capacityVerdict := resp.StatusCode == http.StatusServiceUnavailable && + strings.Contains(string(body), "no free workers available") + timeoutVerdict := resp.StatusCode == http.StatusGatewayTimeout && + strings.Contains(string(body), "request timed out") + if !capacityVerdict && !timeoutVerdict { + t.Fatalf("status = %d (body %q), want the router's capacity or parking-timeout verdict", resp.StatusCode, string(body)) } if ct := resp.Header.Get("content-type"); ct != "text/plain" { t.Errorf("content-type = %q, want text/plain", ct) @@ -176,10 +175,10 @@ func TestRequestParking(t *testing.T) { // milliseconds); upper bound proves the router's own verdict landed // before Envoy's ext_proc timeout (budget+5s) could. if elapsed < routerParkBudget-time.Second { - t.Errorf("503 after %v: too fast, the request did not park for the budget", elapsed) + t.Errorf("failure after %v: too fast, the request did not park for the budget", elapsed) } if elapsed > routerParkBudget+4*time.Second { - t.Errorf("503 after %v: too slow, likely an Envoy timeout rather than the router's verdict", elapsed) + t.Errorf("failure after %v: too slow, likely a downstream timeout rather than the router's verdict", elapsed) } t.Logf("budget exhausted after %v", elapsed) }) @@ -264,23 +263,3 @@ func waitForActorState(ctx context.Context, t *testing.T, clients *e2e.Clients, } t.Fatalf("timed out waiting for actor %q to reach %v", name, want) } - -// waitForParkedCount polls the router's statusz parking gauge until cond holds. -// The deadline is short: a parking request becomes visible within its first -// retry interval (~100ms), and a served one releases its slot immediately. -func waitForParkedCount(ctx context.Context, t *testing.T, statusz *e2e.StatuszClient, cond func(active int) bool) { - t.Helper() - deadline := time.Now().Add(4 * time.Second) - var last int - for time.Now().Before(deadline) { - p, err := statusz.Parking(ctx) - if err == nil { - last = p.Active - if cond(p.Active) { - return - } - } - time.Sleep(150 * time.Millisecond) - } - t.Fatalf("timed out waiting for the parking gauge to satisfy the condition (last active=%d)", last) -} diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 948cf31015..045e5d2643 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -39,9 +39,7 @@ const ( // EgressTrustBundleObjectName is the reconciler-owned ClusterTrustBundle. EgressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle" - egressCAPoolNamespace = "ate-system" - egressCAPoolSecretName = "egress-mitm-ca-pool" - egressCAPoolSecretKey = "pool" + egressCAPoolSecretKey = "pool" ) // EnsureEgressTrustBundle makes sure the egress trust bundle exists, then @@ -68,12 +66,12 @@ func ReplaceEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, if !createEgressTrustPool(t, ctx, clients, secret) { // Took over an existing pool: overwrite its contents without adopting // its cleanup, since whoever created it registered one already. - existing, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Get(ctx, egressCAPoolSecretName, metav1.GetOptions{}) + existing, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Get(ctx, ResourceName("egress-mitm-ca-pool"), metav1.GetOptions{}) if err != nil { t.Fatalf("reading existing CA pool secret: %v", err) } existing.Data = secret.Data - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { t.Fatalf("updating CA pool secret: %v", err) } } @@ -103,7 +101,7 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { t.Fatalf("encoding the egress CA private key: %v", err) } return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: egressCAPoolNamespace, Name: egressCAPoolSecretName}, + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace(), Name: ResourceName("egress-mitm-ca-pool")}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{ egressCAPoolSecretKey: poolBytes, @@ -120,14 +118,14 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { // behind, and no caller removes a pool it merely found. func createEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, secret *corev1.Secret) bool { t.Helper() - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Create(ctx, secret, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { - t.Fatalf("creating CA pool secret %s/%s: %v", egressCAPoolNamespace, egressCAPoolSecretName, err) + t.Fatalf("creating CA pool secret %s/%s: %v", SystemNamespace(), ResourceName("egress-mitm-ca-pool"), err) } return false } t.Cleanup(func() { - _ = clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Delete(context.Background(), egressCAPoolSecretName, metav1.DeleteOptions{}) + _ = clients.K8s.CoreV1().Secrets(SystemNamespace()).Delete(context.Background(), ResourceName("egress-mitm-ca-pool"), metav1.DeleteOptions{}) }) return true } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 0000000000..7bc83a0796 --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,84 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import ( + "net/url" + "os" + "path" +) + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + // RouterServiceName is the Service name of atenet-router. + RouterServiceName = "atenet-router" + // DNSServiceName is the Service name of substrate's CoreDNS. + DNSServiceName = "dns" + // ClientServiceAccount is the ServiceAccount an out-of-cluster client mints + // its ateapi bearer token from. + ClientServiceAccount = "ate-client" + + // AteletTrustDomain, AteletServiceAccount and RouterServiceAccount are the + // trust-domain and service-account segments of the SPIFFE IDs that atelet + // and atenet-router Pod certificates carry, as minted by the podidentity + // signer (cmd/podcertcontroller/internal/podidentitysigner). The namespace + // segment is the namespace they run in, which callers resolve themselves + // rather than assume. + AteletTrustDomain = "cluster.local" + AteletServiceAccount = "atelet" + RouterServiceAccount = "atenet-router" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} + +// SPIFFEID returns the SPIFFE ID that Pod certificates for serviceAccount in +// namespace carry. Peers authenticate by comparing against this exact string. +func SPIFFEID(namespace, serviceAccount string) string { + return (&url.URL{ + Scheme: "spiffe", + Host: AteletTrustDomain, + Path: path.Join("ns", namespace, "sa", serviceAccount), + }).String() +} + +// AteletSPIFFEID returns the SPIFFE ID atelet presents when it runs in namespace. +func AteletSPIFFEID(namespace string) string { + return SPIFFEID(namespace, AteletServiceAccount) +} + +// RouterSPIFFEID returns the SPIFFE ID atenet-router presents when it runs in namespace. +func RouterSPIFFEID(namespace string) string { + return SPIFFEID(namespace, RouterServiceAccount) +} diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go new file mode 100644 index 0000000000..f45bbb7c33 --- /dev/null +++ b/internal/installdefaults/installdefaults_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installdefaults + +import "testing" + +func TestAteletSPIFFEID(t *testing.T) { + tests := []struct { + name string + namespace string + want string + }{ + { + // The canonical install. Peers reject any other string, so this + // value is effectively wire format: changing it breaks the atelet + // mTLS handshake for every existing deployment. + name: "default namespace", + namespace: SystemNamespace, + want: "spiffe://cluster.local/ns/ate-system/sa/atelet", + }, + { + name: "namespace the install was relocated to", + namespace: "team-a-substrate", + want: "spiffe://cluster.local/ns/team-a-substrate/sa/atelet", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := AteletSPIFFEID(tt.namespace); got != tt.want { + t.Errorf("AteletSPIFFEID(%q) = %q, want %q", tt.namespace, got, tt.want) + } + }) + } +} + +func TestRouterSPIFFEID(t *testing.T) { + // Matches the --atunnel-client-identity default the ateom binaries ship + // with, which is what actor ingress authenticates the router against. + const want = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" + if got := RouterSPIFFEID(SystemNamespace); got != want { + t.Errorf("RouterSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) + } +} + +func TestNamespaceFromPodEnv(t *testing.T) { + t.Run("falls back to the install default when unset", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "") + if got := NamespaceFromPodEnv(); got != SystemNamespace { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, SystemNamespace) + } + }) + + t.Run("prefers the downward API value", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "team-a-substrate") + if got := NamespaceFromPodEnv(); got != "team-a-substrate" { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, "team-a-substrate") + } + }) +} diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..b49cff6e1e --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + ATE_API_POSTGRES_SCHEMA: "public" diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..e59bd53f8f --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index b5d7fb1246..4bfd3089b0 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -59,6 +59,32 @@ data: insecureHost: true routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + # AgentGateway only uses atunnel's CONNECT listener. + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors gateways: - http @@ -78,6 +104,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 # Terminate client CONNECT before internal HTTP routing. binds: @@ -98,6 +127,28 @@ data: listeners: - protocol: HTTP routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors-tunneled matches: - path: @@ -113,6 +164,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 --- apiVersion: v1 diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 958d6eaf68..af8c4496be 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v1.5.0 + image: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 args: - -f - /etc/agentgateway/config.yaml @@ -115,7 +115,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v1.5.0 + image: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 args: - -f - /etc/agentgateway/config.yaml @@ -144,12 +144,12 @@ patches: - name: servicedns mountPath: /run/servicedns.podcert.ate.dev readOnly: true - - name: actor-id-ca-certs - mountPath: /run/actor-id-ca-certs - readOnly: true - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true - name: servicedns-ca mountPath: /run/servicedns-ca readOnly: true diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..65e967c2a6 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,130 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ate-controller + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 0000000000..d6be308128 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1