diff --git a/.github/workflows/detect-changes.yml b/.github/workflows/detect-changes.yml index 25efa5233..159423efa 100644 --- a/.github/workflows/detect-changes.yml +++ b/.github/workflows/detect-changes.yml @@ -64,7 +64,9 @@ jobs: path.startsWith("components/execd/") || path.startsWith("components/egress/") || path.startsWith("sdks/") || - path.startsWith("tests/"), + path.startsWith("tests/") || + path === "scripts/python-execd-init-e2e.sh" || + path === "scripts/python-execd-hardening-e2e.sh", egress: (path) => workflowChanged(path, "egress-test.yml") || path.startsWith("components/egress/") || @@ -74,6 +76,11 @@ jobs: path === "scripts/python-k8s-e2e.sh" || path === "scripts/python-k8s-e2e-ingress.sh" || path === "scripts/common/kubernetes-e2e.sh" || + path === "scripts/python-k8s-execd-init-e2e.sh" || + path === "tests/python/tests/test_execd_init_e2e.py" || + path === "tests/python/tests/test_execd_hardening_e2e.py" || + path === "server/opensandbox_server/examples/e2e.batchsandbox-template.yaml" || + path.startsWith("components/execd/") || path.startsWith("kubernetes/charts/"), ingress: (path) => workflowChanged(path, "ingress-test.yaml") || diff --git a/.github/workflows/execd-test.yml b/.github/workflows/execd-test.yml index 73a881714..37a3649bb 100644 --- a/.github/workflows/execd-test.yml +++ b/.github/workflows/execd-test.yml @@ -55,6 +55,11 @@ jobs: run: | go test -v -coverpkg=./... -coverprofile=coverage.out -covermode=atomic ./pkg/... + - name: Run execd-ebpf variant tests + working-directory: components/execd + run: | + CGO_ENABLED=1 go test -tags ebpf -count=1 ./pkg/ebpf/ + - name: Calculate coverage and generate summary working-directory: components/execd id: coverage @@ -130,6 +135,14 @@ jobs: chmod +x components/execd/tests/smoke_bwrap.sh bash components/execd/tests/smoke_bwrap.sh + - name: Init-mode container regression (OSEP-0018) + if: matrix.os == 'ubuntu-latest' + shell: bash + timeout-minutes: 30 + run: | + chmod +x components/execd/tests/init_container.sh + bash components/execd/tests/init_container.sh + - name: Show logs if: always() run: | @@ -224,6 +237,17 @@ jobs: working-directory: components/execd run: sudo -E env "PATH=$PATH" go test -tags="linux,bwrap" -v -count=1 -timeout=5m ./pkg/runtime/bwrap_test/ + - name: Run isolated session init-mode reaper integration test + working-directory: components/execd + run: | + # bwrap lifecycle under init-mode reaper dispatch (OSEP-0018 R-o): + # the pre-reap barrier must serialize process-group teardown with + # the reaper's observe/consume while execd owns wait4. + sudo -E env "PATH=$PATH" go test \ + -tags="linux,bwrap" -v -count=1 -timeout=3m \ + -run '^TestIsolatedSessionWithInitReaper$' \ + ./pkg/runtime + required: name: Execd CI if: always() diff --git a/.github/workflows/kubernetes-nightly-build.yml b/.github/workflows/kubernetes-nightly-build.yml index a6f3bccad..efc6f8457 100644 --- a/.github/workflows/kubernetes-nightly-build.yml +++ b/.github/workflows/kubernetes-nightly-build.yml @@ -112,9 +112,82 @@ jobs: run: | kind delete cluster --name "${KIND_CLUSTER}" || true + execd-init-e2e: + name: Execd-Init + Hardening E2E + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: ubuntu-latest + env: + KIND_CLUSTER: opensandbox-e2e + KIND_K8S_VERSION: v1.30.4 + KUBECONFIG_PATH: /tmp/opensandbox-kind-kubeconfig + KUBECONFIG: /tmp/opensandbox-kind-kubeconfig + OPENSANDBOX_E2E_SANDBOX_CPU: 250m + OPENSANDBOX_E2E_SANDBOX_MEMORY: 512Mi + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: "1.25.0" + + - name: Add Go bin to PATH + run: echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + + - name: Set up kubectl + uses: azure/setup-kubectl@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Run execd-init + hardening E2E + run: bash scripts/python-k8s-execd-init-e2e.sh + + - name: Dump kind diagnostics + if: always() + run: | + kubectl get pods -A -o wide || true + kubectl get batchsandboxes -A || true + kubectl describe deployment -n opensandbox-system opensandbox-server || true + + - name: Eval in-cluster server logs + if: always() + run: | + kubectl logs -n opensandbox-system deployment/opensandbox-server || true + cat /tmp/opensandbox-server-port-forward.log || true + + - name: Upload Python test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: execd-init-e2e-logs + path: | + /tmp/opensandbox-server-port-forward.log + /tmp/opensandbox-e2e-pods.yaml + /tmp/opensandbox-e2e-batchsandboxes.yaml + if-no-files-found: ignore + retention-days: 5 + + - name: Clean up Kind cluster + if: always() + run: | + kind delete cluster --name "${KIND_CLUSTER}" || true + publish-nightly-latest: name: Publish latest (${{ matrix.component }} nightly) - needs: k8s-mini-e2e + needs: [k8s-mini-e2e, execd-init-e2e] if: github.event_name != 'pull_request' runs-on: ubuntu-latest strategy: @@ -179,7 +252,7 @@ jobs: required: name: Kubernetes Mini E2E CI if: ${{ always() && github.event_name == 'pull_request' }} - needs: [changes, k8s-mini-e2e] + needs: [changes, k8s-mini-e2e, execd-init-e2e] runs-on: ubuntu-latest steps: - name: Verify required jobs @@ -187,13 +260,14 @@ jobs: RELEVANT: ${{ needs.changes.outputs.relevant }} CHANGES_RESULT: ${{ needs.changes.result }} E2E_RESULT: ${{ needs.k8s-mini-e2e.result }} + EXECD_INIT_RESULT: ${{ needs.execd-init-e2e.result }} run: | if [[ "$CHANGES_RESULT" != "success" ]]; then echo "Change detection failed: $CHANGES_RESULT" exit 1 fi if [[ "$RELEVANT" == "true" ]]; then - [[ "$E2E_RESULT" == "success" ]] + [[ "$E2E_RESULT" == "success" && "$EXECD_INIT_RESULT" == "success" ]] else - [[ "$RELEVANT" == "false" && "$E2E_RESULT" == "skipped" ]] + [[ "$RELEVANT" == "false" && "$E2E_RESULT" == "skipped" && "$EXECD_INIT_RESULT" == "skipped" ]] fi diff --git a/.github/workflows/kubernetes-test.yml b/.github/workflows/kubernetes-test.yml index 646264f93..55f21764e 100644 --- a/.github/workflows/kubernetes-test.yml +++ b/.github/workflows/kubernetes-test.yml @@ -94,6 +94,11 @@ jobs: run: | make lint + - name: Lint Helm charts + working-directory: kubernetes + run: | + make helm-lint + - name: Build binary working-directory: kubernetes run: | diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index a40c4b995..812b4fa99 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -13,13 +13,17 @@ on: - opensandbox - opensandbox-node-agent default: 'opensandbox-controller' + chart_version: + description: 'Chart version to release (without v prefix, e.g., 0.1.0)' + required: true + default: '0.1.0' app_version: description: 'App version (without v prefix, e.g., 0.1.0)' required: true default: '0.1.0' push: tags: - - 'helm/**' # Format: helm//, e.g., helm/opensandbox-controller/0.1.0 + - 'helm/**' # Format: helm//, e.g., helm/opensandbox-controller/0.1.0 jobs: release-preflight: @@ -63,12 +67,13 @@ jobs: VERSION=${VERSION#v} echo "component=$COMPONENT" >> $GITHUB_OUTPUT - echo "app_version=$VERSION" >> $GITHUB_OUTPUT + echo "chart_version=$VERSION" >> $GITHUB_OUTPUT echo "release_tag=$TAG_PATH" >> $GITHUB_OUTPUT else echo "component=${{ inputs.component }}" >> $GITHUB_OUTPUT + echo "chart_version=${{ inputs.chart_version }}" >> $GITHUB_OUTPUT echo "app_version=${{ inputs.app_version }}" >> $GITHUB_OUTPUT - echo "release_tag=helm/${{ inputs.component }}/${{ inputs.app_version }}" >> $GITHUB_OUTPUT + echo "release_tag=helm/${{ inputs.component }}/${{ inputs.chart_version }}" >> $GITHUB_OUTPUT fi - name: Verify release tag on origin @@ -120,10 +125,18 @@ jobs: run: | CHART_PATH="${{ steps.chart_path.outputs.path }}" CHART_VERSION=$(grep '^version:' $CHART_PATH/Chart.yaml | awk '{print $2}') + EXPECTED_CHART_VERSION="${{ steps.parse_tag.outputs.chart_version }}" + + if [ "$CHART_VERSION" != "$EXPECTED_CHART_VERSION" ]; then + echo "::error::Chart.yaml version '$CHART_VERSION' does not match requested chart version '$EXPECTED_CHART_VERSION'." + exit 1 + fi + echo "version=$CHART_VERSION" >> $GITHUB_OUTPUT - echo "Chart version: $CHART_VERSION" + echo "Verified chart version: $CHART_VERSION" - - name: Update Chart.yaml with app version + - name: Update Chart.yaml with app version for manual release + if: ${{ github.event_name == 'workflow_dispatch' }} run: | APP_VERSION="${{ steps.parse_tag.outputs.app_version }}" CHART_PATH="${{ steps.chart_path.outputs.path }}" @@ -134,6 +147,14 @@ jobs: echo "Updated Chart.yaml:" cat $CHART_PATH/Chart.yaml + - name: Get app version from Chart.yaml + id: app_version + run: | + CHART_PATH="${{ steps.chart_path.outputs.path }}" + APP_VERSION=$(grep '^appVersion:' $CHART_PATH/Chart.yaml | awk '{print $2}' | tr -d '"') + echo "version=$APP_VERSION" >> $GITHUB_OUTPUT + echo "App version: $APP_VERSION" + - name: Build dependencies (for opensandbox all-in-one chart) if: ${{ steps.parse_tag.outputs.component == 'opensandbox' }} run: | @@ -160,12 +181,12 @@ jobs: uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.parse_tag.outputs.release_tag }} - name: Helm Chart ${{ steps.parse_tag.outputs.component }} ${{ steps.chart_version.outputs.version }} (App v${{ steps.parse_tag.outputs.app_version }}) + name: Helm Chart ${{ steps.parse_tag.outputs.component }} ${{ steps.chart_version.outputs.version }} (App v${{ steps.app_version.outputs.version }}) body: | ## ${{ steps.parse_tag.outputs.component }} Helm Chart **Chart Version:** ${{ steps.chart_version.outputs.version }} - **App Version:** ${{ steps.parse_tag.outputs.app_version }} + **App Version:** ${{ steps.app_version.outputs.version }} ### Installation @@ -195,7 +216,7 @@ jobs: ### What's Changed - Chart version: ${{ steps.chart_version.outputs.version }} - - App version: ${{ steps.parse_tag.outputs.app_version }} + - App version: ${{ steps.app_version.outputs.version }} files: | ${{ steps.parse_tag.outputs.component }}-*.tgz draft: false diff --git a/.github/workflows/real-e2e.yml b/.github/workflows/real-e2e.yml index ee2cb509c..ab9ad6f30 100644 --- a/.github/workflows/real-e2e.yml +++ b/.github/workflows/real-e2e.yml @@ -127,6 +127,138 @@ jobs: docker rm -f opensandbox-e2e-redis || true docker run --rm -v /tmp:/host_tmp alpine rm -rf /host_tmp/opensandbox-e2e || true + python-execd-init-e2e: + name: Python Execd-Init E2E (docker bridge) + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Clean Docker runner cache + run: bash scripts/ci-docker-cleanup.sh + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Verify uv + run: | + uv --version + uv run python --version + + - name: Clean up previous E2E resources + run: | + docker ps -aq --filter "label=opensandbox" | xargs -r docker rm -f || true + docker rm -f opensandbox-e2e-redis || true + docker run --rm -v /tmp:/host_tmp alpine rm -rf /host_tmp/opensandbox-e2e || true + docker image prune -f || true + + - name: Build local egress image + run: docker build -t opensandbox/egress:local -f components/egress/Dockerfile . + + - name: Run execd-init E2E + run: | + set -e + + cat < ~/.sandbox.toml + [server] + host = "127.0.0.1" + port = 8080 + api_key = "" + [log] + level = "INFO" + [runtime] + type = "docker" + execd_image = "opensandbox/execd:local" + execd_run_as_init = true + [egress] + image = "opensandbox/egress:local" + mode = "dns+nft" + [docker] + network_mode = "bridge" + [storage] + allowed_host_paths = ["/tmp/opensandbox-e2e"] + [renew_intent] + enabled = true + min_interval_seconds = 60 + EOF + + bash scripts/python-execd-init-e2e.sh + + - name: Eval server logs + if: ${{ always() }} + run: cat server/server.log + + - name: Upload execd logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: execd-log-for-execd-init-e2e + path: /tmp/opensandbox-e2e/logs/ + retention-days: 5 + + - name: Clean up after E2E + if: always() + run: | + docker ps -aq --filter "label=opensandbox" | xargs -r docker rm -f || true + docker rm -f opensandbox-e2e-redis || true + docker run --rm -v /tmp:/host_tmp alpine rm -rf /host_tmp/opensandbox-e2e || true + + python-execd-hardening-e2e: + name: Python Execd-Hardening E2E (docker bridge) + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Clean Docker runner cache + run: bash scripts/ci-docker-cleanup.sh + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Verify uv + run: | + uv --version + uv run python --version + + - name: Clean up previous E2E resources + run: | + docker ps -aq --filter "label=opensandbox" | xargs -r docker rm -f || true + docker rm -f opensandbox-e2e-redis || true + docker run --rm -v /tmp:/host_tmp alpine rm -rf /host_tmp/opensandbox-e2e || true + docker image prune -f || true + + - name: Build local egress image + run: docker build -t opensandbox/egress:local -f components/egress/Dockerfile . + + - name: Run execd-hardening E2E + run: | + set -e + bash scripts/python-execd-hardening-e2e.sh + + - name: Eval server logs + if: ${{ always() }} + run: cat server/server.log + + - name: Upload execd logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: execd-log-for-execd-hardening-e2e + path: /tmp/opensandbox-e2e/logs/ + retention-days: 5 + + - name: Clean up after E2E + if: always() + run: | + docker ps -aq --filter "label=opensandbox" | xargs -r docker rm -f || true + docker rm -f opensandbox-e2e-redis || true + docker run --rm -v /tmp:/host_tmp alpine rm -rf /host_tmp/opensandbox-e2e || true + java-e2e: name: Java E2E (docker bridge) needs: changes diff --git a/.gitignore b/.gitignore index 24d9ecf51..e7737c48d 100644 --- a/.gitignore +++ b/.gitignore @@ -269,6 +269,19 @@ nbdist/ generated/ **/generated/** +# fleets FastPath gRPC stubs are committed intentionally (deterministic protoc +# output; the server package needs them at install time, so they are checked +# in rather than regenerated). +!server/opensandbox_server/services/fleets/generated/ +!server/opensandbox_server/services/fleets/generated/** +server/opensandbox_server/services/fleets/generated/__pycache__/ + +# execd eBPF kernel BTF dumps (bpf2go build-time only, never committed โ€” +# see components/execd/pkg/ebpf/prog/audit.bpf.c; regenerate with: +# bpftool btf dump file /sys/kernel/btf/vmlinux format c > .../vmlinux.h) +components/execd/pkg/ebpf/prog/vmlinux.h +components/execd/pkg/ebpf/prog/vmlinux_*.h + # gVisor runtime binaries (downloaded dynamically) kubernetes/test/kind/gvisor/runsc kubernetes/test/kind/gvisor/containerd-shim-runsc-v1 diff --git a/README.md b/README.md index fea5b837d..5c28b05f4 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,20 @@ OpenSandbox is a **general-purpose sandbox platform** for AI applications, offer - ๐Ÿ”‘ **Credential Vault**: Secure credential injection for sandbox outbound requests without exposing real secrets to workloads. See [Credential Vault](docs/guides/credential-vault.md). - ๐Ÿฐ **Strong Isolation**: Supports secure container runtimes like gVisor, Kata Containers, and Firecracker microVM for enhanced isolation between sandbox workloads and the host. See [Secure Container Runtime Guide](docs/guides/secure-container.md) for details. +## Official Container Images + +OpenSandbox release images are published under the same component name in +three official registries: + +- Docker Hub: `docker.io/opensandbox/` +- GitHub Container Registry: `ghcr.io/opensandbox-group/opensandbox/` +- Alibaba Cloud Container Registry: `sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/` + +Tagged release images are signed keylessly with Cosign and include provenance +attestations. Pin production images by digest and follow the +[release verification guide](docs/community/release-verification.md) to verify +the image against the OpenSandbox GitHub Actions identity before deployment. + ## SDKs Python: @@ -224,7 +238,7 @@ OpenSandbox provides examples covering SDK usage, agent integrations, browser au #### ๐Ÿค– Coding Agent Integrations -- **Coding CLIs** โ€” [Claude Code](docs/examples/claude-code.md), [Gemini CLI](docs/examples/gemini-cli.md), [OpenAI Codex CLI](docs/examples/codex-cli.md), [Qwen Code](docs/examples/qwen-code.md), [Kimi CLI](docs/examples/kimi-cli.md): run each vendor CLI inside OpenSandbox. +- **Coding CLIs** โ€” [Claude Code](docs/examples/claude-code.md), [Gemini CLI](docs/examples/gemini-cli.md), [OpenAI Codex CLI](docs/examples/codex-cli.md), [OpenCode](docs/examples/opencode.md), [Qwen Code](docs/examples/qwen-code.md), [Kimi CLI](docs/examples/kimi-cli.md): run each CLI inside OpenSandbox. - **[langgraph](docs/examples/langgraph.md)** - LangGraph state-machine workflow that creates/runs a sandbox job with fallback retry. - **[google-adk](docs/examples/google-adk.md)** - Google ADK agent using OpenSandbox tools to write/read files and run commands. - **[openclaw](docs/examples/openclaw.md)** - Launch an OpenClaw Gateway inside a sandbox. diff --git a/ROADMAP.md b/ROADMAP.md index 61dcf88ea..bff4a9a10 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,7 +27,7 @@ Proposals (OSEPs). |------|--------|----------|-------| | Local lightweight sandbox | Planned | TBD | Lightweight sandbox runtime for AI tools running directly on PCs. | | Persistent volumes | Implementing | [OSEP-0003](oseps/0003-volume-and-volumebinding-support.md) | Close remaining runtime/backend gaps from OSEP-0003 before treating volume support as mature. | -| Secure container runtime | Maturing | [OSEP-0004](oseps/0004-secure-container-runtime.md), [secure container guide](docs/guides/secure-container.md) | Continue hardening isolation guidance and deployment practices. | +| Secure container runtime | Implemented / maturing | [OSEP-0004](oseps/0004-secure-container-runtime.md), [secure container guide](docs/guides/secure-container.md) | Continue hardening isolation guidance and deployment practices. | | Pause and resume via rootfs snapshot | Implementing | [OSEP-0008](oseps/0008-pause-resume-rootfs-snapshot.md) | Improve lifecycle support for stateful sandbox workflows. | | Secure endpoint access | Implemented / maturing | [OSEP-0011](oseps/0011-secure-access-endpoint.md) | Keep endpoint security behavior aligned across server, SDKs, and docs. | @@ -36,7 +36,7 @@ Proposals (OSEPs). | Area | Status | Tracking | Notes | |------|--------|----------|-------| | SDK parity | Ongoing | [sdks/](sdks/), [specs/](specs/README.md) | Keep Python, Go, Kotlin, JavaScript/TypeScript, and C# SDKs aligned with public specs. | -| Client-side sandbox pool | Implementing / maturing | [OSEP-0005](oseps/0005-client-side-sandbox-pool.md) | Expand behavior consistency, tests, and documentation where practical. | +| Client-side sandbox pool | Implemented / maturing | [OSEP-0005](oseps/0005-client-side-sandbox-pool.md) | Expand behavior consistency, tests, and documentation where practical. | | CLI usability | Planned | [cli/](cli/README.md) | Improve common sandbox lifecycle workflows and developer ergonomics. | | Developer console | Implementable | [OSEP-0006](oseps/0006-developer-console.md) | Provide a clearer operational surface for sandbox users and maintainers. | diff --git a/cli/README.md b/cli/README.md index 441d6b7a1..ad9490229 100644 --- a/cli/README.md +++ b/cli/README.md @@ -166,6 +166,17 @@ osb sandbox metrics osb sandbox metrics --watch -o raw ``` +### Pause a sandbox + +Pause is asynchronous. `Pause request accepted` confirms that the server accepted +the request, not that the sandbox has reached the `Paused` state. Poll the sandbox +until the transition finishes: + +```bash +osb sandbox pause +osb sandbox get -o json +``` + ### Expose a service ```bash @@ -252,21 +263,23 @@ credential values as command-line flags; keep them in the payload stream or file Use the stable diagnostics commands for API-backed log and event descriptors. ```bash -osb diagnostics events --scope lifecycle -o raw osb diagnostics events --scope runtime -o raw +osb diagnostics events --scope all -o raw osb diagnostics logs --scope container -o raw -osb diagnostics logs --scope lifecycle -o json +osb diagnostics logs --scope all -o json osb diagnostics events --scope runtime -o json osb diagnostics logs --scope container -o yaml ``` -`--scope` is required for stable diagnostics. Common scopes are `lifecycle` and -`container` for logs, and `lifecycle` and `runtime` for events. Raw output -prints inline diagnostic text, or the content URL when diagnostics are -delivered as a temporary URL. Structured CLI output follows the SDK/Python field -style, for example `content_url`, `content_length`, and `expires_at`. -Some server builds may return `DIAGNOSTICS_NOT_IMPLEMENTED` for scoped -diagnostics until the stable backend implementation is enabled. +`--scope` is required for stable diagnostics. The built-in server supports +`container` and `all` for logs, and `runtime` and `all` for events. It returns +`DIAGNOSTICS_SCOPE_UNSUPPORTED` for unavailable scopes, including lifecycle events. +Best-effort scopes may include a `warnings` field when the backend can only +provide a subset. Raw output prints inline +diagnostic text, or the content URL when diagnostics are delivered as a +temporary URL. Structured CLI output follows the SDK/Python field style, for +example `content_url`, `content_length`, and `expires_at`. Older server builds +may still return `DIAGNOSTICS_NOT_IMPLEMENTED` for scoped diagnostics. Legacy DevOps diagnostics remain experimental. Prefer `osb diagnostics logs/events` for stable API-backed log and event collection. diff --git a/cli/src/opensandbox_cli/commands/diagnostics.py b/cli/src/opensandbox_cli/commands/diagnostics.py index 369aea6a9..0ead16685 100644 --- a/cli/src/opensandbox_cli/commands/diagnostics.py +++ b/cli/src/opensandbox_cli/commands/diagnostics.py @@ -46,13 +46,19 @@ def render_diagnostic_content( ) if output.fmt == "raw": + if content.truncated: + click.echo("Warning: diagnostic content was truncated.", err=True) + for warning in content.warnings or []: + click.echo(f"Warning: {warning}", err=True) if content.delivery == "inline": click.echo(content.content or "") return if content.content_url: click.echo(content.content_url) return - raise click.ClickException("Diagnostic response did not include inline content or a content URL.") + raise click.ClickException( + "Diagnostic response did not include inline content or a content URL." + ) output.print_dict(_diagnostic_to_dict(content), title=title) @@ -72,8 +78,8 @@ def diagnostics_group(ctx: click.Context) -> None: "-s", required=True, help=( - "Diagnostic log scope. Common scopes: lifecycle for manager logs, " - "container for sandbox stdout; other scopes are server-defined." + "Diagnostic log scope. Built-in server scopes: container and all; " + "other scopes are server-defined." ), ) @output_option( @@ -104,8 +110,8 @@ def diagnostics_logs( "-s", required=True, help=( - "Diagnostic event scope. Common scopes: lifecycle for audit events, " - "runtime for scheduler/container events; other scopes are server-defined." + "Diagnostic event scope. Built-in server scopes: runtime and all; " + "other scopes are server-defined." ), ) @output_option( diff --git a/cli/src/opensandbox_cli/commands/sandbox.py b/cli/src/opensandbox_cli/commands/sandbox.py index 4053261fb..80a498b56 100644 --- a/cli/src/opensandbox_cli/commands/sandbox.py +++ b/cli/src/opensandbox_cli/commands/sandbox.py @@ -371,7 +371,7 @@ def sandbox_pause(obj: ClientContext, sandbox_id: str, output_format: str | None mgr = obj.get_manager() with obj.output.spinner("Pausing sandbox..."): mgr.pause_sandbox(sandbox_id) - obj.output.success(f"Sandbox paused: {sandbox_id}") + obj.output.success(f"Pause request accepted: {sandbox_id}") # ---- resume --------------------------------------------------------------- diff --git a/cli/src/opensandbox_cli/skills/opensandbox-sandbox-lifecycle.md b/cli/src/opensandbox_cli/skills/opensandbox-sandbox-lifecycle.md index 69307d4cb..3a1227f7d 100644 --- a/cli/src/opensandbox_cli/skills/opensandbox-sandbox-lifecycle.md +++ b/cli/src/opensandbox_cli/skills/opensandbox-sandbox-lifecycle.md @@ -226,6 +226,7 @@ Rules: - `sandbox list --state` accepts known lifecycle states case-insensitively - use `renew` before long-running work instead of waiting for expiry - use `pause` only when the workload can tolerate suspension +- treat the pause result as request acceptance and poll `sandbox get` until the state is `Paused` or `Failed` - use `kill` when cleanup is the real goal; do not leave orphaned sandboxes behind ## Runtime Notes @@ -233,6 +234,7 @@ Rules: - `renew` resets the expiration to approximately `now + timeout`; treat it as a fresh TTL, not a simple additive extension to the old timestamp - `create --timeout none` means no automatic expiration; cleanup becomes an explicit `kill` responsibility - `create` without `--timeout` does not mean manual cleanup; it uses `defaults.timeout` first and otherwise leaves TTL selection to the SDK/server default +- `pause` is asynchronous; `Pause request accepted` does not mean the sandbox is already paused - `pause` and `resume` may depend on the underlying runtime; if the runtime does not support them, avoid promising they will work - host-path volumes depend on server-side allowed host path configuration - if creation fails or the sandbox never becomes healthy, switch to `sandbox-troubleshooting` instead of adding more create flags blindly diff --git a/cli/src/opensandbox_cli/skills/opensandbox-sandbox-troubleshooting.md b/cli/src/opensandbox_cli/skills/opensandbox-sandbox-troubleshooting.md index e3e5d1a31..cf2d51c86 100644 --- a/cli/src/opensandbox_cli/skills/opensandbox-sandbox-troubleshooting.md +++ b/cli/src/opensandbox_cli/skills/opensandbox-sandbox-troubleshooting.md @@ -67,7 +67,6 @@ Use this order by default: ```bash osb sandbox get -o json osb sandbox health -o json -osb diagnostics events --scope lifecycle -o raw osb diagnostics events --scope runtime -o raw osb diagnostics logs --scope container -o raw ``` @@ -75,7 +74,6 @@ osb diagnostics logs --scope container -o raw Then drill down only where the stable diagnostics point: ```bash -osb diagnostics logs --scope lifecycle -o raw osb diagnostics events --scope all -o raw osb diagnostics logs --scope all -o raw ``` @@ -86,9 +84,9 @@ Important properties of the diagnostics commands: - `diagnostics events` and `diagnostics logs` are stable API-backed commands - `--scope` is required for stable diagnostics; requests without scope use deprecated plain-text DevOps behavior -- if the server returns `DIAGNOSTICS_NOT_IMPLEMENTED`, state that stable diagnostics are unavailable on this server and stop diagnostics collection -- use known supported scopes first: `events:lifecycle`, `events:runtime`, `logs:lifecycle`, and `logs:container` -- `--scope all` is useful when the server supports aggregate diagnostics; if it is empty, retry concrete supported scopes +- older server builds may return `DIAGNOSTICS_NOT_IMPLEMENTED`; state that stable diagnostics are unavailable on that server and stop diagnostics collection +- use built-in server scopes first: `events:runtime`, `events:all`, `logs:container`, and `logs:all` +- best-effort scopes may include `warnings` when the backend contributes only a subset; preserve those warnings in the evidence - other scopes such as `network` or `process` are server-defined and may be empty on some deployments - `-o raw` prints inline diagnostic text directly, or a content URL when the server returns URL delivery - `-o json` / `-o yaml` prints the CLI descriptor including `delivery`, `content_url`, `expires_at`, `truncated`, and `warnings` @@ -96,16 +94,16 @@ Important properties of the diagnostics commands: Use: -- `osb diagnostics events --scope lifecycle -o raw` for sandbox actions such as `CREATE`, `RENEW`, `DELETE`, `PAUSE`, `RESUME`, and `FORK` - `osb diagnostics events --scope runtime -o raw` for scheduler and container events such as `Scheduled`, `Pulling`, `Pulled`, `Created`, `Started`, and `ContainerDied` -- `osb diagnostics logs --scope lifecycle -o raw` for manager server logs related to create, renew, delete, callbacks, request IDs, and server-side failures +- `osb diagnostics events --scope all -o raw` for the best-effort event aggregate available from the server; check `warnings` before treating it as complete - `osb diagnostics logs --scope container -o raw` for sandbox main-process stdout, including application errors, missing binaries, bad entrypoints, startup hangs, and health-check failures +- `osb diagnostics logs --scope all -o raw` for the best-effort aggregate available from the server; check `warnings` before treating it as complete ## Evidence Semantics - `sandbox get` shows control-plane state; it does not prove the workload is healthy - `sandbox health` shows readiness or endpoint health; it can fail even when the sandbox is running -- lifecycle diagnostics explain OpenSandbox manager behavior; container logs explain the user workload +- the built-in server does not expose lifecycle audit events; use external control-plane or audit sources when current state and runtime events are insufficient - runtime events are platform facts and usually outrank application logs for scheduling, image pull, restart, and kill reasons - empty diagnostics do not prove there is no issue; the scope may be unsupported, expired, or outside retention - `truncated: true` means the evidence is incomplete; lower confidence and mention the truncation @@ -117,7 +115,7 @@ Use: - with `delivery: url`, `-o raw` prints the diagnostic content URL; fetch it only if you need the diagnostic body - `content_url` in structured CLI output is a diagnostic artifact URL, not a sandbox service endpoint - check `expires_at`; container log URLs may expire quickly, so request diagnostics again if the URL is stale -- do not forward diagnostic URLs or lifecycle logs to unrelated people because they may contain sensitive troubleshooting data +- do not forward diagnostic URLs or logs to unrelated people because they may contain sensitive troubleshooting data ## Symptom To Command Mapping @@ -125,7 +123,7 @@ Use the first command that best matches the reported symptom: | Symptom | First command | What to confirm next | | --- | --- | --- | -| pending forever or stuck creating | `osb diagnostics events --scope runtime -o raw` | image pull errors, scheduling failures, admission errors, then lifecycle logs | +| pending forever or stuck creating | `osb diagnostics events --scope runtime -o raw` | image pull errors, scheduling failures, admission errors, then `sandbox get` and external control-plane logs | | image pull failure | `osb diagnostics events --scope runtime -o raw` | image name, tag, registry auth | | crash loop or repeated restarts | `osb diagnostics logs --scope container -o raw` | `osb diagnostics events --scope runtime -o raw` for restarts or kill signals | | suspected OOM or exit code issue | `osb diagnostics events --scope runtime -o raw` | kill signals, restart events, resource pressure messages | @@ -169,7 +167,6 @@ CLI-first troubleshooting: ```bash osb sandbox get -o json osb sandbox health -o json -osb diagnostics events --scope lifecycle -o raw osb diagnostics events --scope runtime -o raw osb diagnostics logs --scope container -o raw ``` diff --git a/cli/tests/test_cli_help.py b/cli/tests/test_cli_help.py index 0f61ef9b0..8d6b5fb08 100644 --- a/cli/tests/test_cli_help.py +++ b/cli/tests/test_cli_help.py @@ -221,17 +221,19 @@ def test_diagnostics_subcommand_help(self, runner: CliRunner, subcmd: str) -> No assert "--scope" in result.output assert "content URL" in result.output - def test_diagnostics_logs_help_describes_common_scopes(self, runner: CliRunner) -> None: + def test_diagnostics_logs_help_describes_builtin_scopes(self, runner: CliRunner) -> None: result = runner.invoke(cli, ["diagnostics", "logs", "--help"]) assert result.exit_code == 0 - assert "lifecycle" in result.output assert "container" in result.output + assert "all" in result.output + assert "lifecycle" not in result.output - def test_diagnostics_events_help_describes_common_scopes(self, runner: CliRunner) -> None: + def test_diagnostics_events_help_describes_builtin_scopes(self, runner: CliRunner) -> None: result = runner.invoke(cli, ["diagnostics", "events", "--help"]) assert result.exit_code == 0 - assert "lifecycle" in result.output assert "runtime" in result.output + assert "all" in result.output + assert "lifecycle" not in result.output # --------------------------------------------------------------------------- diff --git a/cli/tests/test_commands.py b/cli/tests/test_commands.py index 462b561a4..36ce10652 100644 --- a/cli/tests/test_commands.py +++ b/cli/tests/test_commands.py @@ -569,12 +569,12 @@ def test_kill_multiple(self, runner: CliRunner) -> None: class TestSandboxPause: - def test_pause_calls_manager(self, runner: CliRunner) -> None: + def test_pause_reports_request_accepted(self, runner: CliRunner) -> None: mock_mgr = MagicMock() result = _invoke(runner, ["sandbox", "pause", "sb-123"], manager=mock_mgr) assert result.exit_code == 0 mock_mgr.pause_sandbox.assert_called_once_with("sb-123") - assert "Sandbox paused: sb-123" in result.output + assert "Pause request accepted: sb-123" in result.output class TestSandboxResume: @@ -1380,6 +1380,32 @@ def test_logs_raw_prints_inline_content(self, runner: CliRunner) -> None: assert "line 1\nline 2" in result.output manager.get_diagnostic_logs.assert_called_once_with("sb-1", scope="container") + def test_logs_raw_surfaces_diagnostic_notices_on_stderr( + self, runner: CliRunner + ) -> None: + manager = MagicMock() + manager.get_diagnostic_logs.return_value = DiagnosticContent( + sandboxId="sb-1", + kind="logs", + scope="all", + delivery="inline", + contentType="text/plain; charset=utf-8", + content="container logs", + truncated=True, + warnings=["Only container logs are available."], + ) + + result = _invoke( + runner, + ["diagnostics", "logs", "sb-1", "--scope", "all", "-o", "raw"], + manager=manager, + ) + + assert result.exit_code == 0 + assert result.stdout == "container logs\n" + assert "Warning: diagnostic content was truncated." in result.stderr + assert "Warning: Only container logs are available." in result.stderr + def test_events_json_prints_descriptor(self, runner: CliRunner) -> None: manager = MagicMock() manager.get_diagnostic_events.return_value = DiagnosticContent( diff --git a/cli/tests/test_skills.py b/cli/tests/test_skills.py index f8719b343..753e20aa6 100644 --- a/cli/tests/test_skills.py +++ b/cli/tests/test_skills.py @@ -528,9 +528,13 @@ def test_sandbox_troubleshooting_keeps_triage_and_diagnostics_contract(self) -> assert "## Triage Order" in content assert "osb sandbox get -o json" in content - assert "osb diagnostics events --scope lifecycle -o raw" in content + assert "osb diagnostics events --scope lifecycle" not in content assert "osb diagnostics events --scope runtime -o raw" in content + assert "osb diagnostics events --scope all -o raw" in content assert "osb diagnostics logs --scope container -o raw" in content + assert "osb diagnostics logs --scope all -o raw" in content + assert "osb diagnostics logs --scope lifecycle" not in content + assert "events:lifecycle" not in content assert "## Diagnostics Streams" in content assert "## Evidence Semantics" in content assert "## URL Delivery" in content diff --git a/components/egress/docs/opentelemetry.md b/components/egress/docs/opentelemetry.md index a7830def8..7f2d738ee 100644 --- a/components/egress/docs/opentelemetry.md +++ b/components/egress/docs/opentelemetry.md @@ -91,10 +91,47 @@ Metric export is enabled only when at least one OTLP endpoint is set. If both are unset, egress keeps metrics local (no OTLP export). +### Automatic Egress Allow Rule + +When an OTLP destination is configured โ€” the endpoint env vars below, or the +exporter fallback node IP (`HOST_IP` / `/etc/hostinfo`) when both are unset โ€” +egress automatically injects an always-allow egress rule for that host +(domain or IP, any port), so telemetry export works under the default deny-all +policy without manually managing allowlist rules. This also covers the egress +sidecar's own metric export, which shares the sandbox network namespace and +would otherwise be blocked by its own egress chain. + +- The rule follows the standard precedence: `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` + wins over `OTEL_EXPORTER_OTLP_ENDPOINT`; the fallback node IP applies only + when neither is set. A set-but-invalid endpoint never falls back (the + exporter does not either), so no rule is injected in that case. +- The endpoint must be a URL (`https://host:4318/v1/metrics`) โ€” the + `otlpmetrichttp` env-var form. Bare `host:port` or `host` values are not + accepted (the exporter parses them as opaque URLs with an empty host); a + trailing root dot on FQDNs is trimmed to match DNS policy normalization. +- The rule lives in the always-allow layer: it survives user `POST`/`PATCH`/`DELETE` + policy updates and always-rule file reloads. Operators can still block the target + with `deny.always`, which takes precedence. +- Rules are host-scoped (any port), matching the egress rule model; ports are not + enforced per rule. + +> **Note**: use a fully-qualified service name or an IP in the endpoint. +> Single-label names (e.g. `otel-collector`) are subject to resolver +> search-domain expansion, and the deny-all DNS proxy answers the expanded +> names (e.g. `otel-collector..svc.cluster.local`) with NXDOMAIN without +> falling back to the bare name, so the auto-generated exact-host allow rule +> would not be reached. + ### Minimal Example ```bash -export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector:4318" +export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector.sandbox.svc.cluster.local:4318" +``` + +An IP endpoint works as well: + +```bash +export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://10.0.0.5:4318" ``` ### Service Name diff --git a/components/egress/main.go b/components/egress/main.go index d74c98c23..d6aeb6c79 100644 --- a/components/egress/main.go +++ b/components/egress/main.go @@ -78,6 +78,7 @@ func main() { if err != nil { log.Fatalf("failed to load always allow/deny rule files: %v", err) } + alwaysAllow = withTelemetryAllow(alwaysAllow) allowIPs := allowIps() mode := parseMode() diff --git a/components/egress/mitmscripts/system.py b/components/egress/mitmscripts/system.py index d45d4d412..f695cdc87 100644 --- a/components/egress/mitmscripts/system.py +++ b/components/egress/mitmscripts/system.py @@ -642,12 +642,18 @@ def requestheaders(flow: http.HTTPFlow) -> None: if vault is None: return - # Reject ambiguous paths before injecting credentials: dot-segments or - # encoded separators on the wire are not produced by legit clients and - # could redirect credentials to a scope the canonical path does not match. - # A single-layer ``%2f`` is tolerated here (npm scoped packages send - # ``/@scope%2fname``); the next check rejects it if it crosses a binding - # boundary. + # Requests outside credential binding scope are ordinary egress traffic. + # Leave them untouched, including paths whose encoding would be ambiguous + # for credential injection, because no secret is at risk. + binding = _select_binding(flow, vault) + if not binding: + return + + # Reject ambiguous paths only for requests that would receive credentials: + # dot-segments or encoded separators could redirect credentials to a scope + # the canonical path does not match. A single-layer ``%2f`` is tolerated + # here (npm scoped packages send ``/@scope%2fname``); the next check rejects + # it if it crosses a binding boundary. raw_path = flow.request.path or "/" if _path_is_ambiguous(raw_path, allow_single_encoded_slash=True): _reject_request(flow, b"request path contains ambiguous segments\n") @@ -669,9 +675,6 @@ def requestheaders(flow: http.HTTPFlow) -> None: ) return - binding = _select_binding(flow, vault) - if not binding: - return flow.metadata[FLOW_BINDING_KEY] = binding # Persist the redactions of the matched revision: body substitutions run # later in the request hook, and reloading the vault there could return a diff --git a/components/egress/policy_server.go b/components/egress/policy_server.go index ac510f3cc..b6a05ff1e 100644 --- a/components/egress/policy_server.go +++ b/components/egress/policy_server.go @@ -713,6 +713,8 @@ func (s *policyServer) reloadAlwaysRules() (bool, error) { if !changed { return false, nil } + allow = withTelemetryAllow(allow) + s.setAlwaysRules(deny, allow) s.proxy.UpdateAlwaysRules(deny, allow) return true, nil } diff --git a/components/egress/telemetry_allow.go b/components/egress/telemetry_allow.go new file mode 100644 index 000000000..1c8f3b298 --- /dev/null +++ b/components/egress/telemetry_allow.go @@ -0,0 +1,67 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/alibaba/opensandbox/egress/pkg/log" + "github.com/alibaba/opensandbox/egress/pkg/policy" + inttelemetry "github.com/alibaba/opensandbox/internal/telemetry" +) + +// telemetryAllowRules returns an always-allow egress rule for the OTLP +// destination the exporter will dial, so metric export works under the default +// deny-all policy without operator-provided allowlist rules. The destination +// is the endpoint env var +// (OTEL_EXPORTER_OTLP_METRICS_ENDPOINT / OTEL_EXPORTER_OTLP_ENDPOINT, URL form +// as required by otlpmetrichttp) or, only when neither is set, the exporter +// fallback node IP (HOST_IP / /etc/hostinfo). A set-but-unparseable endpoint +// is not treated as unset: the exporter never falls back to the node IP in +// that case, so no rule is injected. The rule targets the host (any port), +// matching the egress rule model. Operators can still block the target via +// deny.always, which takes precedence. Returns nil when no OTLP destination +// is configured. +func telemetryAllowRules() []policy.EgressRule { + host, port, ok := inttelemetry.OTLPEndpointHostPort() + if !ok { + if inttelemetry.OTLPEndpointEnvSet() { + log.Warnf("telemetry: configured OTLP endpoint is not a valid URL; skipping auto egress allow") + return nil + } + host, port, ok = inttelemetry.OTLPEndpointFallbackHostPort() + } + if !ok { + return nil + } + rule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, host) + if err != nil { + log.Warnf("telemetry: skipping auto egress allow for OTLP endpoint host %q: %v", host, err) + return nil + } + log.Infof("telemetry: auto-allowing egress to OTLP endpoint %s:%s (deny.always can override)", host, port) + return []policy.EgressRule{rule} +} + +// withTelemetryAllow appends the auto-generated OTLP allow rule(s) to the +// always-allow list so every effective-policy merge (startup, policy updates, +// always-file reloads) keeps telemetry egress open. +func withTelemetryAllow(allow []policy.EgressRule) []policy.EgressRule { + rules := telemetryAllowRules() + if len(rules) == 0 { + return allow + } + out := make([]policy.EgressRule, 0, len(allow)+len(rules)) + out = append(out, allow...) + return append(out, rules...) +} diff --git a/components/egress/telemetry_allow_test.go b/components/egress/telemetry_allow_test.go new file mode 100644 index 000000000..cce684944 --- /dev/null +++ b/components/egress/telemetry_allow_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/alibaba/opensandbox/egress/pkg/policy" + "github.com/stretchr/testify/require" +) + +func TestTelemetryAllowRulesUnconfigured(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + require.Nil(t, telemetryAllowRules()) + + existing := []policy.EgressRule{{Action: policy.ActionAllow, Target: "a.example.com"}} + require.Equal(t, existing, withTelemetryAllow(existing), "no telemetry rules must not mutate the input") +} + +func TestTelemetryAllowRulesFromMetricsEndpoint(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://collector.example:4318/v1/metrics") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + rules := telemetryAllowRules() + require.Len(t, rules, 1) + require.Equal(t, policy.ActionAllow, rules[0].Action) + require.Equal(t, "collector.example", rules[0].Target) + + merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules) + require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example."), "domain rule must allow DNS resolution") + allowV4, allowV6, _, _ := merged.StaticIPSets() + require.Empty(t, allowV4) + require.Empty(t, allowV6) +} + +func TestTelemetryAllowRulesFallbackEndpoint(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel-collector:4318") + rules := telemetryAllowRules() + require.Len(t, rules, 1) + require.Equal(t, "otel-collector", rules[0].Target) +} + +func TestTelemetryAllowRulesFallbackNodeIP(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("HOST_IP", "10.0.0.9") + rules := telemetryAllowRules() + require.Len(t, rules, 1) + require.Equal(t, "10.0.0.9", rules[0].Target) + + merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules) + allowV4, allowV6, _, _ := merged.StaticIPSets() + require.Equal(t, []string{"10.0.0.9"}, allowV4, "fallback node IP must land in the static allow v4 set") + require.Empty(t, allowV6) +} + +func TestTelemetryAllowRulesFQDNTrailingDot(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://otel-collector.ns.svc.cluster.local.:4318") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + rules := telemetryAllowRules() + require.Len(t, rules, 1) + require.Equal(t, "otel-collector.ns.svc.cluster.local", rules[0].Target) + + merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules) + require.Equal(t, policy.ActionAllow, merged.Evaluate("otel-collector.ns.svc.cluster.local."), "trailing-dot host must match DNS policy normalization") + require.Equal(t, policy.ActionDeny, merged.Evaluate("other.ns.svc.cluster.local.")) +} + +func TestTelemetryAllowRulesIPEndpoint(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://10.0.0.5:4317") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + rules := telemetryAllowRules() + require.Len(t, rules, 1) + require.Equal(t, "10.0.0.5", rules[0].Target) + + merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules) + allowV4, allowV6, _, _ := merged.StaticIPSets() + require.Equal(t, []string{"10.0.0.5"}, allowV4, "IP target must land in the static allow v4 set") + require.Empty(t, allowV6) +} + +func TestTelemetryAllowRulesInvalidEndpoint(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + require.Nil(t, telemetryAllowRules()) +} + +func TestTelemetryAllowRulesInvalidEndpointSkipsNodeIPFallback(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("HOST_IP", "10.0.0.9") + require.Nil(t, telemetryAllowRules(), "configured-but-invalid endpoint must not open node-IP egress") + + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + rules := telemetryAllowRules() + require.Len(t, rules, 1, "unset endpoint should fall back to the node IP") + require.Equal(t, "10.0.0.9", rules[0].Target) +} + +func TestWithTelemetryAllowAppends(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://collector.example:4318") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + existingRule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, "a.example.com") + require.NoError(t, err) + existing := []policy.EgressRule{existingRule} + rules := withTelemetryAllow(existing) + require.Len(t, rules, 2) + require.Equal(t, "a.example.com", rules[0].Target) + require.Equal(t, "collector.example", rules[1].Target) + + merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules) + require.Equal(t, policy.ActionDeny, merged.Evaluate("other.example.com.")) + require.Equal(t, policy.ActionAllow, merged.Evaluate("a.example.com.")) + require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example.")) +} diff --git a/components/egress/tests/test_mitmproxy_runtime.py b/components/egress/tests/test_mitmproxy_runtime.py index d6339bf99..d4cbcc828 100644 --- a/components/egress/tests/test_mitmproxy_runtime.py +++ b/components/egress/tests/test_mitmproxy_runtime.py @@ -55,7 +55,7 @@ "schemes": ["http"], "hosts": ["code.example.com"], "methods": ["POST"], - "paths": ["/v1/chat/completions"], + "paths": ["/v1/chat/*"], }, "headers": [{"name": "x-api-key", "value": "secret-api-key"}], } diff --git a/components/egress/tests/test_mitmscripts_system.py b/components/egress/tests/test_mitmscripts_system.py index 88bd90a8d..0982293c1 100644 --- a/components/egress/tests/test_mitmscripts_system.py +++ b/components/egress/tests/test_mitmscripts_system.py @@ -611,7 +611,7 @@ def test_encoded_slash_rejected(self) -> None: """%2f encoded path separator must be rejected.""" system = self._make_system_with_vault() flow = _Flow() - flow.request.path = "/api/v8/projects/123%2f..%2f456/variables" + flow.request.path = "/api/v8/projects/123/%2f..%2f456/variables" system.requestheaders(flow) @@ -650,6 +650,35 @@ def test_normal_path_outside_scope_no_injection(self) -> None: self.assertNotIn("Private-Token", flow.request.headers._values) + def test_double_encoded_path_outside_binding_scope_is_allowed(self) -> None: + """Ambiguous paths pass through when no credential binding matches.""" + system = self._make_system_with_vault() + flow = _Flow() + flow.request.pretty_host = "packages.example.com" + flow.request.host = "packages.example.com" + flow.request.path = ( + "/1/pypi/simple/pyyaml/" + "%252Fcentral-pypi-proxy%252Fpackages%252F8b%252F9d/wheel.whl" + ) + + system.requestheaders(flow) + + self.assertFalse(flow.killed) + self.assertNotEqual(403, getattr(flow.response, "status_code", None)) + self.assertNotIn("Private-Token", flow.request.headers._values) + + def test_ambiguous_path_on_bound_host_outside_path_scope_is_allowed(self) -> None: + """A host match alone does not put a request in credential scope.""" + system = self._make_system_with_vault() + flow = _Flow() + flow.request.path = "/downloads/%252Fartifacts%252Fwheel.whl" + + system.requestheaders(flow) + + self.assertFalse(flow.killed) + self.assertNotEqual(403, getattr(flow.response, "status_code", None)) + self.assertNotIn("Private-Token", flow.request.headers._values) + def test_dot_dot_substring_not_rejected(self) -> None: """'..' not as a complete segment (e.g. '/.../') must NOT be blocked.""" system = self._make_system_with_vault() @@ -682,7 +711,7 @@ def test_raw_backslash_rejected(self) -> None: """Raw backslash in path must be rejected.""" system = self._make_system_with_vault() flow = _Flow() - flow.request.path = "/api/v8/projects/123\\..\\456/variables" + flow.request.path = "/api/v8/projects/123/\\..\\456/variables" system.requestheaders(flow) @@ -887,7 +916,7 @@ def _make_vault_with_large_body_binding(self, system): "match": { "hosts": ["code.example.com"], "methods": ["POST"], - "paths": ["/v1/chat/completions"], + "paths": ["/v1/chat/*"], }, "headers": [{"name": "x-api-key", "value": "secret-api-key"}], "substitutions": [ diff --git a/components/execd/Dockerfile b/components/execd/Dockerfile index 64a049347..97e7d1ebb 100644 --- a/components/execd/Dockerfile +++ b/components/execd/Dockerfile @@ -79,6 +79,9 @@ RUN apk add --no-cache git musl-dev meson ninja gcc libcap-dev libcap-static pkg COPY components/execd/native/session-gate.c /build/session-gate.c RUN gcc -Os -static -s -Wall -Wextra -Werror \ -o /build/opensandbox-session-gate /build/session-gate.c +COPY components/execd/native/launcher.c /build/launcher.c +RUN gcc -Os -static -s -Wall -Wextra -Werror \ + -o /build/opensandbox-launcher /build/launcher.c RUN git clone --depth 1 --branch v0.11.2 \ https://github.com/containers/bubblewrap /build/bwrap WORKDIR /build/bwrap @@ -93,14 +96,44 @@ RUN rm /usr/lib/libcap.so /usr/lib/libcap.so.2 && \ ninja -C builddir && \ cp builddir/bwrap /build/bwrap/bwrap +# execd-ebpf: observation variant binary (OSEP-0018 ยง5). The default image +# below carries both the minimal control-plane binary and this variant; who +# runs which is decided at launch time (bootstrap.sh honors the EXECD env, +# defaulting to /opt/opensandbox/execd). The variant attaches +# exec/connect/privilege audit hooks and needs CAP_BPF + CAP_PERFMON in the +# container ceiling. +FROM golang:1.25.9 AS ebpf-builder +ARG VERSION=dev +ARG GIT_COMMIT=unknown +ARG BUILD_TIME=unknown +RUN apt-get update && apt-get install -y --no-install-recommends clang +WORKDIR /build +COPY components/internal/go.mod components/internal/go.sum ./components/internal/ +COPY components/execd/go.mod components/execd/go.sum ./components/execd/ +RUN cd components/internal && go mod download +RUN cd components/execd && go mod download +COPY components/internal ./components/internal +COPY components/execd ./components/execd +WORKDIR /build/components/execd +# cilium/ebpf is pure Go, so the variant builds fully static. +RUN CGO_ENABLED=0 go build -tags ebpf -trimpath -buildvcs=false \ + -ldflags "-buildid= -B none \ + -X 'github.com/alibaba/opensandbox/internal/version.Version=${VERSION}' \ + -X 'github.com/alibaba/opensandbox/internal/version.BuildTime=${BUILD_TIME}' \ + -X 'github.com/alibaba/opensandbox/internal/version.GitCommit=${GIT_COMMIT}'" \ + -o /build/execd-ebpf ./main.go + FROM alpine:latest COPY --from=bwrap-builder /build/bwrap/bwrap /usr/local/bin/bwrap COPY --from=bwrap-builder --chown=0:0 --chmod=0555 /build/opensandbox-session-gate /usr/local/libexec/opensandbox-session-gate COPY --from=bwrap-builder --chown=0:0 --chmod=0555 /build/opensandbox-session-gate /opt/opensandbox/opensandbox-session-gate +COPY --from=bwrap-builder --chown=0:0 --chmod=0555 /build/opensandbox-launcher /usr/local/libexec/opensandbox-launcher +COPY --from=bwrap-builder --chown=0:0 --chmod=0555 /build/opensandbox-launcher /opt/opensandbox/opensandbox-launcher COPY --from=builder /build/execd . COPY --from=builder /build/execd.exe ./execd.exe COPY --from=builder /build/opensandbox-supervisor ./opensandbox-supervisor +COPY --from=ebpf-builder /build/execd-ebpf ./execd-ebpf COPY components/execd/bootstrap.sh ./bootstrap.sh COPY components/execd/install.bat ./install.bat diff --git a/components/execd/Makefile b/components/execd/Makefile index a90b8ab1a..ebbcf3dac 100644 --- a/components/execd/Makefile +++ b/components/execd/Makefile @@ -35,6 +35,9 @@ SESSION_GATE_SOURCE_INSTALL_DIR := /usr/local/libexec SESSION_GATE_RUNTIME_DIR := /opt/opensandbox SESSION_GATE_CFLAGS ?= $(CFLAGS) -O2 -Wall -Wextra -Werror SESSION_GATE_LDFLAGS ?= -static -s +LAUNCHER_BINARY := bin/opensandbox-launcher +LAUNCHER_SOURCE := native/launcher.c +LAUNCHER_RUNTIME_DIR := /opt/opensandbox INSTALL ?= install DESTDIR ?= ifeq ($(strip $(DESTDIR)),) @@ -49,7 +52,7 @@ PROJECT_LDFLAGS := -buildid= -B none -X 'github.com/alibaba/opensandbox/internal GO_BUILD_FLAGS := $(strip $(GOFLAGS) $(PROJECT_GOFLAGS)) GO_LDFLAGS := $(strip $(LDFLAGS) $(PROJECT_LDFLAGS)) -.PHONY: build-session-gate install-session-gate +.PHONY: build-session-gate install-session-gate build-launcher install-launcher build-session-gate: @set -eu; \ host_goos="$$(go env GOHOSTOS)"; \ @@ -99,11 +102,62 @@ install-session-gate: $(INSTALL) $(SESSION_GATE_INSTALL_OWNER_ARGS) -m 0555 "$(SESSION_GATE_BINARY)" \ "$(DESTDIR)$(SESSION_GATE_RUNTIME_DIR)/opensandbox-session-gate" +build-launcher: + @set -eu; \ + host_goos="$$(go env GOHOSTOS)"; \ + host_goarch="$$(go env GOHOSTARCH)"; \ + target_goos="$(if $(GOOS),$(GOOS),$$(go env GOOS))"; \ + target_goarch="$(if $(GOARCH),$(GOARCH),$$(go env GOARCH))"; \ + if [ "$$target_goos" != "linux" ]; then \ + echo "Skipping launcher: hardening requires Linux (target=$$target_goos/$$target_goarch)"; \ + exit 0; \ + fi; \ + if [ "$$host_goos/$$host_goarch" != "$$target_goos/$$target_goarch" ]; then \ + echo "launcher cross-build is unsupported (host=$$host_goos/$$host_goarch, target=$$target_goos/$$target_goarch)" >&2; \ + echo "use the execd Docker build for multi-architecture Linux artifacts" >&2; \ + exit 1; \ + fi; \ + mkdir -p bin; \ + $(CC) $(CPPFLAGS) $(SESSION_GATE_CFLAGS) "$(LAUNCHER_SOURCE)" \ + $(SESSION_GATE_LDFLAGS) -o "$(LAUNCHER_BINARY).tmp"; \ + mv -f "$(LAUNCHER_BINARY).tmp" "$(LAUNCHER_BINARY)" + +install-launcher: + @if [ "$$(uname -s)" != "Linux" ]; then \ + echo "install-launcher requires Linux" >&2; \ + exit 1; \ + fi + @if [ ! -x "$(LAUNCHER_BINARY)" ]; then \ + echo "$(LAUNCHER_BINARY) is missing; run make build-launcher first" >&2; \ + exit 1; \ + fi + @if [ -z "$(DESTDIR)" ] && [ "$$(id -u)" -ne 0 ]; then \ + echo "install-launcher requires root unless DESTDIR is set" >&2; \ + exit 1; \ + fi + @umask 022; mkdir -p "$(DESTDIR)$(LAUNCHER_RUNTIME_DIR)" + @if [ -z "$(DESTDIR)" ]; then \ + chown root:root "$(DESTDIR)$(LAUNCHER_RUNTIME_DIR)"; \ + fi + chmod go-w "$(DESTDIR)$(LAUNCHER_RUNTIME_DIR)" + $(INSTALL) $(SESSION_GATE_INSTALL_OWNER_ARGS) -m 0555 "$(LAUNCHER_BINARY)" \ + "$(DESTDIR)$(LAUNCHER_RUNTIME_DIR)/opensandbox-launcher" + .PHONY: build -build: vet build-session-gate ## Build execd and the Linux session gate. +build: vet build-session-gate build-launcher ## Build execd and the Linux native helpers. @mkdir -p bin go build $(GO_BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -o bin/execd main.go +.PHONY: build-ebpf +build-ebpf: ## Build the execd-ebpf observation variant (CGO + cilium/ebpf). + @if [ "$$(uname -s 2>/dev/null || echo non-linux)" != "Linux" ]; then \ + echo "execd-ebpf requires Linux (BPF attachable host)" >&2; \ + exit 1; \ + fi + @mkdir -p bin + CGO_ENABLED=1 go build -tags ebpf $(GO_BUILD_FLAGS) -ldflags "$(GO_LDFLAGS)" -o bin/execd-ebpf main.go + @echo "built bin/execd-ebpf" + .PHONY: test-integration test-integration: ## Run integration tests (Linux + bwrap required). go test -v -tags="linux,bwrap" -run Integration ./pkg/runtime/bwrap_test/ diff --git a/components/execd/README.md b/components/execd/README.md index 1102e6ac1..43291cc59 100644 --- a/components/execd/README.md +++ b/components/execd/README.md @@ -2,3 +2,14 @@ Documentation: [docs/components/execd.md](../../docs/components/execd.md) +## Known issue / TODO: execd-ebpf selection is not wired end to end + +The default image ships both the minimal `execd` binary and the +`execd-ebpf` observation variant (root `/execd` and `/execd-ebpf`), but the +**server-side selection is not implemented yet**: nothing in the server +injects the `EXECD` env var (which `bootstrap.sh` honors to pick a binary, +defaulting to `/opt/opensandbox/execd`), and the Docker / K8s +distribution paths only install `/execd` into `/opt/opensandbox`. Until +that lands, using `execd-ebpf` requires manually staging the binary (or +overriding `EXECD`) โ€” do not rely on it in production. + diff --git a/components/execd/bootstrap.sh b/components/execd/bootstrap.sh index fecc217f0..363c162f9 100755 --- a/components/execd/bootstrap.sh +++ b/components/execd/bootstrap.sh @@ -310,8 +310,6 @@ if [ -n "${EXECD_BOOTSTRAP_PRE_SCRIPT:-}" ]; then fi echo "starting OpenSandbox Execd daemon at $EXECD." -$EXECD & -EXECD_PID=$! # Allow chained shell commands (e.g., /test1.sh && /test2.sh) # Usage: @@ -337,17 +335,27 @@ if [ -z "$SHELL_BIN" ]; then fi fi +# Resolve the user command into a concrete argv shared by both branches. if [ "$CMD" != "" ]; then - "$SHELL_BIN" -c "$CMD" & - CMD_PID=$! + set -- "$SHELL_BIN" -c "$CMD" elif [ $# -eq 0 ]; then - "$SHELL_BIN" & - CMD_PID=$! -else - "$@" & - CMD_PID=$! + set -- "$SHELL_BIN" +fi + +# Init mode (OSEP-0018): exec into execd so it becomes the sandbox init (PID 1 +# on the direct paths) and supervises the user command itself. The shell must +# exec, never background, or execd would run as a subreaper without the kernel +# signal shield. +if is_truthy "${EXECD_INIT:-}"; then + exec "$EXECD" --init -- "$@" fi +"$EXECD" & +EXECD_PID=$! + +"$@" & +CMD_PID=$! + set +e wait "$CMD_PID" 2>/dev/null CMD_STATUS=$? diff --git a/components/execd/configs/isolation.example.toml b/components/execd/configs/isolation.example.toml index a56a67710..1397bcbb1 100644 --- a/components/execd/configs/isolation.example.toml +++ b/components/execd/configs/isolation.example.toml @@ -82,3 +82,53 @@ allowed_writable = ["/workspace", "/mnt", "/media", "/data"] # # Other potentially dangerous # "userfaultfd", "kexec_load", "kexec_file_load", "acct", # ] + +# Hardening floor (OSEP-0018 ยง4). Default OFF โ€” omit the section for today's +# behavior. When enabled, every user-code process (entrypoint, /command, +# /code, PTY) is launched through the opensandbox-launcher native helper with +# reduced capabilities, no_new_privs, and the seccomp floor; combined with +# init mode via bootstrap.sh EXECD_INIT + --init. +# +# [hardening] +# enabled = true +# +# # Capabilities the workload retains (raised in the ambient set). +# # Default: drop all. Names use the CAP_ prefix. +# keep_capabilities = [] +# +# NOTE: with [hardening] enabled, [seccomp] deny must NOT list "execve" โ€” +# it is reserved for the launcher's final exec (execveat stays allowed). +# Missing runtime support (no CAP_SETPCAP, launcher binary absent) is +# reported on GET /v1/isolated/capabilities under "hardening" and skipped, +# never fatal. + +# Landlock filesystem confinement (OSEP-0018 ยง5) on top of [hardening]. +# Default OFF. When enabled, user-code processes are allowlisted: system +# paths read+exec, /proc/self and well-known read-only proc files, the +# needed /dev device files, /tmp, /run, and allowed_writable โ€” everything +# else is denied. A kernel without Landlock (ABI < 1) degrades to +# "unsupported" and is skipped. +# +# [landlock] +# enabled = true +# +# # Extra writable paths beyond the built-in set (read+write+create). +# extra_writable = ["/var/cache"] +# +# # Extra read-only paths beyond the built-in set (read+exec). +# extra_readable = ["/opt/readonly"] + +# eBPF observation (OSEP-0018 ยง5). Default OFF. Requires the execd-ebpf +# build variant (CGO + cilium/ebpf), CAP_BPF + CAP_PERFMON, and a +# BTF-capable kernel (5.10+ with CONFIG_DEBUG_INFO_BTF). Observation is +# scoped to the sandbox cgroup and written as JSONL to a rotating audit +# file. Missing prerequisites degrade to "unsupported" and are skipped. +# +# [ebpf] +# enabled = true +# +# # Event kinds to record. Default: all three. +# observe = ["exec", "connect", "privilege"] +# +# # Append-only JSONL audit sink (rotated). Default below. +# audit_file = "/var/log/opensandbox/ebpf-audit.jsonl" diff --git a/components/execd/configs/isolation.hardened.toml b/components/execd/configs/isolation.hardened.toml new file mode 100644 index 000000000..31e36b8ac --- /dev/null +++ b/components/execd/configs/isolation.hardened.toml @@ -0,0 +1,28 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hardened isolation configuration for the execd-as-init e2e (OSEP-0018, +# R-i). Baked into the e2e image variant (Dockerfile.hardened-e2e) and +# injected into sandboxes via [docker] sandbox_env +# (EXECD_ISOLATION_CONFIG=/etc/opensandbox/isolation.toml) so the whole +# server -> sandbox -> execd path runs with the floor on. +# +# This is the "common cases" config from the OSEP: one line each for the +# floor and for filesystem confinement. Everything else stays built-in. + +[hardening] +enabled = true + +[landlock] +enabled = true diff --git a/components/execd/go.mod b/components/execd/go.mod index 14e4e8b5b..af9301eca 100644 --- a/components/execd/go.mod +++ b/components/execd/go.mod @@ -30,6 +30,7 @@ require ( github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.16.0 github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -68,13 +69,14 @@ require ( go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.52.0 // indirect + golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect diff --git a/components/execd/go.sum b/components/execd/go.sum index cfcf6cab5..1ea60dc56 100644 --- a/components/execd/go.sum +++ b/components/execd/go.sum @@ -10,6 +10,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -44,6 +46,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688= github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -61,6 +65,10 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -77,6 +85,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -156,6 +168,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= +golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -167,6 +181,8 @@ golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/components/execd/main.go b/components/execd/main.go index 3792a66ea..310718589 100644 --- a/components/execd/main.go +++ b/components/execd/main.go @@ -31,6 +31,7 @@ import ( _ "go.uber.org/automaxprocs/maxprocs" "github.com/alibaba/opensandbox/execd/pkg/clone3compat" + "github.com/alibaba/opensandbox/execd/pkg/ebpf" "github.com/alibaba/opensandbox/execd/pkg/flag" "github.com/alibaba/opensandbox/execd/pkg/isolation" "github.com/alibaba/opensandbox/execd/pkg/log" @@ -69,6 +70,22 @@ func run() int { return 1 } + // Activate the pre-exec hardening floor ([hardening] enabled, OSEP-0018). + // Config errors (unknown capability, reserved execve) are fatal; missing + // runtime support degrades and is reported on the capabilities endpoint. + if err := runtime.InitHardening(isoCfg); err != nil { + log.Error("hardening: %v", err) + return 1 + } + + // Start the eBPF observation layer ([ebpf] enabled, OSEP-0018 ยง5). + // The stub build reports disabled; the execd-ebpf variant attaches the + // exec/connect/privilege hooks. + { + ebpfState, ebpfMessage := ebpf.Init(isoCfg.Ebpf, os.Getenv("OPENSANDBOX_ID")) + runtime.SetEbpfState(runtime.LayerState{State: ebpfState, Message: ebpfMessage}) + } + // Probe isolation runtime capabilities. isolationProbe := isolation.Probe(isolation.ProbeConfig{ UpperRoot: isoCfg.UpperRoot, @@ -79,6 +96,13 @@ func run() int { log.Init(flag.ServerLogLevel) + if flag.InitMode { + // OSEP-0018: execd is the sandbox init. Must start after the startup + // probes (which run short-lived children via cmd.Run) so the reaper is + // the only wait4 caller from here on. + runtime.StartInitMode(flag.Args()) + } + ctrl := controller.InitCodeRunner() // Always store probe result for capabilities endpoint. @@ -134,10 +158,16 @@ func run() int { return 1 } log.Info("execd listening on %s (IPv4)", addr) + // In init mode SIGTERM belongs to the init lifecycle (forward + graceful + // shutdown with the entrypoint's exit status); only SIGINT cancels the + // HTTP server there. + ctxSignals := []os.Signal{os.Interrupt} + if !flag.InitMode { + ctxSignals = append(ctxSignals, syscall.SIGTERM) + } serverCtx, stopSignals := signal.NotifyContext( context.Background(), - os.Interrupt, - syscall.SIGTERM, + ctxSignals..., ) defer stopSignals() if err := serveHTTPUntilShutdown(serverCtx, listener, engine); err != nil { diff --git a/components/execd/native/launcher.c b/components/execd/native/launcher.c new file mode 100644 index 000000000..32ad79827 --- /dev/null +++ b/components/execd/native/launcher.c @@ -0,0 +1,519 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * opensandbox-launcher is the pre-exec hardening prelude (OSEP-0018 ยง4). + * + * execd execs it as the child's argv[0]; it applies the privilege floor in + * the child between fork and exec, then execve(2)s the real user command: + * + * 1. unset execd's credential env vars + * 2. prctl(PR_SET_KEEPCAPS) (keep caps across the uid change) + * 3. drop every bounding-set cap not kept (needs CAP_SETPCAP) + * 4. prctl(PR_SET_NO_NEW_PRIVS) + * 5. setgroups + setgid + setuid (identity drop) + * 6. capset permitted/effective to the kept caps; PR_CAP_AMBIENT_RAISE each + * 7. seccomp BPF filter (SECCOMP_MODE_FILTER) โ€” LAST, so it never blocks + * the launcher's own setup syscalls above + * 8. execve(user argv) + * + * The order is pinned by Linux semantics (see the OSEP). Every step is + * best-effort and fail-open: a missing prerequisite is logged to stderr and + * skipped, never fatal โ€” matching execd's degradation contract. The policy is + * read from an inherited descriptor; any malformed policy exits without + * executing the workload. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define POLICY_MAGIC 0x4f534258u /* "OSBX" */ +#define POLICY_VERSION 1u + +#define FLAG_UID_DROP 0x1u +#define FLAG_CAP_DROP 0x2u + +#define LAUNCH_FAILURE 125 +#define EXEC_FAILURE 126 + +/* Keep in sync with policyHeader in hardening_linux.go. */ +struct policy_header { + uint32_t magic; + uint32_t version; + uint32_t flags; + uint32_t uid; + uint32_t gid; + uint32_t n_groups; + uint32_t n_keepcaps; + uint32_t n_env; + uint32_t seccomp_len; + uint32_t landlock_len; +}; + +#define MAX_CAPS 64 + +/* + * Landlock ABI (stable since Linux 5.13): syscalls 444-446 and the fs access + * bits below are part of the kernel UAPI and are defined here so the helper + * does not depend on a specific linux/landlock.h. + */ +#define LL_EXECUTE (1ULL << 0) +#define LL_WRITE_FILE (1ULL << 1) +#define LL_READ_FILE (1ULL << 2) +#define LL_READ_DIR (1ULL << 3) +#define LL_REMOVE_DIR (1ULL << 4) +#define LL_REMOVE_FILE (1ULL << 5) +#define LL_MAKE_CHAR (1ULL << 6) +#define LL_MAKE_DIR (1ULL << 7) +#define LL_MAKE_REG (1ULL << 8) +#define LL_MAKE_SOCK (1ULL << 9) +#define LL_MAKE_FIFO (1ULL << 10) +#define LL_MAKE_BLOCK (1ULL << 11) +#define LL_MAKE_SYM (1ULL << 12) +#define LL_REFER (1ULL << 13) /* ABI >= 2 */ +#define LL_TRUNCATE (1ULL << 14) /* ABI >= 3 */ + +#define LANDLOCK_CREATE_RULESET_VERSION 1 +#define LANDLOCK_CREATE_RULESET 0 +#define LANDLOCK_RULE_PATH_BENEATH 1 + +struct ll_ruleset_attr { + uint64_t handled_access_fs; +}; + +struct ll_path_beneath_attr { + uint64_t allowed_access; + int32_t parent_fd; +}; + +/* Landlock rule from the policy: a path plus the access bits to grant. + * required rules must all install, or confinement is skipped entirely. */ +struct ll_rule { + uint64_t access; + const char *path; + int required; +}; + +static void log_err(const char *msg, int err) +{ + fprintf(stderr, "opensandbox-launcher: %s: %s\n", msg, strerror(err)); +} + +static void fail(int fd, const char *msg) +{ + if (fd >= 0) + (void)close(fd); + fprintf(stderr, "opensandbox-launcher: %s\n", msg); + _exit(LAUNCH_FAILURE); +} + +/* Read exactly n bytes from fd, retrying on EINTR and short reads. */ +static int read_exact(int fd, void *buf, size_t n) +{ + uint8_t *p = buf; + size_t left = n; + + while (left > 0) { + ssize_t got = read(fd, p, left); + if (got < 0) { + if (errno == EINTR) + continue; + return -1; + } + if (got == 0) + return -1; /* EOF before the expected length */ + p += got; + left -= (size_t)got; + } + return 0; +} + +/* Capability ABI v3: two data blocks covering caps 0..63. */ +struct cap_header { + uint32_t version; + int pid; +}; + +struct cap_data { + uint32_t effective; + uint32_t permitted; + uint32_t inheritable; +}; + +static int capset_all(uint64_t kept) +{ + struct cap_header header; + struct cap_data data[2]; + + memset(&header, 0, sizeof(header)); + memset(data, 0, sizeof(data)); + header.version = _LINUX_CAPABILITY_VERSION_3; + header.pid = 0; + /* Capability ABI v3 uses two 32-bit words; caps above 31 (e.g. + * CAP_PERFMON, CAP_BPF) live in the second one. */ + data[0].effective = (uint32_t)kept; + data[0].permitted = (uint32_t)kept; + data[0].inheritable = (uint32_t)kept; + data[1].effective = (uint32_t)(kept >> 32); + data[1].permitted = (uint32_t)(kept >> 32); + data[1].inheritable = (uint32_t)(kept >> 32); + return syscall(SYS_capset, &header, data); +} + +static int raise_ambient(uint32_t cap) +{ + return prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, (unsigned long)cap, 0, 0); +} + +/* Trim an access mask to what the detected Landlock ABI supports. */ +static uint64_t ll_trim_access(uint64_t access, long abi) +{ + if (abi < 2) + access &= ~(uint64_t)LL_REFER; + if (abi < 3) + access &= ~(uint64_t)LL_TRUNCATE; + return access; +} + +/* Apply the Landlock filesystem confinement from the policy (fail-open). */ +static void apply_landlock(const struct ll_rule *rules, size_t n_rules) +{ + long abi; + int ruleset = -1; + uint64_t handled; + + if (n_rules == 0) + return; + + abi = syscall(444, NULL, 0, + LANDLOCK_CREATE_RULESET_VERSION); + if (abi < 1) { + log_err("landlock unavailable (kernel ABI < 1)", ENOSYS); + return; + } + + handled = LL_EXECUTE | LL_WRITE_FILE | LL_READ_FILE | LL_READ_DIR | + LL_REMOVE_DIR | LL_REMOVE_FILE | LL_MAKE_CHAR | LL_MAKE_DIR | + LL_MAKE_REG | LL_MAKE_SOCK | LL_MAKE_FIFO | LL_MAKE_BLOCK | + LL_MAKE_SYM | LL_REFER | LL_TRUNCATE; + handled = ll_trim_access(handled, abi); + + { + struct ll_ruleset_attr attr = { .handled_access_fs = handled }; + + ruleset = (int)syscall(444, &attr, + sizeof(attr), LANDLOCK_CREATE_RULESET); + if (ruleset < 0) { + log_err("landlock_create_ruleset", errno); + return; + } + } + + { + int rule_failed = 0; + + for (size_t i = 0; i < n_rules; i++) { + int fd; + uint64_t access = ll_trim_access(rules[i].access, abi); + + if (access == 0) + continue; + /* O_PATH: no read/write rights needed to build the rule. */ + fd = open(rules[i].path, O_PATH | O_CLOEXEC); + if (fd < 0) { + fprintf(stderr, + "opensandbox-launcher: landlock: skip %s: %s\n", + rules[i].path, strerror(errno)); + rule_failed |= rules[i].required; + continue; + } + { + struct ll_path_beneath_attr path_attr = { + .allowed_access = access, + .parent_fd = fd, + }; + + if (syscall(445, ruleset, + LANDLOCK_RULE_PATH_BENEATH, &path_attr, 0) != 0) { + fprintf(stderr, + "opensandbox-launcher: landlock: add_rule %s: %s\n", + rules[i].path, strerror(errno)); + rule_failed |= rules[i].required; + } + } + close(fd); + } + + if (rule_failed) { + /* Fail closed per launch: a missing required rule would silently + * deny access the operator explicitly granted. Skip confinement + * entirely and report instead of restricting with a narrower + * policy. Best-effort (mount-expansion) failures are logged + * above and do not abort. */ + fprintf(stderr, + "opensandbox-launcher: landlock: rule installation failed; " + "skipping filesystem confinement for this launch\n"); + close(ruleset); + return; + } + } + + if (syscall(446, ruleset, 0) != 0) { + log_err("landlock_restrict_self", errno); + return; + } + close(ruleset); + /* Irrevocable: from here on the process is confined to the granted set. */ +} + +int main(int argc, char **argv) +{ + int policy_fd; + struct policy_header hdr; + uint32_t keepcaps[MAX_CAPS]; + uint32_t groups[MAX_CAPS]; + char *env_names[MAX_CAPS]; + int n_env; + size_t env_budget; + struct sock_filter *filter = NULL; + struct sock_fprog prog; + struct ll_rule *ll_rules = NULL; + size_t n_ll_rules = 0; + + if (argc < 4 || strcmp(argv[2], "--") != 0) + fail(-1, "usage: opensandbox-launcher -- "); + + { + char *end = NULL; + long parsed; + + errno = 0; + parsed = strtol(argv[1], &end, 10); + if (errno != 0 || end == argv[1] || *end != '\0' || + parsed < 3 || parsed > INT32_MAX) + fail(-1, "invalid policy descriptor"); + policy_fd = (int)parsed; + } + + if (fcntl(policy_fd, F_GETFD) < 0) + fail(policy_fd, "policy descriptor is not open"); + + if (read_exact(policy_fd, &hdr, sizeof(hdr)) != 0) + fail(policy_fd, "truncated policy header"); + if (hdr.magic != POLICY_MAGIC || hdr.version != POLICY_VERSION) + fail(policy_fd, "invalid policy header"); + + if (hdr.n_groups > MAX_CAPS) + fail(policy_fd, "too many supplementary groups"); + if (hdr.n_keepcaps > MAX_CAPS) + fail(policy_fd, "too many kept capabilities"); + if (hdr.n_env > MAX_CAPS) + fail(policy_fd, "too many environment names"); + if (hdr.seccomp_len % sizeof(struct sock_filter) != 0) + fail(policy_fd, "seccomp filter length is not a multiple of sock_filter"); + + if (hdr.n_groups > 0 && + read_exact(policy_fd, groups, hdr.n_groups * sizeof(uint32_t)) != 0) + fail(policy_fd, "truncated group list"); + if (hdr.n_keepcaps > 0 && + read_exact(policy_fd, keepcaps, hdr.n_keepcaps * sizeof(uint32_t)) != 0) + fail(policy_fd, "truncated capability list"); + + /* Env names are NUL-terminated strings packed back to back. Bound the + * total budget so a malicious policy cannot exhaust the stack. */ + env_budget = hdr.n_env * 64u; + if (env_budget > 4096u) + fail(policy_fd, "environment names exceed the policy budget"); + n_env = 0; + while (n_env < (int)hdr.n_env) { + static char env_buf[4096]; + size_t off = 0; + + while (off + 1 < sizeof(env_buf)) { + if (read(policy_fd, &env_buf[off], 1) != 1) + fail(policy_fd, "truncated environment name"); + if (env_buf[off] == '\0') + break; + off++; + } + if (off + 1 >= sizeof(env_buf) && env_buf[off] != '\0') + fail(policy_fd, "environment name too long"); + env_buf[off] = '\0'; + env_names[n_env++] = strdup(env_buf); + if (env_names[n_env - 1] == NULL) + fail(policy_fd, "out of memory for environment name"); + } + + if (hdr.seccomp_len > 0) { + filter = (struct sock_filter *)malloc(hdr.seccomp_len); + if (filter == NULL) + fail(policy_fd, "out of memory for seccomp filter"); + if (read_exact(policy_fd, filter, hdr.seccomp_len) != 0) + fail(policy_fd, "truncated seccomp filter"); + } + + /* Landlock rules: repeated { u8 required; u64 access; u16 pathlen; + * path bytes }. Allocated dynamically: mount-heavy pods can exceed any + * fixed cap after the mount-expansion in the policy. */ + if (hdr.landlock_len > 0) { + size_t left = hdr.landlock_len; + size_t max_rules = hdr.landlock_len / (sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint16_t)); + + ll_rules = (struct ll_rule *)calloc(max_rules + 1, sizeof(struct ll_rule)); + if (ll_rules == NULL) + fail(policy_fd, "out of memory for landlock rules"); + + while (left > 0) { + uint8_t required; + uint64_t access; + uint16_t pathlen; + char *path; + + if (left < sizeof(required) + sizeof(access) + sizeof(pathlen)) + fail(policy_fd, "truncated landlock rule header"); + if (read_exact(policy_fd, &required, sizeof(required)) != 0 || + read_exact(policy_fd, &access, sizeof(access)) != 0 || + read_exact(policy_fd, &pathlen, sizeof(pathlen)) != 0) + fail(policy_fd, "truncated landlock rule header"); + left -= sizeof(required) + sizeof(access) + sizeof(pathlen); + if (pathlen == 0 || pathlen > 4096) + fail(policy_fd, "invalid landlock path length"); + if (left < pathlen) + fail(policy_fd, "truncated landlock path"); + path = malloc((size_t)pathlen + 1); + if (path == NULL) + fail(policy_fd, "out of memory for landlock path"); + if (read_exact(policy_fd, path, pathlen) != 0) + fail(policy_fd, "truncated landlock path"); + path[pathlen] = '\0'; + left -= pathlen; + ll_rules[n_ll_rules].access = access; + ll_rules[n_ll_rules].path = path; + ll_rules[n_ll_rules].required = required != 0; + n_ll_rules++; + } + if (left > 0) + fail(policy_fd, "truncated landlock rules"); + } + + if (close(policy_fd) != 0) + _exit(LAUNCH_FAILURE); + + /* 1. Strip execd's credential/config env from the workload. */ + for (int i = 0; i < n_env; i++) { + unsetenv(env_names[i]); + free(env_names[i]); + } + + if (hdr.flags & FLAG_CAP_DROP) { + int caps_dropped = 0; + + /* 2. Keep caps across the identity change (step 5 clears them). */ + if (prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) != 0) + log_err("PR_SET_KEEPCAPS", errno); + + /* 3. Trim the bounding set while CAP_SETPCAP is still held. */ + for (int cap = 0; cap <= CAP_LAST_CAP; cap++) { + int keep = 0; + + for (uint32_t k = 0; k < hdr.n_keepcaps; k++) { + if ((uint32_t)cap == keepcaps[k]) { + keep = 1; + break; + } + } + if (!keep && prctl(PR_CAPBSET_DROP, (unsigned long)cap, 0, 0, 0) == 0) + caps_dropped++; + else if (!keep && errno != EPERM && errno != EINVAL) + log_err("PR_CAPBSET_DROP", errno); + } + (void)caps_dropped; + } + + /* 4. No new privileges: nothing below can regain what the launcher drops. */ + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + log_err("PR_SET_NO_NEW_PRIVS", errno); + + if (hdr.flags & FLAG_UID_DROP) { + /* 5. Identity change. The requested identity is part of the launch + * contract: a same-uid re-apply always succeeds, but a foreign + * target that cannot be applied must abort the launch (os/exec + * would fail) instead of silently running as the wrong user. */ + if (hdr.n_groups > 0) { + gid_t *gids = (gid_t *)malloc(hdr.n_groups * sizeof(gid_t)); + + if (gids == NULL) + fail(-1, "out of memory for supplementary groups"); + for (uint32_t g = 0; g < hdr.n_groups; g++) + gids[g] = (gid_t)groups[g]; + if (setgroups(hdr.n_groups, gids) != 0) + fail(-1, "setgroups: cannot apply requested supplementary groups"); + free(gids); + } else if (setgroups(0, NULL) != 0) { + fail(-1, "setgroups: cannot clear supplementary groups"); + } + if (setgid((gid_t)hdr.gid) != 0) + fail(-1, "setgid: cannot apply requested gid"); + if (setuid((uid_t)hdr.uid) != 0) + fail(-1, "setuid: cannot apply requested uid"); + } + + if (hdr.flags & FLAG_CAP_DROP) { + /* 6. Final cap sets + ambient raise so kept caps survive execve. */ + uint64_t kept = 0; + + for (uint32_t k = 0; k < hdr.n_keepcaps; k++) + kept |= (UINT64_C(1) << keepcaps[k]); + if (capset_all(kept) != 0) + log_err("capset", errno); + for (uint32_t k = 0; k < hdr.n_keepcaps; k++) { + if (raise_ambient(keepcaps[k]) != 0) + log_err("PR_CAP_AMBIENT_RAISE", errno); + } + } + + /* 6. Landlock filesystem confinement, before seccomp. */ + apply_landlock(ll_rules, n_ll_rules); + free(ll_rules); + + /* 7. Seccomp LAST: it must never block the setup above, and execve + * (which the Go side reserves from the deny list) is still allowed. */ + if (filter != NULL) { + prog.len = (unsigned short)(hdr.seccomp_len / sizeof(struct sock_filter)); + prog.filter = filter; + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog) != 0) + log_err("PR_SET_SECCOMP", errno); + free(filter); + } + + /* 8. Exec the real workload. */ + execvp(argv[3], &argv[3]); + fprintf(stderr, "opensandbox-launcher: exec %s: %s\n", argv[3], strerror(errno)); + _exit(EXEC_FAILURE); +} diff --git a/components/execd/pkg/ebpf/audit.go b/components/execd/pkg/ebpf/audit.go new file mode 100644 index 000000000..a6338f586 --- /dev/null +++ b/components/execd/pkg/ebpf/audit.go @@ -0,0 +1,442 @@ +//go:build ebpf + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// OSEP-0018 ยง5: opt-in eBPF observation of exec / connect / privilege +// events, scoped to the sandbox cgroup, written as JSONL to a rotating +// audit file. Compiled only into the execd-ebpf build variant (CGO + +// cilium/ebpf); the default static image never contains this code. + +package ebpf + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/ringbuf" + "github.com/cilium/ebpf/rlimit" + "gopkg.in/natefinch/lumberjack.v2" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" + "github.com/alibaba/opensandbox/execd/pkg/log" +) + +const ( + defaultAuditFile = "/var/log/opensandbox/ebpf-audit.jsonl" + + // Capability numbers (linux/capability.h). + capBpf = 39 + capPerfmon = 38 +) + +// Event is the JSONL record: a stable common envelope plus per-kind fields +// (OSEP-0018 ยง5). +type Event struct { + TS string `json:"ts"` + Event string `json:"event"` // exec | connect | privilege + SandboxID string `json:"sandbox_id"` + PID uint32 `json:"pid"` + Comm string `json:"comm"` + + // exec + Filename string `json:"filename,omitempty"` + PPID uint32 `json:"ppid,omitempty"` + + // connect + DstIP string `json:"dst_ip,omitempty"` + DstPort uint16 `json:"dst_port,omitempty"` + Proto string `json:"proto,omitempty"` + + // privilege + OldUID uint32 `json:"old_uid,omitempty"` + NewUID uint32 `json:"new_uid,omitempty"` + OldGID uint32 `json:"old_gid,omitempty"` + NewGID uint32 `json:"new_gid,omitempty"` + CapAdded []string `json:"cap_added,omitempty"` +} + +// Packed BPF event sizes (must match audit.bpf.c). +const ( + sizeEventExec = 4 + 4 + 16 + 64 + sizeEventConnect = 4 + 16 + 16 + 2 + sizeEventPrivilege = 4 + 16 + 4*4 + 8 +) + +// Observer consumes ringbuf events and appends them to the audit file. +type Observer struct { + mu sync.Mutex + logger *lumberjack.Logger + sandboxID string + kinds map[string]bool + reader *ringbuf.Reader + objs *auditObjects + links []link.Link + closed chan struct{} + closeOnce sync.Once +} + +// Init activates the observer from the isolation config. The returned state +// and message describe what is actually enforced (for the capabilities +// endpoint). +func Init(cfg *isolation.EbpfConfig, sandboxID string) (state, message string) { + disabled := func(msg string) (string, string) { + return "disabled", msg + } + if cfg == nil || !cfg.Enabled { + return disabled("eBPF observation is not enabled ([ebpf] enabled = false)") + } + if sandboxID == "" { + return "unsupported", + "eBPF observation cannot attribute audit records: OPENSANDBOX_ID is not set " + + "(pool fast-path allocations without a task template cannot inject it); " + + "set it via the runtime env to enable sandbox_id attribution" + } + if !effectiveCapsHave(capBpf) || !effectiveCapsHave(capPerfmon) { + return "unsupported", + "eBPF observation requires CAP_BPF + CAP_PERFMON (execd-ebpf build with a privileged container)" + } + if _, err := os.Stat("/sys/kernel/btf/vmlinux"); err != nil { + return "unsupported", + "eBPF observation requires a BTF-capable kernel (no /sys/kernel/btf/vmlinux); under gVisor/Kata the host kernel is not attachable" + } + cgroupID, err := currentCgroupID() + if err != nil { + return "degraded", fmt.Sprintf("eBPF observation cannot scope to the sandbox cgroup: %v", err) + } + + observer, err := newObserver(cfg, sandboxID, cgroupID) + if err != nil { + return "degraded", fmt.Sprintf("eBPF observation failed to start: %v", err) + } + observer.start() + return "active", fmt.Sprintf("eBPF observation active (cgroup %d, audit file %s)", cgroupID, observer.logger.Filename) +} + +func newObserver(cfg *isolation.EbpfConfig, sandboxID string, cgroupID uint64) (*Observer, error) { + _ = rlimit.RemoveMemlock() + + spec, err := loadAudit() + if err != nil { + return nil, fmt.Errorf("load audit programs: %w", err) + } + // Pin the cgroup filter; events outside the sandbox are dropped. + if err := spec.RewriteConstants(map[string]interface{}{ + "target_cgroup": cgroupID, + }); err != nil { + return nil, fmt.Errorf("set audit cgroup filter: %w", err) + } + + var objs auditObjects + if err := spec.LoadAndAssign(&objs, nil); err != nil { + return nil, fmt.Errorf("assign audit programs: %w", err) + } + + auditFile := cfg.AuditFile + if auditFile == "" { + auditFile = defaultAuditFile + } + if err := os.MkdirAll(dirOf(auditFile), 0o755); err != nil { + objs.Close() + return nil, fmt.Errorf("create audit dir: %w", err) + } + logger := &lumberjack.Logger{ + Filename: auditFile, + MaxSize: 100, // MB + MaxBackups: 3, + MaxAge: 7, // days + } + + var links []link.Link + attached := map[string]bool{} + attach := func(kind string, prog *ebpf.Program, attachFn func() (link.Link, error)) { + if prog == nil { + return + } + l, err := attachFn() + if err != nil { + log.Warn("ebpf: attach %s: %v", kind, err) + return + } + links = append(links, l) + attached[kind] = true + } + attach("exec", objs.OnExec, func() (link.Link, error) { + return link.Tracepoint("sched", "sched_process_exec", objs.OnExec, nil) + }) + attach("connect", objs.OnConnect, func() (link.Link, error) { + return link.Tracepoint("sock", "inet_sock_set_state", objs.OnConnect, nil) + }) + attach("privilege", objs.OnCommitCreds, func() (link.Link, error) { + return link.Kprobe("commit_creds", objs.OnCommitCreds, nil) + }) + + reader, err := ringbuf.NewReader(objs.Events) + if err != nil { + for _, l := range links { + _ = l.Close() + } + objs.Close() + return nil, fmt.Errorf("ringbuf reader: %w", err) + } + + kinds := map[string]bool{} + for _, kind := range cfg.Observe { + kinds[kind] = true + } + if len(kinds) == 0 { + for _, kind := range []string{"exec", "connect", "privilege"} { + kinds[kind] = true + } + } + for kind := range kinds { + if !attached[kind] { + // Release every already-attached hook and the ringbuf reader so + // partially attached programs do not stay live after the + // degraded report. + for _, l := range links { + _ = l.Close() + } + reader.Close() + objs.Close() + return nil, fmt.Errorf("requested observer hook %q could not be attached", kind) + } + } + + return &Observer{ + logger: logger, + sandboxID: sandboxID, + kinds: kinds, + reader: reader, + objs: &objs, + links: links, + closed: make(chan struct{}), + }, nil +} + +func (o *Observer) start() { + go func() { + defer o.Close() + for { + record, err := o.reader.Read() + if err != nil { + if err == ringbuf.ErrClosed { + return + } + log.Warn("ebpf: ringbuf read: %v", err) + continue + } + o.handleRecord(record.RawSample) + } + }() +} + +// Close stops the observer and releases all BPF resources. +func (o *Observer) Close() { + o.closeOnce.Do(func() { + close(o.closed) + _ = o.reader.Close() + for _, l := range o.links { + _ = l.Close() + } + if o.objs != nil { + o.objs.Close() + } + _ = o.logger.Close() + }) +} + +func (o *Observer) handleRecord(raw []byte) { + event, ok := decodeEvent(raw) + if !ok { + log.Warn("ebpf: unknown event size %d", len(raw)) + return + } + if !o.kinds[event.Event] { + return + } + event.SandboxID = o.sandboxID + line, err := json.Marshal(event) + if err != nil { + log.Warn("ebpf: marshal event: %v", err) + return + } + o.mu.Lock() + if _, err := o.logger.Write(append(line, '\n')); err != nil { + log.Error("ebpf: audit write failed: %v", err) + } + o.mu.Unlock() +} + +func decodeEvent(raw []byte) (Event, bool) { + now := time.Now().UTC().Format(time.RFC3339) + switch len(raw) { + case sizeEventExec: + ev := Event{TS: now, Event: "exec", PID: binary.LittleEndian.Uint32(raw[0:4])} + ev.PPID = binary.LittleEndian.Uint32(raw[4:8]) + ev.Comm = cstring(raw[8:24]) + ev.Filename = cstring(raw[24:88]) + return ev, true + case sizeEventConnect: + ev := Event{TS: now, Event: "connect", PID: binary.LittleEndian.Uint32(raw[0:4])} + ev.Comm = cstring(raw[4:20]) + ip := raw[20:36] + ev.DstIP = formatIP(ip) + ev.DstPort = binary.BigEndian.Uint16(raw[36:38]) + ev.Proto = "tcp" + return ev, true + case sizeEventPrivilege: + ev := Event{TS: now, Event: "privilege", PID: binary.LittleEndian.Uint32(raw[0:4])} + ev.Comm = cstring(raw[4:20]) + ev.OldUID = binary.LittleEndian.Uint32(raw[20:24]) + ev.NewUID = binary.LittleEndian.Uint32(raw[24:28]) + ev.OldGID = binary.LittleEndian.Uint32(raw[28:32]) + ev.NewGID = binary.LittleEndian.Uint32(raw[32:36]) + ev.CapAdded = capsFromBits(binary.LittleEndian.Uint64(raw[36:44])) + return ev, true + default: + return Event{}, false + } +} + +func cstring(b []byte) string { + if i := strings.IndexByte(string(b), 0); i >= 0 { + return string(b[:i]) + } + return string(b) +} + +func formatIP(raw []byte) string { + // IPv4 is stored in the last 4 bytes of the 16-byte field. + if raw[0] == 0 && raw[1] == 0 && raw[2] == 0 && raw[3] == 0 && + raw[4] == 0 && raw[5] == 0 && raw[6] == 0 && raw[7] == 0 && + raw[8] == 0 && raw[9] == 0 && raw[10] == 0xff && raw[11] == 0xff { + return net.IPv4(raw[12], raw[13], raw[14], raw[15]).String() + } + return net.IP(raw).String() +} + +var capNames = []string{ + "CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", + "CAP_FSETID", "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", "CAP_NET_RAW", "CAP_IPC_LOCK", "CAP_IPC_OWNER", + "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", + "CAP_LEASE", "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", + "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN", "CAP_SYSLOG", "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ", "CAP_PERFMON", "CAP_BPF", + "CAP_CHECKPOINT_RESTORE", +} + +func capsFromBits(bits uint64) []string { + var caps []string + for i, name := range capNames { + if bits&(1<= 3 && fields[2] == "cgroup2" { + return fields[1], nil + } + } + return "", fmt.Errorf("no cgroup v2 mount found") +} + +func dirOf(path string) string { + if i := strings.LastIndexByte(path, '/'); i > 0 { + return path[:i] + } + return "." +} diff --git a/components/execd/pkg/ebpf/audit_bpfeb.go b/components/execd/pkg/ebpf/audit_bpfeb.go new file mode 100644 index 000000000..8bdb46ca5 --- /dev/null +++ b/components/execd/pkg/ebpf/audit_bpfeb.go @@ -0,0 +1,125 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build ebpf && (mips || mips64 || ppc64 || s390x) + +package ebpf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadAudit returns the embedded CollectionSpec for audit. +func loadAudit() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_AuditBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load audit: %w", err) + } + + return spec, err +} + +// loadAuditObjects loads audit and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *auditObjects +// *auditPrograms +// *auditMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadAuditObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadAudit() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// auditSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditSpecs struct { + auditProgramSpecs + auditMapSpecs +} + +// auditSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditProgramSpecs struct { + OnCommitCreds *ebpf.ProgramSpec `ebpf:"on_commit_creds"` + OnConnect *ebpf.ProgramSpec `ebpf:"on_connect"` + OnExec *ebpf.ProgramSpec `ebpf:"on_exec"` +} + +// auditMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditMapSpecs struct { + Events *ebpf.MapSpec `ebpf:"events"` +} + +// auditObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditObjects struct { + auditPrograms + auditMaps +} + +func (o *auditObjects) Close() error { + return _AuditClose( + &o.auditPrograms, + &o.auditMaps, + ) +} + +// auditMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditMaps struct { + Events *ebpf.Map `ebpf:"events"` +} + +func (m *auditMaps) Close() error { + return _AuditClose( + m.Events, + ) +} + +// auditPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditPrograms struct { + OnCommitCreds *ebpf.Program `ebpf:"on_commit_creds"` + OnConnect *ebpf.Program `ebpf:"on_connect"` + OnExec *ebpf.Program `ebpf:"on_exec"` +} + +func (p *auditPrograms) Close() error { + return _AuditClose( + p.OnCommitCreds, + p.OnConnect, + p.OnExec, + ) +} + +func _AuditClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed audit_bpfeb.o +var _AuditBytes []byte diff --git a/components/execd/pkg/ebpf/audit_bpfeb.o b/components/execd/pkg/ebpf/audit_bpfeb.o new file mode 100644 index 000000000..8a62674b7 Binary files /dev/null and b/components/execd/pkg/ebpf/audit_bpfeb.o differ diff --git a/components/execd/pkg/ebpf/audit_bpfel.go b/components/execd/pkg/ebpf/audit_bpfel.go new file mode 100644 index 000000000..b750c5b0a --- /dev/null +++ b/components/execd/pkg/ebpf/audit_bpfel.go @@ -0,0 +1,125 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build ebpf && (386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64) + +package ebpf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadAudit returns the embedded CollectionSpec for audit. +func loadAudit() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_AuditBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load audit: %w", err) + } + + return spec, err +} + +// loadAuditObjects loads audit and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *auditObjects +// *auditPrograms +// *auditMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadAuditObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadAudit() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// auditSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditSpecs struct { + auditProgramSpecs + auditMapSpecs +} + +// auditSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditProgramSpecs struct { + OnCommitCreds *ebpf.ProgramSpec `ebpf:"on_commit_creds"` + OnConnect *ebpf.ProgramSpec `ebpf:"on_connect"` + OnExec *ebpf.ProgramSpec `ebpf:"on_exec"` +} + +// auditMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type auditMapSpecs struct { + Events *ebpf.MapSpec `ebpf:"events"` +} + +// auditObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditObjects struct { + auditPrograms + auditMaps +} + +func (o *auditObjects) Close() error { + return _AuditClose( + &o.auditPrograms, + &o.auditMaps, + ) +} + +// auditMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditMaps struct { + Events *ebpf.Map `ebpf:"events"` +} + +func (m *auditMaps) Close() error { + return _AuditClose( + m.Events, + ) +} + +// auditPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadAuditObjects or ebpf.CollectionSpec.LoadAndAssign. +type auditPrograms struct { + OnCommitCreds *ebpf.Program `ebpf:"on_commit_creds"` + OnConnect *ebpf.Program `ebpf:"on_connect"` + OnExec *ebpf.Program `ebpf:"on_exec"` +} + +func (p *auditPrograms) Close() error { + return _AuditClose( + p.OnCommitCreds, + p.OnConnect, + p.OnExec, + ) +} + +func _AuditClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed audit_bpfel.o +var _AuditBytes []byte diff --git a/components/execd/pkg/ebpf/audit_bpfel.o b/components/execd/pkg/ebpf/audit_bpfel.o new file mode 100644 index 000000000..6a5836bae Binary files /dev/null and b/components/execd/pkg/ebpf/audit_bpfel.o differ diff --git a/components/execd/pkg/ebpf/audit_stub.go b/components/execd/pkg/ebpf/audit_stub.go new file mode 100644 index 000000000..86462e78b --- /dev/null +++ b/components/execd/pkg/ebpf/audit_stub.go @@ -0,0 +1,34 @@ +//go:build !ebpf + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// eBPF observation is compiled only into the execd-ebpf variant (CGO + +// cilium/ebpf). The default static build reports it as not configured. + +package ebpf + +import "github.com/alibaba/opensandbox/execd/pkg/isolation" + +// Init reports the observation state for this build. A non-ebpf build can +// never provide the hooks, so a requested [ebpf] enabled is "unsupported", +// not merely "disabled" (not configured). +func Init(cfg *isolation.EbpfConfig, sandboxID string) (state, message string) { + if cfg != nil && cfg.Enabled { + return "unsupported", + "eBPF observation requested but this build lacks the execd-ebpf variant (CGO + cilium/ebpf); default image unchanged" + } + return "disabled", + "eBPF observation is not enabled ([ebpf] enabled = false)" +} diff --git a/components/execd/pkg/ebpf/audit_test.go b/components/execd/pkg/ebpf/audit_test.go new file mode 100644 index 000000000..535e892a4 --- /dev/null +++ b/components/execd/pkg/ebpf/audit_test.go @@ -0,0 +1,115 @@ +//go:build ebpf + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebpf + +import ( + "encoding/binary" + "testing" +) + +func TestDecodeExecEvent(t *testing.T) { + raw := make([]byte, sizeEventExec) + binary.LittleEndian.PutUint32(raw[0:4], 42) + binary.LittleEndian.PutUint32(raw[4:8], 1) + copy(raw[8:24], "python3") + copy(raw[24:88], "/usr/bin/python3") + ev, ok := decodeEvent(raw) + if !ok { + t.Fatal("decode failed") + } + if ev.Event != "exec" || ev.PID != 42 || ev.PPID != 1 { + t.Fatalf("exec envelope = %+v", ev) + } + if ev.Comm != "python3" || ev.Filename != "/usr/bin/python3" { + t.Fatalf("exec fields = %q/%q", ev.Comm, ev.Filename) + } + if ev.TS == "" || ev.SandboxID != "" { + t.Fatalf("envelope ts/sandbox_id = %q/%q", ev.TS, ev.SandboxID) + } +} + +func TestDecodeConnectEvent(t *testing.T) { + raw := make([]byte, sizeEventConnect) + binary.LittleEndian.PutUint32(raw[0:4], 7) + copy(raw[4:20], "curl") + // IPv4-mapped ::ffff:5d:b8:d8:22 (93.184.216.34) + copy(raw[20:24], []byte{0, 0, 0, 0}) + copy(raw[24:28], []byte{0, 0, 0, 0}) + copy(raw[28:32], []byte{0, 0, 0xff, 0xff}) + copy(raw[32:36], []byte{93, 184, 216, 34}) + binary.BigEndian.PutUint16(raw[36:38], 443) + + ev, ok := decodeEvent(raw) + if !ok { + t.Fatal("decode failed") + } + if ev.Event != "connect" || ev.PID != 7 || ev.Comm != "curl" { + t.Fatalf("connect envelope = %+v", ev) + } + if ev.DstIP != "93.184.216.34" || ev.DstPort != 443 || ev.Proto != "tcp" { + t.Fatalf("connect dst = %s:%d/%s", ev.DstIP, ev.DstPort, ev.Proto) + } +} + +func TestDecodeConnectEventIPv6(t *testing.T) { + raw := make([]byte, sizeEventConnect) + binary.LittleEndian.PutUint32(raw[0:4], 9) + copy(raw[4:20], "curl") + // 2606:4700::6810:84e5 stored big-endian + ip := []byte{0x26, 0x06, 0x47, 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0x68, 0x10, 0x84, 0xe5} + copy(raw[20:36], ip) + binary.BigEndian.PutUint16(raw[36:38], 80) + + ev, ok := decodeEvent(raw) + if !ok { + t.Fatal("decode failed") + } + if ev.DstIP != "2606:4700::6810:84e5" || ev.DstPort != 80 { + t.Fatalf("ipv6 dst = %s:%d", ev.DstIP, ev.DstPort) + } +} + +func TestDecodePrivilegeEvent(t *testing.T) { + raw := make([]byte, sizeEventPrivilege) + binary.LittleEndian.PutUint32(raw[0:4], 57) + copy(raw[4:20], "sudo") + binary.LittleEndian.PutUint32(raw[20:24], 1000) + binary.LittleEndian.PutUint32(raw[24:28], 0) + binary.LittleEndian.PutUint32(raw[28:32], 1000) + binary.LittleEndian.PutUint32(raw[32:36], 0) + binary.LittleEndian.PutUint64(raw[36:44], 1<<7) + + ev, ok := decodeEvent(raw) + if !ok { + t.Fatal("decode failed") + } + if ev.Event != "privilege" || ev.PID != 57 || ev.Comm != "sudo" { + t.Fatalf("privilege envelope = %+v", ev) + } + if ev.OldUID != 1000 || ev.NewUID != 0 { + t.Fatalf("uid change = %d -> %d", ev.OldUID, ev.NewUID) + } + if len(ev.CapAdded) != 1 || ev.CapAdded[0] != "CAP_SETUID" { + t.Fatalf("cap_added = %v", ev.CapAdded) + } +} + +func TestDecodeUnknownSize(t *testing.T) { + if _, ok := decodeEvent(make([]byte, 3)); ok { + t.Fatal("unknown size decoded") + } +} diff --git a/components/execd/pkg/ebpf/prog/audit.bpf.c b/components/execd/pkg/ebpf/prog/audit.bpf.c new file mode 100644 index 000000000..2ce37d5c9 --- /dev/null +++ b/components/execd/pkg/ebpf/prog/audit.bpf.c @@ -0,0 +1,185 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// OSEP-0018 ยง5: opt-in, observation-only audit of exec / connect / +// privilege events, scoped to the sandbox cgroup. Compiles with bpf2go +// (CO-RE); the generated bytecode is embedded in the execd-ebpf build +// variant only. + +// vmlinux.h provides every kernel type used below (including __u32/__u64), +// so it must be included before the libbpf helper headers. It is a +// build-time-only artifact โ€” a kernel BTF dump for CO-RE โ€” regenerated with +// `bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h` on a +// target kernel before running bpf2go, and is NOT committed (the generated +// bytecode embedded in the audit_bpf*.go bindings is what ships). +#include "vmlinux.h" +#include +#include +#include + +char LICENSE[] SEC("license") = "GPL"; + +#define AF_INET 2 +#define TCP_SYN_SENT 2 + +// Events ring buffer, consumed by the execd-ebpf process. +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 20); +} events SEC(".maps"); + +// Set from userspace to the sandbox cgroup id; events outside it are +// dropped (execd shares the sandbox cgroup with every workload process). +const volatile uint64_t target_cgroup; + +struct event_exec { + uint32_t pid; + uint32_t ppid; + char comm[16]; + char filename[64]; +} __attribute__((packed)); + +struct event_connect { + uint32_t pid; + char comm[16]; + uint32_t dst_ip[4]; // big-endian; IPv4 is stored in dst_ip[3] + uint16_t dst_port; // network byte order +} __attribute__((packed)); + +struct event_privilege { + uint32_t pid; + char comm[16]; + uint32_t old_uid; + uint32_t new_uid; + uint32_t old_gid; + uint32_t new_gid; + uint64_t cap_added; // caps in new_cred not in old_cred +} __attribute__((packed)); + +static __always_inline int emit(void *event, uint32_t size) +{ + if (target_cgroup && + bpf_get_current_cgroup_id() != target_cgroup) + return 0; + return bpf_ringbuf_output(&events, event, size, 0) == 0; +} + +SEC("tracepoint/sched/sched_process_exec") +int on_exec(struct trace_event_raw_sched_process_exec *ctx) +{ + struct event_exec ev = {}; + struct task_struct *task; + struct task_struct *parent; + + ev.pid = bpf_get_current_pid_tgid() >> 32; + task = (struct task_struct *)bpf_get_current_task(); + bpf_core_read(&parent, sizeof(parent), &task->real_parent); + bpf_core_read(&ev.ppid, sizeof(ev.ppid), &parent->pid); + bpf_get_current_comm(&ev.comm, sizeof(ev.comm)); + + // The raw trace event layout changed in 5.16: filename moved from an + // inline 1024-byte array to a __data_loc string. The tracepoint payload + // does not carry an argv array, so exec events report the filename only. + if (bpf_core_field_exists(ctx->__data_loc_filename)) { + uint32_t filename_loc = BPF_CORE_READ(ctx, __data_loc_filename); + + bpf_probe_read_str(&ev.filename, sizeof(ev.filename), + (void *)ctx + (filename_loc & 0xffff)); + } else { + // 5.10-5.15: filename is inline at a fixed offset after + // trace_entry(8) + pid(4) + old_pid(4). + bpf_probe_read_str(&ev.filename, sizeof(ev.filename), (void *)ctx + 16); + } + + emit(&ev, sizeof(ev)); + return 0; +} + +SEC("tracepoint/sock/inet_sock_set_state") +int on_connect(struct trace_event_raw_inet_sock_set_state *ctx) +{ + struct event_connect ev = {}; + struct sock *sk; + struct sock_common *skc; + + // Emit connect attempts (TCP_SYN_SENT) only. + if (BPF_CORE_READ(ctx, newstate) != TCP_SYN_SENT) + return 0; + + ev.pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&ev.comm, sizeof(ev.comm)); + + bpf_core_read(&sk, sizeof(sk), &ctx->skaddr); + skc = &sk->__sk_common; + if (BPF_CORE_READ(skc, skc_family) == AF_INET) { + __be32 daddr; + + bpf_core_read(&daddr, sizeof(daddr), &skc->skc_daddr); + // Store the IPv4-mapped ::ffff:a.b.c.d form so the userspace + // decoder can format plain dotted IPv4. + ev.dst_ip[0] = 0; + ev.dst_ip[1] = 0; + // On little-endian targets the u32 0xffff0000 occupies bytes + // 00 00 ff ff, i.e. the ::ffff: prefix at bytes 10-11 the Go + // decoder looks for. + ev.dst_ip[2] = 0xffff0000; + ev.dst_ip[3] = daddr; + } else { + bpf_core_read(ev.dst_ip, sizeof(ev.dst_ip), + &skc->skc_v6_daddr.in6_u.u6_addr32); + } + bpf_core_read(&ev.dst_port, sizeof(ev.dst_port), &skc->skc_dport); + + emit(&ev, sizeof(ev)); + return 0; +} + +SEC("kprobe/commit_creds") +int BPF_KPROBE(on_commit_creds, struct cred *new) +{ + struct event_privilege ev = {}; + struct task_struct *task; + struct cred *old; + + task = (struct task_struct *)bpf_get_current_task(); + bpf_core_read(&old, sizeof(old), &task->real_cred); + + ev.pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&ev.comm, sizeof(ev.comm)); + bpf_core_read(&ev.old_uid, sizeof(ev.old_uid), &old->uid.val); + bpf_core_read(&ev.old_gid, sizeof(ev.old_gid), &old->gid.val); + bpf_core_read(&ev.new_uid, sizeof(ev.new_uid), &new->uid.val); + bpf_core_read(&ev.new_gid, sizeof(ev.new_gid), &new->gid.val); + + // kernel_cap_t is 8 bytes in every kernel but its shape changed in + // 6.3 ({u32 cap[2]} -> {u64 val}); reading the field as raw 8 bytes + // is layout-agnostic and works on both. + { + uint64_t old_caps; + uint64_t new_caps; + + bpf_core_read(&old_caps, sizeof(old_caps), &old->cap_effective); + bpf_core_read(&new_caps, sizeof(new_caps), &new->cap_effective); + ev.cap_added = new_caps & ~old_caps; + } + + // Emit when identity or effective capabilities change (cap-only + // transitions such as file caps or ambient raises are audit-relevant). + if (ev.old_uid == ev.new_uid && ev.old_gid == ev.new_gid && + ev.cap_added == 0) + return 0; + + emit(&ev, sizeof(ev)); + return 0; +} diff --git a/components/execd/pkg/flag/flags.go b/components/execd/pkg/flag/flags.go index b55f0d530..af66dab13 100644 --- a/components/execd/pkg/flag/flags.go +++ b/components/execd/pkg/flag/flags.go @@ -42,4 +42,10 @@ var ( // IsolationConfigPath points to the TOML isolation config file. // Empty means use built-in defaults. IsolationConfigPath string + + // InitMode runs execd as the sandbox init: reap children, forward + // signals, and own the container lifecycle. Topology (PID 1 vs + // subreaper) is decided by bootstrap.sh via EXECD_INIT, which passes + // this flag when it execs into execd. + InitMode bool ) diff --git a/components/execd/pkg/flag/parser.go b/components/execd/pkg/flag/parser.go index f92f33a6d..685444d65 100644 --- a/components/execd/pkg/flag/parser.go +++ b/components/execd/pkg/flag/parser.go @@ -42,6 +42,7 @@ func InitFlags() { ApiGracefulShutdownTimeout = time.Second * 1 JupyterIdlePollInterval = 100 * time.Millisecond IsolationConfigPath = "" + InitMode = false // First, set default values from environment variables if jupyterFromEnv := os.Getenv(jupyterHostEnv); jupyterFromEnv != "" { @@ -95,6 +96,10 @@ func InitFlags() { } flag.StringVar(&IsolationConfigPath, "isolation-config", IsolationConfigPath, "Path to isolation TOML config file (default: built-in defaults)") + // Init mode must be enabled explicitly; bootstrap.sh passes it together + // with EXECD_INIT so the shell's exec/background decision stays in lockstep. + flag.BoolVar(&InitMode, "init", false, "Run as the sandbox init: reap children, forward signals, own the container lifecycle") + // Parse flags - these will override environment variables if provided flag.Parse() if JupyterIdlePollInterval <= 0 { @@ -106,3 +111,9 @@ func InitFlags() { log.Info("Jupyter server host is: %s", JupyterServerHost) log.Info("Jupyter server token is: %s", log.MaskToken(JupyterServerToken)) } + +// Args returns the non-flag arguments after flag.Parse โ€” in init mode this is +// the user command passed after "--" (e.g. `execd --init -- sh -c "..."`). +func Args() []string { + return flag.Args() +} diff --git a/components/execd/pkg/isolation/bwrap.go b/components/execd/pkg/isolation/bwrap.go index 09c11b483..69597475e 100644 --- a/components/execd/pkg/isolation/bwrap.go +++ b/components/execd/pkg/isolation/bwrap.go @@ -282,16 +282,6 @@ func bwrapWorkspaceSegment(opts WrapOptions) ([]string, error) { } } -// execdConfigEnvBlacklist enumerates execd's own configuration env vars. -// They are always stripped so execd's credentials never leak into the sandbox. -var execdConfigEnvBlacklist = []string{ - "EXECD_ACCESS_TOKEN", - "JUPYTER_HOST", - "JUPYTER_TOKEN", - "EXECD_ISOLATION_CONFIG", - "EXECD_ENVS", -} - func unsetExecdConfigEnv() []string { argv := make([]string, 0, 2*len(execdConfigEnvBlacklist)) for _, key := range execdConfigEnvBlacklist { diff --git a/components/execd/pkg/isolation/config.go b/components/execd/pkg/isolation/config.go index ddadf0d08..e9f0ac7b0 100644 --- a/components/execd/pkg/isolation/config.go +++ b/components/execd/pkg/isolation/config.go @@ -34,7 +34,39 @@ type Config struct { // Seccomp overrides the built-in syscall denylist. When nil (i.e. the // [seccomp] section is absent), the built-in denylist is used. When // present, Deny completely replaces the built-in list โ€” no merging. + // With [hardening] enabled, the same list becomes the workload's seccomp + // floor (the launcher's exec syscall, execve, is reserved and rejected). Seccomp *SeccompOverride `toml:"seccomp"` + + // Hardening enables the pre-exec privilege floor (OSEP-0018 ยง4): every + // user-code process is launched through the native launcher with reduced + // capabilities, no_new_privs, and the seccomp floor. Defaults to off. + Hardening *HardeningConfig `toml:"hardening"` + + // Landlock adds filesystem confinement (OSEP-0018 ยง5) on top of the + // hardening floor. Defaults to off. + Landlock *LandlockConfig `toml:"landlock"` + + // Ebpf enables exec/connect/privilege observation (OSEP-0018 ยง5), + // written as JSONL to a rotating audit file. Requires the execd-ebpf + // build variant. Defaults to off. + Ebpf *EbpfConfig `toml:"ebpf"` +} + +// execdConfigEnvBlacklist enumerates execd's own configuration env vars. +// They are always stripped so execd's credentials never leak into the +// workload; the hardening launcher unsets the same set before execve. +var execdConfigEnvBlacklist = []string{ + "EXECD_ACCESS_TOKEN", + "JUPYTER_HOST", + "JUPYTER_TOKEN", + "EXECD_ISOLATION_CONFIG", + "EXECD_ENVS", +} + +// ExecdConfigEnvBlacklist returns a copy of the execd config env names. +func ExecdConfigEnvBlacklist() []string { + return append([]string(nil), execdConfigEnvBlacklist...) } // SeccompOverride specifies a custom syscall denylist that replaces the @@ -43,6 +75,43 @@ type SeccompOverride struct { Deny []string `toml:"deny"` } +// HardeningConfig controls the pre-exec hardening floor. +type HardeningConfig struct { + // Enabled turns the floor on: init + cap-drop + no_new_privs + seccomp + // for every user-code launch, via the opensandbox-launcher helper. + Enabled bool `toml:"enabled"` + // KeepCapabilities lists capabilities the workload retains (raised in + // the ambient set). Default: drop all. + KeepCapabilities []string `toml:"keep_capabilities"` +} + +// LandlockConfig controls Landlock filesystem confinement (OSEP-0018 ยง5). +type LandlockConfig struct { + // Enabled applies a Landlock allowlist to user-code processes, on top + // of the [hardening] floor. + Enabled bool `toml:"enabled"` + // ExtraWritable grants read+write (and file creation) beneath extra + // paths beyond the built-in set (system paths, /proc/self, /tmp, /run, + // allowed_writable). + ExtraWritable []string `toml:"extra_writable"` + // ExtraReadable grants read+exec beneath extra paths beyond the + // built-in read set. + ExtraReadable []string `toml:"extra_readable"` +} + +// EbpfConfig controls the eBPF observation layer (OSEP-0018 ยง5). +type EbpfConfig struct { + // Enabled turns observation on (requires the execd-ebpf build variant + // and CAP_BPF + CAP_PERFMON). + Enabled bool `toml:"enabled"` + // Observe lists the event kinds to record: "exec" | "connect" | + // "privilege". Default: all three. + Observe []string `toml:"observe"` + // AuditFile is the append-only JSONL audit sink (rotated). Default: + // /var/log/opensandbox/ebpf-audit.jsonl. + AuditFile string `toml:"audit_file"` +} + // DefaultConfig returns the built-in defaults used when no config file is // provided or when individual fields are missing from the file. func DefaultConfig() Config { @@ -52,6 +121,9 @@ func DefaultConfig() Config { DiffMaxBytes: 4 * 1024 * 1024 * 1024, // 4 GiB AllowedWritable: []string{"/workspace", "/mnt", "/media", "/data"}, Seccomp: nil, // use built-in denylist + Hardening: nil, // floor off + Landlock: nil, // confinement off + Ebpf: nil, // observation off } } diff --git a/components/execd/pkg/isolation/seccomp_gen.go b/components/execd/pkg/isolation/seccomp_gen.go index 099ba5e51..b4d4aa360 100644 --- a/components/execd/pkg/isolation/seccomp_gen.go +++ b/components/execd/pkg/isolation/seccomp_gen.go @@ -70,6 +70,12 @@ var denylistSyscalls = []string{ "acct", } +// GenerateSeccompDenyBPF returns BPF bytecode for a default-allow, +// deny-listed syscall filter (exported for the hardening floor launcher). +func GenerateSeccompDenyBPF(override *SeccompOverride) ([]byte, error) { + return generateSeccompDenyBPF(override) +} + // generateSeccompDenyBPF returns BPF bytecode for a default-allow, // deny-listed syscall filter. The returned bytes are in struct sock_filter // format (8 bytes per instruction, native endian). diff --git a/components/execd/pkg/runtime/bash_session.go b/components/execd/pkg/runtime/bash_session.go index 563eecbc6..4ddc9f551 100644 --- a/components/execd/pkg/runtime/bash_session.go +++ b/components/execd/pkg/runtime/bash_session.go @@ -32,11 +32,21 @@ import ( "github.com/google/uuid" + "github.com/alibaba/opensandbox/execd/pkg/isolation" "github.com/alibaba/opensandbox/execd/pkg/jupyter/execute" "github.com/alibaba/opensandbox/execd/pkg/log" "github.com/alibaba/opensandbox/execd/pkg/util/pathutil" ) +func containsStr(list []string, s string) bool { + for _, item := range list { + if item == s { + return true + } + } + return false +} + const ( envDumpStartMarker = "__ENV_DUMP_START__" envDumpEndMarker = "__ENV_DUMP_END__" @@ -117,9 +127,14 @@ func newBashSession(cwd string) *bashSession { StartupTimeout: 5 * time.Second, } + // The session env snapshot is exported into the wrapped script at the + // top, after the launcher has stripped the process environment โ€” so it + // must not carry execd's own config/credential vars or a session user + // could recover them with a plain `echo`. + blacklist := isolation.ExecdConfigEnvBlacklist() env := make(map[string]string) for _, kv := range os.Environ() { - if k, v, ok := splitEnvPair(kv); ok { + if k, v, ok := splitEnvPair(kv); ok && !containsStr(blacklist, k) { env[k] = v } } @@ -211,20 +226,29 @@ func (s *bashSession) run(ctx context.Context, request *ExecuteCodeRequest) erro // Do not pass envSnapshot via cmd.Env to avoid "argument list too long" when session env is large. // Child inherits parent env (nil => default in Go). The script file already has "export K=V" for // all session vars at the top, so the session environment is applied when the script runs. - stdout, err := cmd.StdoutPipe() + stdoutR, stdoutW, err := os.Pipe() if err != nil { return fmt.Errorf("stdout pipe: %w", err) } - cmd.Stderr = cmd.Stdout + cmd.Stdout = stdoutW + cmd.Stderr = stdoutW - if err := cmd.Start(); err != nil { + mp, err := launchManaged(cmd) + if err != nil { + _ = stdoutR.Close() + _ = stdoutW.Close() log.Error("start %s session failed: %v (command: %q)", shell, err, log.SanitizeCommand(request.Code)) return fmt.Errorf("start %s: %w", shell, err) } + // The child holds its own copy of the write end; closing ours lets the + // scanner below see EOF as soon as the child (and its descendants that + // inherited stdout) exit. + _ = stdoutW.Close() + defer stdoutR.Close() defer s.untrackCurrentProcess() s.trackCurrentProcess(cmd.Process.Pid) - scanner := bufio.NewScanner(stdout) + scanner := bufio.NewScanner(stdoutR) scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) var ( @@ -259,7 +283,7 @@ func (s *bashSession) run(ctx context.Context, request *ExecuteCodeRequest) erro } scanErr := scanner.Err() - waitErr := cmd.Wait() + waitErr := mp.Wait() if scanErr != nil { log.Error("read stdout failed: %v (command: %q)", scanErr, log.SanitizeCommand(request.Code)) @@ -271,9 +295,9 @@ func (s *bashSession) run(ctx context.Context, request *ExecuteCodeRequest) erro return fmt.Errorf("timeout after %s", wait) } - if exitCode == nil && cmd.ProcessState != nil { - code := cmd.ProcessState.ExitCode() //nolint:staticcheck - exitCode = &code //nolint:ineffassign + if exitCode == nil && mp.ExitCode() >= 0 { + code := mp.ExitCode() + exitCode = &code } updatedEnv := parseExportDump(envLines) @@ -286,8 +310,8 @@ func (s *bashSession) run(ctx context.Context, request *ExecuteCodeRequest) erro } s.mu.Unlock() - var exitErr *exec.ExitError - if waitErr != nil && !errors.As(waitErr, &exitErr) { + var exitCodeErr exitCoder + if waitErr != nil && !errors.As(waitErr, &exitCodeErr) { log.Error("command wait failed: %v (command: %q)", waitErr, log.SanitizeCommand(request.Code)) return waitErr } diff --git a/components/execd/pkg/runtime/command.go b/components/execd/pkg/runtime/command.go index 2fa819278..f8696a610 100644 --- a/components/execd/pkg/runtime/command.go +++ b/components/execd/pkg/runtime/command.go @@ -49,6 +49,27 @@ var forwardSignals = []os.Signal{ syscall.SIGWINCH, } +// subscribeCommandSignals sets up the classic-mode subscription that +// forwards application signals to a running /command process group, and +// returns the signal channel plus a stop function. In init mode +// (OSEP-0018) nothing is subscribed and the channel is nil: application +// signals are owned by forwardInitSignals, which forwards them to the +// entrypoint group (and SIGTERM triggers the shutdown sequence). An +// additional subscription here would split each in-namespace signal +// between two channels and leak HUP/USR*/WINCH into whatever /command +// happens to be running. +func subscribeCommandSignals() (chan os.Signal, func()) { + if initModeActive() { + return nil, func() {} + } + signals := make(chan os.Signal, len(forwardSignals)+1) + signal.Notify(signals, forwardSignals...) + return signals, func() { + signal.Stop(signals) + close(signals) + } +} + // getShell returns "bash" if available, otherwise "sh". The result is cached // for the process lifetime; tests that mutate PATH must call // resetShellCacheForTest. @@ -125,10 +146,8 @@ func buildCredential(uid, gid *uint32) (*syscall.Credential, error) { func (c *Controller) runCommand(ctx context.Context, request *ExecuteCodeRequest) error { session := c.newContextID() - signals := make(chan os.Signal, len(forwardSignals)+1) - defer close(signals) - signal.Notify(signals, forwardSignals...) - defer signal.Stop(signals) + signals, stopSignals := subscribeCommandSignals() + defer stopSignals() stdout, stderr, err := c.stdLogDescriptor(session) if err != nil { @@ -177,7 +196,7 @@ func (c *Controller) runCommand(ctx context.Context, request *ExecuteCodeRequest c.tailStdPipe(stderrPath, request.Hooks.OnExecuteStderr, done) }) - err = cmd.Start() + mp, err := launchManaged(cmd) if err != nil { close(done) wg.Wait() @@ -245,7 +264,7 @@ func (c *Controller) runCommand(ctx context.Context, request *ExecuteCodeRequest } }) - err = cmd.Wait() + err = mp.Wait() close(done) wg.Wait() if err != nil { @@ -253,9 +272,9 @@ func (c *Controller) runCommand(ctx context.Context, request *ExecuteCodeRequest var eCode int var traceback []string - var exitError *exec.ExitError - if errors.As(err, &exitError) { - exitCode := exitError.ExitCode() + var exitCodeErr exitCoder + if errors.As(err, &exitCodeErr) { + exitCode := exitCodeErr.ExitCode() eName = "CommandExecError" eValue = strconv.Itoa(exitCode) eCode = exitCode @@ -295,10 +314,10 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca stdoutPath := c.combinedOutputFileName(session) stderrPath := c.combinedOutputFileName(session) - signals := make(chan os.Signal, len(forwardSignals)+1) - defer close(signals) - signal.Notify(signals, forwardSignals...) - defer signal.Stop(signals) + // Classic-mode signal subscription (no-op in init mode; the channel is + // never consumed, keeping today's behavior of not dying on SIGHUP etc.). + _, stopSignals := subscribeCommandSignals() + defer stopSignals() startAt := time.Now() log.Info("received command: %v", log.SanitizeCommand(request.Code)) @@ -334,7 +353,7 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca defer devNull.Close() } - err = cmd.Start() + mp, err := launchManaged(cmd) kernel := &commandKernel{ pid: -1, stdoutPath: stdoutPath, @@ -363,14 +382,14 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca safego.Go(func() { defer pipe.Close() - err = cmd.Wait() + err = mp.Wait() cancel() if err != nil { log.Error("CommandExecError: error running commands: %v", err) exitCode := 1 - var exitError *exec.ExitError - if errors.As(err, &exitError) { - exitCode = exitError.ExitCode() + var exitCodeErr exitCoder + if errors.As(err, &exitCodeErr) { + exitCode = exitCodeErr.ExitCode() } c.markCommandFinished(session, exitCode, err.Error()) return diff --git a/components/execd/pkg/runtime/exit_error.go b/components/execd/pkg/runtime/exit_error.go new file mode 100644 index 000000000..e550dcdab --- /dev/null +++ b/components/execd/pkg/runtime/exit_error.go @@ -0,0 +1,22 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +// exitCoder is implemented by both os/exec.ExitError and the init-mode +// processExitError, so callers can read the exit code uniformly across init +// and non-init mode. +type exitCoder interface { + ExitCode() int +} diff --git a/components/execd/pkg/runtime/hardening_common.go b/components/execd/pkg/runtime/hardening_common.go new file mode 100644 index 000000000..034822768 --- /dev/null +++ b/components/execd/pkg/runtime/hardening_common.go @@ -0,0 +1,32 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +// LayerState reports whether a hardening layer is actually enforced. +type LayerState struct { + State string // "active" | "disabled" | "degraded" | "unsupported" + Message string +} + +// HardeningReport describes the hardening state for the capabilities +// endpoint (OSEP-0018 ยง6). +type HardeningReport struct { + InitMode string + SignalShield bool + CapDrop LayerState + Seccomp LayerState + Landlock LayerState + Ebpf LayerState +} diff --git a/components/execd/pkg/runtime/hardening_linux.go b/components/execd/pkg/runtime/hardening_linux.go new file mode 100644 index 000000000..ef9a757ea --- /dev/null +++ b/components/execd/pkg/runtime/hardening_linux.go @@ -0,0 +1,705 @@ +//go:build linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Hardening floor (OSEP-0018 ยง4): when [hardening] is enabled, every +// user-code launch is routed through the opensandbox-launcher native helper, +// which applies the privilege floor between fork and exec (env strip, +// bounding-set trim, no_new_privs, identity drop, ambient caps, seccomp +// last). The launcher's exec syscall (execve) is reserved and rejected at +// config time. Everything is fail-open: a missing prerequisite is reported +// on the capabilities endpoint and the launch proceeds without that layer. +// +// Isolated sessions are exempt from the launcher: their workload is already +// reduced inside the bwrap namespace (bwrap --seccomp + session-gate), and +// applying the floor to the bwrap process itself would deny the unshare/ +// setns syscalls and strip the capabilities bwrap needs to build the +// namespace. + +package runtime + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "sync/atomic" + + "golang.org/x/sys/unix" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" + "github.com/alibaba/opensandbox/execd/pkg/log" +) + +const ( + launcherRuntimePath = "/opt/opensandbox/opensandbox-launcher" + + layerStateDisabled = "disabled" + + policyMagic = 0x4f534258 // "OSBX" + policyVersion = 1 + flagUIDDrop = 0x1 + flagCapDrop = 0x2 + + // capSetpcap is the capability number required to trim bounding sets. + capSetpcap = 8 + + // Landlock fs access bits (stable kernel UAPI, linux/landlock.h). + llExecute uint64 = 1 << 0 + llWriteFile uint64 = 1 << 1 + llReadFile uint64 = 1 << 2 + llReadDir uint64 = 1 << 3 + llRemoveDir uint64 = 1 << 4 + llRemoveFile uint64 = 1 << 5 + llMakeChar uint64 = 1 << 6 + llMakeDir uint64 = 1 << 7 + llMakeReg uint64 = 1 << 8 + llMakeSock uint64 = 1 << 9 + llMakeFifo uint64 = 1 << 10 + llMakeBlock uint64 = 1 << 11 + llMakeSym uint64 = 1 << 12 + llRefer uint64 = 1 << 13 // ABI >= 2 + llTruncate uint64 = 1 << 14 // ABI >= 3 + + // llRwAccess is the full writable-subtree mask (creation, removal, + // rename, truncate); the launcher trims bits its kernel ABI lacks. + llRwAccess = llReadFile | llWriteFile | llReadDir | llMakeChar | + llMakeDir | llMakeReg | llMakeSock | llMakeFifo | llMakeBlock | + llMakeSym | llRemoveDir | llRemoveFile | llRefer | llTruncate +) + +// landlockRule grants access beneath path (OSEP-0018 ยง5). Rules only grant +// access; everything else under the handled set is denied. Required rules +// must all install or the launch skips confinement entirely (fail closed); +// best-effort rules (mount-expansion duplicates) are logged and skipped. +type landlockRule struct { + Access uint64 + Path string + Required bool +} + +// buildLandlockRules assembles the default allowlist. The root rule grants +// EXECUTE only: it covers path traversal and execve of any binary without +// exposing any read access. /proc is deliberately limited to /proc/self and +// /proc/sys โ€” never all of /proc, which would re-expose /proc/1 (and +// execd's credentials) to a same-uid workload. Note that Landlock +// path_beneath rules are scoped to the mount the path sits on, so +// expandMountRules adds a rule per mount point beneath each grant (a bind +// mount like a workspace would otherwise be invisible to the rule). +func buildLandlockRules(cfg isolation.Config) []landlockRule { + bestEffort := func(access uint64, path string) landlockRule { + return landlockRule{Access: access, Path: path, Required: false} + } + // Operator-explicit grants are required: if one cannot be installed the + // layer degrades instead of silently narrowing. The default set stays + // best-effort โ€” images legitimately differ (e.g. alpine has no /lib64, + // minimal images lack /workspace). + required := func(access uint64, path string) landlockRule { + return landlockRule{Access: access, Path: path, Required: true} + } + var rules []landlockRule + + rules = append(rules, bestEffort(llExecute, "/")) + + readExec := llReadFile | llReadDir | llExecute + for _, p := range []string{"/usr", "/bin", "/lib", "/lib64", "/etc", "/opt"} { + rules = append(rules, bestEffort(readExec, p)) + } + // Only directory paths are usable here: the kernel rejects path_beneath + // rules whose parent is a regular file (e.g. /proc/cpuinfo). + for _, p := range []string{"/proc/self", "/proc/sys"} { + rules = append(rules, bestEffort(readExec, p)) + } + + deviceRW := llReadFile | llWriteFile + for _, p := range []string{ + "/dev/null", "/dev/zero", "/dev/full", "/dev/random", + "/dev/urandom", "/dev/tty", + } { + rules = append(rules, bestEffort(deviceRW, p)) + } + // The controlling terminal lives beneath /dev/pts. + rules = append(rules, bestEffort(deviceRW, "/dev/pts")) + + for _, p := range []string{"/tmp", "/run"} { + rules = append(rules, bestEffort(llRwAccess, p)) + } + // The workspace family (allowed_writable) must additionally be + // executable: workloads compile/run scripts there, and the e2e contract + // asserts it. Landlock anchors a rule on the mount the path resolves to, + // so granting llExecute here covers the workspace even when the + // mount-expansion rules below are incomplete (e.g. a mount not present + // in /proc/self/mounts at policy-build time). One rule per path: a + // duplicate entry would shadow the combined access in rule matching. + for _, p := range cfg.AllowedWritable { + rules = append(rules, bestEffort(llRwAccess|llExecute, p)) + } + if cfg.Landlock != nil { + for _, p := range cfg.Landlock.ExtraWritable { + rules = append(rules, required(llRwAccess, p)) + } + for _, p := range cfg.Landlock.ExtraReadable { + rules = append(rules, required(readExec, p)) + } + } + return expandMountRules(rules) +} + +// expandMountRules duplicates every rule onto each mount point beneath the +// rule path. Landlock path_beneath rules only cover the mount the path +// belongs to, so a bind-mounted workspace (a separate mount) is invisible +// to a rule on its parent path โ€” without expansion, execve and file access +// on bind mounts would be denied. +func expandMountRules(rules []landlockRule) []landlockRule { + mounts := readMountPoints() + if len(mounts) == 0 { + return rules + } + expanded := append([]landlockRule(nil), rules...) + for _, mount := range mounts { + access, ok := ruleForPath(rules, mount) + if !ok { + continue + } + expanded = append(expanded, landlockRule{Access: access, Path: mount, Required: false}) + } + return expanded +} + +// ruleForPath returns the merged access of every rule that covers path +// (path == rule.Path or path is beneath it). A mount point may be beneath +// several grants (e.g. /mnt beneath both / for EXECUTE and the /mnt +// writable grant); merging keeps both, so a bind-mounted workspace keeps +// execute access. +func ruleForPath(rules []landlockRule, path string) (uint64, bool) { + var access uint64 + found := false + for _, rule := range rules { + if !pathBeneath(rule.Path, path) { + continue + } + access |= rule.Access + found = true + } + return access, found +} + +// pathBeneath reports whether path == parent or path is beneath parent +// (boundary-aware prefix match). +func pathBeneath(parent, path string) bool { + if parent == "/" { + return strings.HasPrefix(path, "/") + } + if path == parent { + return true + } + return strings.HasPrefix(path, parent+"/") +} + +// missingRequiredRulePaths reports required rule paths that cannot be +// opened with O_PATH (the same check the launcher performs before +// restrict_self). +func missingRequiredRulePaths(rules []landlockRule) []string { + var missing []string + for _, rule := range rules { + if !rule.Required { + continue + } + fd, err := unix.Open(rule.Path, unix.O_PATH|unix.O_CLOEXEC, 0) + if err != nil { + missing = append(missing, rule.Path) + continue + } + _ = unix.Close(fd) + } + return missing +} + +// readMountPoints parses /proc/self/mounts and returns the mount points +// (escape-decoded). +func readMountPoints() []string { + data, err := os.ReadFile("/proc/self/mounts") + if err != nil { + return nil + } + var mounts []string + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + mounts = append(mounts, decodeMountPath(fields[1])) + } + return mounts +} + +// decodeMountPath decodes the /proc/self/mounts escaping (\040, \011, +// \012, \134). +func decodeMountPath(s string) string { + if !strings.ContainsRune(s, '\\') { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+3 < len(s) { + switch s[i+1 : i+4] { + case "040": + b.WriteByte(' ') + i += 3 + continue + case "011": + b.WriteByte('\t') + i += 3 + continue + case "012": + b.WriteByte('\n') + i += 3 + continue + case "134": + b.WriteByte('\\') + i += 3 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +// landlockABI probes the kernel Landlock ABI version (0 = unavailable). +func landlockABI() int64 { + abi, _, errno := unix.Syscall(unix.SYS_LANDLOCK_CREATE_RULESET, 0, 0, 1) + if errno != 0 { + return 0 + } + return int64(abi) +} + +// hardeningPolicy is the serialized policy handed to the launcher over a +// memfd. Field order must stay in sync with struct policy_header in +// native/launcher.c. +type hardeningPolicy struct { + flags uint32 + uid uint32 + gid uint32 + groups []uint32 + keepcaps []uint32 + stripEnv []string + seccomp []byte + landlock []landlockRule +} + +type policyHeader struct { + Magic uint32 + Version uint32 + Flags uint32 + UID uint32 + GID uint32 + NGroups uint32 + NKeepCaps uint32 + NEnv uint32 + SeccompLen uint32 + LandlockLen uint32 +} + +var hardening struct { + enabled atomic.Bool + launcherPath string + policy *hardeningPolicy + capDrop atomic.Pointer[LayerState] + seccomp atomic.Pointer[LayerState] + landlock atomic.Pointer[LayerState] + ebpf atomic.Pointer[LayerState] +} + +// SetEbpfState records the eBPF observation state reported by the observer +// (execd-ebpf variant) for the capabilities endpoint. +func SetEbpfState(state LayerState) { + hardening.ebpf.Store(&state) +} + +// InitHardening activates the floor from the isolation config. It returns an +// error only for invalid configuration (unknown capability name, or the +// launcher's reserved execve in [seccomp] deny); missing runtime support +// degrades to a reported, non-fatal state. +func InitHardening(cfg isolation.Config) error { + setLayer := func(dst *atomic.Pointer[LayerState], s LayerState) { + dst.Store(&s) + } + disabled := func(msg string) LayerState { + return LayerState{State: layerStateDisabled, Message: msg} + } + degraded := func(msg string) LayerState { + return LayerState{State: "degraded", Message: msg} + } + active := LayerState{State: "active"} + + setLayer(&hardening.capDrop, disabled("hardening not enabled")) + setLayer(&hardening.seccomp, disabled("hardening not enabled")) + setLayer(&hardening.landlock, disabled("landlock not enabled")) + + if cfg.Hardening == nil || !cfg.Hardening.Enabled { + return nil + } + + if cfg.Seccomp != nil { + for _, name := range cfg.Seccomp.Deny { + if name == "execve" { + return fmt.Errorf( + "hardening: [seccomp] deny lists execve, which is reserved for the launcher's final exec; " + + "use execveat if you need to deny that syscall", + ) + } + } + } + + keepcaps, err := parseKeepCapabilities(cfg.Hardening.KeepCapabilities) + if err != nil { + return err + } + + path := findLauncher() + if path == "" { + msg := "opensandbox-launcher not found (searched: /opt/opensandbox/opensandbox-launcher, $PATH)" + log.Warn("hardening: %s", msg) + setLayer(&hardening.capDrop, degraded(msg)) + setLayer(&hardening.seccomp, degraded(msg)) + setLayer(&hardening.landlock, degraded(msg)) + return nil + } + + seccompBPF, err := isolation.GenerateSeccompDenyBPF(cfg.Seccomp) + if err != nil { + return fmt.Errorf("hardening: generate seccomp floor: %w", err) + } + + var landlockRules []landlockRule + if cfg.Landlock != nil && cfg.Landlock.Enabled { + if abi := landlockABI(); abi < 1 { + msg := fmt.Sprintf( + "landlock unavailable: kernel ABI < 1 (needs >= 5.13, detected %d); FS confinement skipped", + abi, + ) + log.Warn("hardening: %s", msg) + setLayer(&hardening.landlock, LayerState{State: "unsupported", Message: msg}) + } else { + landlockRules = buildLandlockRules(cfg) + // Preflight the required (operator-explicit) grants: a missing + // one would make every launch skip confinement (launcher + // fail-closed), so report degraded and do not enable the layer. + if missing := missingRequiredRulePaths(landlockRules); len(missing) > 0 { + msg := fmt.Sprintf( + "landlock degraded: required paths missing: %s; FS confinement disabled", + strings.Join(missing, ", "), + ) + log.Warn("hardening: %s", msg) + setLayer(&hardening.landlock, LayerState{State: "degraded", Message: msg}) + landlockRules = nil + } else { + msg := fmt.Sprintf("landlock active (kernel ABI %d, %d rules)", abi, len(landlockRules)) + log.Info("hardening: %s", msg) + setLayer(&hardening.landlock, LayerState{State: "active", Message: msg}) + } + } + } + + hardening.launcherPath = path + hardening.policy = &hardeningPolicy{ + uid: uint32(os.Getuid()), + gid: uint32(os.Getgid()), + keepcaps: keepcaps, + stripEnv: isolation.ExecdConfigEnvBlacklist(), + seccomp: seccompBPF, + landlock: landlockRules, + } + // The identity drop is only meaningful when execd is root (a non-root + // execd already runs as the image's user). + if os.Geteuid() == 0 { + hardening.policy.flags |= flagUIDDrop + } + hardening.policy.flags |= flagCapDrop + + hasSetpcap := effectiveCapsHave(capSetpcap) + if hasSetpcap { + setLayer(&hardening.capDrop, active) + } else { + msg := "cap_drop skipped: execd lacks CAP_SETPCAP (bounding-set trim impossible); seccomp/identity still apply" + log.Warn("hardening: %s", msg) + setLayer(&hardening.capDrop, degraded(msg)) + } + if len(seccompBPF) == 0 { + msg := "seccomp floor skipped: deny list is empty" + log.Warn("hardening: %s", msg) + setLayer(&hardening.seccomp, degraded(msg)) + } else { + setLayer(&hardening.seccomp, active) + } + + hardening.enabled.Store(true) + log.Info("hardening: enabled (launcher=%s uid=%d gid=%d keep_caps=%v seccomp=%d bytes)", + path, hardening.policy.uid, hardening.policy.gid, + cfg.Hardening.KeepCapabilities, len(seccompBPF)) + return nil +} + +var launcherSearchPaths = []string{launcherRuntimePath} + +func findLauncher() string { + // Trusted runtime path first: a user-controlled image must not be able + // to substitute its own launcher on PATH and bypass the floor. PATH is + // only a fallback for developer/source builds. + for _, p := range launcherSearchPaths { + if _, err := os.Stat(p); err == nil { + return p + } + } + if path, err := exec.LookPath("opensandbox-launcher"); err == nil { + return path + } + return "" +} + +func effectiveCapsHave(capNum uint32) bool { + data, err := os.ReadFile("/proc/self/status") + if err != nil { + return false + } + for _, line := range strings.Split(string(data), "\n") { + if !strings.HasPrefix(line, "CapEff:") { + continue + } + value, err := strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "CapEff:")), 16, 64) + if err != nil { + return false + } + return value&(1<= 64 || strings.ContainsRune(name, '\x00') { + return nil, fmt.Errorf("invalid env-strip name %q", name) + } + } + var landlockBuf bytes.Buffer + for _, rule := range p.landlock { + if len(rule.Path) == 0 || len(rule.Path) > 4096 { + return nil, fmt.Errorf("invalid landlock path %q", rule.Path) + } + required := byte(0) + if rule.Required { + required = 1 + } + if err := landlockBuf.WriteByte(required); err != nil { + return nil, err + } + if err := binary.Write(&landlockBuf, binary.LittleEndian, rule.Access); err != nil { + return nil, err + } + if err := binary.Write(&landlockBuf, binary.LittleEndian, uint16(len(rule.Path))); err != nil { + return nil, err + } + landlockBuf.WriteString(rule.Path) + } + buf := new(bytes.Buffer) + hdr := policyHeader{ + Magic: policyMagic, + Version: policyVersion, + Flags: p.flags, + UID: p.uid, + GID: p.gid, + NGroups: uint32(len(p.groups)), + NKeepCaps: uint32(len(p.keepcaps)), + NEnv: uint32(len(p.stripEnv)), + SeccompLen: uint32(len(p.seccomp)), + LandlockLen: uint32(landlockBuf.Len()), + } + if err := binary.Write(buf, binary.LittleEndian, &hdr); err != nil { + return nil, err + } + for _, g := range p.groups { + if err := binary.Write(buf, binary.LittleEndian, g); err != nil { + return nil, err + } + } + for _, capNum := range p.keepcaps { + if err := binary.Write(buf, binary.LittleEndian, capNum); err != nil { + return nil, err + } + } + for _, name := range p.stripEnv { + buf.WriteString(name) + buf.WriteByte(0) + } + buf.Write(p.seccomp) + buf.Write(landlockBuf.Bytes()) + return buf.Bytes(), nil +} + +func createPolicyMemfd(policy []byte) (int, error) { + fd, err := unix.MemfdCreate("launcher-policy", unix.MFD_CLOEXEC) + if err != nil { + return -1, fmt.Errorf("memfd_create: %w", err) + } + if _, err := unix.Write(fd, policy); err != nil { + unix.Close(fd) + return -1, fmt.Errorf("write launcher policy: %w", err) + } + if _, err := unix.Seek(fd, 0, 0); err != nil { + unix.Close(fd) + return -1, fmt.Errorf("seek launcher policy: %w", err) + } + return fd, nil +} + +// HardeningReport returns the current hardening enforcement state for the +// capabilities endpoint. +func ReportHardening() HardeningReport { + mode, shield := InitModeReport() + report := HardeningReport{ + InitMode: mode, + SignalShield: shield, + CapDrop: LayerState{State: layerStateDisabled, Message: "hardening not enabled"}, + Seccomp: LayerState{State: layerStateDisabled, Message: "hardening not enabled"}, + Landlock: LayerState{ + State: layerStateDisabled, + Message: "landlock confinement is not enabled", + }, + Ebpf: LayerState{ + State: layerStateDisabled, + Message: "eBPF observation is not enabled", + }, + } + if cs := hardening.capDrop.Load(); cs != nil { + report.CapDrop = *cs + } + if ss := hardening.seccomp.Load(); ss != nil { + report.Seccomp = *ss + } + if ls := hardening.landlock.Load(); ls != nil { + report.Landlock = *ls + } + if es := hardening.ebpf.Load(); es != nil { + report.Ebpf = *es + } + // Without init mode (classic background-and-wait topology), the image + // entrypoint โ€” and any /code kernels it spawns โ€” is launched by the + // bootstrap shell, not by execd, so it never passes through the launcher. + // The layer states above only cover execd-spawned commands/sessions; say + // so instead of letting the endpoint claim full enforcement. Key the + // correction off hardening being enabled (not off cap_drop's state: the + // layer can be degraded while seccomp/Landlock are active, or cap_drop + // can be active while a configured layer is disabled) and only touch the + // layers that are actually in effect. + if mode == "none" && hardening.enabled.Load() { + msg := "hardening enabled but execd is not the sandbox init (EXECD_INIT unset): " + + "the image entrypoint and its /code kernels are not wrapped; only " + + "execd-spawned commands/sessions are reduced. Enable " + + "runtime.execd_run_as_init for full coverage" + if report.CapDrop.State != layerStateDisabled { + report.CapDrop = LayerState{State: "degraded", Message: msg} + } + if report.Seccomp.State != layerStateDisabled { + report.Seccomp = LayerState{State: "degraded", Message: msg} + } + if report.Landlock.State != layerStateDisabled { + report.Landlock = LayerState{State: "degraded", Message: msg} + } + } + return report +} diff --git a/components/execd/pkg/runtime/hardening_linux_test.go b/components/execd/pkg/runtime/hardening_linux_test.go new file mode 100644 index 000000000..c72b0425d --- /dev/null +++ b/components/execd/pkg/runtime/hardening_linux_test.go @@ -0,0 +1,506 @@ +//go:build linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" +) + +var ( + launcherOnce sync.Once + launcherBuilt string + launcherErr error +) + +func buildLauncher(t *testing.T) string { + t.Helper() + launcherOnce.Do(func() { + cc, err := exec.LookPath("cc") + if err != nil { + launcherErr = err + return + } + dir, err := os.MkdirTemp("", "launcher-test-*") + if err != nil { + launcherErr = err + return + } + launcherBuilt = filepath.Join(dir, "opensandbox-launcher") + src := filepath.Join("..", "..", "native", "launcher.c") + cmd := exec.Command(cc, "-O2", "-Wall", "-Wextra", "-Werror", + "-o", launcherBuilt, src) + if out, err := cmd.CombinedOutput(); err != nil { + launcherErr = err + t.Logf("launcher build output: %s", out) + } + }) + if launcherErr != nil { + t.Skipf("cannot build opensandbox-launcher: %v", launcherErr) + } + return launcherBuilt +} + +func resetHardening() { + hardening.enabled.Store(false) + hardening.launcherPath = "" + hardening.policy = nil + hardening.capDrop.Store(nil) + hardening.seccomp.Store(nil) + hardening.landlock.Store(nil) + launcherSearchPaths = []string{launcherRuntimePath} +} + +func initHardeningForTest(t *testing.T, cfg isolation.Config) { + t.Helper() + t.Cleanup(resetHardening) + if err := InitHardening(cfg); err != nil { + t.Fatalf("InitHardening: %v", err) + } +} + +func hardenedCfg(keepCaps ...string) isolation.Config { + return isolation.Config{ + Hardening: &isolation.HardeningConfig{ + Enabled: true, + KeepCapabilities: keepCaps, + }, + } +} + +// childStatus launches a command through the floor and returns its combined +// output. The command may exit non-zero (e.g. a seccomp-denied syscall); +// assertions run against the output. +func childStatus(t *testing.T, cfg isolation.Config, script string) string { + t.Helper() + initHardeningForTest(t, cfg) + var out bytes.Buffer + cmd := exec.Command("sh", "-c", script) + cmd.Stdout = &out + cmd.Stderr = &out + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + if err := mp.Wait(); err != nil { + t.Logf("command exited with error: %v", err) + } + return out.String() +} + +func TestHardeningDisabledByDefault(t *testing.T) { + initHardeningForTest(t, isolation.Config{}) + report := ReportHardening() + if report.CapDrop.State != "disabled" || report.Seccomp.State != "disabled" { + t.Fatalf("hardening states = %q/%q, want disabled/disabled", + report.CapDrop.State, report.Seccomp.State) + } + out := childStatus(t, isolation.Config{}, "echo hi") + if out != "hi\n" { + t.Fatalf("output = %q, want hi (launch must be unmodified)", out) + } +} + +func TestHardeningAppliesFloor(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + out := childStatus(t, hardenedCfg(), `grep -E "^CapEff:|^CapPrm:|^NoNewPrivs:|^Uid:" /proc/self/status`) + if !strings.Contains(out, "NoNewPrivs: 1") { + t.Fatalf("NoNewPrivs not set: %q", out) + } + if strings.Contains(out, "CapEff: 0000000000000000") || os.Geteuid() != 0 { + return + } + t.Fatalf("CapEff not dropped to zero: %q", out) +} + +func TestHardeningKeepsExecdPrivileges(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + + before, err := os.ReadFile("/proc/self/status") + if err != nil { + t.Fatal(err) + } + out := childStatus(t, hardenedCfg(), "true") + after, err := os.ReadFile("/proc/self/status") + if err != nil { + t.Fatal(err) + } + capOf := func(status []byte) string { + for _, line := range strings.Split(string(status), "\n") { + if strings.HasPrefix(line, "CapEff:") { + return line + } + } + return "" + } + if capOf(before) != capOf(after) { + t.Fatalf("execd CapEff changed across a hardened launch:\n before=%s\n after =%s\n child=%q", + capOf(before), capOf(after), out) + } +} + +func TestHardeningSeccompBlocksDeniedSyscall(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + // The filter is installed (Seccomp: 2 = SECCOMP_MODE_FILTER) before the + // workload execs; a behavioral probe is unreliable because the container + // runtime's own seccomp profile already blocks syscalls like mount. + out := childStatus(t, hardenedCfg(), `grep "^Seccomp:" /proc/self/status`) + if !strings.Contains(out, "Seccomp: 2") { + t.Fatalf("seccomp filter not active in the child: %q", out) + } +} + +func TestHardeningEnvStrip(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + initHardeningForTest(t, hardenedCfg()) + + var out bytes.Buffer + cmd := exec.Command("sh", "-c", "env") + cmd.Stdout = &out + cmd.Stderr = &out + cmd.Env = append(os.Environ(), + "EXECD_ACCESS_TOKEN=supersecret", + "JUPYTER_TOKEN=anothersecret", + ) + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + _ = mp.Wait() + for _, secret := range []string{"supersecret", "anothersecret"} { + if strings.Contains(out.String(), secret) { + t.Fatalf("execd credential env leaked into the workload: %q", out.String()) + } + } +} + +func TestHardeningKeepCapabilities(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root to raise capabilities") + } + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + out := childStatus(t, hardenedCfg("CAP_NET_RAW"), `grep "^CapEff:" /proc/self/status`) + // CAP_NET_RAW = 13 โ†’ 0x2000. + if !strings.Contains(out, "CapEff: 0000000000002000") { + t.Fatalf("kept capability not raised: %q", out) + } +} + +func TestHardeningRejectsReservedExecve(t *testing.T) { + cfg := isolation.Config{ + Seccomp: &isolation.SeccompOverride{Deny: []string{"execve", "mount"}}, + Hardening: &isolation.HardeningConfig{ + Enabled: true, + }, + } + if err := InitHardening(cfg); err == nil || !strings.Contains(err.Error(), "execve") { + t.Fatalf("InitHardening error = %v, want execve rejection", err) + } + resetHardening() +} + +func TestHardeningRejectsUnknownCapability(t *testing.T) { + cfg := isolation.Config{ + Hardening: &isolation.HardeningConfig{ + Enabled: true, + KeepCapabilities: []string{"CAP_DOES_NOT_EXIST"}, + }, + } + if err := InitHardening(cfg); err == nil { + t.Fatal("InitHardening error = nil, want unknown capability rejection") + } + resetHardening() +} + +func TestHardeningDegradesWhenLauncherMissing(t *testing.T) { + launcherSearchPaths = nil + initHardeningForTest(t, hardenedCfg()) + report := ReportHardening() + if report.CapDrop.State != "degraded" || report.Seccomp.State != "degraded" { + t.Fatalf("states = %q/%q, want degraded/degraded", + report.CapDrop.State, report.Seccomp.State) + } + // Fail-open: the launch still works without the floor. + out := childStatus(t, hardenedCfg(), "echo still-works") + if out != "still-works\n" { + t.Fatalf("output = %q, want still-works", out) + } +} + +func TestHardeningReportLayers(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + startReaperForTest(t) + initHardeningForTest(t, hardenedCfg()) + report := ReportHardening() + if report.Ebpf.State != "disabled" { + t.Fatalf("ebpf layer = %q, want disabled", report.Ebpf.State) + } + if report.CapDrop.State != "active" && report.CapDrop.State != "degraded" { + t.Fatalf("cap_drop state = %q, want active or degraded (root)", report.CapDrop.State) + } + if report.Seccomp.State != "active" { + t.Fatalf("seccomp state = %q, want active", report.Seccomp.State) + } +} + +func TestHardeningReportDegradesWithoutInitMode(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + + // Without init topology the entrypoint and /code kernels never pass + // through the launcher; every enabled layer must report degraded, and a + // layer that is not configured must stay disabled rather than being + // dragged into the degradation. + initHardeningForTest(t, hardenedCfg()) + report := ReportHardening() + if report.InitMode != "none" { + t.Fatalf("InitMode = %q, want none (no reaper in this test)", report.InitMode) + } + for _, layer := range []struct{ name, state string }{ + {"cap_drop", report.CapDrop.State}, + {"seccomp", report.Seccomp.State}, + } { + if layer.state != "degraded" { + t.Fatalf("%s state = %q, want degraded (hardening enabled without init mode)", layer.name, layer.state) + } + } + if report.Landlock.State != "disabled" { + t.Fatalf("landlock state = %q, want disabled (not configured)", report.Landlock.State) + } + if !strings.Contains(report.CapDrop.Message, "EXECD_INIT") { + t.Fatalf("cap_drop message = %q, want EXECD_INIT guidance", report.CapDrop.Message) + } + + // An enabled Landlock layer must be degraded too, regardless of the + // underlying kernel state. + resetHardening() + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + cfg := isolation.DefaultConfig() + cfg.Hardening = &isolation.HardeningConfig{Enabled: true} + cfg.Landlock = &isolation.LandlockConfig{Enabled: true} + initHardeningForTest(t, cfg) + report = ReportHardening() + if report.Landlock.State != "degraded" { + t.Fatalf("landlock state = %q, want degraded (enabled without init mode)", report.Landlock.State) + } + if !strings.Contains(report.Landlock.Message, "EXECD_INIT") { + t.Fatalf("landlock message = %q, want EXECD_INIT guidance", report.Landlock.Message) + } +} + +func TestLandlockDisabledByDefault(t *testing.T) { + initHardeningForTest(t, isolation.Config{}) + if report := ReportHardening(); report.Landlock.State != "disabled" { + t.Fatalf("landlock state = %q, want disabled", report.Landlock.State) + } +} + +func TestLandlockActiveOrUnsupported(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + startReaperForTest(t) + cfg := isolation.DefaultConfig() + cfg.Hardening = &isolation.HardeningConfig{Enabled: true} + cfg.Landlock = &isolation.LandlockConfig{ + Enabled: true, + ExtraWritable: []string{"/cache"}, + ExtraReadable: []string{"/opt/data"}, + } + initHardeningForTest(t, cfg) + + report := ReportHardening() + switch report.Landlock.State { + case "active": + if report.Landlock.Message == "" { + t.Fatalf("landlock active but no message") + } + rules := buildLandlockRules(cfg) + assertLandlockRule(t, rules, "/", llExecute) + assertLandlockRule(t, rules, "/usr", llReadFile|llReadDir|llExecute) + assertLandlockRule(t, rules, "/proc/self", llReadFile|llReadDir|llExecute) + assertLandlockRule(t, rules, "/tmp", llRwAccess) + // allowed_writable paths carry execute on the default rule: the + // mount-expansion rule is the backup, not the only source of the + // workspace exec grant. + assertLandlockRule(t, rules, "/workspace", llRwAccess|llExecute) + assertLandlockRule(t, rules, "/cache", llRwAccess) + assertLandlockRule(t, rules, "/opt/data", llReadFile|llReadDir|llExecute) + for _, rule := range rules { + if rule.Path == "/proc" && rule.Access&llReadFile != 0 { + t.Fatalf("all of /proc must not be granted read access: %+v", rule) + } + } + case "degraded": + if !strings.Contains(report.Landlock.Message, "/cache") || + !strings.Contains(report.Landlock.Message, "/opt/data") { + t.Fatalf("landlock degraded message = %q, want the missing extra paths", report.Landlock.Message) + } + case "unsupported": + t.Logf("landlock unsupported on this kernel; skipping rule assertions") + default: + t.Fatalf("landlock state = %q, want active, degraded or unsupported", report.Landlock.State) + } +} + +func TestPathBeneath(t *testing.T) { + tests := []struct { + parent, path string + want bool + }{ + {"/", "/mnt/test/hardened.sh", true}, + {"/mnt", "/mnt/test", true}, + {"/mnt", "/mnt/test/hardened.sh", true}, + {"/mnt", "/mnt", true}, + {"/mnt", "/mntx", false}, + {"/mnt", "/", false}, + {"/usr", "/usr/bin/bash", true}, + } + for _, tt := range tests { + if got := pathBeneath(tt.parent, tt.path); got != tt.want { + t.Fatalf("pathBeneath(%q, %q) = %v, want %v", tt.parent, tt.path, got, tt.want) + } + } +} + +func TestRuleForPathMergesMatches(t *testing.T) { + rules := []landlockRule{ + {Access: llExecute, Path: "/"}, + {Access: llRwAccess, Path: "/mnt"}, + } + // A bind-mounted workspace beneath /mnt must keep execute access from + // the "/" rule merged with the writable grant. + access, ok := ruleForPath(rules, "/mnt/test/hardened.sh") + if !ok || access != llRwAccess|llExecute { + t.Fatalf("ruleForPath(/mnt/test/hardened.sh) = %#x/%v, want rw+exec", access, ok) + } + access, ok = ruleForPath(rules, "/etc/passwd") + if !ok || access != llExecute { + t.Fatalf("ruleForPath(/etc/passwd) = %#x/%v, want llExecute", access, ok) + } + if _, ok := ruleForPath(rules, "relative"); ok { + t.Fatal("ruleForPath accepted a relative path") + } +} + +func TestDecodeMountPath(t *testing.T) { + if got := decodeMountPath(`/mnt/test\040dir`); got != "/mnt/test dir" { + t.Fatalf("decode = %q", got) + } + if got := decodeMountPath(`/a\134b`); got != `a\\b` && got != `/a\b` { + t.Fatalf("decode backslash = %q", got) + } + if got := decodeMountPath("/plain"); got != "/plain" { + t.Fatalf("decode plain = %q", got) + } +} + +func assertLandlockRule(t *testing.T, rules []landlockRule, path string, access uint64) { + t.Helper() + for _, rule := range rules { + if rule.Path == path { + if rule.Access != access { + t.Fatalf("landlock rule %s access = %#x, want %#x", path, rule.Access, access) + } + return + } + } + t.Fatalf("landlock rule for %s missing", path) +} + +// TestHardeningPTYSessions verifies the PTY launch paths pass through the +// hardening floor (OSEP-0018 R-n): StartPTY and StartPipe both route through +// the opensandbox-launcher, whose argv[0]-replacement execve must preserve the +// pty/session semantics (setsid/Setctty by creack/pty, the pty fds) while +// applying the floor to the final workload. The reaper is started so the launch +// also exercises reaper dispatch of launcher-exec'd children. +func TestHardeningPTYSessions(t *testing.T) { + buildLauncher(t) + launcherSearchPaths = append(launcherSearchPaths, launcherBuilt) + startReaperForTest(t) + initHardeningForTest(t, hardenedCfg()) + requireBash(t) + + // The launcher strips execd's credential env from the workload; seed it in + // the test process so the session's inherited environment would leak it + // without the strip. + t.Setenv("EXECD_ACCESS_TOKEN", "pty-session-secret") + + // The session shell is the launcher-exec'd workload: read the floor from + // its own /proc/self/status and verify the credential env was stripped. + probe := "grep -E '^CapEff:|^Seccomp:|^NoNewPrivs:' /proc/self/status; " + + "if env | grep -q '^EXECD_ACCESS_TOKEN='; then echo token_leaked; else echo token_stripped; fi" + + assertFloor := func(mode string, data string) { + t.Helper() + for _, want := range []string{ + "Seccomp:\t2", + "NoNewPrivs:\t1", + "token_stripped", + } { + if !strings.Contains(data, want) { + t.Fatalf("%s session output missing %q:\n%s", mode, want, data) + } + } + // CapEff is only meaningful to assert when execd runs as root (the + // launcher drops caps it holds; a non-root test process has none). + if os.Geteuid() == 0 && !strings.Contains(data, "CapEff:\t0000000000000000") { + t.Fatalf("%s session CapEff not dropped:\n%s", mode, data) + } + } + + t.Run("pipe", func(t *testing.T) { + s := newPTYSession(uuidString(), "", probe) + if err := s.StartPipe(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.close() }) + + if !replayContains(t, s, "token_stripped", 10*time.Second) { + t.Fatal("pipe session did not produce the floor probe output") + } + data, _ := s.replay.ReadFrom(0) + assertFloor("pipe", string(data)) + }) + + t.Run("pty", func(t *testing.T) { + s := newPTYSession(uuidString(), "", probe) + if err := s.StartPTY(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.close() }) + + if !replayContains(t, s, "token_stripped", 10*time.Second) { + t.Fatal("pty session did not produce the floor probe output") + } + data, _ := s.replay.ReadFrom(0) + assertFloor("pty", string(data)) + }) +} diff --git a/components/execd/pkg/runtime/hardening_other.go b/components/execd/pkg/runtime/hardening_other.go new file mode 100644 index 000000000..7746570b0 --- /dev/null +++ b/components/execd/pkg/runtime/hardening_other.go @@ -0,0 +1,76 @@ +//go:build !linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Hardening is Linux-only (OSEP-0018); on other platforms it is a no-op. + +package runtime + +import ( + "sync/atomic" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" +) + +// Requested-state flags so the capabilities endpoint can distinguish a +// configured layer that this platform cannot provide ("unsupported") from +// an opt-out deployment ("disabled"). +var ( + otherHardeningRequested atomic.Bool + otherLandlockRequested atomic.Bool + otherEbpfRequested atomic.Bool +) + +// InitHardening is a no-op off Linux; it only records which layers were +// requested so ReportHardening can report them as unsupported instead of +// silently claiming they are disabled. +func InitHardening(cfg isolation.Config) error { + otherHardeningRequested.Store(cfg.Hardening != nil && cfg.Hardening.Enabled) + otherLandlockRequested.Store(cfg.Landlock != nil && cfg.Landlock.Enabled) + return nil +} + +// SetEbpfState records whether eBPF observation was requested off Linux. +func SetEbpfState(state LayerState) { + otherEbpfRequested.Store(state.State != "disabled") +} + +// HardeningReport reports that no hardening layer is in effect, marking +// configured layers as unsupported rather than disabled. +func ReportHardening() HardeningReport { + initMode, shield := InitModeReport() + report := HardeningReport{ + InitMode: initMode, + SignalShield: shield, + CapDrop: LayerState{State: "disabled", Message: "hardening is Linux-only"}, + Seccomp: LayerState{State: "disabled", Message: "hardening is Linux-only"}, + Landlock: LayerState{State: "disabled", Message: "hardening is Linux-only"}, + Ebpf: LayerState{State: "disabled", Message: "hardening is Linux-only"}, + } + if otherHardeningRequested.Load() { + msg := "hardening requested but unavailable on this platform (Linux-only)" + report.CapDrop = LayerState{State: "unsupported", Message: msg} + report.Seccomp = LayerState{State: "unsupported", Message: msg} + } + if otherLandlockRequested.Load() { + report.Landlock = LayerState{State: "unsupported", + Message: "landlock requested but unavailable on this platform (Linux-only)"} + } + if otherEbpfRequested.Load() { + report.Ebpf = LayerState{State: "unsupported", + Message: "eBPF observation requested but unavailable on this platform (Linux-only)"} + } + return report +} diff --git a/components/execd/pkg/runtime/initmode_barrier_unix.go b/components/execd/pkg/runtime/initmode_barrier_unix.go new file mode 100644 index 000000000..a97745449 --- /dev/null +++ b/components/execd/pkg/runtime/initmode_barrier_unix.go @@ -0,0 +1,23 @@ +//go:build !linux && !windows + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +// waitManagedWithBarrier mirrors waitCommandWithExitBarrier, which exists on +// this platform set. +func waitManagedWithBarrier(mp *managedProcess, mark func(error)) error { + return waitCommandWithExitBarrier(mp.cmd, mark) +} diff --git a/components/execd/pkg/runtime/initmode_barrier_windows.go b/components/execd/pkg/runtime/initmode_barrier_windows.go new file mode 100644 index 000000000..aa3135b30 --- /dev/null +++ b/components/execd/pkg/runtime/initmode_barrier_windows.go @@ -0,0 +1,25 @@ +//go:build windows + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +// waitManagedWithBarrier mirrors waitCommandWithExitBarrier's semantics on +// platforms where that helper does not exist. +func waitManagedWithBarrier(mp *managedProcess, mark func(error)) error { + err := mp.Wait() + mark(nil) + return err +} diff --git a/components/execd/pkg/runtime/initmode_linux.go b/components/execd/pkg/runtime/initmode_linux.go new file mode 100644 index 000000000..260c70c27 --- /dev/null +++ b/components/execd/pkg/runtime/initmode_linux.go @@ -0,0 +1,591 @@ +//go:build linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Init mode (OSEP-0018, phase 1): execd is the sandbox init. It reaps every +// child through a single reaper, forwards application signals to the user +// entrypoint, and owns the container lifecycle (entrypoint exit code is +// propagated to the runtime). +// +// The reaper is the only wait4-family caller in init mode, so execd never +// calls os/exec.Cmd.Wait for its own children. Callers build the process with +// exec.Command as usual but launch and wait through the managedProcess +// abstraction, which reproduces the pipe teardown Cmd.Wait would otherwise +// perform. +// +// The reaper registry lock spans child start and registration: a child is +// added to the owned map before any concurrent drain can observe it, so the +// start/register race is closed structurally. Any child observed that is not +// owned is a reparented orphan and is reaped and logged. + +package runtime + +import ( + "errors" + "fmt" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" + "unsafe" + + "github.com/alibaba/opensandbox/internal/safego" + "golang.org/x/sys/unix" + + "github.com/alibaba/opensandbox/execd/pkg/log" +) + +var ( + initShutdownGrace = 10 * time.Second + initReaper *reaper + reaperSweepInterval = 200 * time.Millisecond + initForwardedSignals = []os.Signal{ + syscall.SIGTERM, + syscall.SIGHUP, + syscall.SIGUSR1, + syscall.SIGUSR2, + syscall.SIGWINCH, + } +) + +// siginfoWait mirrors the kernel siginfo_t fields waitid fills. The vendored +// x/sys Siginfo only exposes the first three fields, and the union member +// offsets are arch-specific, so this struct is the 64-bit Linux layout +// (verified on amd64 and arm64): the union is 8-byte aligned, so si_pid sits +// at offset 16, si_uid at 20, si_status at 24. waitid's si_status is the raw +// exit code or signal number, not the wait4 status encoding. +type siginfoWait struct { + signo int32 + errno int32 + code int32 + _ int32 + pid int32 + uid uint32 + status int32 + _ [104]byte +} + +// waitStatus converts waitid's si_status/si_code into the syscall.WaitStatus +// encoding the rest of the code base understands. si_code values are the +// stable UAPI CLD_* constants (linux/siginfo.h). +func (i *siginfoWait) waitStatus() syscall.WaitStatus { + switch i.code { + case 1: // CLD_EXITED + return syscall.WaitStatus(uint32(i.status&0xff) << 8) + case 2, 3: // CLD_KILLED, CLD_DUMPED + return syscall.WaitStatus(uint32(i.status & 0x7f)) + default: + return 0 + } +} + +func waitidObserve(info *siginfoWait) error { + _, _, errno := unix.Syscall6(unix.SYS_WAITID, + uintptr(unix.P_ALL), 0, + uintptr(unsafe.Pointer(info)), + uintptr(unix.WEXITED|unix.WNOHANG|unix.WNOWAIT), + 0, 0) + if errno == 0 { + return nil + } + if errno == unix.EINTR { + return unix.EINTR + } + return errno +} + +func waitidConsume(pid int) (syscall.WaitStatus, error) { + var info siginfoWait + _, _, errno := unix.Syscall6(unix.SYS_WAITID, + uintptr(unix.P_PID), uintptr(pid), + uintptr(unsafe.Pointer(&info)), + uintptr(unix.WEXITED|unix.WNOHANG), + 0, 0) + if errno == 0 { + return info.waitStatus(), nil + } + if errno == unix.EINTR { + return 0, unix.EINTR + } + return 0, errno +} + +// reaper is the single wait4-family caller while init mode is active. +type reaper struct { + mu sync.Mutex + owned map[int]*managedProcess + sigchld chan os.Signal + quit chan struct{} + quitOnce sync.Once //nolint:unused // test-only lifecycle; see stop + done chan struct{} +} + +func newReaper() *reaper { + return &reaper{ + owned: map[int]*managedProcess{}, + quit: make(chan struct{}), + done: make(chan struct{}), + } +} + +// start registers the SIGCHLD notification synchronously so no child can +// exit before the handler exists (a lost SIGCHLD would strand its status). +func (r *reaper) start() { + r.sigchld = make(chan os.Signal, 1) + signal.Notify(r.sigchld, syscall.SIGCHLD) +} + +// stop terminates the reaper and waits until its signal subscription is +// removed. Test-only in practice; execd runs one reaper for its lifetime. +// +//nolint:unused // test-only lifecycle; execd runs one reaper for its lifetime +func (r *reaper) stop() { + r.quitOnce.Do(func() { close(r.quit) }) + <-r.done +} + +func (r *reaper) run() { + defer func() { + signal.Stop(r.sigchld) + close(r.done) + }() + // The ticker is a backstop: SIGCHLD may be coalesced or (in edge cases) + // lost, so a periodic drain keeps the process table bounded regardless. + sweep := time.NewTicker(reaperSweepInterval) + defer sweep.Stop() + for { + select { + case <-r.quit: + return + case <-r.sigchld: + r.drain() + case <-sweep.C: + r.drain() + } + } +} + +func (r *reaper) drain() { + r.mu.Lock() + defer r.mu.Unlock() + for { + var info siginfoWait + if err := waitidObserve(&info); err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + if !errors.Is(err, unix.ECHILD) { + log.Warn("init: reaper observe: %v", err) + } + return + } + if info.pid == 0 { + return + } + pid := int(info.pid) + if mp := r.owned[pid]; mp != nil { + // The pre-reap barrier runs between the WNOWAIT observe and the + // consuming wait, while the kernel still reserves the PID/PGID + // (isolated sessions rely on this to avoid signalling a recycled + // process group). + if mp.preReap != nil { + mp.preReap() + } + ws, err := waitidConsume(pid) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + log.Error("init: reaper consume pid %d: %v", pid, err) + return + } + // Drop the child from the registry once reaped: stale entries + // would grow without bound and shutdown could signal a recycled + // process group. + delete(r.owned, pid) + mp.deliver(ws) + continue + } + // Unknown child: reparented orphan. Reap it so the process table + // stays bounded; its status is not delivered to anyone. + ws, err := waitidConsume(pid) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + log.Error("init: reaper consume orphan pid %d: %v", pid, err) + return + } + log.Info("init: reaped orphan pid=%d status=%s", pid, ws) + } +} + +// managedProcess wraps an exec.Cmd whose status is delivered by the reaper. +// In non-init mode it falls back to plain Cmd.Start/Cmd.Wait, so callers +// share one launch path regardless of mode. +type managedProcess struct { + cmd *exec.Cmd + preReap func() + noHardening bool + stripEnv []string // nil = default blacklist; explicit list overrides + done chan struct{} + once sync.Once + ws syscall.WaitStatus + exitErr error +} + +func newManagedProcess(cmd *exec.Cmd) *managedProcess { + return &managedProcess{cmd: cmd, done: make(chan struct{})} +} + +func (mp *managedProcess) pid() int { + return mp.cmd.Process.Pid +} + +func (mp *managedProcess) deliver(ws syscall.WaitStatus) { + mp.once.Do(func() { + mp.ws = ws + mp.exitErr = exitStatusError(ws) + close(mp.done) + }) +} + +func (mp *managedProcess) Wait() error { + if initReaper == nil { + return mp.cmd.Wait() + } + <-mp.done + return mp.exitErr +} + +// ExitCode returns the process exit code, or -1 if it has not exited (or was +// killed by a signal), matching os.ProcessState.ExitCode semantics. +func (mp *managedProcess) ExitCode() int { + if initReaper == nil { + return mp.cmd.ProcessState.ExitCode() + } + select { + case <-mp.done: + return mp.ws.ExitStatus() + default: + return -1 + } +} + +func (mp *managedProcess) exitStatus() syscall.WaitStatus { + return mp.ws +} + +type launchOption func(*managedProcess) + +func withPreReap(fn func()) launchOption { + return func(mp *managedProcess) { + mp.preReap = fn + } +} + +// withoutHardening exempts a launch from the hardening floor. Used for the +// bwrap process of isolated sessions, whose workload is already reduced +// inside the namespace and whose own syscalls (unshare) the floor would deny. +func withoutHardening() launchOption { + return func(mp *managedProcess) { + mp.noHardening = true + } +} + +// bootstrapEnv overrides the env strip for the user entrypoint: the image's +// own entrypoint scripts may need JUPYTER_TOKEN/EXECD_ENVS to configure +// themselves (e.g. the code-interpreter entrypoint), so those survive โ€” but +// EXECD_ACCESS_TOKEN is execd's control-plane credential and must never +// reach the long-lived entrypoint (its Jupyter kernels are user code). +func bootstrapEnv() launchOption { + return func(mp *managedProcess) { + mp.stripEnv = []string{"EXECD_ACCESS_TOKEN"} + } +} + +// launchManagedWith starts the command and registers it with the reaper. +// startFn is called under the reaper lock so the child cannot be observed +// (and misclassified as an orphan) before registration. When the hardening +// floor is active, cmd is first rewritten to exec through the launcher. +func launchManagedWith(cmd *exec.Cmd, startFn func() error, opts ...launchOption) (*managedProcess, error) { + mp := newManagedProcess(cmd) + for _, o := range opts { + o(mp) + } + policyFile, err := hardenCmd(cmd, mp.noHardening, mp.stripEnv) + if err != nil { + return nil, err + } + if policyFile != nil { + defer policyFile.Close() + } + if initReaper == nil { + if err := startFn(); err != nil { + return nil, err + } + return mp, nil + } + initReaper.mu.Lock() + defer initReaper.mu.Unlock() + if err := startFn(); err != nil { + return nil, err + } + initReaper.owned[cmd.Process.Pid] = mp + return mp, nil +} + +func launchManaged(cmd *exec.Cmd, opts ...launchOption) (*managedProcess, error) { + return launchManagedWith(cmd, cmd.Start, opts...) +} + +// waitManagedWithBarrier mirrors waitCommandWithExitBarrier: in init mode the +// pre-reap barrier was registered at launch and runs inside the reaper, so +// this just waits; otherwise the original WNOWAIT barrier path applies. +func waitManagedWithBarrier(mp *managedProcess, mark func(error)) error { + if initReaper == nil { + return waitCommandWithExitBarrier(mp.cmd, mark) + } + return mp.Wait() +} + +// processExitError is the error returned by managedProcess.Wait in init mode +// when the child did not exit cleanly. It mirrors exec.ExitError's contract +// without needing a constructed os.ProcessState. +type processExitError struct { + code int + msg string +} + +func (e *processExitError) Error() string { + return e.msg +} + +func (e *processExitError) ExitCode() int { + return e.code +} + +func exitStatusError(ws syscall.WaitStatus) error { + if ws.Exited() { + if code := ws.ExitStatus(); code == 0 { + return nil + } else { + return &processExitError{code: code, msg: fmt.Sprintf("exit status %d", code)} + } + } + return &processExitError{code: -1, msg: fmt.Sprintf("signal: %v", ws.Signal())} +} + +// StartInitMode activates init duties: non-dumpable self, subreaper fallback +// when not PID 1, the reaper, the user entrypoint, signal forwarding, and the +// container lifecycle owner. It returns once the entrypoint is launched; the +// process is torn down via os.Exit when the entrypoint exits or SIGTERM +// arrives. +func StartInitMode(entryArgs []string) { + if err := unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil { + log.Warn("init: PR_SET_DUMPABLE(0) failed: %v", err) + } + if os.Getpid() != 1 { + // Pool path (or a misconfigured background launch): execd is not the + // kernel init, so orphaned descendants reparent to it only if it is a + // subreaper. The kernel signal shield is lost in this mode. + if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil { + log.Warn("init: PR_SET_CHILD_SUBREAPER failed: %v", err) + } + } + initReaper = newReaper() + initReaper.start() + safego.Go(initReaper.run) + log.Info("init: execd is the sandbox init (pid=%d mode=%s)", os.Getpid(), initModeName()) + + // Register the application-signal subscription before the entrypoint + // starts: an early SIGTERM must reach the forwarding loop instead of + // hitting the runtime default handler. + sigCh := make(chan os.Signal, 8) + signal.Notify(sigCh, initForwardedSignals...) + + entry := launchEntrypoint(entryArgs) + if entry == nil { + signal.Stop(sigCh) + return + } + safego.Go(func() { forwardInitSignals(entry, sigCh) }) + safego.Go(func() { waitEntrypointExit(entry) }) +} + +func launchEntrypoint(args []string) *managedProcess { + if len(args) == 0 { + log.Warn("init: --init set but no user command provided; no entrypoint to supervise") + return nil + } + cmd := exec.Command(args[0], args[1:]...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + mp, err := launchManaged(cmd, bootstrapEnv()) + if err != nil { + log.Error("init: failed to start user entrypoint %q: %v", args[0], err) + os.Exit(1) + } + log.Info("init: user entrypoint started pid=%d argv=%v", mp.pid(), args) + return mp +} + +// waitEntrypointExit owns the container lifecycle: when the entrypoint exits, +// the other children are stopped gracefully and execd exits with the +// entrypoint's status so Docker/kubelet observe it. +func waitEntrypointExit(entry *managedProcess) { + entryErr := entry.Wait() + code := initExitCode(entry) + log.Info("init: user entrypoint exited: code=%d err=%v", code, entryErr) + stopChildrenExcept(entry) + log.Info("init: exiting with entrypoint status %d", code) + os.Exit(code) +} + +// initExitCode converts a delivered status into the container exit code, +// following the shell convention of 128+signal for signalled processes. +func initExitCode(mp *managedProcess) int { + ws := mp.exitStatus() + if ws.Signaled() { + return 128 + int(ws.Signal()) + } + if ws.Exited() { + return ws.ExitStatus() + } + return 1 +} + +// forwardInitSignals forwards application signals to the entrypoint process +// group. SIGTERM additionally starts the graceful shutdown sequence, matching +// the runtime-initiated container stop contract (Docker/K8s send SIGTERM to +// PID 1). +func forwardInitSignals(entry *managedProcess, ch <-chan os.Signal) { + for sig := range ch { + s, ok := sig.(syscall.Signal) + if !ok { + continue + } + if s == syscall.SIGTERM { + log.Info("init: received SIGTERM; forwarding to workload and shutting down") + terminateInit(entry) + return + } + log.Info("init: forwarding %v to workload", s) + if err := killGroup(entry.pid(), s); err != nil { + log.Warn("init: forward %v to entrypoint group: %v", s, err) + } + } +} + +// terminateInit performs the SIGTERM shutdown: forward TERM to the entrypoint +// tree, stop the other children, then exit once the entrypoint is reaped +// (SIGKILL after a bounded grace). +func terminateInit(entry *managedProcess) { + if err := killGroup(entry.pid(), syscall.SIGTERM); err != nil && !errors.Is(err, syscall.ESRCH) { + log.Warn("init: SIGTERM entrypoint group: %v", err) + } + stopChildrenExcept(entry) + deadline := time.After(initShutdownGrace) + select { + case <-entry.done: + case <-deadline: + if err := killGroup(entry.pid(), syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + log.Warn("init: SIGKILL entrypoint group: %v", err) + } + select { + case <-entry.done: + case <-time.After(5 * time.Second): + } + } + log.Info("init: exiting after SIGTERM shutdown") + os.Exit(initExitCode(entry)) +} + +// stopChildrenExcept signals every other tracked child group with SIGTERM, +// waits up to the shutdown grace (total budget across all children), then +// SIGKILLs the survivors. Reaping is done by the reaper; the kernel reaps +// anything left when execd exits. +func stopChildrenExcept(keep *managedProcess) { + // SIGTERM and the final SIGKILL pass are sent while holding the reaper + // lock: the pid is verified against the owned map and the reaper cannot + // consume (and release) the PID/PGID between verification and kill, so + // a recycled process group can never be signalled. + others := initReaper.signalOthers(keep, syscall.SIGTERM) + if len(others) == 0 { + return + } + deadline := time.Now().Add(initShutdownGrace) + for _, mp := range others { + select { + case <-mp.done: + case <-time.After(time.Until(deadline)): + } + } + initReaper.signalOthers(keep, syscall.SIGKILL) +} + +// signalOthers delivers sig to every still-tracked child group except keep, +// while holding the reaper lock. It returns the targets that were signalled. +func (r *reaper) signalOthers(keep *managedProcess, sig syscall.Signal) []*managedProcess { + r.mu.Lock() + defer r.mu.Unlock() + var others []*managedProcess + for pid, mp := range r.owned { + if mp == keep { + continue + } + others = append(others, mp) + if err := killGroup(pid, sig); err != nil && !errors.Is(err, syscall.ESRCH) { + log.Warn("init: %v child group %d: %v", sig, pid, err) + } + } + return others +} + +// killGroup sends sig to the child's process group; all managed children are +// launched with Setpgid, so the group id equals the child pid. +func killGroup(pid int, sig syscall.Signal) error { + return syscall.Kill(-pid, sig) +} + +func initModeName() string { + if os.Getpid() == 1 { + return "pid1" + } + return "subreaper" +} + +// InitModeReport reports the init mode actually in effect for the +// capabilities endpoint. +func InitModeReport() (mode string, signalShield bool) { + if initReaper == nil { + return "none", false + } + if os.Getpid() == 1 { + return "pid1", true + } + return "subreaper", false +} + +// initModeActive reports whether the init-mode signal/runtime ownership is in +// effect (execd started with --init). Shared launch paths consult it to avoid +// competing with forwardInitSignals. +func initModeActive() bool { + return initReaper != nil +} diff --git a/components/execd/pkg/runtime/initmode_linux_test.go b/components/execd/pkg/runtime/initmode_linux_test.go new file mode 100644 index 000000000..5437059b5 --- /dev/null +++ b/components/execd/pkg/runtime/initmode_linux_test.go @@ -0,0 +1,370 @@ +//go:build linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "errors" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +func startReaperForTest(t *testing.T) { + t.Helper() + if initReaper != nil { + t.Fatal("init reaper already running") + } + initReaper = newReaper() + initReaper.start() + go initReaper.run() + t.Cleanup(func() { + initReaper.stop() + initReaper = nil + }) +} + +func waitForFile(t *testing.T, path string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("file %s did not appear within %s", path, timeout) +} + +// exitStatusByPid observes a child's status with WNOWAIT without consuming it. +func exitStatusByPid(t *testing.T, pid int) (syscall.WaitStatus, bool) { + t.Helper() + var info siginfoWait + _, _, errno := syscall.RawSyscall6(syscall.SYS_WAITID, + uintptr(unix.P_PID), uintptr(pid), + uintptr(unsafe.Pointer(&info)), + uintptr(unix.WEXITED|unix.WNOHANG|unix.WNOWAIT), + 0, 0) + if errno == syscall.ECHILD { + return 0, false + } + if errno != 0 { + t.Fatalf("waitid observe: %v", errno) + } + return info.waitStatus(), true +} + +func TestManagedProcessExitStatus(t *testing.T) { + startReaperForTest(t) + + cmd := exec.Command("sh", "-c", "exit 7") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + err = mp.Wait() + var exitCodeErr exitCoder + if !errors.As(err, &exitCodeErr) || exitCodeErr.ExitCode() != 7 { + t.Fatalf("Wait error = %v, want exit status 7", err) + } + if mp.ExitCode() != 7 { + t.Fatalf("ExitCode = %d, want 7", mp.ExitCode()) + } +} + +func TestManagedProcessExitZero(t *testing.T) { + startReaperForTest(t) + + cmd := exec.Command("sh", "-c", "exit 0") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + if err := mp.Wait(); err != nil { + t.Fatalf("Wait error = %v, want nil", err) + } +} + +func TestManagedProcessKilledBySignal(t *testing.T) { + startReaperForTest(t) + + cmd := exec.Command("sh", "-c", "kill -TERM $$") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + if err := mp.Wait(); err == nil { + t.Fatal("Wait error = nil, want signal error") + } + if mp.ExitCode() != -1 { + t.Fatalf("ExitCode = %d, want -1 for signalled process", mp.ExitCode()) + } +} + +func TestManagedProcessDispatchIsPerPid(t *testing.T) { + startReaperForTest(t) + + codes := []int{0, 1, 7, 42, 255} + var mps []*managedProcess + for _, code := range codes { + cmd := exec.Command("sh", "-c", exitScript(code)) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + mps = append(mps, mp) + } + // Wait in reverse order: a cross-delivered status would surface here. + for i := len(mps) - 1; i >= 0; i-- { + mp := mps[i] + if err := mp.Wait(); err != nil && i == 0 { + t.Fatalf("child %d (exit 0): Wait error = %v", i, err) + } + if got := mp.ExitCode(); got != codes[i] { + t.Fatalf("child %d exit code = %d, want %d", i, got, codes[i]) + } + } +} + +func exitScript(code int) string { + return "exit " + strconv.Itoa(code) +} + +func TestReaperReapsOrphan(t *testing.T) { + startReaperForTest(t) + + cmd := exec.Command("sh", "-c", "exit 3") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + pid := cmd.Process.Pid + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, ok := exitStatusByPid(t, pid); !ok { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("orphan pid %d was not reaped by the reaper", pid) +} + +// TestReaperSweepBackstop verifies the periodic sweep drains children even +// when no SIGCHLD notification can reach the run loop โ€” the lost/coalesced +// signal case the sweep ticker backstops (OSEP-0018 R-t). +// +// The reaper's signal.Notify subscription stays registered (which also stops +// the Go runtime from auto-reaping children), but the run loop is severed +// from the subscribed channel before it starts, so the kernel-delivered +// SIGCHLD goes nowhere. Only the ticker can reap the exiting child. +func TestReaperSweepBackstop(t *testing.T) { + oldInterval := reaperSweepInterval + reaperSweepInterval = 50 * time.Millisecond + defer func() { reaperSweepInterval = oldInterval }() + + r := newReaper() + r.start() + // Replace the subscribed channel before the run loop reads it: the kernel + // keeps signalling the subscribed channel (unread), so the loop below can + // only ever drain via the sweep ticker. Keep the original channel so + // cleanup can stop the subscription โ€” signal.Stop on the replacement + // would leave the process-global SIGCHLD handler registered. + subscribed := r.sigchld + r.sigchld = make(chan os.Signal, 1) + initReaper = r + t.Cleanup(func() { + initReaper.stop() + signal.Stop(subscribed) + initReaper = nil + }) + go r.run() + + cmd := exec.Command("sh", "-c", "exit 0") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + pid := cmd.Process.Pid + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, ok := exitStatusByPid(t, pid); !ok { + return // reaped by the sweep ticker + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("child pid %d was not reaped by the sweep backstop", pid) +} + +func TestPreReapBarrierRunsBeforeWaitReturns(t *testing.T) { + startReaperForTest(t) + + barrierRan := make(chan struct{}) + cmd := exec.Command("sh", "-c", "exit 0") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd, withPreReap(func() { close(barrierRan) })) + if err != nil { + t.Fatal(err) + } + if err := mp.Wait(); err != nil { + t.Fatal(err) + } + select { + case <-barrierRan: + default: + t.Fatal("pre-reap barrier did not run before Wait returned") + } +} + +func TestInitExitCode(t *testing.T) { + startReaperForTest(t) + + tests := []struct { + script string + want int + }{ + {"exit 7", 7}, + {"kill -TERM $$", 128 + 15}, + {"exit 0", 0}, + } + for _, tt := range tests { + cmd := exec.Command("sh", "-c", tt.script) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + _ = mp.Wait() + if got := initExitCode(mp); got != tt.want { + t.Fatalf("initExitCode(%q) = %d, want %d", tt.script, got, tt.want) + } + } +} + +func TestForwardSignalToWorkload(t *testing.T) { + startReaperForTest(t) + + dir := t.TempDir() + marker := filepath.Join(dir, "usr1") + installed := filepath.Join(dir, "installed") + cmd := exec.Command("sh", "-c", "trap 'touch \"$MARKER\"' USR1; touch \"$INSTALLED\"; while :; do sleep 0.05; done") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Env = append(os.Environ(), "MARKER="+marker, "INSTALLED="+installed) + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = killGroup(mp.pid(), syscall.SIGKILL) + _ = mp.Wait() + }() + + // Wait until the trap is installed; a signal sent earlier is dropped while + // the shell initializes (dash blocks signals during startup). + waitForFile(t, installed, 3*time.Second) + if err := killGroup(mp.pid(), syscall.SIGUSR1); err != nil { + t.Fatal(err) + } + waitForFile(t, marker, 3*time.Second) +} + +func TestStopChildrenGraceThenKill(t *testing.T) { + startReaperForTest(t) + oldGrace := initShutdownGrace + initShutdownGrace = 500 * time.Millisecond + defer func() { initShutdownGrace = oldGrace }() + + dir := t.TempDir() + termMarker := filepath.Join(dir, "term") + termInstalled := filepath.Join(dir, "term-installed") + termCmd := exec.Command("sh", "-c", + "trap 'touch \"$MARKER\"; exit 0' TERM; touch \"$INSTALLED\"; while :; do sleep 0.05; done") + termCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + termCmd.Env = append(os.Environ(), "MARKER="+termMarker, "INSTALLED="+termInstalled) + termMp, err := launchManaged(termCmd) + if err != nil { + t.Fatal(err) + } + + // This child ignores SIGTERM and must be SIGKILLed after the grace period. + stubbornInstalled := filepath.Join(dir, "stubborn-installed") + stubbornCmd := exec.Command("sh", "-c", + "trap '' TERM; touch \"$INSTALLED\"; while :; do sleep 0.05; done") + stubbornCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stubbornCmd.Env = append(os.Environ(), "INSTALLED="+stubbornInstalled) + stubbornMp, err := launchManaged(stubbornCmd) + if err != nil { + t.Fatal(err) + } + + waitForFile(t, termInstalled, 3*time.Second) + waitForFile(t, stubbornInstalled, 3*time.Second) + + stopChildrenExcept(nil) + + waitForFile(t, termMarker, 2*time.Second) + for _, mp := range []*managedProcess{termMp, stubbornMp} { + select { + case <-mp.done: + case <-time.After(2 * time.Second): + t.Fatalf("child pid %d was not stopped", mp.pid()) + } + } + if ws := termMp.exitStatus(); !ws.Exited() || ws.ExitStatus() != 0 { + t.Fatalf("term-trapping child status = %#v, want clean exit", ws) + } +} + +func TestInitModeReport(t *testing.T) { + if mode, _ := InitModeReport(); mode != "none" { + t.Fatalf("InitModeReport before init = %q, want none", mode) + } + startReaperForTest(t) + if mode, shield := InitModeReport(); mode != "subreaper" || shield { + t.Fatalf("InitModeReport after init = %q/%v, want subreaper/false (test is not PID 1)", mode, shield) + } +} + +func TestManagedProcessWithoutReaperUsesCmdWait(t *testing.T) { + if initReaper != nil { + t.Fatal("reaper unexpectedly running") + } + cmd := exec.Command("sh", "-c", "exit 4") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + mp, err := launchManaged(cmd) + if err != nil { + t.Fatal(err) + } + if err := mp.Wait(); err == nil { + t.Fatal("Wait error = nil, want exit status 4") + } + if mp.ExitCode() != 4 { + t.Fatalf("ExitCode = %d, want 4", mp.ExitCode()) + } +} diff --git a/components/execd/pkg/runtime/initmode_other.go b/components/execd/pkg/runtime/initmode_other.go new file mode 100644 index 000000000..fdb0ffe0d --- /dev/null +++ b/components/execd/pkg/runtime/initmode_other.go @@ -0,0 +1,81 @@ +//go:build !linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Init mode is Linux-only (OSEP-0018). On other platforms the managedProcess +// abstraction degrades to plain Cmd.Start/Cmd.Wait so the shared launch paths +// keep compiling unchanged. + +package runtime + +import ( + "os/exec" + + "github.com/alibaba/opensandbox/execd/pkg/log" +) + +type managedProcess struct { + cmd *exec.Cmd +} + +func newManagedProcess(cmd *exec.Cmd) *managedProcess { + return &managedProcess{cmd: cmd} +} + +func (mp *managedProcess) Wait() error { + return mp.cmd.Wait() +} + +func (mp *managedProcess) ExitCode() int { + return mp.cmd.ProcessState.ExitCode() +} + +type launchOption func(*managedProcess) + +func withPreReap(fn func()) launchOption { + return func(*managedProcess) {} +} + +// withoutHardening is a no-op off Linux (the floor never applies there). +func withoutHardening() launchOption { + return func(*managedProcess) {} +} + +func launchManagedWith(cmd *exec.Cmd, startFn func() error, opts ...launchOption) (*managedProcess, error) { + if err := startFn(); err != nil { + return nil, err + } + return newManagedProcess(cmd), nil +} + +func launchManaged(cmd *exec.Cmd, opts ...launchOption) (*managedProcess, error) { + return launchManagedWith(cmd, cmd.Start, opts...) +} + +// StartInitMode is unsupported off Linux; execd keeps today's behavior. +func StartInitMode(entryArgs []string) { + log.Warn("init mode is unsupported on this platform; continuing without init duties") +} + +// InitModeReport reports the init mode actually in effect for the +// capabilities endpoint. +func InitModeReport() (mode string, signalShield bool) { + return "none", false +} + +// initModeActive is always false off Linux: init mode never runs there. +func initModeActive() bool { + return false +} diff --git a/components/execd/pkg/runtime/isolated_session.go b/components/execd/pkg/runtime/isolated_session.go index e4a6c1d12..df7fd1b3a 100644 --- a/components/execd/pkg/runtime/isolated_session.go +++ b/components/execd/pkg/runtime/isolated_session.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "os" "os/exec" "sync" "sync/atomic" @@ -191,29 +192,48 @@ func (s *isolatedSession) start() error { } s.lifecycle = lifecycle - stdin, err := cmd.StdinPipe() + stdinR, stdinW, err := os.Pipe() if err != nil { closeCommandExtraFiles(cmd) return s.failStartup(err) } - s.stdin = stdin - stdout, err := cmd.StdoutPipe() + s.stdin = stdinW + stdoutR, stdoutW, err := os.Pipe() if err != nil { + _ = stdinR.Close() + _ = stdinW.Close() closeCommandExtraFiles(cmd) return s.failStartup(err) } - s.stdout = stdout - cmd.Stderr = cmd.Stdout - - if err := cmd.Start(); err != nil { + s.stdout = stdoutR + cmd.Stdin = stdinR + cmd.Stdout = stdoutW + cmd.Stderr = stdoutW + + mp, err := launchManaged( + cmd, + withPreReap(func() { s.markProcessExitedBeforeReap(nil) }), + // bwrap needs unshare/mount + capabilities to build the namespace; + // its workload is already reduced inside by bwrap's own seccomp and + // the session gate. + withoutHardening(), + ) + if err != nil { + _ = stdinR.Close() + _ = stdinW.Close() + _ = stdoutR.Close() + _ = stdoutW.Close() closeCommandExtraFiles(cmd) return s.failStartup(fmt.Errorf("start %s: %w", shell, err)) } + // Close the child-side ends in the parent โ€” the child has its own copies. + _ = stdinR.Close() + _ = stdoutW.Close() closeCommandExtraFiles(cmd) go func() { - _ = waitCommandWithExitBarrier(cmd, s.markProcessExitedBeforeReap) + _ = waitManagedWithBarrier(mp, s.markProcessExitedBeforeReap) // Publish process reaping before waiting for lifecycle accounting. // Once Wait returns, the numeric PID/PGID may be reused and must never // be signalled again. diff --git a/components/execd/pkg/runtime/isolated_session_initmode_linux_test.go b/components/execd/pkg/runtime/isolated_session_initmode_linux_test.go new file mode 100644 index 000000000..624b48733 --- /dev/null +++ b/components/execd/pkg/runtime/isolated_session_initmode_linux_test.go @@ -0,0 +1,141 @@ +//go:build linux && bwrap + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" +) + +// Integration test: isolated sessions (bwrap) under init-mode reaper dispatch +// (OSEP-0018 R-o). The bwrap process is launched through launchManaged with +// withoutHardening and a pre-reap barrier; with the reaper active, the barrier +// runs inside the reaper's drain between the WNOWAIT observe and the consuming +// wait. These tests run the real bwrap lifecycle (create / run / delete, and +// delete racing a running workload) with the reaper owning wait4. +func TestIsolatedSessionWithInitReaper(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("bwrap init-mode integration requires root") + } + startReaperForTest(t) + + ctrl := NewController("", "") + cfg := isolation.Config{ + UpperRoot: t.TempDir(), + UpperMaxBytes: 1 << 30, + AllowedWritable: []string{"/tmp"}, + } + iso := isolation.NewBwrap(cfg) + if !iso.Available() { + t.Skip("bwrap isolator unavailable") + } + runner, err := NewIsolatedRunner(ctrl, iso, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := runner.Close(); err != nil { + t.Errorf("close isolated runner: %v", err) + } + }) + + opts := &IsolatedSessionOptions{ + Profile: string(isolation.ProfileStrict), + WorkspacePath: t.TempDir(), + WorkspaceMode: string(isolation.WorkspaceRW), + } + id, err := runner.CreateIsolatedSession(opts) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Runs execute while the reaper owns wait4. + var lines []string + if err := runner.RunInIsolatedSession(ctx, id, "echo init_reaper_ok", nil, func(line string) { + lines = append(lines, line) + }); err != nil { + t.Fatal(err) + } + if len(lines) != 1 || lines[0] != "init_reaper_ok" { + t.Fatalf("run output = %v, want [init_reaper_ok]", lines) + } + + // Exit codes still propagate through the reaper-delivered status. + if err := runner.RunInIsolatedSession(ctx, id, "bash -c 'exit 13'", nil, nil); err == nil || + !strings.Contains(err.Error(), "13") { + t.Fatalf("exit-code run error = %v, want exit 13", err) + } + + // Delete racing a running workload: stop signals the process group while + // the reaper holds the pid (WNOWAIT observe -> pre-reap barrier -> consume), + // so the PGID-reuse protection must serialize with reaper dispatch. + runCtx, runCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer runCancel() + runErrCh := make(chan error, 1) + started := make(chan struct{}, 1) + go func() { + runErrCh <- runner.RunInIsolatedSession( + runCtx, + id, + "echo run_started; sleep 30", + nil, + func(line string) { + if strings.Contains(line, "run_started") { + select { + case started <- struct{}{}: + default: + } + } + }, + ) + }() + select { + case <-started: + case err := <-runErrCh: + t.Fatalf("run ended before start: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("run did not start") + } + + begin := time.Now() + if err := runner.DeleteIsolatedSession(id); err != nil { + t.Fatalf("delete during run: %v", err) + } + if elapsed := time.Since(begin); elapsed > 10*time.Second { + t.Fatalf("delete during run took %v", elapsed) + } + select { + case err := <-runErrCh: + if err == nil { + t.Fatal("in-flight run unexpectedly succeeded after session deletion") + } + case <-time.After(10 * time.Second): + t.Fatal("in-flight run did not terminate after session deletion") + } + + if _, err := runner.GetIsolatedSession(id); err == nil { + t.Fatal("deleted session still present") + } +} diff --git a/components/execd/pkg/runtime/isolated_session_lifecycle_test.go b/components/execd/pkg/runtime/isolated_session_lifecycle_test.go index 32158f827..e708de1d9 100644 --- a/components/execd/pkg/runtime/isolated_session_lifecycle_test.go +++ b/components/execd/pkg/runtime/isolated_session_lifecycle_test.go @@ -19,7 +19,6 @@ package runtime import ( "context" "errors" - "io" "os" "os/exec" "strings" @@ -667,18 +666,6 @@ func TestIsolatedSessionPreStartFailuresCleanLifecycleAndDescriptors(t *testing. configure func(*exec.Cmd) }{ {name: "wrap", wrapErr: wrapErr}, - { - name: "stdin pipe", - configure: func(cmd *exec.Cmd) { - cmd.Stdin = strings.NewReader("") - }, - }, - { - name: "stdout pipe", - configure: func(cmd *exec.Cmd) { - cmd.Stdout = io.Discard - }, - }, { name: "command start", configure: func(cmd *exec.Cmd) { diff --git a/components/execd/pkg/runtime/pty_session.go b/components/execd/pkg/runtime/pty_session.go index 875cef41a..36496e887 100644 --- a/components/execd/pkg/runtime/pty_session.go +++ b/components/execd/pkg/runtime/pty_session.go @@ -86,6 +86,7 @@ type ptySession struct { lastExitCode int // exit code; -1 until process exits doneCh chan struct{} // closed when process exits (non-nil after Start*) outputDoneCh chan struct{} // closed after output broadcasters finish writing to replay + proc *managedProcess // Stdin (PTY master in PTY mode; write end of os.Pipe in pipe mode) stdin io.WriteCloser @@ -268,7 +269,12 @@ func (s *ptySession) StartPTY() error { // Do NOT set Setpgid: pty.StartWithSize sets Setsid+Setctty internally. // Combining Setsid+Setpgid causes EPERM (setpgid is illegal for a session leader). - ptmx, err := pty.StartWithSize(cmd, &pty.Winsize{Cols: 80, Rows: 24}) + var ptmx *os.File + mp, err := launchManagedWith(cmd, func() error { + var perr error + ptmx, perr = pty.StartWithSize(cmd, &pty.Winsize{Cols: 80, Rows: 24}) + return perr + }) if err != nil { return fmt.Errorf("pty.StartWithSize: %w", err) } @@ -276,6 +282,7 @@ func (s *ptySession) StartPTY() error { s.ptmx = ptmx s.isPTY = true s.pid = cmd.Process.Pid + s.proc = mp s.doneCh = make(chan struct{}) outputDoneCh := make(chan struct{}) s.outputDoneCh = outputDoneCh @@ -285,7 +292,7 @@ func (s *ptySession) StartPTY() error { defer close(outputDoneCh) s.broadcastPTY(ptmx) }) - safego.Go(func() { s.waitAndExit(cmd, ptmx) }) + safego.Go(func() { s.waitAndExit(mp, ptmx) }) return nil } @@ -332,7 +339,8 @@ func (s *ptySession) StartPipe() error { cmd.Stdout = stdoutW cmd.Stderr = stderrW - if err := cmd.Start(); err != nil { + mp, err := launchManagedWith(cmd, cmd.Start) + if err != nil { _ = stdinR.Close() _ = stdinW.Close() _ = stdoutR.Close() @@ -349,6 +357,7 @@ func (s *ptySession) StartPipe() error { s.isPTY = false s.pid = cmd.Process.Pid + s.proc = mp s.doneCh = make(chan struct{}) outputDoneCh := make(chan struct{}) s.outputDoneCh = outputDoneCh @@ -368,7 +377,7 @@ func (s *ptySession) StartPipe() error { outputWg.Wait() close(outputDoneCh) }) - safego.Go(func() { s.waitAndExitPipe(cmd, stdinW, stdoutR, stderrR) }) + safego.Go(func() { s.waitAndExitPipe(mp, stdinW, stdoutR, stderrR) }) return nil } @@ -429,17 +438,14 @@ func (s *ptySession) writeAndFanout(chunk []byte, isStdout bool) { } // waitAndExit waits for the PTY-mode process and updates session state on exit. -func (s *ptySession) waitAndExit(cmd *exec.Cmd, ptmx *os.File) { - _ = cmd.Wait() +func (s *ptySession) waitAndExit(mp *managedProcess, ptmx *os.File) { + _ = mp.Wait() // Close the PTY master to unblock the broadcast goroutine. _ = ptmx.Close() s.mu.Lock() - exitCode := 0 - if cmd.ProcessState != nil { - exitCode = cmd.ProcessState.ExitCode() - } + exitCode := mp.ExitCode() s.lastExitCode = exitCode s.pid = 0 doneCh := s.doneCh @@ -449,17 +455,14 @@ func (s *ptySession) waitAndExit(cmd *exec.Cmd, ptmx *os.File) { } // waitAndExitPipe waits for the pipe-mode process and updates session state on exit. -func (s *ptySession) waitAndExitPipe(cmd *exec.Cmd, stdinW, stdoutR, stderrR *os.File) { - _ = cmd.Wait() +func (s *ptySession) waitAndExitPipe(mp *managedProcess, stdinW, stdoutR, stderrR *os.File) { + _ = mp.Wait() // Close stdin write-end so the child (if still running) sees EOF. _ = stdinW.Close() s.mu.Lock() - exitCode := 0 - if cmd.ProcessState != nil { - exitCode = cmd.ProcessState.ExitCode() - } + exitCode := mp.ExitCode() s.lastExitCode = exitCode s.pid = 0 doneCh := s.doneCh diff --git a/components/execd/pkg/web/controller/isolated_session.go b/components/execd/pkg/web/controller/isolated_session.go index 7f78e8bc6..dabe548af 100644 --- a/components/execd/pkg/web/controller/isolated_session.go +++ b/components/execd/pkg/web/controller/isolated_session.go @@ -443,11 +443,33 @@ func (c *IsolatedSessionController) Commit() { // Capabilities handles GET /v1/isolated/capabilities. func (c *IsolatedSessionController) Capabilities() { + hardeningReport := runtime.ReportHardening() + hardening := &model.HardeningStatus{ + InitMode: hardeningReport.InitMode, + SignalShield: hardeningReport.SignalShield, + CapDrop: &model.HardeningLayerState{ + State: hardeningReport.CapDrop.State, + Message: hardeningReport.CapDrop.Message, + }, + Seccomp: &model.HardeningLayerState{ + State: hardeningReport.Seccomp.State, + Message: hardeningReport.Seccomp.Message, + }, + Landlock: &model.HardeningLayerState{ + State: hardeningReport.Landlock.State, + Message: hardeningReport.Landlock.Message, + }, + Ebpf: &model.HardeningLayerState{ + State: hardeningReport.Ebpf.State, + Message: hardeningReport.Ebpf.Message, + }, + } if isolatedRunner == nil { resp := model.CapabilitiesResponse{ Available: false, CommitSupported: false, DiffSupported: false, + Hardening: hardening, } if isolatedProbeResult != nil { resp.Isolator = isolatedProbeResult.Isolator @@ -468,6 +490,7 @@ func (c *IsolatedSessionController) Capabilities() { UsernsAvailable: caps.UsernsAvailable, CommitSupported: caps.CommitSupported, DiffSupported: caps.DiffSupported, + Hardening: hardening, } // Probe results indicate overlay capability, not diff/commit implementation. // Diff and commit are Phase 2; do not advertise them as supported. diff --git a/components/execd/pkg/web/controller/pty_ws.go b/components/execd/pkg/web/controller/pty_ws.go index d369884d6..6e846c4f5 100644 --- a/components/execd/pkg/web/controller/pty_ws.go +++ b/components/execd/pkg/web/controller/pty_ws.go @@ -460,6 +460,8 @@ const ptyViewerReadOnlyViolationLimit = 5 // ptyViewerClientReadLoop accepts ping frames but rejects every operation that // could mutate the session. It closes a connection that repeatedly sends // mutating frames to bound server-to-client error traffic. +// +//nolint:gocognit // pre-existing complexity on main; not part of OSEP-0018 func ptyViewerClientReadLoop( conn *websocket.Conn, writeJSON func(any) error, @@ -500,38 +502,60 @@ func ptyViewerClientReadLoop( switch msgType { case websocket.BinaryMessage: - if len(data) > 0 && data[0] == model.BinStdin { - if !readOnlyError() { - return - } + if !ptyViewerHandleBinaryMessage(data, readOnlyError) { + return } case websocket.TextMessage: - var frame model.ClientFrame - if json.Unmarshal(data, &frame) != nil { - continue - } - switch frame.Type { - case "stdin", "signal", "resize": - if !readOnlyError() { - return - } - case "ping": - if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil { - cancelOnce() - } - default: - if err := writeJSON(model.ServerFrame{ - Type: "error", - Code: model.WSErrCodeInvalidFrame, - Error: fmt.Sprintf("unknown frame type %q", frame.Type), - }); err != nil { - cancelOnce() - } + if !ptyViewerHandleTextMessage(data, writeJSON, readOnlyError, cancelOnce) { + return } } } } +// ptyViewerHandleBinaryMessage reports stdin payloads on a read-only viewer; +// returns false when the read loop should exit. +func ptyViewerHandleBinaryMessage(data []byte, readOnlyError func() bool) bool { + if len(data) > 0 && data[0] == model.BinStdin { + return readOnlyError() + } + return true +} + +// ptyViewerHandleTextMessage handles client frames on a read-only viewer; +// returns false when the read loop should exit. +func ptyViewerHandleTextMessage(data []byte, writeJSON func(any) error, readOnlyError func() bool, cancelOnce func()) bool { + var frame model.ClientFrame + if json.Unmarshal(data, &frame) != nil { + return true + } + switch frame.Type { + case "stdin", "signal", "resize": + return readOnlyError() + case "ping": + ptyViewerReplyPong(writeJSON, cancelOnce) + default: + ptyViewerReplyInvalidFrame(writeJSON, cancelOnce, frame.Type) + } + return true +} + +func ptyViewerReplyPong(writeJSON func(any) error, cancelOnce func()) { + if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil { + cancelOnce() + } +} + +func ptyViewerReplyInvalidFrame(writeJSON func(any) error, cancelOnce func(), frameType string) { + if err := writeJSON(model.ServerFrame{ + Type: "error", + Code: model.WSErrCodeInvalidFrame, + Error: fmt.Sprintf("unknown frame type %q", frameType), + }); err != nil { + cancelOnce() + } +} + // ptyPingLoop sends periodic WebSocket pings until cancelCh is closed. func ptyPingLoop(conn *websocket.Conn, connMu *sync.Mutex, cancelCh <-chan struct{}, cancelOnce func()) { t := time.NewTicker(wsPingInterval) diff --git a/components/execd/pkg/web/model/hardening.go b/components/execd/pkg/web/model/hardening.go new file mode 100644 index 000000000..4780f66e9 --- /dev/null +++ b/components/execd/pkg/web/model/hardening.go @@ -0,0 +1,37 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +// HardeningLayerState reports whether one hardening layer is actually +// enforced (OSEP-0018 ยง6). +type HardeningLayerState struct { + // State is "active" | "disabled" (not configured) | "degraded" + // (configured but a prerequisite is missing) | "unsupported". + State string `json:"state"` + Message string `json:"message,omitempty"` +} + +// HardeningStatus reports which execd init-mode controls are in effect +// (OSEP-0018). This is execd-global state (execd as the sandbox init / PID 1), +// not an isolation/bwrap capability; it is reported on the capabilities +// endpoint so operators see what is actually enforced in one place. +type HardeningStatus struct { + InitMode string `json:"init_mode"` // "pid1" | "subreaper" | "none" + SignalShield bool `json:"signal_shield"` // kernel PID 1 signal shield active + CapDrop *HardeningLayerState `json:"cap_drop"` // bounding-set/capability reduction + Seccomp *HardeningLayerState `json:"seccomp"` // seccomp floor on user code + Landlock *HardeningLayerState `json:"landlock"` // filesystem confinement on user code + Ebpf *HardeningLayerState `json:"ebpf"` // eBPF exec/connect/privilege observation +} diff --git a/components/execd/pkg/web/model/isolated_session.go b/components/execd/pkg/web/model/isolated_session.go index 19a0a205e..b2208ee95 100644 --- a/components/execd/pkg/web/model/isolated_session.go +++ b/components/execd/pkg/web/model/isolated_session.go @@ -195,12 +195,13 @@ type ListIsolatedSessionsResponse struct { // CapabilitiesResponse is returned by GET /v1/isolated/capabilities. type CapabilitiesResponse struct { - Available bool `json:"available"` - Isolator string `json:"isolator,omitempty"` - Version string `json:"version,omitempty"` - Message string `json:"message,omitempty"` - SetprivAvailable bool `json:"setpriv_available"` - UsernsAvailable bool `json:"userns_available"` - CommitSupported bool `json:"commit_supported"` - DiffSupported bool `json:"diff_supported"` + Available bool `json:"available"` + Isolator string `json:"isolator,omitempty"` + Version string `json:"version,omitempty"` + Message string `json:"message,omitempty"` + SetprivAvailable bool `json:"setpriv_available"` + UsernsAvailable bool `json:"userns_available"` + CommitSupported bool `json:"commit_supported"` + DiffSupported bool `json:"diff_supported"` + Hardening *HardeningStatus `json:"hardening,omitempty"` } diff --git a/components/execd/tests/init_container.sh b/components/execd/tests/init_container.sh new file mode 100755 index 000000000..adf0e3a3b --- /dev/null +++ b/components/execd/tests/init_container.sh @@ -0,0 +1,356 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Regression test: execd as the sandbox init (OSEP-0018) in a real container. +# +# Builds the execd image and runs it with EXECD_INIT=1 so bootstrap.sh execs +# into execd (PID 1), then verifies the init-mode contract end to end: +# - execd is PID 1 and the workload is its direct child +# - orphaned children are reaped (no zombie accumulation under PID 1) +# - in-namespace `kill -9 1` is inert (kernel signal shield) +# - the entrypoint exit code is propagated to the container exit code +# - runtime SIGTERM is forwarded and the workload's status is preserved +# - with [hardening] enabled, the floor applies (caps/no_new_privs/seccomp/ +# env strip) and the capabilities endpoint reports pid1 + active layers +# - on a non-PID-1 topology execd degrades to subreaper and says so +# +# Prerequisites: docker (root or in the docker group) +# +# Usage: +# bash components/execd/tests/init_container.sh +# +# Set EXECD_TEST_IMAGE to an existing image (with /bootstrap.sh, /execd and +# /opt/opensandbox/opensandbox-launcher) to skip the Dockerfile build โ€” handy +# for local iteration; unset builds the full image in CI. +# +# Exit 0 on success, non-zero on failure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +IMAGE="${EXECD_TEST_IMAGE:-execd-init-container-test:test}" +PREFIX="execd-init-$$" +TESTDIR="$(mktemp -d)" +# mktemp creates mode 700; with hardening the workload has no +# CAP_DAC_OVERRIDE, so the host-side test dir must be fully accessible to +# the container workload (traverse AND write for the hardened.out / +# caps.json the workload produces). Docker Desktop's lax mount permissions +# hide this locally; native Linux mounts do not. +chmod 777 "${TESTDIR}" +RUNNERS=() + +cleanup() { + echo ">> Cleaning up..." + for c in "${RUNNERS[@]:-}"; do + docker rm -f "${c}" >/dev/null 2>&1 || true + done + rm -rf "${TESTDIR}" + if [ -z "${EXECD_TEST_IMAGE:-}" ]; then + docker rmi -f "${IMAGE}" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +fail() { + echo "FAIL: $1" >&2 + exit 1 +} + +wait_file() { + local path="$1" + local i + for i in $(seq 1 60); do + [ -f "$path" ] && return 0 + sleep 0.5 + done + return 1 +} + +dump_container_logs() { + local c="$1" + echo ">> Container logs for ${c}:" + docker logs "$c" 2>&1 | grep -E "launcher|landlock|FAIL|init:|hardening|exited" | tail -30 || true +} + +echo "=========================================" +echo " Init-mode container regression (OSEP-0018)" +echo "=========================================" + +# ------------------------------------------------------------------- +# Build the execd image (unless an existing image was provided). +# ------------------------------------------------------------------- +if [ -n "${EXECD_TEST_IMAGE:-}" ]; then + echo ">> Using prebuilt image ${IMAGE} (EXECD_TEST_IMAGE set)" +else + echo ">> Building image ${IMAGE}..." + cd "${REPO_ROOT}" + docker build \ + -f components/execd/Dockerfile \ + -t "${IMAGE}" \ + --build-arg VERSION=init-container-test \ + . >/dev/null + echo ">> Image built." +fi + +# ------------------------------------------------------------------- +# Test 1: execd is PID 1, workload is its child, orphans are reaped, +# and in-namespace kill -9 1 is inert. +# ------------------------------------------------------------------- +echo "" +echo ">> Test 1: PID 1 handoff, orphan reaping, signal shield" + +cat > "${TESTDIR}/verify_pid1.sh" <<'SCRIPT' +#!/bin/sh +out=/mnt/test/pid1.out +: > "$out" +comm=$(cat /proc/1/comm) +echo "pid1_comm=$comm" >> "$out" +[ "$comm" = "execd" ] || { echo "FAIL: pid1 is $comm" >> "$out"; exit 90; } +ppid=$(awk '{print $4}' /proc/$$/stat) +echo "workload_ppid=$ppid" >> "$out" +[ "$ppid" = "1" ] || { echo "FAIL: workload ppid=$ppid" >> "$out"; exit 91; } + +for i in $(seq 1 10); do ( sleep 0.1 ) & done +sleep 3 +zombies=0 +for p in /proc/[0-9]*; do + stat=$(cat "$p/stat" 2>/dev/null) || continue + stat=${stat#*)} # strip "pid (comm)" + set -- $stat + [ "$1" = "Z" ] && [ "$2" = "1" ] && zombies=$((zombies+1)) +done +echo "zombies_ppid1=$zombies" >> "$out" +[ "$zombies" = "0" ] || { echo "FAIL: $zombies zombies under pid 1" >> "$out"; exit 92; } + +kill -9 1 +echo "kill9_1_inert=yes" >> "$out" +exit 0 +SCRIPT +chmod +x "${TESTDIR}/verify_pid1.sh" + +C1="${PREFIX}-t1" +RUNNERS+=("$C1") +docker run -d --name "$C1" \ + --entrypoint /bootstrap.sh \ + -e EXECD=/execd \ + -e EXECD_INIT=1 \ + -v "${TESTDIR}:/mnt/test" \ + "${IMAGE}" \ + /mnt/test/verify_pid1.sh >/dev/null +if ! wait_file "${TESTDIR}/pid1.out"; then + dump_container_logs "$C1" + fail "test 1: container did not produce pid1.out" +fi +RC=$(docker wait "$C1") +[ "$RC" = "0" ] || fail "test 1: container exited $RC: $(cat "${TESTDIR}/pid1.out")" +grep -q "kill9_1_inert=yes" "${TESTDIR}/pid1.out" || fail "test 1: kill -9 1 was not inert" +grep -q "zombies_ppid1=0" "${TESTDIR}/pid1.out" || fail "test 1: zombies accumulated" +docker rm -f "$C1" >/dev/null +echo "PASS: pid1 handoff, orphan reaping, signal shield" + +# ------------------------------------------------------------------- +# Test 2: entrypoint exit code propagation. +# ------------------------------------------------------------------- +echo "" +echo ">> Test 2: entrypoint exit code propagation" + +C2="${PREFIX}-t2" +RUNNERS+=("$C2") +set +e +docker run --rm --name "$C2" \ + --entrypoint /bootstrap.sh \ + -e EXECD=/execd \ + -e EXECD_INIT=1 \ + -v "${TESTDIR}:/mnt/test" \ + "${IMAGE}" \ + /bin/sh -c 'exit 7' +RC=$? +set -e +[ "$RC" = "7" ] || fail "test 2: container exit code = $RC, want 7" +echo "PASS: entrypoint exit code propagated ($RC)" + +# ------------------------------------------------------------------- +# Test 3: SIGTERM is forwarded; workload status is preserved. +# ------------------------------------------------------------------- +echo "" +echo ">> Test 3: SIGTERM graceful shutdown" + +cat > "${TESTDIR}/sigterm.sh" <<'SCRIPT' +#!/bin/sh +trap 'touch /mnt/test/sigterm_received; exit 7' TERM +while :; do sleep 0.5; done +SCRIPT +chmod +x "${TESTDIR}/sigterm.sh" + +C3="${PREFIX}-t3" +RUNNERS+=("$C3") +docker run -d --name "$C3" \ + --entrypoint /bootstrap.sh \ + -e EXECD=/execd \ + -e EXECD_INIT=1 \ + -v "${TESTDIR}:/mnt/test" \ + "${IMAGE}" \ + /mnt/test/sigterm.sh >/dev/null + +sleep 3 +docker stop -t 10 "$C3" >/dev/null +RC=$(docker wait "$C3") +[ "$RC" = "7" ] || fail "test 3: container exit code after SIGTERM = $RC, want 7" +[ -f "${TESTDIR}/sigterm_received" ] || fail "test 3: workload did not receive SIGTERM" +docker rm -f "$C3" >/dev/null +echo "PASS: SIGTERM forwarded, status preserved ($RC)" + +# ------------------------------------------------------------------- +# Test 4: hardening floor in a real PID-1 sandbox. +# ------------------------------------------------------------------- +echo "" +echo ">> Test 4: hardening floor ([hardening] enabled)" + +cat > "${TESTDIR}/isolation.toml" <<'TOML' +[hardening] +enabled = true + +[landlock] +enabled = true +TOML + +cat > "${TESTDIR}/hardened.sh" <<'SCRIPT' +#!/bin/sh +out=/mnt/test/hardened.out +: > "$out" +# /proc/self is read via the entrypoint process itself (a forked descendant +# resolves its own /proc/ dir, which the Landlock allowlist does not +# cover โ€” the documented /proc/self limitation of OSEP-0018). +nnp=""; sec=""; capeff="" +while IFS= read -r line; do + case "$line" in + NoNewPrivs:*) nnp="${line#*: }" ;; + Seccomp:*) sec="${line#*: }" ;; + CapEff:*) capeff="${line#*: }" ;; + esac +done < /proc/self/status +echo "nnp=$nnp sec=$sec capeff=$capeff" >> "$out" + +[ "$nnp" = "1" ] || { echo "FAIL: no_new_privs not set ($nnp)" >> "$out"; exit 93; } +[ "$sec" = "2" ] || { echo "FAIL: seccomp not in filter mode ($sec)" >> "$out"; exit 94; } +[ "$capeff" = "0000000000000000" ] || { echo "FAIL: CapEff=$capeff, want zero" >> "$out"; exit 95; } +# The entrypoint keeps bootstrap env (JUPYTER_TOKEN) but never execd's +# control-plane credential (EXECD_ACCESS_TOKEN) โ€” its Jupyter kernels are +# user code and must not see it. +[ -z "${EXECD_ACCESS_TOKEN:-}" ] || { echo "FAIL: execd credential leaked to entrypoint" >> "$out"; exit 96; } +[ "${JUPYTER_TOKEN:-}" = "jt-secret" ] || { echo "FAIL: bootstrap env not preserved" >> "$out"; exit 95; } + +# The execd API is guarded by the token we injected via the container env +# (which the floor stripped from this workload's own environment). +for i in $(seq 1 30); do + wget -qO- --header="X-EXECD-ACCESS-TOKEN: supersecret" \ + http://127.0.0.1:44772/v1/isolated/capabilities > /mnt/test/caps.json 2>/dev/null && break + sleep 0.5 +done +grep -q '"init_mode":"pid1"' /mnt/test/caps.json || { echo "FAIL: init_mode != pid1" >> "$out"; exit 97; } +grep -q '"cap_drop":{"state":"active"' /mnt/test/caps.json || { echo "FAIL: cap_drop not active" >> "$out"; exit 98; } +grep -q '"seccomp":{"state":"active"' /mnt/test/caps.json || { echo "FAIL: seccomp not active" >> "$out"; exit 99; } +if grep -q '"landlock":{"state":"active"' /mnt/test/caps.json; then + echo "landlock=active" >> "$out" + if cat /proc/1/environ >/dev/null 2>&1; then + echo "FAIL: /proc/1/environ readable despite landlock" >> "$out"; exit 90 + fi +else + # Kernel without Landlock (e.g. some CI VM kernels): the launcher fails + # open and the layer reports unsupported. + grep -q '"landlock":{"state":"unsupported"' /mnt/test/caps.json || { echo "FAIL: landlock neither active nor unsupported" >> "$out"; exit 91; } + echo "landlock=unsupported (skipped)" >> "$out" +fi +# The default image has no eBPF code; the layer must report disabled. +grep -q '"ebpf":{"state":"disabled"' /mnt/test/caps.json || { echo "FAIL: ebpf not disabled in the default image" >> "$out"; exit 92; } +echo "hardened_ok=yes" >> "$out" +exit 0 +SCRIPT +chmod +x "${TESTDIR}/hardened.sh" + +C4="${PREFIX}-t4" +RUNNERS+=("$C4") +docker run -d --name "$C4" \ + --entrypoint /bootstrap.sh \ + -e EXECD=/execd \ + -e EXECD_INIT=1 \ + -e EXECD_ISOLATION_CONFIG=/mnt/test/isolation.toml \ + -e EXECD_ACCESS_TOKEN=supersecret \ + -e JUPYTER_TOKEN=jt-secret \ + -v "${TESTDIR}:/mnt/test" \ + "${IMAGE}" \ + /mnt/test/hardened.sh >/dev/null +if ! wait_file "${TESTDIR}/hardened.out"; then + dump_container_logs "$C4" + fail "test 4: container did not produce hardened.out" +fi +RC=$(docker wait "$C4") +[ "$RC" = "0" ] || fail "test 4: container exited $RC: $(cat "${TESTDIR}/hardened.out")" +grep -q "hardened_ok=yes" "${TESTDIR}/hardened.out" || fail "test 4: floor assertions failed: $(cat "${TESTDIR}/hardened.out")" +docker rm -f "$C4" >/dev/null +echo "PASS: hardening floor active in a PID-1 sandbox" + +# ------------------------------------------------------------------- +# Test 5: non-PID-1 topology degrades to subreaper. +# ------------------------------------------------------------------- +echo "" +echo ">> Test 5: subreaper mode on the Pool-style topology" + +cat > "${TESTDIR}/subreaper.sh" <<'SCRIPT' +#!/bin/sh +out=/mnt/test/subreaper.out +: > "$out" +for i in $(seq 1 30); do + wget -qO- http://127.0.0.1:44772/v1/isolated/capabilities > /mnt/test/subreaper.json 2>/dev/null && break + sleep 0.5 +done +grep -q '"init_mode":"subreaper"' /mnt/test/subreaper.json || { echo "FAIL: init_mode != subreaper" >> "$out"; exit 97; } +echo "subreaper_ok=yes" >> "$out" +exit 0 +SCRIPT +chmod +x "${TESTDIR}/subreaper.sh" + +C5="${PREFIX}-t5" +RUNNERS+=("$C5") +# Pool-style: bootstrap.sh is not the container entrypoint; a shell is PID 1 +# and bootstrap (with EXECD_INIT) runs backgrounded, so execd degrades to +# subreaper mode. The background "&" is essential: `sh -c 'cmd'` would exec +# the command and make execd PID 1. +docker run -d --name "$C5" \ + --entrypoint /bin/sh \ + -e EXECD=/execd \ + -e EXECD_INIT=1 \ + -v "${TESTDIR}:/mnt/test" \ + "${IMAGE}" \ + -c 'EXECD_INIT=1 /bootstrap.sh /mnt/test/subreaper.sh & wait' >/dev/null +if ! wait_file "${TESTDIR}/subreaper.out"; then + dump_container_logs "$C5" + fail "test 5: container did not produce subreaper.out" +fi +RC=$(docker wait "$C5") +[ "$RC" = "0" ] || fail "test 5: container exited $RC: $(cat "${TESTDIR}/subreaper.out")" +grep -q "subreaper_ok=yes" "${TESTDIR}/subreaper.out" || fail "test 5: subreaper assertions failed" +docker rm -f "$C5" >/dev/null +echo "PASS: subreaper degradation reported" + +# ------------------------------------------------------------------- +echo "" +echo "=========================================" +echo " Init-mode container regression PASSED" +echo "=========================================" +echo " image: ${IMAGE}" +echo " cases: pid1 handoff / reaping / signal shield /" +echo " exit propagation / SIGTERM / hardening floor / subreaper" diff --git a/components/execd/tests/init_mode.sh b/components/execd/tests/init_mode.sh new file mode 100644 index 000000000..638eedecb --- /dev/null +++ b/components/execd/tests/init_mode.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: bootstrap.sh EXECD_INIT mode (OSEP-0018). +# +# With EXECD_INIT set, bootstrap.sh must exec into execd (--init -- ) +# instead of backgrounding it, so execd becomes the sandbox init. The user +# command is passed through the concrete argv forms bootstrap.sh supports: +# BOOTSTRAP_CMD, the "-c" form, plain positional args, and the default shell. +# +# Usage: +# cd components/execd +# bash tests/init_mode.sh + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +BOOTSTRAP="$ROOT_DIR/bootstrap.sh" + +TESTDIR="$(mktemp -d)" +cleanup() { + rm -rf "$TESTDIR" +} +trap cleanup EXIT + +# Stub execd: records its argv and pid, then sleeps so the test can verify the +# process tree (bootstrap must have replaced itself, not spawned a child). +EXECD_STUB="$TESTDIR/execd_stub.sh" +cat > "$EXECD_STUB" << 'STUB' +#!/bin/sh +printf '%s\n' "$*" > "$EXECD_ARGV_FILE" +printf '%s\n' "$$" > "$EXECD_PID_FILE" +while true; do sleep 1; done +STUB +chmod +x "$EXECD_STUB" + +run_bootstrap() { + ARGV_FILE="$1" + PID_FILE="$2" + shift 2 + EXECD="$EXECD_STUB" \ + EXECD_ARGV_FILE="$ARGV_FILE" \ + EXECD_PID_FILE="$PID_FILE" \ + EXECD_INIT=1 \ + "$BOOTSTRAP" "$@" & + BOOTSTRAP_PID=$! + + for i in $(seq 1 50); do + [ -f "$PID_FILE" ] && break + sleep 0.1 + done + if [ ! -f "$PID_FILE" ]; then + echo "FAIL: execd stub did not start" + kill "$BOOTSTRAP_PID" 2>/dev/null || true + wait "$BOOTSTRAP_PID" 2>/dev/null || true + exit 1 + fi +} + +stop_stub() { + kill "$BOOTSTRAP_PID" 2>/dev/null || true + wait "$BOOTSTRAP_PID" 2>/dev/null || true +} + +# 1. BOOTSTRAP_CMD form: execd must receive --init -- bash -c '' +ARGV_FILE="$TESTDIR/argv1" +PID_FILE="$TESTDIR/pid1" +BOOTSTRAP_CMD="echo init-works" run_bootstrap "$ARGV_FILE" "$PID_FILE" -c "ignored" +STUB_PID="$(cat "$PID_FILE")" +if [ "$STUB_PID" != "$BOOTSTRAP_PID" ]; then + echo "FAIL: bootstrap did not exec execd (bootstrap pid $BOOTSTRAP_PID != stub pid $STUB_PID)" + stop_stub + exit 1 +fi +if ! grep -q '^--init -- /[^ ]* -c echo init-works$' "$ARGV_FILE"; then + echo "FAIL: BOOTSTRAP_CMD argv = $(cat "$ARGV_FILE"), want --init -- -c " + stop_stub + exit 1 +fi +stop_stub +echo "PASS: EXECD_INIT=1 + BOOTSTRAP_CMD execs execd --init -- -c " + +# 2. Positional form: execd must receive --init -- +ARGV_FILE="$TESTDIR/argv2" +PID_FILE="$TESTDIR/pid2" +run_bootstrap "$ARGV_FILE" "$PID_FILE" sh -c 'echo positional' +STUB_PID="$(cat "$PID_FILE")" +if [ "$STUB_PID" != "$BOOTSTRAP_PID" ]; then + echo "FAIL: bootstrap did not exec execd (bootstrap pid $BOOTSTRAP_PID != stub pid $STUB_PID)" + stop_stub + exit 1 +fi +if ! grep -q '^--init -- sh -c echo positional$' "$ARGV_FILE"; then + echo "FAIL: positional argv = $(cat "$ARGV_FILE"), want --init -- sh -c echo positional" + stop_stub + exit 1 +fi +stop_stub +echo "PASS: EXECD_INIT=1 + positional args pass through unchanged" + +# 3. No command: execd must receive --init -- +ARGV_FILE="$TESTDIR/argv3" +PID_FILE="$TESTDIR/pid3" +run_bootstrap "$ARGV_FILE" "$PID_FILE" +if ! grep -q '^--init -- /' "$ARGV_FILE"; then + echo "FAIL: default-shell argv = $(cat "$ARGV_FILE"), want --init -- " + stop_stub + exit 1 +fi +stop_stub +echo "PASS: EXECD_INIT=1 with no command passes the default shell" + +echo "=== all init-mode bootstrap contract tests passed ===" diff --git a/components/ingress/go.mod b/components/ingress/go.mod index 72968833c..85ed2af07 100644 --- a/components/ingress/go.mod +++ b/components/ingress/go.mod @@ -70,7 +70,7 @@ require ( golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.10.0 // indirect + golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect diff --git a/components/ingress/go.sum b/components/ingress/go.sum index fdf2e015d..fd06c6b8a 100644 --- a/components/ingress/go.sum +++ b/components/ingress/go.sum @@ -182,8 +182,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/components/internal/telemetry/endpoint.go b/components/internal/telemetry/endpoint.go new file mode 100644 index 000000000..00ae587c9 --- /dev/null +++ b/components/internal/telemetry/endpoint.go @@ -0,0 +1,89 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "net/url" + "strings" +) + +// OTLPEndpointHostPort returns the host and port of the configured OTLP +// endpoint. Endpoint precedence matches the exporters: +// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT. +// The value must be a URL (scheme://host[:port][/path]), matching the +// otlpmetrichttp env-var form; bare host:port or host values are invalid +// because the exporter parses them as opaque URLs with an empty host. +// A missing port falls back to the scheme default (https->443, http->80). +// Domain hosts are returned without the trailing dot, matching DNS policy +// normalization. ok is false when no endpoint is configured or it cannot +// be parsed. +func OTLPEndpointHostPort() (host, port string, ok bool) { + raw := otlpEndpointFromEnv() + if raw == "" { + return "", "", false + } + return parseOTLPEndpoint(raw) +} + +// OTLPEndpointEnvSet reports whether any OTEL endpoint env var is non-blank, +// regardless of whether it parses. Callers that also use +// OTLPEndpointFallbackHostPort need this to distinguish "unset" from +// "configured but invalid": the exporter never falls back to the node IP once +// an endpoint env var is set, so neither should the auto-allow logic. +func OTLPEndpointEnvSet() bool { + return otlpEndpointFromEnv() != "" +} + +// OTLPEndpointFallbackHostPort returns the exporter fallback destination used +// only when no OTEL endpoint env var is set: the resolved node IP +// (HOST_IP, then /etc/hostinfo) on the default OTLP/HTTP port 4318. ok is +// false when no node IP can be resolved. +func OTLPEndpointFallbackHostPort() (host, port string, ok bool) { + ip, ok := resolveNodeIP() + if !ok { + return "", "", false + } + return ip, otlpHTTPPort, true +} + +func parseOTLPEndpoint(raw string) (host, port string, ok bool) { + raw = strings.TrimSpace(raw) + if !strings.Contains(raw, "://") { + return "", "", false + } + u, err := url.Parse(raw) + if err != nil { + return "", "", false + } + host = strings.TrimRight(strings.TrimSpace(u.Hostname()), ".") + if host == "" { + return "", "", false + } + port = u.Port() + if port == "" { + port = defaultPortForScheme(u.Scheme) + } + return host, port, true +} + +func defaultPortForScheme(scheme string) string { + switch strings.ToLower(scheme) { + case "https": + return "443" + case "http": + return "80" + } + return "" +} diff --git a/components/internal/telemetry/endpoint_test.go b/components/internal/telemetry/endpoint_test.go new file mode 100644 index 000000000..d4ebbfe73 --- /dev/null +++ b/components/internal/telemetry/endpoint_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import "testing" + +func TestParseOTLPEndpoint(t *testing.T) { + cases := []struct { + name string + raw string + host string + port string + ok bool + }{ + {name: "empty", raw: "", ok: false}, + {name: "whitespace", raw: " ", ok: false}, + {name: "url with port and path", raw: "https://collector.example:4318/v1/metrics", host: "collector.example", port: "4318", ok: true}, + {name: "url without port", raw: "https://collector.example/v1/metrics", host: "collector.example", port: "443", ok: true}, + {name: "http url without port", raw: "http://collector.example/v1/metrics", host: "collector.example", port: "80", ok: true}, + {name: "ip url", raw: "http://10.0.0.1:4317", host: "10.0.0.1", port: "4317", ok: true}, + {name: "ipv6 url", raw: "http://[::1]:4318/v1/metrics", host: "::1", port: "4318", ok: true}, + {name: "host port without scheme", raw: "collector.example:4318", ok: false}, + {name: "ip port without scheme", raw: "10.0.0.1:4318", ok: false}, + {name: "bare host", raw: "collector.example", ok: false}, + {name: "bare ip", raw: "10.0.0.1", ok: false}, + {name: "fqdn url trailing dot", raw: "http://otel-collector.ns.svc.cluster.local.:4318", host: "otel-collector.ns.svc.cluster.local", port: "4318", ok: true}, + {name: "fqdn trailing dot without scheme", raw: "otel-collector.ns.svc.cluster.local.:4318", ok: false}, + {name: "bare fqdn trailing dot", raw: "collector.example.", ok: false}, + {name: "scheme only", raw: "http://", ok: false}, + {name: "malformed url", raw: "https://:443", ok: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + host, port, ok := parseOTLPEndpoint(tc.raw) + if ok != tc.ok { + t.Fatalf("parseOTLPEndpoint(%q) ok=%v, want %v", tc.raw, ok, tc.ok) + } + if host != tc.host || port != tc.port { + t.Fatalf("parseOTLPEndpoint(%q) = (%q, %q), want (%q, %q)", tc.raw, host, port, tc.host, tc.port) + } + }) + } +} + +func TestOTLPEndpointHostPortPrecedence(t *testing.T) { + t.Setenv(envOTLPMetricsEndpoint, "") + t.Setenv(envOTLPEndpoint, "") + host, _, ok := OTLPEndpointHostPort() + if ok { + t.Fatal("expected no endpoint when both env vars are unset") + } + if host != "" { + t.Fatalf("expected empty host, got %q", host) + } + + t.Setenv(envOTLPEndpoint, "https://fallback.example:4318") + host, port, ok := OTLPEndpointHostPort() + if !ok || host != "fallback.example" || port != "4318" { + t.Fatalf("fallback endpoint parsed as (%q, %q, %v)", host, port, ok) + } + + t.Setenv(envOTLPMetricsEndpoint, "https://primary.example:4317/v1/metrics") + host, port, ok = OTLPEndpointHostPort() + if !ok || host != "primary.example" || port != "4317" { + t.Fatalf("metrics endpoint should win; parsed as (%q, %q, %v)", host, port, ok) + } + + t.Setenv(envOTLPMetricsEndpoint, " ") + host, port, ok = OTLPEndpointHostPort() + if !ok || host != "fallback.example" || port != "4318" { + t.Fatalf("blank metrics endpoint should fall back; parsed as (%q, %q, %v)", host, port, ok) + } +} + +func TestOTLPEndpointEnvSet(t *testing.T) { + t.Setenv(envOTLPMetricsEndpoint, "") + t.Setenv(envOTLPEndpoint, "") + if OTLPEndpointEnvSet() { + t.Fatal("expected env unset") + } + + t.Setenv(envOTLPEndpoint, "http://") + if !OTLPEndpointEnvSet() { + t.Fatal("expected env set even when unparseable") + } + + t.Setenv(envOTLPEndpoint, "") + t.Setenv(envOTLPMetricsEndpoint, "https://collector.example:4318") + if !OTLPEndpointEnvSet() { + t.Fatal("expected metrics endpoint env set") + } +} + +func TestOTLPEndpointFallbackHostPort(t *testing.T) { + t.Setenv(envHostIP, "10.0.0.9") + host, port, ok := OTLPEndpointFallbackHostPort() + if !ok || host != "10.0.0.9" || port != otlpHTTPPort { + t.Fatalf("fallback from HOST_IP parsed as (%q, %q, %v)", host, port, ok) + } + + t.Setenv(envHostIP, " ") + host, _, ok = OTLPEndpointFallbackHostPort() + if ok { + t.Fatalf("expected no fallback without a resolvable node IP, got %q", host) + } +} diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index d06107619..7559e79ba 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -99,6 +99,7 @@ export default defineConfig({ { text: "Windows Sandbox", link: "/guides/windows-sandbox" }, { text: "Client Pool", link: "/guides/client-pool" }, { text: "SDK Telemetry", link: "/guides/sdk-telemetry" }, + { text: "SDK Tracing (Pool Warmup)", link: "/guides/sdk-tracing" }, ], }, ], @@ -186,6 +187,7 @@ export default defineConfig({ { text: "Claude Code", link: "/examples/claude-code" }, { text: "Gemini CLI", link: "/examples/gemini-cli" }, { text: "Codex CLI", link: "/examples/codex-cli" }, + { text: "OpenCode", link: "/examples/opencode" }, { text: "Qwen Code", link: "/examples/qwen-code" }, { text: "Kimi CLI", link: "/examples/kimi-cli" }, { text: "LangGraph", link: "/examples/langgraph" }, diff --git a/docs/architecture/index.md b/docs/architecture/index.md index c7d90ebf6..0e34c1d89 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -156,7 +156,11 @@ The lifecycle endpoint API returns the reachable address for a service port insi - A Kubernetes ingress gateway endpoint. - A server-proxied URL under `/sandboxes/{sandboxId}/proxy/{port}` when `use_server_proxy=true`. -The server proxy supports HTTP and WebSocket traffic and is also integrated with optional renew-on-access behavior. +Header-routed ingress endpoints include `OpenSandbox-Ingress-To` in their endpoint metadata. +When the endpoint is rewritten to a server-proxied URL, the server removes only this +ingress routing header and preserves other required endpoint headers. + +The server proxy supports HTTP and WebSocket traffic and is also integrated with optional renew-on-access behavior. For HTTP responses, it strips hop-by-hop headers and the backend `Server` header while preserving an origin `Date`; the server adds a current `Date` only when the response does not already contain one. A root-relative `Location` value that starts with a single `/` is rebased under the same sandbox proxy route, while absolute URLs, network-path references (`//host/path`), and ordinary path-relative values are forwarded unchanged. ## 4. Runtime Backends diff --git a/docs/cli/index.md b/docs/cli/index.md index b9331e0e9..7f1fecda9 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -258,18 +258,22 @@ Use `--file -` to read a JSON/YAML payload from stdin. Do not pass plaintext cre Use the stable diagnostics commands for API-backed log and event descriptors. ```bash -osb diagnostics events --scope lifecycle -o raw osb diagnostics events --scope runtime -o raw +osb diagnostics events --scope all -o raw osb diagnostics logs --scope container -o raw -osb diagnostics logs --scope lifecycle -o json +osb diagnostics logs --scope all -o json osb diagnostics events --scope runtime -o json osb diagnostics logs --scope container -o yaml ``` -`--scope` is required for stable diagnostics. Common scopes are `lifecycle` and -`container` for logs, and `lifecycle` and `runtime` for events. Raw output -prints inline diagnostic text, or the content URL when diagnostics are -delivered as a temporary URL. +`--scope` is required for stable diagnostics. The built-in server supports +`container` and `all` for logs, and `runtime` and `all` for events. It returns +`DIAGNOSTICS_SCOPE_UNSUPPORTED` for unavailable scopes, including lifecycle events. +Best-effort scopes may include a `warnings` field when the backend can only +provide a subset. Raw output prints inline +diagnostic text, or the content URL when diagnostics are delivered as a +temporary URL. Older server builds may still return +`DIAGNOSTICS_NOT_IMPLEMENTED` for scoped diagnostics. ::: info Legacy DevOps diagnostics remain experimental. Prefer `osb diagnostics logs/events` for stable API-backed log and event collection. diff --git a/docs/community/oseps.md b/docs/community/oseps.md index 719ed013a..56ceaba98 100644 --- a/docs/community/oseps.md +++ b/docs/community/oseps.md @@ -16,8 +16,8 @@ See the [OSEP contributing guide](https://github.com/opensandbox-group/OpenSandb | [OSEP-0001](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0001-fqdn-based-egress-control.md) | FQDN-based Egress Control | implemented | 2026-01-22 | | [OSEP-0002](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0002-kubernetes-sigs-agent-sandbox-support.md) | kubernetes-sigs/agent-sandbox Support | implemented | 2026-01-23 | | [OSEP-0003](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0003-volume-and-volumebinding-support.md) | Volume Support | implementing | 2026-02-11 | -| [OSEP-0004](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0004-secure-container-runtime.md) | Pluggable Secure Container Runtime Support | implemented | 2026-02-09 | -| [OSEP-0005](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0005-client-side-sandbox-pool.md) | Client-Side Sandbox Pool | implementing | 2026-03-09 | +| [OSEP-0004](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0004-secure-container-runtime.md) | Pluggable Secure Container Runtime Support | implemented | 2026-07-27 | +| [OSEP-0005](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0005-client-side-sandbox-pool.md) | Client-Side Sandbox Pool | implemented | 2026-07-27 | | [OSEP-0006](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0006-developer-console.md) | Developer Console for Sandbox Operations | implementable | 2026-03-06 | | [OSEP-0007](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0007-fast-sandbox-runtime-support.md) | Fast Sandbox Runtime Support | provisional | 2026-02-08 | | [OSEP-0008](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0008-pause-resume-rootfs-snapshot.md) | Pause and Resume via Rootfs Snapshot | implementing | 2026-03-13 | @@ -26,7 +26,7 @@ See the [OSEP contributing guide](https://github.com/opensandbox-group/OpenSandb | [OSEP-0011](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0011-secure-access-endpoint.md) | Secure Access on GetEndpoint and Signed Endpoint | implemented | 2026-04-25 | | [OSEP-0012](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0012-credential-vault.md) | Credential Vault and Credential Proxy | implemented | 2026-06-23 | | [OSEP-0013](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0013-isolated-execution-api.md) | Isolated Execution API | implementing | 2026-06-23 | -| [OSEP-0014](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0014-multi-tenancy.md) | Multi-Tenancy Support for Kubernetes Runtime | draft | 2026-04-29 | +| [OSEP-0014](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0014-multi-tenancy.md) | Multi-Tenancy Support for Kubernetes Runtime | implemented | 2026-07-27 | | [OSEP-0015](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0015-pod-snapshot.md) | Spec-Driven Pod Snapshot for Pause and Resume | draft | 2026-06-27 | | [OSEP-0016](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0016-unified-umbrella-release-governance.md) | Unified Umbrella Release Governance | draft | 2026-07-21 | | [OSEP-0017](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0017-resilient-sdk-transport.md) | Resilient SDK Transport | implementing | 2026-07-22 | diff --git a/docs/community/release-verification.md b/docs/community/release-verification.md index 3ba5c6169..13eaae323 100644 --- a/docs/community/release-verification.md +++ b/docs/community/release-verification.md @@ -19,9 +19,10 @@ OpenSandbox uses these signing paths: - Source code releases: the Generic Release workflow uploads an explicit `opensandbox-.tar.gz` source archive and `SHA256SUMS` file to the GitHub Release, then creates GitHub/Sigstore provenance attestations for both files. -- Container images: the component and server image workflows sign Docker Hub - and ACR image digests with `cosign` keyless signing, and publish provenance - attestations to the registries. +- Container images: the component and server image workflows sign Docker Hub, + GitHub Container Registry (GHCR), and Alibaba Cloud Container Registry (ACR) + image digests with `cosign` keyless signing, and publish provenance + attestations to all three registries. - Python and CLI packages: wheels and source distributions are attested before `uv publish`. - JavaScript packages: the workflow runs `pnpm pack`, attests the generated npm @@ -83,9 +84,9 @@ If you run the release workflows from a downstream fork, replace `opensandbox-group/OpenSandbox` in the verification commands with that fork's `owner/repository` identity. -Private signing material is not stored in GitHub Releases, Docker Hub, ACR, -PyPI, npm, Maven Central, NuGet, or Helm chart downloads. Java/Kotlin Maven -Central signing keys are held only in GitHub Actions secrets. +Private signing material is not stored in GitHub Releases, Docker Hub, GHCR, +ACR, PyPI, npm, Maven Central, NuGet, or Helm chart downloads. Java/Kotlin +Maven Central signing keys are held only in GitHub Actions secrets. ## Verify Source Releases @@ -136,6 +137,19 @@ provenance `source-ref` is the ref selected when the workflow was dispatched Install `cosign` and `gh`, then resolve the image digest. Always verify by digest, not by mutable tag alone. +Release images are published with the same component name and digest in all +three official registries: + +| Registry | Image name pattern | +| --- | --- | +| Docker Hub | `docker.io/opensandbox/` | +| GitHub Container Registry | `ghcr.io/opensandbox-group/opensandbox/` | +| Alibaba Cloud Container Registry | `sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/` | + +The component can be `execd`, `code-interpreter`, `ingress`, `egress`, +`controller`, `task-executor`, `image-committer`, or `nodeagent`. The server +image uses the component name `server`. + ```bash IMAGE="docker.io/opensandbox/execd" TAG="v1.0.15" @@ -173,10 +187,12 @@ cosign verify "$IMAGE_REF" \ --certificate-identity-regexp "^${WORKFLOW_REPOSITORY_URL}/.github/workflows/publish-server.yml@refs/tags/server/v[0-9].*$" ``` -ACR images use the same digest and identity checks with the ACR image name, for -example: +GHCR and ACR images use the same digest and identity checks with their +respective image names, for example: ```bash +IMAGE="ghcr.io/opensandbox-group/opensandbox/execd" +# or IMAGE="sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd" ``` diff --git a/docs/components/execd.md b/docs/components/execd.md index 23ecc4479..45202bcc7 100644 --- a/docs/components/execd.md +++ b/docs/components/execd.md @@ -196,6 +196,7 @@ override it. | `--graceful-shutdown-timeout` | `1s` | SSE tail-drain wait window before closing. | | `--jupyter-idle-poll-interval` | `100ms` | Poll interval after Jupyter reports idle. | | `--isolation-config` | `""` | Path to the isolation TOML config (see below). | +| `--init` | `false` | Run as the sandbox init (OSEP-0018): reap children, forward signals, own the container lifecycle. Set together with `EXECD_INIT`; see [Init mode](#init-mode). | ### Environment Variables @@ -207,11 +208,12 @@ override it. | `EXECD_API_GRACE_SHUTDOWN` | Same as `--graceful-shutdown-timeout`. | | `EXECD_JUPYTER_IDLE_POLL_INTERVAL` | Same as `--jupyter-idle-poll-interval`. | | `EXECD_ISOLATION_CONFIG` | Same as `--isolation-config`. | +| `EXECD_INIT` | Init-mode switch read by `bootstrap.sh`: when truthy (`1`/`true`/`yes`/`on`), the script `exec`s `execd --init -- ` so execd becomes PID 1; see [Init mode](#init-mode). Unset preserves the classic background-and-wait topology. | | `EXECD_CLONE3_COMPAT` | Linux clone3 compatibility switch (see below). | | `EXECD_LOG_FILE` | Optional log output file path; default is stdout. | | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Preferred OTLP metrics endpoint. | | `OTEL_EXPORTER_OTLP_ENDPOINT` | Fallback OTLP endpoint when metrics-specific endpoint is unset. | -| `OPENSANDBOX_ID` | Optional `sandbox_id` metric/resource attribute. | +| `OPENSANDBOX_ID` | Authoritative sandbox id stamped into eBPF audit records (`sandbox_id`) and metrics; the server injects it on Docker/Kubernetes task-template paths. Kubernetes pool allocations that skip the task template (default entrypoint, no env, no init mode) cannot inject it, and the eBPF layer reports `unsupported` attribution on that path. | | `OPENSANDBOX_EXECD_METRICS_EXTRA_ATTRS` | Optional extra metric attrs (`k=v,k2=v2`). | ### Isolation Config File @@ -230,6 +232,105 @@ upper_root = "/var/lib/execd/isolation" allowed_writable = ["/workspace", "/mnt", "/media", "/data"] ``` +### Hardening Floor + +The pre-exec privilege floor (OSEP-0018 ยง4) is off by default. Enable it in +the same isolation TOML: + +```toml +[hardening] +enabled = true + +# Capabilities the workload keeps (raised in the ambient set). +# Default: drop all. Names use the CAP_ prefix. +keep_capabilities = [] + +# Optional: replace the built-in syscall denylist. With hardening enabled, +# "execve" is reserved for the launcher's final exec and is rejected at +# startup ("execveat" stays allowed). +[seccomp] +deny = ["mount", "ptrace", "bpf", "seccomp"] + +# Optional: Landlock filesystem confinement on top of the floor. +[landlock] +enabled = true +extra_writable = [] # writable paths beyond the built-in set +extra_readable = [] # read-only paths beyond the built-in set +``` + +When enabled, every user-code process (entrypoint, `/command`, `/code`, +PTY) is launched through the `opensandbox-launcher` native helper, which +applies the floor between fork and exec: execd credential env vars are +stripped, the bounding set is trimmed to `keep_capabilities` (none by +default), `no_new_privs` is set, the identity is dropped to the image's +user, kept caps are raised in the ambient set, and the seccomp filter is +installed last. Isolated-session workloads are already reduced inside the +bwrap namespace and are not additionally wrapped. + +Everything is fail-open and reported on `GET /v1/isolated/capabilities` +under `hardening.cap_drop` / `hardening.seccomp` / `hardening.landlock` +(`active` | `degraded` | `unsupported` | `disabled` with a reason message). +Missing `CAP_SETPCAP` degrades the cap drop but keeps seccomp; a missing +launcher binary disables the floor; a kernel without Landlock (ABI < 1) +reports `unsupported` and skips FS confinement. + +With `[landlock] enabled`, user-code processes are allowlisted to: system +paths (`/usr`, `/bin`, `/lib`, `/lib64`, `/etc`) read+exec, `/proc/self` +and `/proc/sys` read+exec (never all of `/proc`, which would re-expose +`/proc/1` and execd's credentials), the needed `/dev` device files and the +controlling tty, `/tmp`, `/run`, `allowed_writable`, plus +`extra_writable`/`extra_readable`. Everything else is denied. Note that +only the initial workload process keeps `/proc/self` access (a Landlock +rule is inode-based); forked descendants lose their own `/proc/self` โ€” +tooling that needs it should be run as the entrypoint process. + +Two Landlock kernel behaviors shape the policy: + +- `path_beneath` rules are scoped to the mount the path belongs to, so at + startup execd expands every rule onto each mount point beneath it โ€” + bind-mounted workspaces (a separate mount) get the same access as their + parent path. +- rules only accept directory parents, so per-file grants are impossible; + well-known proc files (`/proc/cpuinfo`, `/proc/meminfo`, โ€ฆ) are not + individually readable under Landlock. + +Recommended container ceiling (operator side): keep `CAP_SETPCAP`, +`CAP_SETUID`, `CAP_SETGID` so execd can reduce children; drop the rest +(`NET_RAW`, `SYS_MODULE`, `SYS_TIME`, `SYS_TTY_CONFIG`, `AUDIT_WRITE`, +`MKNOD`). + +### eBPF Observation + +Opt-in exec/connect/privilege audit (OSEP-0018 ยง5), off by default: + +```toml +[ebpf] +enabled = true +observe = ["exec", "connect", "privilege"] # default: all three +audit_file = "/var/log/opensandbox/ebpf-audit.jsonl" # rotated JSONL +``` + +Requires the `execd-ebpf` build variant (CGO + `cilium/ebpf`), a +container with `CAP_BPF` + `CAP_PERFMON`, and a BTF-capable kernel โ€” +Linux โ‰ฅ 5.10 with `CONFIG_DEBUG_INFO_BTF` (5.10โ€“5.15 kernels use the +inline-`filename` trace event layout, which the BPF program detects via +CO-RE; 5.16+ use the `__data_loc` layout). Events are scoped to the +sandbox cgroup, so only this sandbox's processes are observed; they are +written as JSONL (one object per line) with a stable common envelope +(`ts`, `event`, `sandbox_id`, `pid`, `comm`) plus per-kind fields +(`filename`/`ppid` for `exec`, `dst_ip`/`dst_port`/`proto` for +`connect`, uid/gid deltas and `cap_added` for `privilege`). Under +gVisor/Kata the host kernel is not attachable, and the layer reports +`unsupported`. Missing prerequisites never block startup. + +The default image ships both binaries: `execd` (the static default variant, +without eBPF code) and `execd-ebpf` (the observation variant with CGO + +cilium/ebpf, built in the Dockerfile's `ebpf-builder` stage and copied +alongside `execd`). `make build-ebpf` produces the standalone +`bin/execd-ebpf` variant. Server-side selection of the observation binary +based on `[ebpf] enabled` is not wired up yet, so run the `execd-ebpf` +binary explicitly when observation is required. + ## Observability ### OpenTelemetry Metrics @@ -244,8 +345,74 @@ OTLP metrics export is enabled when either endpoint is set: - `GET /metrics`: point-in-time host metrics snapshot - `GET /metrics/watch`: SSE stream (1s cadence) -## Linux clone3 Compatibility +## Init mode + +[OSEP-0018](https://github.com/opensandbox-group/OpenSandbox/blob/main/oseps/0018-execd-as-sandbox-init.md) makes execd the sandbox +init: it becomes the parent of the user entrypoint, reaps every child through +a single reaper, forwards application signals, and propagates the entrypoint +exit code to the container runtime. + +Init mode is **off by default** and gated by two settings set in lockstep: + +- `EXECD_INIT` (read by `bootstrap.sh`): decides the process topology โ€” the + script `exec`s into `execd --init -- ` so execd inherits PID 1 + (Docker / K8s Batch paths), instead of backgrounding execd and the user + command as siblings. +- `--init` (read by execd): activates the init duties (reaper, signal + forwarding, lifecycle). If execd is not PID 1 (e.g. the K8s Pool task path, + or a stray `&`), it degrades to subreaper mode: orphan reaping works, but + the kernel PID 1 signal shield does not. + +Behavioral contract in init mode: + +- The user entrypoint owns the container lifecycle: when it exits, execd + stops the remaining children (`SIGTERM` โ†’ grace โ†’ `SIGKILL`) and exits with + the entrypoint's status. +- `HUP`/`USR1`/`USR2`/`WINCH` are forwarded to the entrypoint process group. +- `SIGTERM` (runtime-initiated container stop) is forwarded to the workload + and starts the graceful shutdown sequence. +- In-namespace `kill -9 1` is inert (kernel signal shield). A workload + `kill 1` (SIGTERM) is treated like a runtime stop; the trusted out-of-band + stop channel is a follow-up (see the OSEP, ยง3). +- The actual mode is reported on `GET /v1/isolated/capabilities` under + `hardening.init_mode` (`pid1` | `subreaper` | `none`). + +### Pool (pre-warmed) sandboxes + +Pool tasks are executed by the task-executor with `bootstrap.sh `. +With `execd_run_as_init` enabled, the generated task no longer backgrounds +bootstrap: the task-executor's shim shell execs bootstrap, which execs +`execd --init`, so execd becomes the root of the task process tree โ€” +orphaned task children are reaped (subreaper mode, since the task process is +not the container's PID 1) and the entrypoint exit code propagates back to +the shim and the task status. + +To make execd the *container's* PID 1 in pooled pods, the operator's Pool pod +template should start the main container with `bootstrap.sh` plus a +keep-alive entrypoint and `EXECD_INIT=1`: + +```yaml +spec: + template: + spec: + containers: + - name: sandbox + image: opensandbox/execd:latest + command: ["/bootstrap.sh", "/bin/sh", "-c", "while :; do sleep 3600; done"] + env: + - name: EXECD_INIT + value: "1" + - name: EXECD + value: /execd +``` + +execd then stays alive as PID 1 (reaping + kernel signal shield) while the +task-executor keeps running user tasks in the pod. The K8s Restart recycle +strategy (`kill 1` via pod exec) keeps working against init-mode execd: the +signal is forwarded and execd exits with the workload's status, so the +kubelet restarts the container. +## Linux clone3 Compatibility Some sandbox environments fail on `clone3(2)`. Set `EXECD_CLONE3_COMPAT` in sandbox env to force fallback behavior: diff --git a/docs/components/server.md b/docs/components/server.md index 7b8f7b1b0..6192dc226 100644 --- a/docs/components/server.md +++ b/docs/components/server.md @@ -70,6 +70,29 @@ opensandbox-server init-config ~/.sandbox.toml --example docker Topics covered there include: Docker `network_mode` / `host_ip` (e.g. server in Docker Compose), `[egress]` when clients send `networkPolicy`, `[ingress]`, `[secure_runtime]`, Kubernetes `workload_provider` / `batchsandbox_template_file`, `[agent_sandbox]`, TTL caps, `[renew_intent]`. The server-wide persistence backend is configured under `[store]`; by default OpenSandbox uses a local SQLite database at `~/.opensandbox/opensandbox.db` for server-managed metadata such as snapshot records. +### OpenTelemetry metrics + +The Server can export metrics through OTLP when `[otel].enabled = true`. It uses +the configured OTLP HTTP endpoint and does not expose a Prometheus `/metrics` +listener. + +| Metric | Type | Unit | Attributes | +|---|---|---|---| +| `server.http.request.duration` | Histogram | `ms` | `http_method`, `http_route`, `http_status_code` | +| `opensandbox.sandbox.create.duration` | Histogram | `ms` | `sdk.language`, `sdk.version`, `success` | + +HTTP metrics use matched route templates such as `/v1/sandboxes/{sandbox_id}`, +not raw paths. Requests that do not reach a matched route, including early +authentication failures, use `http_route=unknown`. Sandbox IDs, tenant IDs, +API keys, bodies, and query strings are never metric attributes. Standard HTTP +methods are recorded in uppercase, while extension methods use +`http_method=OTHER` to keep attribute cardinality bounded. + +The HTTP histogram's sample count can be used for request rate, its status-code +attribute for error rate, and its buckets for latency percentiles. See the +[Server configuration reference](https://github.com/opensandbox-group/OpenSandbox/blob/main/server/configuration.md#otel) +for the complete `[otel]` settings. + ### Run the server ```bash @@ -219,6 +242,39 @@ For Kubernetes-backed sandboxes, pause/resume is implemented via `BatchSandbox.s +------------+ +--------+ ``` +### Failure recovery and `resume` + +`resume` is a pause-state operation, not a general restart operation. The API +accepts it only for a sandbox in `Paused` and returns `409 Conflict` for a +container or workload that has already exited into `Terminated` or `Failed`. +It does not restart an externally stopped Docker container. + +Runtime restart behavior is configured below the Lifecycle API: + +- **Docker**: OpenSandbox does not set a Docker restart policy on sandbox + containers. If the entrypoint exits or an operator stops the container, the + sandbox becomes `Terminated` for exit code 0 or `Failed` for a non-zero exit. +- **Kubernetes BatchSandbox**: container restart behavior follows the effective + Pod template's `restartPolicy`. The example Linux template uses `Never`, but + operators can supply a different template when its lifecycle and state-loss + tradeoffs are acceptable. OpenSandbox reports the resulting workload state; + it does not turn `resume` into a restart of a failed Pod. + +To continue after a terminal failure, create a replacement sandbox and update +the caller to use its new sandbox ID. Ordinary recreation starts from the +configured image and persistent volume contents; it does not recover process +memory or unpersisted container filesystem changes. For an intentional +state-preserving suspension, call `pause` while the sandbox is healthy and then +`resume`. Kubernetes pause/resume keeps the same sandbox ID and restores the +captured root filesystem, but not running processes or memory; see +[Pause and Resume](/guides/pause-resume). + +TTL is an absolute expiration time. Runtime or server restarts do not reset it: +the Docker server restores timers for managed containers after a server +restart, and Kubernetes keeps `spec.expireTime` on the workload. A newly +created replacement receives its own ID and expiration time from its new create +request. + ## Experimental Features Optional experimental behavior; off by default. See release notes before production. diff --git a/docs/examples/agent-sandbox.md b/docs/examples/agent-sandbox.md index ebeed2398..62bd4b77c 100644 --- a/docs/examples/agent-sandbox.md +++ b/docs/examples/agent-sandbox.md @@ -28,7 +28,7 @@ opensandbox-server init-config ~/.sandbox.toml --example docker ```toml [runtime] type = "kubernetes" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [kubernetes] namespace = "default" diff --git a/docs/examples/aks-kata.md b/docs/examples/aks-kata.md index 0f9fbdb0c..4a9a35ce6 100644 --- a/docs/examples/aks-kata.md +++ b/docs/examples/aks-kata.md @@ -130,7 +130,7 @@ kubectl rollout status deploy/opensandbox-controller-manager \ ``` ::: tip -This example clears `controller.snapshot.containerdSocketPath` because the pinned controller image (`controller:v0.2.0`) does not accept the `--containerd-socket-path` flag. Current controller builds **do** accept it (see [`kubernetes/cmd/controller/main.go`](https://github.com/opensandbox-group/OpenSandbox/blob/main/kubernetes/cmd/controller/main.go)); if your nodes use a non-default containerd socket and you deploy a controller image that supports the flag, set this value accordingly. +`controller.snapshot.containerdSocketPath` defaults to `""` in the chart, which allows the controller to use its built-in default (`/var/run/containerd/containerd.sock`) without passing the `--containerd-socket-path` flag unless explicitly configured. If your nodes use a non-default containerd socket and you deploy a controller image that supports the flag, set this value accordingly. ::: ## 5. Use `main.py` diff --git a/docs/examples/code-interpreter.md b/docs/examples/code-interpreter.md index 84be66fa5..e3adb52c4 100644 --- a/docs/examples/code-interpreter.md +++ b/docs/examples/code-interpreter.md @@ -112,7 +112,7 @@ spec: - name: opensandbox-bin mountPath: /opt/opensandbox - name: execd-installer - image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21 + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 command: [ "/bin/sh", "-c" ] args: - | @@ -130,7 +130,9 @@ spec: - "/bin/sh" - "-c" - | - /opt/opensandbox/task-executor -listen-addr=0.0.0.0:5758 >/tmp/task-executor.log 2>&1 + /opt/opensandbox/task-executor \ + -listen-addr=0.0.0.0:5758 \ + -log-dir=/tmp env: - name: SANDBOX_MAIN_CONTAINER value: main @@ -152,6 +154,52 @@ spec: poolMin: 0 ``` +#### How Pool entrypoint injection works + +The lifecycle API allocates an already-running Pod from the Pool, so it does not replace that Pod's `command`, `args`, or `env`. When a create request supplies an `entrypoint` or environment variables, the server records them in `BatchSandbox.spec.taskTemplate`. The controller then sends the task to the allocated Pod's IP on port `5758`. + +The Pool template must provide all parts of that execution path: + +- Install and run task-executor, listening on `0.0.0.0:5758`. Set its + `-log-dir` explicitly so the troubleshooting path is deterministic; the + example writes `/tmp/task-executor.log`. +- Install execd and `bootstrap.sh` into the shared volume before the Pod starts. +- Keep `bootstrap.sh` at `/opt/opensandbox/bootstrap.sh`. The server-generated task invokes that exact path. The execd binary can use another path only when the task-executor environment sets `EXECD` accordingly. +- Start execd through `bootstrap.sh` after allocation so request-specific values such as `EXECD_ACCESS_TOKEN` are available. The example above leaves task-executor as the warm Pod's foreground process for this reason. + +The Pod YAML continuing to show the Pool template is therefore expected. Inspect the `BatchSandbox` resource and task-executor instead: + +```shell +# Confirm that the server injected the requested process and environment. +kubectl get batchsandbox -n \ + -o jsonpath='{.spec.taskTemplate}{"\n"}' + +# Find the allocated Pod. The annotation value contains a JSON `pods` array. +kubectl get batchsandbox -n \ + -o jsonpath='{.metadata.annotations.sandbox\.opensandbox\.io/alloc-status}{"\n"}' + +# Replace with the first Pod name from that array. +kubectl exec -n -- \ + sh -c 'test -x /opt/opensandbox/task-executor && test -x /opt/opensandbox/bootstrap.sh' +kubectl exec -n -- \ + tail -n 100 /tmp/task-executor.log + +# Check the executor health endpoint from a second terminal while this runs. +kubectl port-forward pod/ -n 5758:5758 +curl http://127.0.0.1:5758/health +curl http://127.0.0.1:5758/getTasks + +# The lifecycle server uses -0 as the task name. Check the task's +# captured output (adjust the path if task-executor uses a custom data directory). +kubectl exec -n -- \ + sh -c 'tail -n 100 /var/lib/sandbox/tasks/-0/stdout.log; tail -n 100 /var/lib/sandbox/tasks/-0/stderr.log' + +# Check controller logs for delivery failures between the controller and port 5758. +kubectl logs -n opensandbox-system -l control-plane=controller-manager --tail=100 +``` + +If `taskTemplate` exists but the health check cannot reach port `5758`, verify that task-executor is installed and remains running. The generated task intentionally starts `bootstrap.sh` in the background, so its wrapper can report success even when `bootstrap.sh` is missing or the requested entrypoint later fails. Do not rely on `taskFailed` or `taskLastErrorMessage` alone for these failures; inspect the task's captured `stderr.log` and `stdout.log`, then verify the execd or application process directly. + Start the k8s OpenSandbox server: ```shell diff --git a/docs/examples/deep-agents.md b/docs/examples/deep-agents.md new file mode 100644 index 000000000..22c26194b --- /dev/null +++ b/docs/examples/deep-agents.md @@ -0,0 +1,54 @@ +--- +title: Deep Agents +description: Run a Deep Agent with its file and shell tools executing inside an OpenSandbox sandbox. +--- + +# Deep Agents + OpenSandbox Example + +Run a [Deep Agent](https://github.com/langchain-ai/deepagents) whose file and shell tools +execute inside an OpenSandbox sandbox. The +[`langchain-sandbox-opensandbox`](https://pypi.org/project/langchain-sandbox-opensandbox/) +backend adapts the OpenSandbox Python SDK to the Deep Agents `BaseSandbox` interface, so every +`ls` / `read_file` / `write_file` / `glob` / `grep` / command the agent performs is sandboxed. + +The backend passes the full `langchain-tests` `SandboxIntegrationTests` conformance suite +(86/86) against a live server. + +## Start OpenSandbox server [local] + +Start a local OpenSandbox server, logs will be visible in the terminal: + +```shell +uv pip install opensandbox-server +opensandbox-server init-config ~/.sandbox.toml --example docker +opensandbox-server +``` + +## Run the example + +```shell +# Install Deep Agents + the OpenSandbox backend +uv pip install deepagents langchain-sandbox-opensandbox + +# Run the example (requires SANDBOX_DOMAIN / ANTHROPIC_API_KEY) +uv run python examples/deep-agents/main.py +``` + +The agent writes a Python script into the sandbox, runs it, and reports the output โ€” all file +and shell operations happen inside the sandbox rather than on the host. The sandbox is destroyed +on exit (terminated and local resources closed). + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `SANDBOX_DOMAIN` | `localhost:8080` | Sandbox service address (host and optional port) | +| `SANDBOX_PROTOCOL` | `http` | Protocol used to reach the server (`http` or `https`) | +| `SANDBOX_API_KEY` | _(optional for local)_ | API key if your server requires authentication | +| `ANTHROPIC_API_KEY` | _(required)_ | Anthropic API key for the default Deep Agents model | + +## References + +- [Deep Agents](https://github.com/langchain-ai/deepagents) - Agent framework +- [langchain-sandbox-opensandbox](https://pypi.org/project/langchain-sandbox-opensandbox/) - OpenSandbox backend for Deep Agents +- [Source code on GitHub](https://github.com/opensandbox-group/OpenSandbox/tree/main/examples/deep-agents) diff --git a/docs/examples/desktop.md b/docs/examples/desktop.md index ca5d0b8c7..84dd473e3 100644 --- a/docs/examples/desktop.md +++ b/docs/examples/desktop.md @@ -48,11 +48,85 @@ uv run python examples/desktop/main.py ``` The script starts the desktop stack (Xvfb + XFCE + x11vnc) and also launches noVNC/websockify. It prints: -- VNC endpoint (`endpoint.endpoint`) for native VNC clients, password from `VNC_PASSWORD` (default: `opensandbox`) +- VNC endpoint (`endpoint.endpoint`) for native VNC clients when direct endpoint mode is enabled, password from `VNC_PASSWORD` (default: `opensandbox`) - noVNC URL for browsers (`/vnc.html?host=...&port=...&path=...`) The sandbox stays alive for 5 minutes by default; interrupt sooner with Ctrl+C. Uses the prebuilt desktop image by default. +### Embed noVNC in an HTTPS page + +Browsers block an `http://` noVNC iframe inside an HTTPS page. Terminate TLS at +a reverse proxy in front of the OpenSandbox server, then make the example use +that HTTPS origin and the server proxy: + +```shell +export SANDBOX_DOMAIN=sandbox.example.com +export SANDBOX_PROTOCOL=https +export SANDBOX_USE_SERVER_PROXY=true +export VNC_PASSWORD=opensandbox + +uv run python examples/desktop/main.py +``` + +The generated URL uses the same HTTPS origin for both the noVNC page and its +WebSocket, for example: + +```text +https://sandbox.example.com/v1/sandboxes//proxy/6080/vnc.html?host=sandbox.example.com&port=443&path=v1/sandboxes//proxy/6080 +``` + +The TLS reverse proxy must forward normal HTTP requests and WebSocket upgrades +to the OpenSandbox server. For Nginx, the relevant settings are: + +```nginx +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl; + server_name sandbox.example.com; + # Configure ssl_certificate and ssl_certificate_key here. + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} +``` + +This direct browser URL works when the OpenSandbox proxy route does not require +browser-supplied authentication headers, including the default single-tenant +mode. In multi-tenant mode, the proxy route requires +`OPEN-SANDBOX-API-KEY` on both the initial noVNC HTTP request and the WebSocket +upgrade. The SDK adds this header to its own requests, but a browser navigation +or WebSocket cannot add it. + +For multi-tenant browser access, put a trusted, browser-authenticated reverse +proxy in front of OpenSandbox. It must authorize access to the requested +sandbox, map the browser identity to the correct tenant, and inject that +tenant's `OPEN-SANDBOX-API-KEY` header for both HTTP and WebSocket traffic. Do +not put the API key in the noVNC URL or expose a reverse proxy that adds one +shared key to unauthenticated traffic. + +::: warning +Do not expose a direct `http://:` URL in an HTTPS +iframe. That endpoint has no TLS termination and the browser will reject it as +mixed content. This SDK configuration uses one protocol for the management API +and sandbox service requests, so the example requires +`SANDBOX_USE_SERVER_PROXY=true` whenever the management API uses HTTPS. This +keeps lifecycle, execd, noVNC page, and WebSocket traffic on the HTTPS server +origin instead of incorrectly sending HTTPS to a direct sandbox endpoint. +Server proxy mode does not support the raw TCP protocol used by native VNC +clients, so the example omits the native VNC endpoint in this mode. +::: + ![Desktop shell](../public/images/desktop-screenshot-shell.jpg) ![noVNC connect](../public/images/desktop-screenshot-connect.jpg) ![noVNC password](../public/images/desktop-screenshot-password.jpg) @@ -62,8 +136,11 @@ The sandbox stays alive for 5 minutes by default; interrupt sooner with Ctrl+C. | Variable | Default | Description | |----------|---------|-------------| -| `SANDBOX_DOMAIN` | `localhost:8080` | Sandbox service address | -| `SANDBOX_API_KEY` | _(optional for local)_ | API key if your server requires authentication | +| `SANDBOX_DOMAIN` | `localhost:8080` | Sandbox service address; may include an `http://` or `https://` scheme | +| `SANDBOX_PROTOCOL` | Domain scheme, otherwise `http` | Protocol used when `SANDBOX_DOMAIN` has no scheme (`http` or `https`) | +| `SANDBOX_USE_SERVER_PROXY` | `false` | Route sandbox service, noVNC HTTP, and WebSocket traffic through the OpenSandbox server; required with HTTPS | +| `SANDBOX_API_KEY` | _(optional for local)_ | Example-specific API key; takes precedence over `OPEN_SANDBOX_API_KEY` | +| `OPEN_SANDBOX_API_KEY` | _(optional for local)_ | SDK-standard API key fallback when `SANDBOX_API_KEY` is unset | | `SANDBOX_IMAGE` | `opensandbox/desktop:latest` | Sandbox image to use | | `VNC_PASSWORD` | `opensandbox` | Password for VNC access | diff --git a/docs/examples/index.md b/docs/examples/index.md index 9508b2bcb..1a5a4e3f7 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -20,9 +20,11 @@ Run coding CLIs and AI agent frameworks inside isolated sandboxes. | [Claude Code](/examples/claude-code) | Run Claude Code CLI in a sandbox | | [Gemini CLI](/examples/gemini-cli) | Run Gemini CLI in a sandbox | | [Codex CLI](/examples/codex-cli) | Run OpenAI Codex CLI in a sandbox | +| [OpenCode](/examples/opencode) | Run the OpenCode coding agent in a sandbox | | [Qwen Code](/examples/qwen-code) | Run Qwen Code CLI in a sandbox | | [Kimi CLI](/examples/kimi-cli) | Run Kimi CLI (Moonshot AI) in a sandbox | | [LangGraph](/examples/langgraph) | LangGraph state-machine workflow with sandbox | +| [Deep Agents](/examples/deep-agents) | Deep Agents file/shell tools running in a sandbox | | [Google ADK](/examples/google-adk) | Google ADK agent using OpenSandbox tools | | [OpenClaw](/examples/openclaw) | OpenClaw Gateway inside a sandbox | | [NullClaw](/examples/nullclaw) | NullClaw Gateway sandbox integration | diff --git a/docs/examples/kubernetes-pvc-volume-mount.md b/docs/examples/kubernetes-pvc-volume-mount.md index c85f9ff1b..b114c4121 100644 --- a/docs/examples/kubernetes-pvc-volume-mount.md +++ b/docs/examples/kubernetes-pvc-volume-mount.md @@ -107,6 +107,63 @@ python examples/kubernetes-pvc-volume-mount/main.py The script creates a sandbox, writes a marker file under `/mnt/data`, kills it, then creates a second sandbox bound to the same PVC and confirms the marker is still there. +## Pool mode: pre-mount a shared PVC + +Pool pods are created before a sandbox is allocated, so storage shared by a +pool must be part of the Pool pod template. Create the PVC first, then mount it +in every warm pod through `Pool.spec.template`: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shared-workspace-pvc + namespace: opensandbox +spec: + accessModes: [ReadWriteMany] + storageClassName: + resources: + requests: + storage: 100Gi +--- +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: shared-workspace-pool + namespace: opensandbox +spec: + template: + spec: + containers: + - name: sandbox-container + image: python:3.11 + command: ["sleep", "3600"] + volumeMounts: + - name: shared-workspace + mountPath: /workspace + volumes: + - name: shared-workspace + persistentVolumeClaim: + claimName: shared-workspace-pvc + capacitySpec: + bufferMax: 10 + bufferMin: 2 + poolMax: 20 + poolMin: 5 +``` + +Apply the Pool manifest to the cluster before allocating sandboxes from it. +Sandboxes then select the preconfigured Pool with `extensions.poolRef` and do +not pass a per-sandbox `volumes` list. + +::: warning Static pool storage only +The PVC must already exist, and its access mode and storage backend must allow +all scheduled warm pods to mount it. OpenSandbox does not create, mutate, or +delete PVCs referenced by a Pool template. A sandbox creation request cannot +add `volumes` while also using `extensions.poolRef`, because the selected warm +pod already exists. +::: + ## Mode 2: Server-managed PVC Use this when the sandbox should own its storage. The server creates the PVC on demand the first time the claim name is referenced. You control whether the claim survives sandbox termination through `deleteOnSandboxTermination`. @@ -184,7 +241,9 @@ Both paths only match server-labeled PVCs, so BYO and opted-out claims are never ## Important notes ::: warning -- **Pool mode does not support volumes.** Use template mode instead. +- Per-sandbox `volumes` cannot be combined with `extensions.poolRef`. For Pool + mode, pre-mount an existing shared PVC in `Pool.spec.template` as described + above. - Multiple sandboxes can mount the same PVC if the access mode allows (e.g. `ReadWriteMany`) โ€” but only when the PVC is **not** opted into auto-cleanup. PVCs created with `deleteOnSandboxTermination=true` are owned exclusively by the creating sandbox; the server rejects attempts by other sandboxes to mount them with `409 CONFLICT`. - All mounts of the same `claimName` in a single request must agree on `createIfNotExists` and `deleteOnSandboxTermination`; mismatches are rejected with `400 INVALID_PARAMETER`. ::: diff --git a/docs/examples/opencode.md b/docs/examples/opencode.md new file mode 100644 index 000000000..e6c1cd27d --- /dev/null +++ b/docs/examples/opencode.md @@ -0,0 +1,60 @@ +--- +title: OpenCode +description: Run the OpenCode coding agent inside an OpenSandbox container. +--- + +# OpenCode Example + +Run [OpenCode](https://opencode.ai/) inside an OpenSandbox container. The example installs the official CLI and executes a prompt non-interactively with `opencode run`. + +## Start OpenSandbox server [local] + +Pre-pull the code-interpreter image (includes Node.js): + +```shell +docker pull sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.1.0 + +# use Docker Hub +# docker pull opensandbox/code-interpreter:v1.1.0 +``` + +Then start the local OpenSandbox server. Logs will be visible in the terminal: + +```shell +uv pip install opensandbox-server +opensandbox-server init-config ~/.sandbox.toml --example docker +opensandbox-server +``` + +## Create and Access the OpenCode Sandbox + +```shell +# Install the OpenSandbox package +uv pip install opensandbox + +# Run the example. The default free model does not require an API key. +uv run python examples/opencode/main.py +``` + +The script installs OpenCode (`npm install -g opencode-ai@latest`) at runtime, creates an isolated working directory, and runs `opencode run` with a simple prompt. The default model is currently available without authentication. Set `OPENCODE_API_KEY` and `OPENCODE_MODEL` to use an authenticated model from the OpenCode provider instead. + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `SANDBOX_DOMAIN` | `localhost:8080` | Sandbox service address | +| `SANDBOX_API_KEY` | _(optional for local)_ | API key if your server requires authentication | +| `SANDBOX_IMAGE` | `sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.1.0` | Sandbox image to use | +| `OPENCODE_MODEL` | `opencode/deepseek-v4-flash-free` | Model ID in `provider/model` format | +| `OPENCODE_API_KEY` | _(optional)_ | API key for authenticated models from the OpenCode provider | + +::: info +The default free model is offered by OpenCode for a limited time. Do not send private code, credentials, or other confidential data to a free model; review the provider's data-use terms before using it. Override `OPENCODE_MODEL` when you need a different model. Other providers may require additional environment variables or configuration. +::: + +## References + +- [OpenCode CLI](https://opencode.ai/docs/cli/) - Installation and non-interactive `run` command +- [OpenCode Providers](https://opencode.ai/docs/providers) - Provider authentication and configuration +- [OpenCode Zen](https://opencode.ai/docs/zen/) - Model availability and data-use notes +- [Source code on GitHub](https://github.com/opensandbox-group/OpenSandbox/tree/main/examples/opencode) diff --git a/docs/guides/client-pool.md b/docs/guides/client-pool.md index 8b60953ea..ba1b6624c 100644 --- a/docs/guides/client-pool.md +++ b/docs/guides/client-pool.md @@ -19,9 +19,19 @@ your SDK version if you rely on it in production. ## What it actually pools -The pool does **not** pool HTTP connections, and it does **not** pool SDK `Sandbox` -objects. It pools the **IDs of pre-warmed, ready sandboxes** running on the OpenSandbox -server. +The pool does **not** pool SDK `Sandbox` objects. It pools the **IDs of +pre-warmed, ready sandboxes** running on the OpenSandbox server. + +The **Kotlin/Java** SDK additionally gives each `SandboxPool` a pool-wide +shared HTTP connection pool. When the pool's `ConnectionConfig` carries no +custom `connectionPool`, the pool creates one sized by `warmup_concurrency` +(5-minute keep-alive) and uses it for every sandbox it creates โ€” warmup, +direct create, and idle connect โ€” so concurrent warmups reuse TCP connections +instead of each opening fresh ones. At high `warmup_concurrency`, per-sandbox +connection churn otherwise causes intermittent connection resets and retry +amplification. The pool evicts its shared pool on shutdown; a user-provided +pool is never touched. Python and Go pools do not share HTTP connections +across sandboxes today. ![Client pool architecture](/images/client-pool-architecture.svg) @@ -54,6 +64,14 @@ Health is tracked separately as `HEALTHY | DEGRADED | DRAINING | STOPPED`; after exponential backoff before retrying warmup. Callers do not need to observe these states directly โ€” `snapshot()` exposes them for diagnostics. +The Kotlin pool treats HTTP 429 warmup responses as server back-pressure rather than +ordinary create failures. New warmups pause for the server's `Retry-After` duration (capped +at 60 seconds), or 10 seconds when the header is missing, zero, or otherwise non-positive, +while idle maintenance remains active. This local throttle does not increment the degraded +failure count and resets when the pool instance is restarted. While the throttle is active, +`snapshot().backoffActive` is also true so operators can see that creates are paused even +though `failureCount` / `lastError` stay unchanged. + ![Client pool lifecycle state machine](/images/client-pool-lifecycle.svg) ### There is no `release()` @@ -287,43 +305,91 @@ Every SDK exposes read-only accessors: so it is not a safe way to swap creation templates on the same `pool_name`. For that case, retire the whole namespace under a new `pool_name` (see below). +The existing cleanup methods retain their original execution behavior. For opt-in +bounded parallel cleanup, use Python's +`release_all_idle_parallel(max_workers=50)`, Kotlin's +`releaseAllIdle(concurrency)`, or Go's concrete +`(*DefaultSandboxPool).ReleaseAllIdleParallel(ctx, maxWorkers)`. These methods +validate a positive concurrency value and wait for every drained ID to receive a +best-effort kill attempt. The Go method is intentionally outside the +`SandboxPool` interface to preserve compatibility with third-party implementors. + +### Tracing warmups (Kotlin) + +The Kotlin SDK can emit an OpenTelemetry trace per warmup task (`pool.warmup` +root span plus `create` / `prepare` / `renew` / `commit` phases) when +`ConnectionConfig.enableTracing(true)` is set and an OpenTelemetry SDK + +exporter is on the classpath. `trace_id` / `span_id` are published to the +SLF4J MDC, so search your logs for a `sandbox_id` to find the warmup trace and +drill into phase durations. See [SDK Tracing (Pool Warmup)](/guides/sdk-tracing). + ### Retiring an old pool namespace -The retirement procedure differs across SDKs because Go does not currently ship a -`SandboxPoolManager` or the destroy / tombstone primitives that Python and Kotlin -have. - -**Python / Kotlin** โ€” use `SandboxPoolManager.destroy(poolName, options)`. The manager -applies a full `DESTROYING โ†’ DESTROYED` protocol: write a `DESTROYING` fence into the -state store (so any still-running peer instance sees it), best-effort drain and kill -every idle sandbox up to `drain_timeout`, clear the persistent per-pool state, then -write a `DESTROYED` tombstone with `tombstone_ttl` (default 7 days) so future callers -cannot silently rebind to the same `pool_name`. - -**Go** โ€” the Go SDK has no equivalent API and no state-store primitives for -tombstones or fences. The closest safe approximation is an operator-driven, out-of-band -sequence: - -1. Stop every process that instantiates a pool against the old `pool_name`. Call - `pool.Shutdown(ctx, true)` on each. This releases each node's primary lock but - leaves idle entries in the store. -2. From one still-alive pool instance (or a throwaway one bound to the same - `PoolName` + `StateStore`), call `pool.ReleaseAllIdle(ctx)` to drain and - best-effort kill every idle sandbox. -3. Set `store.SetMaxIdle(ctx, poolName, 0)` so any peer that races back in cannot - warm up new sandboxes. -4. Move all future callers to a new `PoolName` (for example - `orders-v2` โ†’ `orders-v3`). This is the Go substitute for the `DESTROYED` - tombstone: without a shared marker, name rotation is the only way to guarantee no - accidental reuse. -5. If you are using the Redis store and want to reclaim keys, delete them directly - with `DEL` / `SCAN` against your Redis instance โ€” the Go SDK does not expose a - destroy helper for this. - -Without a fence, steps 2 and 3 race with any surviving peer that has not yet been -stopped. If you cannot guarantee "all writers stopped" before step 2, the only correct -option is to rotate `PoolName` first (step 4) and let the old namespace's idle entries -expire naturally via `idle_timeout`. +Every SDK exposes a `SandboxPoolManager` with a `destroy` operation that applies the +same `DESTROYING โ†’ DESTROYED` protocol: + +1. Write a `DESTROYING` fence into the state store, so any still-running peer instance + sees it and stops replenishing instead of racing the retirement. +2. Best-effort drain and kill every idle sandbox, bounded by the drain timeout. +3. Clear the persistent per-pool state. +4. Write a `DESTROYED` tombstone with the tombstone TTL (default 7 days) so future + callers cannot silently rebind to the same `pool_name`. + +Destroy is idempotent: calling it on an already-tombstoned namespace reports +`DESTROYED` without draining or killing anything. If the drain or the cleanup cannot +finish, the namespace stays `DESTROYING` and the call reports the destroy as +incomplete; retrying is safe and picks up where it left off. + +**Python / Kotlin** โ€” `SandboxPoolManager.destroy(poolName, options)`, configured +through `PoolDestroyOptions` (`strategy`, `drain_timeout`, `tombstone_ttl`). + +**Go** โ€” `(*SandboxPoolManager).Destroy(ctx, poolName, options)`: + +```go +manager, err := opensandbox.NewSandboxPoolManagerBuilder(). + StateStore(store). + ConnectionConfig(connCfg). + Build() +if err != nil { + return err +} + +result, err := manager.Destroy(ctx, "orders-v2", opensandbox.PoolDestroyOptions{}) +if err != nil { + return err +} +log.Printf("retired %s: drained=%d killed=%d", + result.PoolName, result.DrainedIdleCount, result.KilledIdleCount) +``` + +`PoolDestroyOptions` mirrors the other SDKs. `Strategy` selects the algorithm and only +`PoolDestroyForce` is implemented. `DrainTimeout` and `TombstoneTTL` are `*time.Duration`: +leave them nil for the defaults (30s and 7 days), or set an explicit zero to drain +without a deadline and to write a tombstone that never expires. + +The fence is what makes retirement safe without stopping every writer first, and it +is enforced on two levels. The state store refuses `PutIdle`, `SetMaxIdle` and +`SetIdleEntryTTL` with a `*PoolDestroyedError` and hands out no primary lock, which +stops replenishment. The pool itself also checks the fence when it starts, before every +acquire, again once an acquire holds a live sandbox, and on each reconcile tick: a +surviving peer stops outright on its next tick, an in-flight acquire fails rather +than minting a fresh sandbox into the retired namespace through the direct-create +fallthrough, and a sandbox obtained just before the fence landed is killed instead +of handed out. The post-acquire check matters because the idle take is deliberately +left unfenced so `destroy` can drain: once an ID has been taken, `destroy` can no +longer reach it, so the acquire has to dispose of it itself. +Starting a fresh pool against a tombstoned `PoolName` fails for the same reason, so +rebinding the name requires either waiting out the tombstone TTL or rotating to a new +`PoolName`. + +One deliberate exception: if the state store itself is unreachable, the destroy state +is unknowable, so policies that already fall through to direct create on a store +outage (`DIRECT_CREATE`, `RETRY_NEXT_IDLE_THEN_CREATE`) assume `ACTIVE` and proceed, +matching the existing `try_take_idle` outage behavior in the OSEP-0005 error-code +matrix. `FAIL_FAST` and `RETRY_NEXT_IDLE` surface the outage instead. That relaxation +stops at a sandbox already taken from the idle buffer: there the check is fail-closed +and an unreachable store means the sandbox is killed, because nothing else is tracking +it any more. ## Further reading diff --git a/docs/guides/credential-vault.md b/docs/guides/credential-vault.md index a7681d2f1..435b18ed4 100644 --- a/docs/guides/credential-vault.md +++ b/docs/guides/credential-vault.md @@ -57,10 +57,34 @@ At a high level: substitutions. 6. Secret values are redacted from vault responses and response headers. +Requests that do not match any credential binding are forwarded unchanged. +Credential path-safety checks apply only after a binding matches and the request +would otherwise receive credentials. + The active vault used by the MITM process is served over a local Unix domain socket inside the sidecar. The sandbox workload cannot fetch this active state over the normal server proxy path. +## Persistence Across Pause and Resume + +::: warning In-memory state +Credential Vault entries are process-local memory in the egress sidecar; they +are not part of the sandbox root filesystem or the `BatchSandbox` Pod template. +Kubernetes pause deletes the Pod after snapshotting, so the fresh egress +sidecar created by resume starts with an empty vault. Credential injection does +not resume until a trusted client creates the credentials and bindings again. + +Keep the original vault request or an equivalent secret-manager reference in a +trusted control plane outside the sandbox. After the sandbox returns to +`Running`, call the Credential Vault create API again before allowing work that +depends on those credentials. Do not persist real credential values in sandbox +metadata, environment variables, snapshots, or logs. + +Docker pause/unpause retains the existing container processes, but any egress +sidecar replacement or restart also creates a new in-memory vault and requires +the same re-injection procedure. +::: + ## Service Mesh Compatibility Credential Vault depends on the egress sidecar's transparent redirect and MITM path. If the sandbox pod is also injected with a transparent service-mesh sidecar such as Istio/Envoy, both layers will try to intercept outbound traffic in the same network namespace. OpenSandbox does not currently support that combination for Credential Vault. diff --git a/docs/guides/multi-tenancy.md b/docs/guides/multi-tenancy.md index ef3657418..0a4076066 100644 --- a/docs/guides/multi-tenancy.md +++ b/docs/guides/multi-tenancy.md @@ -136,6 +136,10 @@ Response (401): The HTTP provider caches results per key using the server-suggested `ttl`. On TTL expiry it re-fetches synchronously. If the endpoint is unreachable, stale entries are served up to `max_stale_seconds`, after which requests fail with 503. +::: warning HTTP provider skips startup namespace validation +The HTTP provider resolves tenants per API key and cannot enumerate all tenants at startup, so the OSEP-0014 fail-fast namespace check is skipped for it (a warning is logged instead). The file provider, which loads the full `tenants.toml` at startup, still fails fast when any tenant namespace is missing or inaccessible. With the HTTP provider, ensure namespaces exist and are accessible before issuing tenant API keys. +::: + ## Namespace Setup Before onboarding a tenant, the cluster admin must prepare the target namespace. diff --git a/docs/guides/pause-resume.md b/docs/guides/pause-resume.md index 8f71f824d..4afeeab84 100644 --- a/docs/guides/pause-resume.md +++ b/docs/guides/pause-resume.md @@ -75,9 +75,13 @@ The Lifecycle API exposes only the coarse-grained sandbox states above. For deta | Environment variables | โœ… Yes โ€” from BatchSandbox template | | Running processes / memory | โŒ No โ€” process state is not checkpointed | | Explicit volume mounts | Depends on volume type | +| Credential Vault entries | No - stored only in egress sidecar memory; re-inject from a trusted control plane after resume | Pause/resume is currently single-replica only. The internal pause snapshot records one source Pod's container images and does not store per-replica state, so the Kubernetes controller rejects pause requests unless `BatchSandbox.spec.replicas=1`. +See [Credential Vault](/guides/credential-vault#persistence-across-pause-and-resume) +for the required post-resume credential re-injection procedure. + --- ## Architecture @@ -98,8 +102,8 @@ SandboxSnapshot Controller โ”‚ creates commit Job on the same node โ–ผ commit Job Pod (image-committer) - โ”‚ nerdctl: commit container rootfs โ†’ OCI image - โ”‚ nerdctl: push to registry + โ”‚ containerd API: commit container rootfs โ†’ OCI image + โ”‚ OCI registry resolver: push image โ–ผ SandboxSnapshot.status.phase = Succeed โ”‚ BatchSandbox.status.phase = Paused @@ -152,7 +156,8 @@ Configure the controller manager deployment with snapshot flags: |-----|------|---------|-------------| | `--snapshot-registry` | string | `""` | **Required.** OCI registry prefix. Images are stored as `/-:snap-gen`. | | `--snapshot-registry-insecure` | bool | `false` | Enables insecure registry mode for snapshot push operations. Use only for HTTP or self-signed local registries. | -| `--snapshot-push-secret` | string | `""` | Kubernetes Secret name for pushing snapshots. Must be `kubernetes.io/dockerconfigjson` type. | +| `--snapshot-push-secret` | string | `""` | Kubernetes Secret name for pushing and deleting snapshot images. Must be `kubernetes.io/dockerconfigjson` type, contain inline `auths` credentials for the registry, and permit manifest deletion. `credHelpers`/`credsStore` entries are not usable by the controller. | +| `--image-committer-pod-template-file` | string | `""` | Path to a PodTemplateSpec overlay for commit Job Pods. | | `--resume-pull-secret` | string | `""` | Kubernetes Secret name injected into resumed sandboxes for pulling snapshot images. Can be the same as push secret. | | `--image-committer-image` | string | `"image-committer:dev"` | Image used by commit Jobs. | | `--commit-job-timeout` | duration | `"10m"` | Timeout for commit Jobs. | @@ -162,6 +167,7 @@ Configure the controller manager deployment with snapshot flags: The `opensandbox-controller` Helm chart now exposes the snapshot-related controller values directly: - `controller.snapshot.imageCommitterImage` +- `controller.snapshot.imageCommitterPodTemplate` - `controller.snapshot.commitJobTimeout` - `controller.snapshot.registry` - `controller.snapshot.registryInsecure` @@ -184,6 +190,7 @@ Any OCI-compatible registry works (Docker Hub, GitHub Container Registry, Harbor - **Reachable from cluster nodes** (for the commit Job to push) - **Reachable from the Kubernetes API server / kubelet** (for image pull on resume) +- **Configured to allow manifest deletion** (for snapshot cleanup) ### Step 2: Create the push secret @@ -216,6 +223,9 @@ For development with a cluster-internal `registry:2` deployment: kubectl create deployment docker-registry \ --image=registry:2 --port=5000 +kubectl set env deployment/docker-registry \ + REGISTRY_STORAGE_DELETE_ENABLED=true + kubectl expose deployment docker-registry --port=5000 # No authentication needed for internal registry @@ -314,11 +324,20 @@ The controller creates a short-lived Kubernetes `Job` for each pause: The commit Job mounts the host containerd socket from the source node and runs as UID 0. This gives the `image-committer` image node-level container runtime access. Use only a trusted image, preferably pinned by digest or controlled by an admission policy. -Before pausing containers, `image-committer` attempts to run `sync` inside every running container included in the snapshot. This flushes guest filesystem caches for VM-isolated runtimes such as Kata Containers, where a host-side sync cannot flush the guest kernel. Guest sync is best effort: if it fails, the commit Job logs a warning and continues, so recent guest filesystem writes may be absent from the resulting snapshot. Containers that were already stopped cannot be targeted by `nerdctl exec` and continue through the stopped-container commit path. The Job mounts both the containerd socket and the host `/run/containerd/fifo` directory; the FIFO mount lets the host-side containerd shim access the I/O streams created by `nerdctl exec`. +The built-in image committer uses containerd APIs directly for container lookup, task pause/unpause, writable-snapshot image creation, and registry push. Before committing, it verifies that the node's content store contains every base-image blob for the node platform and fetches missing blobs from the original image digest; this avoids snapshot failures after CRI discards compressed content during image unpack. Source registries use HTTPS by default. For a trusted source registry that requires skipped TLS verification or plain HTTP, inject `SOURCE_IMAGE_REGISTRY_INSECURE=true` into the `commit` container through the image-committer Pod template. The reusable Go contracts and default implementations are exposed in [`pkg/imagecommitter`](https://github.com/opensandbox-group/OpenSandbox/tree/main/kubernetes/pkg/imagecommitter); provider-specific binaries can supply a credential provider and reuse the standard CLI contract. The Job also retains the host `/run/containerd/fifo` mount so compatible implementations can use containerd task exec. Any preparation command used by the built-in implementation is best effort and does not change the commit interface. + +The operator-controlled Pod template can add labels, annotations, a ServiceAccount, scheduling settings, init containers, sidecars, and settings on the required `commit` container such as resources, `env`, and `envFrom`. The template is the merge base, and controller-generated invariants override conflicting template values. The controller reserves the source `nodeName`, restart policy, committer image, image pull policy, command, arguments, security context, termination-message settings, and required environment variables, mounts, and volumes. Environment variables, mounts, and volumes are merged by name, with required controller values winning conflicts. Additional regular sidecars must terminate for the Job to complete. + +Any ServiceAccount or admission configuration referenced by the template must exist in every sandbox namespace. Unpause Jobs do not receive the commit Pod template because they do not access the registry. If the commit Job fails, the controller creates a best-effort `-unpause` Job on the same node to unpause any source containers that may have been left paused by an abrupt committer exit. -Deleting a `SandboxSnapshot` cleans up Kubernetes commit/unpause Jobs, but does not delete pushed OCI images from the registry. Repeated pause cycles create tags such as `snap-gen`; configure registry retention or garbage collection externally. +Deleting a `SandboxSnapshot` stops its commit/unpause Jobs, deletes pushed OCI manifests, and then removes the Kubernetes finalizer. Registry garbage collection may still be required to reclaim unreferenced blob storage. If the registry or credentials are permanently unavailable, remove the finalizer manually only after accepting that the image may need separate registry cleanup: + +```bash +kubectl patch sandboxsnapshot -n --type=merge \ + -p '{"metadata":{"finalizers":[]}}' +``` ### Monitoring @@ -389,7 +408,7 @@ curl http://localhost:8080/v1/sandboxes/{sandbox_id} | `conditions` | list | `Ready` / `Failed` conditions with reason and message | | `sourcePodName` | string | Pod name used for commit | | `sourceNodeName` | string | Node where commit Job runs | -| `containers` | list | `{containerName, imageUri, imageDigest}` per container | +| `containers` | list | `{containerName, imageUri, imageDigest}` per container. `imageDigest` is the image config digest, preserving the image ID semantics of earlier image-committer versions. | | `observedGeneration` | int | Last processed spec generation | --- diff --git a/docs/guides/sdk-tracing.md b/docs/guides/sdk-tracing.md new file mode 100644 index 000000000..5df7d9201 --- /dev/null +++ b/docs/guides/sdk-tracing.md @@ -0,0 +1,182 @@ +--- +title: SDK Tracing (Pool Warmup) +description: How to enable OpenTelemetry tracing for the Kotlin SDK pool warmup path, what spans are produced, and how to query and drill down into warmup traces. +--- + +# SDK Tracing (Pool Warmup) + +The Kotlin/Java SDK (`com.alibaba.opensandbox:sandbox`) can emit +[OpenTelemetry](https://opentelemetry.io/) traces for the client-side +`SandboxPool` warmup path. Each warmup task becomes one trace that covers the +full lifecycle โ€” from the moment the reconcile loop submits the task until the +warmed sandbox is committed to the idle buffer โ€” with per-phase spans so you +can find the actual warmup bottleneck. + +Tracing is **opt-in** (`enableTracing(true)`) and **best-effort**: without an +OpenTelemetry SDK + exporter on the application classpath, all span calls are +no-ops and nothing is exported. Tracing never throws and never affects pool +behavior. + +## Requirements + +| Component | Minimum version | +|-----------|-----------------| +| Kotlin / Java SDK (`com.alibaba.opensandbox:sandbox`) | `1.0.19` | + +Only the Kotlin SDK emits these traces today; the other language SDKs do not +yet support `enableTracing`. + +## Enabling tracing + +### 1. Add an OpenTelemetry SDK + exporter to your application + +The SDK depends only on `opentelemetry-api` (no-op by default). To actually +export traces you bring your own SDK and exporter, for example OTLP over HTTP: + +```kotlin +dependencies { + implementation("io.opentelemetry:opentelemetry-api:1.51.0") + implementation("io.opentelemetry:opentelemetry-sdk:1.51.0") + implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.51.0") +} +``` + +### 2. Configure a global `OpenTelemetry` instance + +Warmup spans use the global instance (`GlobalOpenTelemetry`). Configure it at +application startup, e.g. with `OpenTelemetrySdk`: + +```java +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; + +SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.create( + OtlpGrpcSpanExporter.builder() + .setEndpoint("http://otel-collector:4317") + .build())) + .build(); + +OpenTelemetrySdk sdk = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build(); + +GlobalOpenTelemetry.set(sdk); +``` + +::: tip Propagators +`OpenTelemetrySdk.builder()` defaults to **noop propagators**. If you want the +SDK to inject the W3C `traceparent` header into lifecycle requests (so the +lifecycle server can join the same trace once it supports tracing), configure +W3C propagation explicitly: + +```java +.setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) +``` +::: + +::: tip Sampling +To keep trace volume bounded, use a sampling strategy such as +`parentbased_traceidratio(0.1)` on the `SdkTracerProvider`. Trace-id-ratio +sampling keeps client and server spans consistent for the same warmup. +::: + +### 3. Turn tracing on for the pool + +```java +ConnectionConfig config = ConnectionConfig.builder() + .enableTracing(true) + .build(); + +SandboxPool pool = SandboxPool.builder() + .poolName("demo-pool") + .maxIdle(3) + .stateStore(new InMemoryPoolStateStore()) + .connectionConfig(config) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .build(); +``` + +That is all. No environment variables are involved; `enableTracing` defaults +to `false`. + +## What is traced + +Each warmup task produces **one trace** with a root span and four sequential +phase spans (siblings under the root, so each phase duration stands alone for +comparison): + +| Span name | Covers | +|-----------|--------| +| `pool.warmup` (root) | Task submission โ†’ sandbox committed to idle. Backdated to submission time, so the queue wait before the first phase is visible as the gap before the first child span | +| `pool.warmup.create` | Sandbox create API call, endpoint resolution, and readiness wait | +| `pool.warmup.prepare` | The configured `warmupSandboxPreparer` (user init script / setup work) | +| `pool.warmup.renew` | TTL renewal right before committing the sandbox | +| `pool.warmup.commit` | Primary-lock renewal + `putIdle` against the state store (runs on the pool scheduler thread) | + +Root span attributes (these are your drill-down dimensions): + +| Attribute | Value | +|-----------|-------| +| `pool.name` | Pool name | +| `pool.owner` | Pool owner id | +| `pool.run.generation` | Pool run generation | +| `sandbox.id` | Sandbox id (success only) | +| `sandbox.image` | Creation image (success only) | +| `result` | `success` or `failure` | + +Failures are recorded with `recordException` on the root span plus +`result=failure`; the `pool.warmup.commit` span is not emitted for failed +warmups. + +## Correlating logs to traces + +While a warmup trace is in progress, the pool publishes the trace ids to the +SLF4J [MDC](https://www.slf4j.org/api/org/slf4j/MDC.html): + +| MDC key | Value | +|---------|-------| +| `trace_id` | Current trace id | +| `span_id` | Current span id | + +MDC requires a real SLF4J provider (logback, log4j2, ...). Add the keys to +your log pattern once, and every pool log line carries the trace context: + +```xml +%d %-5level [%thread] %logger{36} trace_id=%X{trace_id} span_id=%X{span_id} - %msg%n +``` + +## Querying traces + +The trace id is random, so a warmup trace cannot be looked up "by pool name" +directly. The reliable paths are: + +1. **Log correlation (recommended).** The pool already logs `pool_name` and + `sandbox_id` on its warmup lines (e.g. `Pool warmup sandbox entered idle`). + Search your logs for a `sandbox_id` โ€” the matching log lines carry + `trace_id`, which you can open directly in your trace backend. +2. **Attribute query in the trace backend.** Filter spans by time window and + attribute, e.g. TraceQL `{ span.pool.name = "demo-pool" }` (Grafana Tempo), + or Jaeger tag search on `pool.name=...`. Backends that derive metrics from + spans (Tempo metrics, Datadog span analytics) let you look at + `pool.warmup` duration percentiles per `pool.name` first, then drill into + slow traces. +3. **Trace-id-ratio sampling.** With sampled traces, `trace_id` in logs and + the backend are consistent for the same warmup. + +### Bottleneck drill-down + +``` +pool.warmup root duration (p50/p95/p99) per pool.name + โ””โ”€ phase spans: pool.warmup.create / prepare / renew / commit + โ””โ”€ single trace: root start gap = queue wait, then each phase duration +``` + +| Symptom | Likely cause | +|---------|--------------| +| Long gap before the first child span | Warmup tasks queued โ€” `warmupConcurrency` too low, or the executor is busy with cleanup kills | +| `pool.warmup.create` slow | Lifecycle server slow (image pull / execd startup) or readiness polling takes long | +| `pool.warmup.prepare` slow | Your `warmupSandboxPreparer` work is the bottleneck | +| `pool.warmup.renew` / `pool.warmup.commit` slow | State store (e.g. Redis) round-trips | diff --git a/docs/kubernetes/deployment.md b/docs/kubernetes/deployment.md index 305248dad..03368e5b1 100644 --- a/docs/kubernetes/deployment.md +++ b/docs/kubernetes/deployment.md @@ -26,13 +26,13 @@ Install the controller and CRDs before the lifecycle server. The server runs in Choose a published `opensandbox-server` chart from [GitHub Releases](https://github.com/opensandbox-group/OpenSandbox/releases?q=helm%2Fopensandbox-server&expanded=true), then set both versions from that release: ```sh -APP_VERSION="" CHART_VERSION="" -CHART_URL="https://github.com/opensandbox-group/OpenSandbox/releases/download/helm/opensandbox-server/${APP_VERSION}/opensandbox-server-${CHART_VERSION}.tgz" +APP_VERSION="" +CHART_URL="https://github.com/opensandbox-group/OpenSandbox/releases/download/helm/opensandbox-server/${CHART_VERSION}/opensandbox-server-${CHART_VERSION}.tgz" ``` ::: info Versioning -The release tag identifies the server application version, while the `.tgz` filename uses the Helm chart version. These versions are independent and are listed on each GitHub Release. +The release tag and `.tgz` filename identify the Helm chart version. The server application version is independent and is listed on each GitHub Release. ::: ### Configure API authentication diff --git a/docs/kubernetes/index.md b/docs/kubernetes/index.md index 567a0c646..7f069fe89 100644 --- a/docs/kubernetes/index.md +++ b/docs/kubernetes/index.md @@ -137,13 +137,16 @@ The snapshot controller supports the following command-line flags: | Flag | Default | Description | |------|---------|-------------| | `--snapshot-registry` | `""` | OCI registry prefix used for snapshot images | -| `--snapshot-push-secret` | `""` | Secret name used by commit Jobs to push snapshots | +| `--snapshot-push-secret` | `""` | Secret name used to push and delete snapshot images; must contain inline `auths` credentials with manifest delete permission | +| `--image-committer-pod-template-file` | `""` | Path to a PodTemplateSpec overlay for image-committer commit Job Pods | | `--resume-pull-secret` | `""` | Secret name injected into resumed sandboxes for image pulls | -| `--image-committer-image` | `image-committer:dev` | Image used for commit operations (must contain `nerdctl` tool) | +| `--image-committer-image` | `image-committer:dev` | Image used for commit operations | | `--commit-job-timeout` | `10m` | Timeout duration for commit jobs | | `--snapshot-registry-insecure` | `false` | Pass insecure registry mode to snapshot commit Jobs | -These flags are configured at controller startup. The `image-committer-image` must be a trusted container image with `nerdctl` to perform rootfs commit and push operations. Commit Jobs mount the host containerd socket on the source node, so the image effectively has node-level runtime access. Pin the image by digest or enforce a trusted registry/admission policy in production. +These flags are configured at controller startup. The built-in image committer uses containerd APIs directly. Any custom `image-committer-image` must implement the documented commit and unpause command contract and must be trusted: commit Jobs mount the host containerd socket on the source node, so the image effectively has node-level runtime access. Pin the image by digest or enforce a trusted registry/admission policy in production. + +The optional Pod template is operator-controlled and can supply identity metadata, a ServiceAccount, resources, scheduling settings, and additional containers while the controller preserves the commit runtime invariants. ### Quick Setup @@ -171,7 +174,7 @@ Then configure the controller manager with: ``` ::: info -Snapshot image retention is registry-managed. Deleting a `SandboxSnapshot` removes the Kubernetes commit/unpause Jobs, but it does not delete pushed OCI images from the registry. Configure registry retention/GC for tags such as `snap-gen` according to your environment. +Deleting a `SandboxSnapshot` stops its commit/unpause Jobs and deletes its pushed OCI images before the controller removes the finalizer. Keep the configured `--snapshot-push-secret` available during cleanup. If the registry is permanently unavailable, remove the finalizer manually with `kubectl patch sandboxsnapshot -n --type=merge -p '{"metadata":{"finalizers":[]}}'`, then clean up the registry image separately. Registry garbage collection may still be required to reclaim unreferenced blob storage. ::: ## Getting Started @@ -400,6 +403,22 @@ spec: Pool pods are created before allocation. The lifecycle API therefore rejects `networkPolicy` together with `extensions.poolRef`; it cannot inject an egress sidecar into an existing pool pod. Configure required network controls in the Pool pod template before pods are created, or use a non-pooled sandbox for per-request policies. ::: +::: tip Entrypoint and environment injection +Pool pods are also created before a lifecycle request supplies its `entrypoint` or environment variables. The server does not rewrite the allocated Pod's `command`, `args`, or `env`; seeing the original Pool template in the Pod YAML is expected. Instead, it writes the request-specific process to `BatchSandbox.spec.taskTemplate`, and the controller sends that task to an in-pod task-executor on port `5758`. + +A Pool used through the lifecycle API with a custom entrypoint or environment must therefore run task-executor and provide an executable `/opt/opensandbox/bootstrap.sh`. See the [Code Interpreter Pool example](/examples/code-interpreter#how-pool-entrypoint-injection-works) for a complete template and troubleshooting commands. +::: + +::: tip Shared storage in Pool mode +Static shared storage follows the same rule: pre-create a PVC (normally with a +`ReadWriteMany`-capable storage class) and mount it in the Pool pod template as +shown in the complete example linked below. The Kubernetes controller preserves +these static mounts when it creates warm pods from the Pool template. +Per-sandbox `volumes` cannot be combined with `extensions.poolRef`, because an +allocated warm pod cannot gain new volumes. See the +[Kubernetes PVC guide](/examples/kubernetes-pvc-volume-mount#pool-mode-pre-mount-a-shared-pvc). +::: + #### Pooled Sandbox with Heterogeneous Tasks Create a batch of sandboxes with process-based heterogeneous tasks. For task execution to work properly, the task-executor must be deployed as a sidecar container in the pool template and share the process namespace with the sandbox container: @@ -459,6 +478,54 @@ kubectl describe pool example-pool kubectl describe batchsandbox example-batch-sandbox ``` +#### Understand BatchSandbox status + +A `BatchSandbox` reports sandbox availability and optional task execution separately. Use `status.phase`, `status.conditions`, and the replica counters for the sandbox runtime. Use the `status.task*` counters for task progress and completion. + +::: warning `Succeed` is not task completion +`status.phase: Succeed` is the steady running phase. The controller sets it after observing at least one non-deleting Pod that is Running and Ready. It is non-terminal and does not mean that a task, Pod, or Kubernetes Job completed successfully. + +For a BatchSandbox with multiple replicas, `Succeed` also does not mean that every desired replica is Ready. +::: + +| `status.phase` | Meaning | +|---|---| +| `Pending` | The controller has not observed a Running and Ready sandbox Pod yet. | +| `Succeed` | At least one sandbox Pod is Running and Ready; the sandbox is available. | +| `Pausing` | A pause operation is in progress. | +| `Paused` | The sandbox is paused and its runtime resources have been released. | +| `Resuming` | The controller is restoring runtime resources after a pause. | +| `Failed` | The controller detected a sandbox runtime failure. Inspect conditions and Pod events for details. | + +The controller records active conditions with `status: "True"`: + +| Condition | Meaning when `True` | +|---|---| +| `Ready` | The phase is `Succeed`; reason `PodsReady` means the sandbox is running. | +| `Progressing` | The sandbox is being created, paused, or resumed. | +| `Paused` | The sandbox is fully paused. | +| `PauseFailed`, `ResumeFailed`, `PodFailed` | The corresponding operation or runtime failed; inspect `reason` and `message`. | + +Treat a condition as satisfied only when the matching entry exists with `status: "True"`. Check `status.observedGeneration` against `metadata.generation` before acting on status after a spec update. + +Inspect runtime and task status together: + +```sh +kubectl get batchsandbox example-batch-sandbox \ + -o custom-columns='NAME:.metadata.name,GEN:.metadata.generation,OBSERVED:.status.observedGeneration,PHASE:.status.phase,READY:.status.ready,DESIRED:.spec.replicas,TASK_RUNNING:.status.taskRunning,TASK_SUCCEED:.status.taskSucceed,TASK_FAILED:.status.taskFailed' + +kubectl get batchsandbox example-batch-sandbox \ + -o jsonpath='{range .status.conditions[*]}{.type}{"="}{.status}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' +``` + +Choose monitoring fields based on the question you need to answer: + +| Monitoring goal | Fields to evaluate | +|---|---| +| At least one sandbox is available | A `Ready=True` condition, normally with `status.phase=Succeed` | +| All desired replicas are Ready | A fresh status where `status.ready == spec.replicas` | +| Tasks have completed | `taskPending`, `taskRunning`, `taskSucceed`, `taskFailed`, and `taskUnknown`, according to the workload's completion policy | + ## Performance When both use resource pools, the total time comparison for delivering 100 Sandboxes: diff --git a/docs/package.json b/docs/package.json index a9f79cf1e..359370b72 100644 --- a/docs/package.json +++ b/docs/package.json @@ -5,7 +5,7 @@ "pnpm": { "overrides": { "rollup": "4.60.2", - "postcss": "8.5.11", + "postcss": "8.5.23", "esbuild": "0.25.2", "vite": "6.4.3" } diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index b1fee4bd0..ae908a853 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: overrides: rollup: 4.60.2 - postcss: 8.5.11 + postcss: 8.5.23 esbuild: 0.25.2 vite: 6.4.3 @@ -16,7 +16,7 @@ importers: devDependencies: vitepress: specifier: ^1.6.4 - version: 1.6.4(@algolia/client-search@5.48.0)(postcss@8.5.11)(search-insights@2.17.3) + version: 1.6.4(@algolia/client-search@5.48.0)(postcss@8.5.23)(search-insights@2.17.3) packages: @@ -690,8 +690,8 @@ packages: mitt@3.0.1: resolution: {integrity: sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==} - nanoid@3.3.11: - resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -708,8 +708,8 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} - postcss@8.5.11: - resolution: {integrity: sha512-5dDj8+lmvA8XB78SmzGI8NlQoksv7IfutGWeVZxiixHbO+p4LDPT3wuG/D9sM/wrjZZ9I+Siy/e117vbFPxSZg==} + postcss@8.5.23: + resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} engines: {node: ^10 || ^12 || >=14} preact@10.28.3: @@ -835,7 +835,7 @@ packages: hasBin: true peerDependencies: markdown-it-mathjax3: ^4 - postcss: 8.5.11 + postcss: 8.5.23 peerDependenciesMeta: markdown-it-mathjax3: optional: true @@ -1254,7 +1254,7 @@ snapshots: '@vue/shared': 3.5.27 estree-walker: 2.0.2 magic-string: 0.30.21 - postcss: 8.5.11 + postcss: 8.5.23 source-map-js: 1.2.1 '@vue/compiler-ssr@3.5.27': @@ -1478,7 +1478,7 @@ snapshots: mitt@3.0.1: {} - nanoid@3.3.11: {} + nanoid@3.3.18: {} oniguruma-to-es@3.1.1: dependencies: @@ -1492,9 +1492,9 @@ snapshots: picomatch@4.0.4: {} - postcss@8.5.11: + postcss@8.5.23: dependencies: - nanoid: 3.3.11 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -1620,13 +1620,13 @@ snapshots: esbuild: 0.25.2 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.11 + postcss: 8.5.23 rollup: 4.60.2 tinyglobby: 0.2.16 optionalDependencies: fsevents: 2.3.3 - vitepress@1.6.4(@algolia/client-search@5.48.0)(postcss@8.5.11)(search-insights@2.17.3): + vitepress@1.6.4(@algolia/client-search@5.48.0)(postcss@8.5.23)(search-insights@2.17.3): dependencies: '@docsearch/css': 3.8.2 '@docsearch/js': 3.8.2(@algolia/client-search@5.48.0)(search-insights@2.17.3) @@ -1647,7 +1647,7 @@ snapshots: vite: 6.4.3 vue: 3.5.27 optionalDependencies: - postcss: 8.5.11 + postcss: 8.5.23 transitivePeerDependencies: - '@algolia/client-search' - '@types/node' diff --git a/docs/sdks/go.md b/docs/sdks/go.md index da90f947a..884b0d8c3 100644 --- a/docs/sdks/go.md +++ b/docs/sdks/go.md @@ -296,6 +296,10 @@ pool, err := opensandbox.NewSandboxPoolBuilder(). - All nodes sharing one pool must use the same creation and warmup definition. If that definition changes, use a new `PoolName` or key prefix and drain the old pool. - `Resize(ctx, maxIdle)` can be called from any node. The call returns after the target is stored in the shared state store; the current primary applies replenish or shrink work during periodic reconcile. - Use `Resize(ctx, 0)` and wait for `Snapshot().IdleCount == 0` to drain a distributed idle buffer. `ReleaseAllIdle()` is only a best-effort cleanup pass in distributed mode. +- `ReleaseAllIdle(ctx)` preserves fire-and-forget kill scheduling. Call + `ReleaseAllIdleParallel(ctx, maxWorkers)` on `*DefaultSandboxPool` for bounded + parallel cleanup that waits for every drained ID to receive a kill attempt. + `maxWorkers` must be positive; the method is not part of the `SandboxPool` interface. - Configure `PrimaryLockTTL` greater than `WarmupReadyTimeout` plus expected warmup preparer time. ::: diff --git a/docs/sdks/kotlin.md b/docs/sdks/kotlin.md index 4f333d2c3..23834ea3d 100644 --- a/docs/sdks/kotlin.md +++ b/docs/sdks/kotlin.md @@ -301,11 +301,21 @@ poolManager.destroy( - Use `warmupSandboxPreparer(...)` if you need to prepare a sandbox after warmup readiness succeeds and before it is put into the idle pool. ::: +::: tip Observing warmup performance +To trace the warmup path, enable `ConnectionConfig.builder().enableTracing(true)` and add an +OpenTelemetry SDK + exporter to your application. Each warmup becomes one trace +(`pool.warmup` root span plus `create` / `prepare` / `renew` / `commit` phases) with +`trace_id` / `span_id` published to the SLF4J MDC, so you can look up a sandbox's +warmup by searching logs for its `sandbox_id`. See [SDK Tracing (Pool Warmup)](/guides/sdk-tracing). +::: + ::: tip Distributed Deployment For distributed deployment, use the optional `com.alibaba.opensandbox:sandbox-pool-redis` module or provide a custom `PoolStateStore` implementation. The Redis module accepts a caller-managed Jedis client, so your application keeps ownership of Redis connection configuration and lifecycle. Nodes sharing the same pool namespace must use the same sandbox creation and warmup definition; use a new `poolName` or namespace when changing that definition. Configure `primaryLockTtl` greater than `warmupReadyTimeout` plus expected warmup preparer time and buffer, otherwise leadership may expire while a node is creating idle sandboxes. In distributed mode, `resize(maxIdle)` can be called from any node. The call returns after the target is stored in the shared state store; the current primary applies replenish or shrink work during periodic reconcile. Use `resize(0)` and wait for `snapshot().idleCount == 0` when you need to drain the distributed idle buffer; `releaseAllIdle()` is only a best-effort cleanup pass. +`releaseAllIdle()` preserves serial cleanup. Use `releaseAllIdle(concurrency)` for bounded parallel cleanup. `concurrency` must be positive, and the overload waits for every drained ID to receive a best-effort kill attempt. + `SandboxPoolManager.destroy(poolName)` is a stronger administrative operation: it writes a `DESTROYING` fence, drains visible idle IDs, best-effort kills idle sandboxes, clears persistent pool state, and then writes a `DESTROYED` tombstone for the configured TTL to prevent old nodes from recreating the same pool namespace. If drain or persistent-state cleanup cannot complete, `destroy()` throws `PoolDestroyIncompleteException` and leaves the namespace fenced as `DESTROYING`; retry `destroy()` to finish cleanup. ::: @@ -327,6 +337,7 @@ The `ConnectionConfig` class manages API server connection settings. | `retryPolicy` | Automatic retry policy for non-streaming requests (see [Automatic retries](#_2-automatic-retries)) | Enabled (`RetryPolicy()`) | - | | `useServerProxy` | Use sandbox server as proxy for execd/endpoint requests (e.g. when client cannot reach the sandbox directly) | `false` | - | | `disableMetrics` | Disable SDK create-latency telemetry (see [SDK Telemetry](/guides/sdk-telemetry)) | `false` | `OPENSANDBOX_DISABLE_METRICS` | +| `enableTracing` | Enable OpenTelemetry tracing for pool warmup (see [SDK Tracing](/guides/sdk-tracing)) | `false` | - | ```java // 1. Basic configuration diff --git a/docs/sdks/python.md b/docs/sdks/python.md index 74b072c23..73b45e352 100644 --- a/docs/sdks/python.md +++ b/docs/sdks/python.md @@ -247,6 +247,10 @@ For async pools, pass a `redis.asyncio` client to `AsyncRedisPoolStateStore`. idle buffer. `release_all_idle()` is only a best-effort cleanup pass in distributed mode because another primary may put new idle sandboxes concurrently unless the shared target has already been reduced. +- `release_all_idle()` preserves serial cleanup. Use + `release_all_idle_parallel(max_workers=50)` for bounded parallel cleanup. The + worker count must be positive, and the call waits for every drained ID to receive + a best-effort kill attempt. - Configure `primary_lock_ttl` greater than `warmup_ready_timeout` plus expected warmup preparer time and buffer. - Redis outages are surfaced as pool state store errors. The pool fails closed; it diff --git a/examples/aks-kata/server-values.yaml b/examples/aks-kata/server-values.yaml index d3ed31963..d97ab5232 100644 --- a/examples/aks-kata/server-values.yaml +++ b/examples/aks-kata/server-values.yaml @@ -41,7 +41,7 @@ configToml: | [runtime] type = "kubernetes" - execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21" + execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" [storage] allowed_host_paths = [] @@ -59,7 +59,7 @@ configToml: | batchsandbox_template_file = "/etc/opensandbox/aks-kata.batchsandbox-template.yaml" [egress] - image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5" + image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" mode = "dns+nft" disable_ipv6 = true diff --git a/examples/deep-agents/README.md b/examples/deep-agents/README.md new file mode 100644 index 000000000..71568b491 --- /dev/null +++ b/examples/deep-agents/README.md @@ -0,0 +1,5 @@ +# Deep Agents + OpenSandbox Example + +Run a [Deep Agent](https://github.com/langchain-ai/deepagents) whose file and shell tools execute inside an OpenSandbox sandbox, via the [`langchain-sandbox-opensandbox`](https://pypi.org/project/langchain-sandbox-opensandbox/) backend. + +> **Full documentation**: [docs/examples/deep-agents.md](../../docs/examples/deep-agents.md) diff --git a/examples/deep-agents/main.py b/examples/deep-agents/main.py new file mode 100644 index 000000000..eb34bdb7c --- /dev/null +++ b/examples/deep-agents/main.py @@ -0,0 +1,81 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deep Agents + OpenSandbox example. + +Runs a Deep Agent whose file and shell tools execute inside an OpenSandbox +sandbox. The `langchain-sandbox-opensandbox` package adapts the OpenSandbox +Python SDK to the Deep Agents `BaseSandbox` interface, so every file read/write +and command the agent runs is sandboxed. + +Prerequisites: + pip install deepagents langchain-sandbox-opensandbox + +Environment: + SANDBOX_DOMAIN Host (and optional port) of the OpenSandbox server. + Defaults to "localhost:8080". + SANDBOX_PROTOCOL "http" (default) or "https". + SANDBOX_API_KEY API key, if the server requires authentication. + ANTHROPIC_API_KEY Required by the default Deep Agents model. +""" + +import os + +from deepagents import create_deep_agent +from langchain_opensandbox import OpenSandboxBackend +from opensandbox import SandboxSync +from opensandbox.config.connection_sync import ConnectionConfigSync + + +def main() -> None: + connection = ConnectionConfigSync( + domain=os.getenv("SANDBOX_DOMAIN", "localhost:8080"), + api_key=os.getenv("SANDBOX_API_KEY"), + protocol=os.getenv("SANDBOX_PROTOCOL", "http"), + ) + sandbox = SandboxSync.create("python:3.12", connection_config=connection) + backend = OpenSandboxBackend(sandbox=sandbox, timeout=300) + + try: + agent = create_deep_agent( + tools=[], + system_prompt=( + "You are a coding assistant. Use the sandbox to write and run " + "Python, and verify your work by executing it." + ), + backend=backend, + ) + + result = agent.invoke( + { + "messages": [ + { + "role": "user", + "content": ( + "Write a script that prints the first 10 Fibonacci " + "numbers, save it as fib.py, run it, and report the " + "output." + ), + } + ] + } + ) + + print(result["messages"][-1].content) + finally: + sandbox.destroy() + + +if __name__ == "__main__": + main() diff --git a/examples/desktop/main.py b/examples/desktop/main.py index 24b5ac71c..69cc52328 100644 --- a/examples/desktop/main.py +++ b/examples/desktop/main.py @@ -16,6 +16,16 @@ import os from datetime import timedelta +from novnc_url import ( + browser_proxy_auth_warning, + build_novnc_url, + normalize_domain, + parse_bool, + resolve_api_key, + resolve_novnc_protocol, + resolve_protocol, + validate_connection_mode, +) from opensandbox import Sandbox from opensandbox.config import ConnectionConfig from opensandbox.models.execd import RunCommandOpts @@ -28,6 +38,13 @@ def _required_env(name: str) -> str: return value +def _bool_env(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return parse_bool(value, name) + + async def _print_logs(label: str, execution) -> None: for msg in execution.logs.stdout: print(f"[{label} stdout] {msg.text}") @@ -38,8 +55,14 @@ async def _print_logs(label: str, execution) -> None: async def main() -> None: - domain = os.getenv("SANDBOX_DOMAIN", "localhost:8080") - api_key = os.getenv("SANDBOX_API_KEY") + domain = normalize_domain(os.getenv("SANDBOX_DOMAIN", "localhost:8080")) + protocol = resolve_protocol(domain, os.getenv("SANDBOX_PROTOCOL")) + use_server_proxy = _bool_env("SANDBOX_USE_SERVER_PROXY") + validate_connection_mode(protocol, use_server_proxy) + api_key = resolve_api_key( + os.getenv("SANDBOX_API_KEY"), + os.getenv("OPEN_SANDBOX_API_KEY"), + ) image = os.getenv( "SANDBOX_IMAGE", "opensandbox/desktop:latest", @@ -49,8 +72,10 @@ async def main() -> None: config = ConnectionConfig( domain=domain, + protocol=protocol, api_key=api_key, request_timeout=timedelta(seconds=60), + use_server_proxy=use_server_proxy, ) sandbox = await Sandbox.create( @@ -79,9 +104,7 @@ async def main() -> None: await _print_logs("xfce", xfce_exec) vnc_exec = await sandbox.commands.run( - "x11vnc -display :0 " - "-passwd \"$VNC_PASSWORD\" " - "-forever -shared -rfbport 5900", + 'x11vnc -display :0 -passwd "$VNC_PASSWORD" -forever -shared -rfbport 5900', opts=RunCommandOpts(background=True), ) await _print_logs("x11vnc", vnc_exec) @@ -93,24 +116,26 @@ async def main() -> None: ) await _print_logs("novnc", novnc_exec) - endpoint_vnc = await sandbox.get_endpoint(5900) endpoint_novnc = await sandbox.get_endpoint(6080) - # Build noVNC URL with host/port/path for routed endpoint, e.g., host:port/proxy/6080 - novnc_host_port, novnc_path = endpoint_novnc.endpoint.split("/", 1) - novnc_host, novnc_port = novnc_host_port.split(":") - novnc_url = ( - f"http://{endpoint_novnc.endpoint}/vnc.html" - f"?host={novnc_host}&port={novnc_port}&path={novnc_path}" - ) + # noVNC uses the page's scheme to select ws:// or wss://. Only server + # proxy endpoints inherit the management API's TLS origin; direct + # websockify endpoints do not terminate TLS. + novnc_protocol = resolve_novnc_protocol(config.protocol, use_server_proxy) + novnc_url = build_novnc_url(endpoint_novnc.endpoint, novnc_protocol) + auth_warning = browser_proxy_auth_warning(use_server_proxy, api_key) - print("\nVNC endpoint (native clients):") - print(f" {endpoint_vnc.endpoint}") - print(f"Password: {vnc_password}") + if not use_server_proxy: + endpoint_vnc = await sandbox.get_endpoint(5900) + print("\nVNC endpoint (native clients):") + print(f" {endpoint_vnc.endpoint}") + print(f"Password: {vnc_password}") print("\nnoVNC (browser):") print(f" {novnc_url}") print(f"Password: {vnc_password}") + if auth_warning: + print(f"Authentication note: {auth_warning}") print("\nKeeping sandbox alive for 5 minutes. Press Ctrl+C to exit sooner.") try: diff --git a/examples/desktop/novnc_url.py b/examples/desktop/novnc_url.py new file mode 100644 index 000000000..a8340d69b --- /dev/null +++ b/examples/desktop/novnc_url.py @@ -0,0 +1,109 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for configuring and building browser-ready noVNC URLs.""" + +from urllib.parse import urlencode, urlsplit, urlunsplit + + +def normalize_domain(domain: str) -> str: + """Normalize whitespace and a case-insensitive HTTP scheme prefix.""" + normalized = domain.strip() + lowered = normalized.lower() + if lowered.startswith("https://"): + return f"https://{normalized[len('https://') :]}" + if lowered.startswith("http://"): + return f"http://{normalized[len('http://') :]}" + return normalized + + +def resolve_protocol(domain: str, configured_protocol: str | None) -> str: + """Resolve the URL protocol, honoring a scheme embedded in the domain.""" + lowered_domain = normalize_domain(domain).lower() + if lowered_domain.startswith("https://"): + return "https" + if lowered_domain.startswith("http://"): + return "http" + return configured_protocol.lower() if configured_protocol else "http" + + +def parse_bool(value: str, name: str) -> bool: + """Parse an environment-style boolean value.""" + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise RuntimeError(f"{name} must be one of: 1, true, yes, on, 0, false, no, off") + + +def validate_connection_mode(management_protocol: str, use_server_proxy: bool) -> None: + """Require proxy mode when the management API uses HTTPS.""" + if management_protocol == "https" and not use_server_proxy: + raise RuntimeError( + "SANDBOX_USE_SERVER_PROXY must be true when the management API uses HTTPS; " + "direct sandbox endpoints do not terminate TLS" + ) + + +def resolve_novnc_protocol(management_protocol: str, use_server_proxy: bool) -> str: + """Use the management scheme only when noVNC shares the server origin.""" + return management_protocol if use_server_proxy else "http" + + +def resolve_api_key( + example_api_key: str | None, + sdk_api_key: str | None, +) -> str | None: + """Prefer the example override, then use the SDK-standard fallback.""" + return example_api_key or sdk_api_key + + +def browser_proxy_auth_warning( + use_server_proxy: bool, + api_key: str | None, +) -> str | None: + """Warn when browser traffic may need authentication headers.""" + if not use_server_proxy or not api_key: + return None + + return ( + "The configured API key authenticates SDK requests only. Browsers cannot attach " + "OPEN-SANDBOX-API-KEY to noVNC HTTP or WebSocket requests. If the server " + "is multi-tenant, use a trusted authenticated reverse proxy that injects " + "the tenant key for both request types." + ) + + +def build_novnc_url(endpoint: str, protocol: str) -> str: + """Build a noVNC page URL whose WebSocket targets the same endpoint.""" + scheme = protocol.lower() + if scheme not in {"http", "https"}: + raise ValueError("protocol must be 'http' or 'https'") + + parsed = urlsplit(f"{scheme}://{endpoint}") + if not parsed.hostname: + raise ValueError("endpoint must contain a hostname") + + port = parsed.port or (443 if scheme == "https" else 80) + proxy_path = parsed.path.strip("/") + page_path = f"{parsed.path.rstrip('/')}/vnc.html" + if not page_path.startswith("/"): + page_path = f"/{page_path}" + + query = urlencode( + {"host": parsed.hostname, "port": port, "path": proxy_path}, + safe="/", + ) + return urlunsplit((scheme, parsed.netloc, page_path, query, "")) diff --git a/examples/desktop/test_novnc_url.py b/examples/desktop/test_novnc_url.py new file mode 100644 index 000000000..88ed3b763 --- /dev/null +++ b/examples/desktop/test_novnc_url.py @@ -0,0 +1,138 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from novnc_url import ( + browser_proxy_auth_warning, + build_novnc_url, + normalize_domain, + parse_bool, + resolve_api_key, + resolve_novnc_protocol, + resolve_protocol, + validate_connection_mode, +) + + +class BuildNoVNCURLTest(unittest.TestCase): + def test_resolves_protocol_from_domain_url(self) -> None: + self.assertEqual(resolve_protocol("https://sandbox.example.com", None), "https") + + def test_domain_url_scheme_overrides_explicit_protocol(self) -> None: + self.assertEqual( + resolve_protocol("https://sandbox.example.com", "HTTP"), + "https", + ) + + def test_normalizes_uppercase_domain_scheme_and_whitespace(self) -> None: + self.assertEqual( + normalize_domain(" HTTPS://sandbox.example.com "), + "https://sandbox.example.com", + ) + + def test_resolves_uppercase_domain_scheme(self) -> None: + self.assertEqual( + resolve_protocol("HTTPS://sandbox.example.com", None), + "https", + ) + + def test_direct_http_endpoint(self) -> None: + self.assertEqual( + build_novnc_url("192.168.0.104:41618/proxy/6080", "http"), + "http://192.168.0.104:41618/proxy/6080/vnc.html" + "?host=192.168.0.104&port=41618&path=proxy/6080", + ) + + def test_https_server_proxy_uses_default_tls_port(self) -> None: + self.assertEqual( + build_novnc_url( + "sandbox.example.com/v1/sandboxes/sbx-123/proxy/6080", "https" + ), + "https://sandbox.example.com/v1/sandboxes/sbx-123/proxy/6080/vnc.html" + "?host=sandbox.example.com&port=443" + "&path=v1/sandboxes/sbx-123/proxy/6080", + ) + + def test_https_server_proxy_preserves_explicit_port(self) -> None: + self.assertIn( + "host=sandbox.example.com&port=8443", + build_novnc_url( + "sandbox.example.com:8443/v1/sandboxes/sbx-123/proxy/6080", + "https", + ), + ) + + def test_rejects_unknown_protocol(self) -> None: + with self.assertRaisesRegex(ValueError, "protocol"): + build_novnc_url("sandbox.example.com/proxy/6080", "ftp") + + +class EnvironmentTest(unittest.TestCase): + def test_example_api_key_takes_precedence(self) -> None: + self.assertEqual(resolve_api_key("example-key", "sdk-key"), "example-key") + + def test_sdk_api_key_is_used_as_fallback(self) -> None: + self.assertEqual(resolve_api_key(None, "sdk-key"), "sdk-key") + + def test_server_proxy_with_sdk_fallback_api_key_warns(self) -> None: + warning = browser_proxy_auth_warning( + True, + resolve_api_key(None, "sdk-key"), + ) + + self.assertIsNotNone(warning) + + def test_bool_error_lists_accepted_values(self) -> None: + with self.assertRaisesRegex( + RuntimeError, + "1, true, yes, on, 0, false, no, off", + ): + parse_bool("maybe", "SANDBOX_USE_SERVER_PROXY") + + def test_direct_novnc_stays_http_with_https_management_api(self) -> None: + self.assertEqual(resolve_novnc_protocol("https", False), "http") + + def test_server_proxy_novnc_inherits_management_protocol(self) -> None: + self.assertEqual(resolve_novnc_protocol("https", True), "https") + + def test_https_management_api_requires_server_proxy(self) -> None: + with self.assertRaisesRegex( + RuntimeError, + "SANDBOX_USE_SERVER_PROXY must be true", + ): + validate_connection_mode("https", False) + + def test_https_management_api_accepts_server_proxy(self) -> None: + validate_connection_mode("https", True) + + def test_http_management_api_accepts_direct_endpoints(self) -> None: + validate_connection_mode("http", False) + + def test_server_proxy_with_api_key_warns_about_browser_auth(self) -> None: + warning = browser_proxy_auth_warning(True, "secret") + + self.assertIsNotNone(warning) + self.assertIn("OPEN-SANDBOX-API-KEY", warning) + self.assertIn("HTTP or WebSocket", warning) + + def test_direct_endpoint_does_not_warn_about_proxy_auth(self) -> None: + self.assertIsNone(browser_proxy_auth_warning(False, "secret")) + + def test_server_proxy_without_api_key_does_not_warn(self) -> None: + self.assertIsNone(browser_proxy_auth_warning(True, None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/opencode/README.md b/examples/opencode/README.md new file mode 100644 index 000000000..252b4104e --- /dev/null +++ b/examples/opencode/README.md @@ -0,0 +1,5 @@ +# OpenCode Example + +Run the OpenCode coding agent in OpenSandbox. + +> **Full documentation**: [docs/examples/opencode.md](../../docs/examples/opencode.md) diff --git a/examples/opencode/main.py b/examples/opencode/main.py new file mode 100644 index 000000000..57ba02e5f --- /dev/null +++ b/examples/opencode/main.py @@ -0,0 +1,76 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +import shlex +from datetime import timedelta + +from opensandbox import Sandbox +from opensandbox.config import ConnectionConfig + + +async def _print_execution_logs(execution) -> None: + for msg in execution.logs.stdout: + print(f"[stdout] {msg.text}") + for msg in execution.logs.stderr: + print(f"[stderr] {msg.text}") + if execution.error: + print(f"[error] {execution.error.name}: {execution.error.value}") + + +async def main() -> None: + domain = os.getenv("SANDBOX_DOMAIN", "localhost:8080") + api_key = os.getenv("SANDBOX_API_KEY") + opencode_api_key = os.getenv("OPENCODE_API_KEY") + opencode_model = os.getenv("OPENCODE_MODEL", "opencode/deepseek-v4-flash-free") + image = os.getenv( + "SANDBOX_IMAGE", + "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.1.0", + ) + + config = ConnectionConfig( + domain=domain, + api_key=api_key, + request_timeout=timedelta(seconds=60), + ) + + env = {"OPENCODE_API_KEY": opencode_api_key} + env = {key: value for key, value in env.items() if value is not None} + + sandbox = await Sandbox.create( + image, + connection_config=config, + env=env, + ) + + try: + # Install OpenCode (Node.js is already in the code-interpreter image). + install_exec = await sandbox.commands.run("npm install -g opencode-ai@latest") + await _print_execution_logs(install_exec) + + # Run OpenCode non-interactively in an isolated working directory. + run_exec = await sandbox.commands.run( + "mkdir -p /tmp/opencode-example && " + "cd /tmp/opencode-example && " + f"opencode run --model {shlex.quote(opencode_model)} " + '"Compute 1+1 and reply with only the final number."' + ) + await _print_execution_logs(run_exec) + finally: + await sandbox.destroy() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/windows/pool-win-example.yaml b/examples/windows/pool-win-example.yaml index f620abedd..3980bfcee 100644 --- a/examples/windows/pool-win-example.yaml +++ b/examples/windows/pool-win-example.yaml @@ -58,7 +58,7 @@ spec: command: - /bin/sh - -c - image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21 + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 name: execd-installer volumeMounts: - mountPath: /opt/opensandbox diff --git a/kubernetes/Dockerfile.image-committer b/kubernetes/Dockerfile.image-committer index c4269ad64..d2fdbd23b 100644 --- a/kubernetes/Dockerfile.image-committer +++ b/kubernetes/Dockerfile.image-committer @@ -26,6 +26,7 @@ RUN GOPROXY=https://goproxy.cn,direct go mod download # Copy source code COPY cmd/image-committer/ cmd/image-committer/ +COPY pkg/imagecommitter/ pkg/imagecommitter/ # Build binary RUN CGO_ENABLED=0 GOOS=linux go build -o /usr/local/bin/image-committer ./cmd/image-committer/ @@ -36,13 +37,8 @@ FROM alpine:3.19 # Use Aliyun mirror for faster downloads in China RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -# Install nerdctl for container operations -# nerdctl is used to find containers, commit rootfs, and push images. -# We use nerdctl directly (not crictl or ctr) to avoid CRI API version issues. -RUN apk add --no-cache \ - curl \ - jq \ - nerdctl +# Registry pushes use the system CA bundle. +RUN apk add --no-cache ca-certificates # Create directory for containerd socket mount RUN mkdir -p /var/run/containerd diff --git a/kubernetes/Makefile b/kubernetes/Makefile index f30bf06fa..330d1d787 100644 --- a/kubernetes/Makefile +++ b/kubernetes/Makefile @@ -54,7 +54,7 @@ OPERATOR_SDK_VERSION ?= v1.42.0 CONTROLLER_IMG ?= controller:dev # TASK_EXECUTOR_IMG defines the image for the task-executor service. TASK_EXECUTOR_IMG ?= task-executor:dev -# IMAGE_COMMITTER_IMG defines the image for the image-committer service. +# IMAGE_COMMITTER_IMG defines the image for the default image-committer service. IMAGE_COMMITTER_IMG ?= image-committer:dev # SNAPSHOT_REGISTRY defines the OCI registry used by the controller for snapshot images. SNAPSHOT_REGISTRY ?= docker-registry.default.svc.cluster.local:5000 @@ -530,9 +530,13 @@ HELM_CHART_PATH ?= charts/opensandbox-controller HELM_CHART_VERSION ?= $(VERSION) .PHONY: helm-lint -helm-lint: ## Lint the Helm chart - @echo "Linting Helm chart..." - helm lint $(HELM_CHART_PATH) +helm-lint: ## Lint all Helm charts and verify dependencies + @echo "Linting Helm charts..." + helm lint charts/opensandbox-controller + helm lint charts/opensandbox-server + helm lint charts/opensandbox-node-agent + helm dependency build charts/opensandbox + helm lint charts/opensandbox .PHONY: helm-template helm-template: ## Generate Kubernetes manifests from Helm chart diff --git a/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go b/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go index afec44013..f48dcfc45 100644 --- a/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go +++ b/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go @@ -46,7 +46,7 @@ type ContainerSnapshot struct { ContainerName string `json:"containerName"` // ImageURI is the snapshot image URI for this container. ImageURI string `json:"imageUri"` - // ImageDigest is the digest of the pushed snapshot image. + // ImageDigest is the config digest of the pushed snapshot image. // +optional ImageDigest string `json:"imageDigest,omitempty"` } diff --git a/kubernetes/charts/opensandbox-controller/Chart.yaml b/kubernetes/charts/opensandbox-controller/Chart.yaml index 2a6cae1c6..8f4e64a23 100644 --- a/kubernetes/charts/opensandbox-controller/Chart.yaml +++ b/kubernetes/charts/opensandbox-controller/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: opensandbox-controller description: A Kubernetes operator for managing sandbox environments with resource pooling and batch delivery type: application -version: 0.2.0 +version: 0.2.1 appVersion: "0.2.0" keywords: diff --git a/kubernetes/charts/opensandbox-controller/README.md b/kubernetes/charts/opensandbox-controller/README.md index 664d8bc21..308aa28ea 100644 --- a/kubernetes/charts/opensandbox-controller/README.md +++ b/kubernetes/charts/opensandbox-controller/README.md @@ -79,6 +79,7 @@ kubectl delete crd sandboxsnapshots.sandbox.opensandbox.io | `controller.kubeClient.qps` | QPS for Kubernetes client rate limiter | `100` | | `controller.kubeClient.burst` | Burst for Kubernetes client rate limiter | `200` | | `controller.snapshot.imageCommitterImage` | Image used by snapshot commit Jobs | `image-committer:dev` | +| `controller.snapshot.imageCommitterPodTemplate` | PodTemplateSpec overlay for snapshot commit Job Pods | `{}` | | `controller.snapshot.commitJobTimeout` | Timeout duration for snapshot commit Jobs | `10m` | | `controller.snapshot.registry` | OCI registry prefix used for snapshot images | `""` | | `controller.snapshot.registryInsecure` | Use insecure registry mode for snapshot pushes | `false` | @@ -169,6 +170,18 @@ The chart exposes the snapshot-related settings below: controller: snapshot: imageCommitterImage: my-registry/image-committer:v0.1.1 + imageCommitterPodTemplate: + metadata: + labels: + identity.example/use: "true" + spec: + serviceAccountName: snapshot-committer + containers: + - name: commit + resources: + requests: + cpu: 100m + memory: 128Mi commitJobTimeout: 15m registry: my-registry/snapshots registryInsecure: false @@ -180,6 +193,7 @@ controller: These values render directly to the controller flags: - `--image-committer-image` +- `--image-committer-pod-template-file` - `--commit-job-timeout` - `--snapshot-registry` - `--snapshot-registry-insecure` diff --git a/kubernetes/charts/opensandbox-controller/templates/_helpers.tpl b/kubernetes/charts/opensandbox-controller/templates/_helpers.tpl index 9fd88ebfd..359ccf37f 100644 --- a/kubernetes/charts/opensandbox-controller/templates/_helpers.tpl +++ b/kubernetes/charts/opensandbox-controller/templates/_helpers.tpl @@ -87,6 +87,14 @@ special tags like 'latest', 'dev', 'main', etc. as-is. {{- printf "%s:%s" .Values.controller.image.repository $finalTag }} {{- end }} +{{/* +Create the image-committer Pod template ConfigMap name. +*/}} +{{- define "opensandbox.imageCommitterPodTemplateName" -}} +{{- $base := include "opensandbox.fullname" . | trunc 34 | trimSuffix "-" -}} +{{- printf "%s-image-committer-pod-template" $base }} +{{- end }} + {{/* Create the name for the leader election role */}} diff --git a/kubernetes/charts/opensandbox-controller/templates/crds/sandboxsnapshots.yaml b/kubernetes/charts/opensandbox-controller/templates/crds/sandboxsnapshots.yaml index 670c40aad..1cbf52401 100644 --- a/kubernetes/charts/opensandbox-controller/templates/crds/sandboxsnapshots.yaml +++ b/kubernetes/charts/opensandbox-controller/templates/crds/sandboxsnapshots.yaml @@ -123,7 +123,7 @@ spec: description: ContainerName is the name of the container. type: string imageDigest: - description: ImageDigest is the digest of the pushed snapshot + description: ImageDigest is the config digest of the pushed snapshot image. type: string imageUri: diff --git a/kubernetes/charts/opensandbox-controller/templates/deployment.yaml b/kubernetes/charts/opensandbox-controller/templates/deployment.yaml index c530a3daa..898bb5ea3 100644 --- a/kubernetes/charts/opensandbox-controller/templates/deployment.yaml +++ b/kubernetes/charts/opensandbox-controller/templates/deployment.yaml @@ -16,6 +16,9 @@ spec: metadata: annotations: kubectl.kubernetes.io/default-container: manager + {{- if .Values.controller.snapshot.imageCommitterPodTemplate }} + checksum/image-committer-pod-template: {{ include (print $.Template.BasePath "/image-committer-pod-template.yaml") . | sha256sum }} + {{- end }} {{- with .Values.controller.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -66,6 +69,9 @@ spec: {{- if .Values.controller.snapshot.imageCommitterImage }} - --image-committer-image={{ .Values.controller.snapshot.imageCommitterImage }} {{- end }} + {{- if .Values.controller.snapshot.imageCommitterPodTemplate }} + - --image-committer-pod-template-file=/etc/opensandbox/image-committer/pod-template.yaml + {{- end }} {{- if .Values.controller.snapshot.containerdSocketPath }} - --containerd-socket-path={{ .Values.controller.snapshot.containerdSocketPath }} {{- end }} @@ -128,17 +134,31 @@ spec: env: {{- toYaml .Values.extraEnv | nindent 8 }} {{- end }} - {{- with .Values.extraVolumeMounts }} + {{- if or .Values.controller.snapshot.imageCommitterPodTemplate .Values.extraVolumeMounts }} volumeMounts: + {{- if .Values.controller.snapshot.imageCommitterPodTemplate }} + - name: image-committer-pod-template + mountPath: /etc/opensandbox/image-committer + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} {{- with .Values.extraContainers }} {{- toYaml . | nindent 6 }} {{- end }} - {{- with .Values.extraVolumes }} + {{- if or .Values.controller.snapshot.imageCommitterPodTemplate .Values.extraVolumes }} volumes: + {{- if .Values.controller.snapshot.imageCommitterPodTemplate }} + - name: image-committer-pod-template + configMap: + name: {{ include "opensandbox.imageCommitterPodTemplateName" . }} + {{- end }} + {{- with .Values.extraVolumes }} {{- toYaml . | nindent 6 }} {{- end }} + {{- end }} {{- with .Values.controller.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/kubernetes/charts/opensandbox-controller/templates/image-committer-pod-template.yaml b/kubernetes/charts/opensandbox-controller/templates/image-committer-pod-template.yaml new file mode 100644 index 000000000..51806eebd --- /dev/null +++ b/kubernetes/charts/opensandbox-controller/templates/image-committer-pod-template.yaml @@ -0,0 +1,13 @@ +{{- with .Values.controller.snapshot.imageCommitterPodTemplate }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensandbox.imageCommitterPodTemplateName" $ }} + namespace: {{ include "opensandbox.namespace" $ }} + labels: + {{- include "opensandbox.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller-manager +data: + pod-template.yaml: | + {{- toYaml . | nindent 4 }} +{{- end }} diff --git a/kubernetes/charts/opensandbox-controller/values.yaml b/kubernetes/charts/opensandbox-controller/values.yaml index cdb1432e3..988898c24 100644 --- a/kubernetes/charts/opensandbox-controller/values.yaml +++ b/kubernetes/charts/opensandbox-controller/values.yaml @@ -57,18 +57,20 @@ controller: # -- Pause/Resume snapshot configuration snapshot: - # -- Image used for commit operations (must contain nerdctl tool) + # -- Image used for commit operations # DockerHub: opensandbox/image-committer:v0.1.1 imageCommitterImage: "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/image-committer:v0.1.1" - # -- Containerd socket path of host - containerdSocketPath: "/var/run/containerd/containerd.sock" + # -- PodTemplateSpec overlay for image-committer commit Job Pods. + imageCommitterPodTemplate: {} + # -- Containerd socket path of host. Defaults to empty so the controller uses its built-in default (/var/run/containerd/containerd.sock) without passing --containerd-socket-path flag. + containerdSocketPath: "" # -- Timeout duration for commit jobs commitJobTimeout: "10m" # -- OCI registry prefix used for snapshot images. registry: "" # -- Use insecure registry mode when pushing snapshot images. registryInsecure: false - # -- Secret name used by commit Jobs to push snapshot images. + # -- Secret name used to push and delete snapshot images. snapshotPushSecret: "" # -- Secret name for pulling the image-committer image in commit Jobs. # Required when imageCommitterImage is stored in a private registry. diff --git a/kubernetes/charts/opensandbox-server/Chart.yaml b/kubernetes/charts/opensandbox-server/Chart.yaml index ca890dad6..12ffd621b 100644 --- a/kubernetes/charts/opensandbox-server/Chart.yaml +++ b/kubernetes/charts/opensandbox-server/Chart.yaml @@ -17,7 +17,7 @@ name: opensandbox-server description: OpenSandbox Lifecycle API server for sandbox creation and management type: application version: 0.1.0 -appVersion: "0.1.0" +appVersion: "0.2.2" # execd bootstrap.sh installed to /opt/opensandbox (flattened from /opt/opensandbox/bin) keywords: diff --git a/kubernetes/charts/opensandbox-server/README.md b/kubernetes/charts/opensandbox-server/README.md index 59a12f6d7..5d1958e27 100644 --- a/kubernetes/charts/opensandbox-server/README.md +++ b/kubernetes/charts/opensandbox-server/README.md @@ -11,12 +11,12 @@ OpenSandbox Lifecycle API server: provides sandbox create/delete and other lifec ## Install from a GitHub Release -Choose a published `opensandbox-server` chart from [GitHub Releases](https://github.com/opensandbox-group/OpenSandbox/releases?q=helm%2Fopensandbox-server&expanded=true). The release tag uses the application version, while the package filename uses the chart version shown in the release notes. +Choose a published `opensandbox-server` chart from [GitHub Releases](https://github.com/opensandbox-group/OpenSandbox/releases?q=helm%2Fopensandbox-server&expanded=true). The release tag and package filename use the chart version shown in the release notes; the application version is listed separately. ```bash -APP_VERSION="" CHART_VERSION="" -CHART_URL="https://github.com/opensandbox-group/OpenSandbox/releases/download/helm/opensandbox-server/${APP_VERSION}/opensandbox-server-${CHART_VERSION}.tgz" +APP_VERSION="" +CHART_URL="https://github.com/opensandbox-group/OpenSandbox/releases/download/helm/opensandbox-server/${CHART_VERSION}/opensandbox-server-${CHART_VERSION}.tgz" helm show values "${CHART_URL}" ``` diff --git a/kubernetes/charts/opensandbox-server/templates/server.yaml b/kubernetes/charts/opensandbox-server/templates/server.yaml index 72ce1da24..4bfb08777 100644 --- a/kubernetes/charts/opensandbox-server/templates/server.yaml +++ b/kubernetes/charts/opensandbox-server/templates/server.yaml @@ -23,6 +23,9 @@ rules: - apiGroups: [""] resources: ["pods", "pods/status", "pods/log", "events", "services", "configmaps"] verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] - apiGroups: [""] resources: ["secrets"] verbs: ["create", "delete", "get"] @@ -89,10 +92,14 @@ spec: {{- with .Values.server.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.server.podAnnotations }} annotations: + # The server parses config.toml once at startup, so an in-place update + # of the ConfigMap leaves the running pod on the previous configuration. + # Hashing the rendered TOML rolls the Deployment whenever it changes. + checksum/config: {{ printf "%s%s" .Values.configToml (include "opensandbox-server.ingressConfigToml" .) | sha256sum }} + {{- with .Values.server.podAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: diff --git a/kubernetes/charts/opensandbox-server/values.yaml b/kubernetes/charts/opensandbox-server/values.yaml index bd48c2438..89ce1fc4b 100644 --- a/kubernetes/charts/opensandbox-server/values.yaml +++ b/kubernetes/charts/opensandbox-server/values.yaml @@ -133,7 +133,7 @@ configToml: | [runtime] type = "kubernetes" - execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21" + execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" [kubernetes] kubeconfig_path = "" @@ -146,5 +146,5 @@ configToml: | batchsandbox_template_file = "/etc/opensandbox/example.batchsandbox-template.yaml" [egress] - image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5" + image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" mode = "dns+nft" diff --git a/kubernetes/charts/opensandbox/Chart.lock b/kubernetes/charts/opensandbox/Chart.lock index a8fa957fb..cdadc4e38 100644 --- a/kubernetes/charts/opensandbox/Chart.lock +++ b/kubernetes/charts/opensandbox/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opensandbox-controller repository: file://../opensandbox-controller - version: 0.2.0 + version: 0.2.1 - name: opensandbox-server repository: file://../opensandbox-server version: 0.1.0 - name: opensandbox-node-agent repository: file://../opensandbox-node-agent version: 0.1.0 -digest: sha256:18ed87f3960e0df808eba7bd27455e416dc6330951281493bba4bb0ab0321512 -generated: "2026-07-30T14:58:20.116343+08:00" +digest: sha256:9a8dfea166e50e4016bd850366a17e02a9052cb73f88375d934558c625dfb956 +generated: "2026-08-18T18:20:43.341958+08:00" diff --git a/kubernetes/charts/opensandbox/Chart.yaml b/kubernetes/charts/opensandbox/Chart.yaml index b8d2a68bd..a0ff495e6 100644 --- a/kubernetes/charts/opensandbox/Chart.yaml +++ b/kubernetes/charts/opensandbox/Chart.yaml @@ -16,8 +16,8 @@ apiVersion: v2 name: opensandbox description: All-in-one Helm chart for deploying OpenSandbox controller and server type: application -version: 0.2.0 -appVersion: "0.2.0" +version: 0.2.2 +appVersion: "0.2.2" keywords: - sandbox @@ -40,7 +40,7 @@ kubeVersion: ">=1.21.1-0" dependencies: - name: opensandbox-controller - version: "0.2.0" + version: "0.2.1" repository: "file://../opensandbox-controller" - name: opensandbox-server version: "0.1.0" diff --git a/kubernetes/charts/opensandbox/README.md b/kubernetes/charts/opensandbox/README.md index 61efe62b7..a1999d213 100644 --- a/kubernetes/charts/opensandbox/README.md +++ b/kubernetes/charts/opensandbox/README.md @@ -70,6 +70,18 @@ opensandbox-controller: registry: my-registry/snapshots registryInsecure: false snapshotPushSecret: registry-snapshot-push-secret + imageCommitterPodTemplate: + metadata: + labels: + identity.example/use: "true" + spec: + serviceAccountName: snapshot-committer + containers: + - name: commit + resources: + requests: + cpu: 100m + memory: 128Mi resumePullSecret: registry-pull-secret opensandbox-server: diff --git a/kubernetes/charts/opensandbox/values.yaml b/kubernetes/charts/opensandbox/values.yaml index 221730ce9..4c72ee928 100644 --- a/kubernetes/charts/opensandbox/values.yaml +++ b/kubernetes/charts/opensandbox/values.yaml @@ -11,6 +11,7 @@ opensandbox-controller: snapshot: # DockerHub: opensandbox/image-committer:v0.1.1 imageCommitterImage: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/image-committer:v0.1.1 + imageCommitterPodTemplate: {} commitJobTimeout: 10m registry: "" registryInsecure: false diff --git a/kubernetes/cmd/controller/main.go b/kubernetes/cmd/controller/main.go index 4482fbd19..01231d334 100644 --- a/kubernetes/cmd/controller/main.go +++ b/kubernetes/cmd/controller/main.go @@ -28,6 +28,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -38,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/yaml" sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1" "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/controller" @@ -202,7 +204,10 @@ func main() { // Image committer var imageCommitterImage string - flag.StringVar(&imageCommitterImage, "image-committer-image", "image-committer:dev", "The image used for commit operations (contains nerdctl tool).") + flag.StringVar(&imageCommitterImage, "image-committer-image", "image-committer:dev", "The image used for commit operations.") + + var imageCommitterPodTemplateFile string + flag.StringVar(&imageCommitterPodTemplateFile, "image-committer-pod-template-file", "", "Path to a PodTemplateSpec overlay for image-committer commit Job Pods.") var containerdSocketPath string flag.StringVar(&containerdSocketPath, "containerd-socket-path", controller.ContainerdSocketPath, "Containerd socket path") @@ -218,7 +223,7 @@ func main() { flag.BoolVar(&snapshotRegistryInsecure, "snapshot-registry-insecure", false, "Use insecure registry mode when pushing snapshot images.") var snapshotPushSecret string - flag.StringVar(&snapshotPushSecret, "snapshot-push-secret", "", "K8s Secret name for pushing snapshots to registry.") + flag.StringVar(&snapshotPushSecret, "snapshot-push-secret", "", "K8s Secret name for pushing and deleting snapshots in the registry.") var imageCommitterPullSecret string flag.StringVar(&imageCommitterPullSecret, "image-committer-pull-secret", "", "K8s Secret name for pulling the image-committer image in commit Jobs. Required when imageCommitterImage is in a private registry.") @@ -248,6 +253,12 @@ func main() { setupLog.Info("Starting controller", "commitID", commitID, "buildDate", buildDate) + imageCommitterPodTemplate, err := loadImageCommitterPodTemplate(imageCommitterPodTemplateFile) + if err != nil { + setupLog.Error(err, "invalid image committer Pod template") + os.Exit(1) + } + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and @@ -455,16 +466,17 @@ func main() { os.Exit(1) } if err := (&controller.SandboxSnapshotReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("sandboxsnapshot-controller"), - ImageCommitterImage: imageCommitterImage, - ContainerdSocketPath: containerdSocketPath, - CommitJobTimeout: commitJobTimeout, - SnapshotRegistry: snapshotRegistry, - SnapshotRegistryInsecure: snapshotRegistryInsecure, - SnapshotPushSecret: snapshotPushSecret, - ImageCommitterPullSecret: imageCommitterPullSecret, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("sandboxsnapshot-controller"), + ImageCommitterImage: imageCommitterImage, + ContainerdSocketPath: containerdSocketPath, + CommitJobTimeout: commitJobTimeout, + SnapshotRegistry: snapshotRegistry, + SnapshotRegistryInsecure: snapshotRegistryInsecure, + SnapshotPushSecret: snapshotPushSecret, + ImageCommitterPullSecret: imageCommitterPullSecret, + ImageCommitterPodTemplate: imageCommitterPodTemplate, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SandboxSnapshot") os.Exit(1) @@ -502,3 +514,18 @@ func main() { os.Exit(1) } } + +func loadImageCommitterPodTemplate(path string) (*corev1.PodTemplateSpec, error) { + if strings.TrimSpace(path) == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read Pod template: %w", err) + } + var template corev1.PodTemplateSpec + if err := yaml.UnmarshalStrict(data, &template); err != nil { + return nil, fmt.Errorf("parse Pod template: %w", err) + } + return &template, nil +} diff --git a/kubernetes/cmd/controller/main_test.go b/kubernetes/cmd/controller/main_test.go new file mode 100644 index 000000000..8ac8ca1fd --- /dev/null +++ b/kubernetes/cmd/controller/main_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadImageCommitterPodTemplate(t *testing.T) { + path := filepath.Join(t.TempDir(), "pod-template.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +metadata: + labels: + identity.example/use: "true" +spec: + serviceAccountName: snapshot-committer + containers: + - name: commit + resources: + requests: + cpu: 100m +`), 0o600)) + + template, err := loadImageCommitterPodTemplate(path) + require.NoError(t, err) + assert.Equal(t, "true", template.Labels["identity.example/use"]) + assert.Equal(t, "snapshot-committer", template.Spec.ServiceAccountName) + require.Len(t, template.Spec.Containers, 1) + assert.Equal(t, "100m", template.Spec.Containers[0].Resources.Requests.Cpu().String()) + + template, err = loadImageCommitterPodTemplate("") + require.NoError(t, err) + assert.Nil(t, template) +} + +func TestLoadImageCommitterPodTemplateRejectsUnknownFields(t *testing.T) { + path := filepath.Join(t.TempDir(), "pod-template.yaml") + require.NoError(t, os.WriteFile(path, []byte("spec:\n unknownField: true\n"), 0o600)) + _, err := loadImageCommitterPodTemplate(path) + require.Error(t, err) +} diff --git a/kubernetes/cmd/image-committer/main.go b/kubernetes/cmd/image-committer/main.go index 7717fdf16..57e69abfc 100644 --- a/kubernetes/cmd/image-committer/main.go +++ b/kubernetes/cmd/image-committer/main.go @@ -15,659 +15,33 @@ package main import ( - "encoding/base64" - "encoding/json" - "errors" + "context" "fmt" "os" - "os/exec" "os/signal" - "path/filepath" - "strconv" - "strings" "syscall" - "time" -) - -var commandCombinedOutput = func(name string, args ...string) ([]byte, error) { - return exec.Command(name, args...).CombinedOutput() -} - -var terminationMessagePath = "/dev/termination-log" - -// containerdSocket returns the containerd socket address from env or default -func containerdSocket() string { - if v := os.Getenv("CONTAINERD_SOCKET"); v != "" { - return v - } - return "/run/containerd/containerd.sock" -} - -// containerdNamespace returns the containerd namespace from env or default -func containerdNamespace() string { - if v := os.Getenv("CONTAINERD_NAMESPACE"); v != "" { - return v - } - return "k8s.io" -} - -// nerdctlBaseArgs returns the base arguments for nerdctl commands -func nerdctlBaseArgs() []string { - return []string{"--address", containerdSocket(), "--namespace", containerdNamespace()} -} - -type ContainerSpec struct { - Name string - URI string -} -type discoveredContainer struct { - ID string - Running bool -} - -type snapshotResult struct { - Containers []snapshotContainerResult `json:"containers"` -} - -type snapshotContainerResult struct { - Name string `json:"name"` - Image string `json:"image"` - Digest string `json:"digest"` -} + "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/imagecommitter" + imagecommittercli "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/imagecommitter/cli" +) -// Global tracking of paused containers for cleanup -var pausedContainerIds []string +const ( + terminationMessagePath = "/dev/termination-log" + registryConfigPath = "/var/run/opensandbox/registry/config.json" +) func main() { - args := os.Args[1:] - - // Set up signal handler to ensure all paused containers are resumed on exit - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - sig := <-c - fmt.Fprintf(os.Stderr, "Received signal %v, cleaning up paused containers...\n", sig) - resumeAllPausedContainers() - os.Exit(1) - }() - - // Defer cleanup in case of panic or early termination - defer func() { - if r := recover(); r != nil { - fmt.Fprintf(os.Stderr, "Panic occurred: %v\n", r) - resumeAllPausedContainers() - panic(r) - } - }() - - if len(args) > 0 && args[0] == "unpause" { - runUnpause(args[1:]) - return - } - - // Parse arguments using unified format: - // [container2:uri2...] - var podName, namespace string - var containerSpecs []ContainerSpec - - if len(args) < 3 { - fmt.Fprintln(os.Stderr, "ERROR: Missing required parameters") - fmt.Fprintln(os.Stderr, "Usage: commit-snapshot [container2:uri2...]") - os.Exit(1) - } - - podName = args[0] - namespace = args[1] - - for i := 2; i < len(args); i++ { - spec, err := parseContainerSpec(args[i]) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(1) - } - containerSpecs = append(containerSpecs, spec) - } - - // Validate required inputs - if len(podName) == 0 { - fmt.Fprintln(os.Stderr, "ERROR: Pod name is required") - os.Exit(1) - } - - if len(namespace) == 0 { - fmt.Fprintln(os.Stderr, "ERROR: Namespace is required") - os.Exit(1) - } - - if len(containerSpecs) == 0 { - fmt.Fprintln(os.Stderr, "ERROR: At least one container specification is required") - fmt.Fprintln(os.Stderr, "Usage: commit-snapshot [container2:uri2...]") - os.Exit(1) - } - - fmt.Println("=== Commit Snapshot Go Program ===") - fmt.Printf("Pod: %s\n", podName) - fmt.Printf("Namespace: %s\n", namespace) - for _, spec := range containerSpecs { - fmt.Printf("Container spec: %s -> %s\n", spec.Name, spec.URI) - } - - // Step 1: Find container IDs via nerdctl (direct containerd API, no CRI dependency) - fmt.Println("\n=== Step 1: Find container IDs via nerdctl ===") - containers := make(map[string]discoveredContainer) // Maps container name to runtime metadata - for _, spec := range containerSpecs { - container, err := getContainerByNerdctl(podName, namespace, spec.Name) - if err != nil { - resumeAllPausedContainers() - fmt.Fprintf(os.Stderr, "ERROR: Failed to find container '%s': %v\n", spec.Name, err) - os.Exit(1) - } - - fmt.Printf("Container '%s' -> ID: %s (running: %t)\n", spec.Name, container.ID, container.Running) - containers[spec.Name] = container - } - - // Step 2: Flush each running container's filesystem from inside its runtime. - // This is required for VM-isolated runtimes such as Kata, where host-side - // sync does not flush the guest kernel's page cache. - fmt.Println("\n=== Step 2: Sync all running containers (best effort) ===") - if err := syncRunningContainerFilesystems(containerSpecs, containers); err != nil { - fmt.Fprintf(os.Stderr, "WARNING: Filesystem sync failed; continuing snapshot as best effort. Recent guest filesystem writes may be missing: %v\n", err) - } - - // Step 3: Pause all containers - fmt.Println("\n=== Step 3: Pause all containers ===") - pauseErrors := 0 - for _, spec := range containerSpecs { - containerID := containers[spec.Name].ID - if err := pauseContainer(containerID); err != nil { - // On pause failure, we still try to continue since commit might work anyway (as in shell script) - fmt.Fprintf(os.Stderr, "WARNING: Could not pause '%s'. Will attempt commit anyway (container may be stopped).\n", spec.Name) - pauseErrors++ - } else { - // Track successfully paused containers for cleanup - pausedContainerIds = append(pausedContainerIds, containerID) - } - } - - // Step 4: Commit all containers - fmt.Println("\n=== Step 4: Commit all containers ===") - committedImages := make(map[string]string) // Maps container name to committed image URI - commitErrors := 0 - for _, spec := range containerSpecs { - containerID := containers[spec.Name].ID - if err := commitContainer(containerID, spec.URI); err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Failed to commit container '%s': %v\n", spec.Name, err) - commitErrors++ - } else { - committedImages[spec.Name] = spec.URI - fmt.Printf("Successfully committed: %s -> %s\n", containerID, spec.URI) - } - } - - // Step 5: Resume all paused containers (regardless of commit success/failure) - fmt.Println("\n=== Step 5: Resume all paused containers ===") - resumeAllPausedContainers() - - // If there were commit errors, exit with failure after cleanup - if commitErrors > 0 { - fmt.Fprintf(os.Stderr, "ERROR: %d container(s) failed to commit. All containers have been resumed.\n", commitErrors) - os.Exit(1) - } - - // Step 6: Push all committed images - fmt.Println("\n=== Step 6: Push all images ===") - pushErrors := 0 - for _, spec := range containerSpecs { - if _, ok := committedImages[spec.Name]; ok { - if err := pushImage(spec.URI); err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Failed to push image for container '%s': %v\n", spec.Name, err) - pushErrors++ - } else { - fmt.Printf("Successfully pushed: %s\n", spec.URI) - } - } - } - - if pushErrors > 0 { - fmt.Fprintf(os.Stderr, "ERROR: %d image(s) failed to push.\n", pushErrors) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + provider := imagecommitter.DockerConfigCredentialProvider{Path: registryConfigPath, ErrorOutput: os.Stderr} + if err := imagecommittercli.Run(ctx, os.Args[1:], imagecommittercli.Config{ + CredentialProvider: provider, + SourceCredentialProvider: provider, + TerminationMessagePath: terminationMessagePath, + Output: os.Stdout, + ErrorOutput: os.Stderr, + }); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) os.Exit(1) } - - // Step 7: Extract digests and output results - fmt.Println("\n=== Step 7: Extract digests ===") - digests := make(map[string]string) // Maps container name to digest - firstDigest := "" - - for _, spec := range containerSpecs { - if _, ok := committedImages[spec.Name]; ok { - digest, err := getImageDigest(spec.URI) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Failed to extract digest for %s: %v\n", spec.URI, err) - os.Exit(1) - } - - digests[spec.Name] = digest - fmt.Printf("Container '%s' digest: %s\n", spec.Name, digest) - - // Capture first digest for legacy output - if firstDigest == "" { - firstDigest = digest - } - } - } - - // Final output - SNAPSHOT_DIGEST_ variables for each container - fmt.Println("\n=== Snapshot completed successfully ===") - for _, spec := range containerSpecs { - if digest, ok := digests[spec.Name]; ok { - upperName := strings.ToUpper(strings.ReplaceAll(spec.Name, "-", "_")) - fmt.Printf("SNAPSHOT_DIGEST_%s=%s\n", upperName, digest) - fmt.Printf(" Image: %s\n", spec.URI) - fmt.Printf(" Digest: %s\n", digest) - } - } - - if err := writeSnapshotResult(containerSpecs, digests); err != nil { - fmt.Fprintf(os.Stderr, "WARNING: Failed to write snapshot result to termination message: %v\n", err) - } - - // Legacy single-digest output for backward compatibility - fmt.Printf("SNAPSHOT_DIGEST=%s\n", firstDigest) -} - -func writeSnapshotResult(containerSpecs []ContainerSpec, digests map[string]string) error { - result := snapshotResult{ - Containers: make([]snapshotContainerResult, 0, len(digests)), - } - for _, spec := range containerSpecs { - digest, ok := digests[spec.Name] - if !ok { - continue - } - result.Containers = append(result.Containers, snapshotContainerResult{ - Name: spec.Name, - Image: spec.URI, - Digest: digest, - }) - } - data, err := json.Marshal(result) - if err != nil { - return err - } - return os.WriteFile(terminationMessagePath, append(data, '\n'), 0644) -} - -func runUnpause(args []string) { - if len(args) < 3 { - fmt.Fprintln(os.Stderr, "ERROR: Missing required parameters") - fmt.Fprintln(os.Stderr, "Usage: image-committer unpause [container_name...]") - os.Exit(1) - } - - podName := args[0] - namespace := args[1] - containerNames := args[2:] - errors := 0 - - for _, containerName := range containerNames { - containerID, err := getContainerIDByNerdctl(podName, namespace, containerName) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Failed to find container '%s': %v\n", containerName, err) - errors++ - continue - } - if err := resumeContainer(containerID); err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Failed to unpause container '%s': %v\n", containerName, err) - errors++ - } - } - - if errors > 0 { - os.Exit(1) - } -} - -// parseContainerSpec parses a "container:uri" string into ContainerSpec -func parseContainerSpec(specStr string) (ContainerSpec, error) { - parts := strings.SplitN(specStr, ":", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return ContainerSpec{}, fmt.Errorf("invalid container spec '%s'. Expected format: container_name:uri", specStr) - } - - return ContainerSpec{ - Name: parts[0], - URI: parts[1], - }, nil -} - -// getContainerIDByNerdctl finds a container ID using nerdctl ps with Kubernetes labels. -// This approach directly queries containerd (k8s.io namespace) without going through -// the CRI API, making it compatible with all containerd versions. -// Kubernetes injects standard labels on all containers: -// - io.kubernetes.pod.name -// - io.kubernetes.pod.namespace -// - io.kubernetes.container.name -func getContainerIDByNerdctl(podName, podNamespace, containerName string) (string, error) { - container, err := getContainerByNerdctl(podName, podNamespace, containerName) - return container.ID, err -} - -// getContainerByNerdctl returns the matching container metadata. Stopped -// containers remain discoverable for commit, but cannot be synchronized with -// nerdctl exec. -func getContainerByNerdctl(podName, podNamespace, containerName string) (discoveredContainer, error) { - containerID, err := lookupContainerIDByNerdctl(podName, podNamespace, containerName, false) - if err != nil { - return discoveredContainer{}, err - } - if containerID != "" { - return discoveredContainer{ID: containerID, Running: true}, nil - } - - containerID, err = lookupContainerIDByNerdctl(podName, podNamespace, containerName, true) - if err != nil { - return discoveredContainer{}, err - } - if containerID != "" { - return discoveredContainer{ID: containerID}, nil - } - - return discoveredContainer{}, fmt.Errorf( - "container '%s' not found in pod %s/%s (nerdctl ps and nerdctl ps -a returned empty)", - containerName, - podNamespace, - podName, - ) -} - -func lookupContainerIDByNerdctl(podName, podNamespace, containerName string, includeStopped bool) (string, error) { - args := append(nerdctlBaseArgs(), "ps") - if includeStopped { - args = append(args, "-a") - } - args = append(args, - "-q", - "--filter", fmt.Sprintf("label=io.kubernetes.pod.name=%s", podName), - "--filter", fmt.Sprintf("label=io.kubernetes.pod.namespace=%s", podNamespace), - "--filter", fmt.Sprintf("label=io.kubernetes.container.name=%s", containerName), - ) - output, err := commandCombinedOutput("nerdctl", args...) - if err != nil { - mode := "nerdctl ps" - if includeStopped { - mode = "nerdctl ps -a" - } - return "", fmt.Errorf( - "%s failed for pod=%s ns=%s container=%s: %v, output: %s", - mode, - podName, - podNamespace, - containerName, - err, - strings.TrimSpace(string(output)), - ) - } - - containerID := strings.TrimSpace(string(output)) - if containerID == "" { - return "", nil - } - - // nerdctl ps -q may return multiple lines; take the first (most recently started) - lines := strings.Split(containerID, "\n") - return strings.TrimSpace(lines[0]), nil -} - -// syncRunningContainerFilesystems attempts to sync every running container, -// collecting failures so one broken container does not prevent the others from -// being flushed. Stopped containers cannot be targeted by nerdctl exec and are -// left on the existing stopped-container commit path. -func syncRunningContainerFilesystems(containerSpecs []ContainerSpec, containers map[string]discoveredContainer) error { - phaseStarted := time.Now() - syncCount := 0 - var syncErrors []error - for _, spec := range containerSpecs { - container := containers[spec.Name] - if !container.Running { - fmt.Printf("Skipping filesystem sync for stopped container '%s'.\n", spec.Name) - continue - } - syncCount++ - if err := syncContainerFilesystem(container.ID); err != nil { - syncErrors = append(syncErrors, fmt.Errorf("container %q: %w", spec.Name, err)) - } - } - fmt.Printf("Filesystem sync phase completed for %d running container(s) in %s.\n", syncCount, time.Since(phaseStarted)) - return errors.Join(syncErrors...) -} - -// syncContainerFilesystem runs sync inside a running container so VM-isolated -// runtimes flush the guest kernel's filesystem page cache before pause/commit. -// TODO: Move guest filesystem synchronization to execd and invoke sync(2) -// directly so snapshots do not depend on a sync binary in the guest image. -func syncContainerFilesystem(containerID string) error { - started := time.Now() - fmt.Printf("Syncing filesystem in container %s...\n", containerID) - args := append(nerdctlBaseArgs(), "exec", containerID, "sync") - output, err := commandCombinedOutput("nerdctl", args...) - duration := time.Since(started) - if err != nil { - return fmt.Errorf("failed to sync container %s after %s: %v, output: %s", containerID, duration, err, strings.TrimSpace(string(output))) - } - fmt.Printf("Filesystem synced successfully: %s (duration: %s)\n", containerID, duration) - return nil -} - -// pauseContainer uses nerdctl to pause a container -func pauseContainer(containerID string) error { - fmt.Printf("Pausing container %s...\n", containerID) - args := append(nerdctlBaseArgs(), "pause", containerID) - cmd := exec.Command("nerdctl", args...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to pause container %s: %v, output: %s", containerID, err, string(output)) - } - fmt.Printf("Paused successfully: %s\n", containerID) - return nil -} - -// resumeContainer uses nerdctl to resume a container -func resumeContainer(containerID string) error { - fmt.Printf("Resuming container %s...\n", containerID) - args := append(nerdctlBaseArgs(), "unpause", containerID) - cmd := exec.Command("nerdctl", args...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to resume container %s: %v, output: %s", containerID, err, string(output)) - } - fmt.Printf("Resumed successfully: %s\n", containerID) - return nil -} - -// resumeAllPausedContainers resumes all paused containers that were tracked -func resumeAllPausedContainers() { - if len(pausedContainerIds) == 0 { - return - } - - fmt.Println("\n=== Cleanup: Resuming all paused containers ===") - - // Process in reverse order to match pause order - for i := len(pausedContainerIds) - 1; i >= 0; i-- { - containerID := pausedContainerIds[i] - err := resumeContainer(containerID) - if err != nil { - fmt.Fprintf(os.Stderr, "WARNING: Could not resume container %s: %v\n", containerID, err) - } - } - - // Clear the paused containers list after cleanup - pausedContainerIds = []string{} -} - -// commitContainer uses nerdctl to commit a container to an image -func commitContainer(containerID, targetImage string) error { - fmt.Printf("Committing container %s to image %s...\n", containerID, targetImage) - args := append(nerdctlBaseArgs(), "commit", containerID, targetImage) - cmd := exec.Command("nerdctl", args...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to commit container %s to %s: %v, output: %s", containerID, targetImage, err, string(output)) - } - return nil -} - -// pushImage uses nerdctl to push the image to the registry. -// nerdctl push does not support --username/--password flags, so we use -// nerdctl login first, then nerdctl push with --insecure-registry. -func pushImage(targetImage string) error { - fmt.Printf("Pushing image %s...\n", targetImage) - - // Parse registry host from target image - imageParts := strings.Split(targetImage, "/") - if len(imageParts) == 0 { - return fmt.Errorf("invalid target image: %s", targetImage) - } - registryHost := imageParts[0] - - isInsecure := shouldUseInsecureRegistry(registryHost) - - // Try to login using credentials from mounted secret - credDir := "/var/run/opensandbox/registry" - configPath := filepath.Join(credDir, "config.json") - if _, err := os.Stat(configPath); err == nil { - fmt.Printf("Found registry credentials at %s\n", configPath) - if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { - fmt.Fprintf(os.Stderr, "WARNING: nerdctl login failed: %v (will attempt push anyway)\n", err) - } - } else { - fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") - } - - // Build push options - pushOpts := append(nerdctlBaseArgs(), "push") - if isInsecure { - pushOpts = append(pushOpts, "--insecure-registry") - } - pushOpts = append(pushOpts, targetImage) - - cmd := exec.Command("nerdctl", pushOpts...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to push image %s: %v, output: %s", targetImage, err, string(output)) - } - - return nil -} - -// nerdctlLogin extracts credentials from a Docker config.json and runs nerdctl login. -func nerdctlLogin(configPath, registryHost string, insecure bool) error { - data, err := os.ReadFile(configPath) - if err != nil { - return fmt.Errorf("failed to read config: %w", err) - } - - var creds map[string]interface{} - if err := json.Unmarshal(data, &creds); err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - auths, ok := creds["auths"].(map[string]interface{}) - if !ok || auths[registryHost] == nil { - return fmt.Errorf("no auth entry for registry %s", registryHost) - } - - authEntry, ok := auths[registryHost].(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid auth entry for registry %s", registryHost) - } - - // Try "auth" field first (base64 encoded), then fall back to username/password fields - var username, password string - if authVal, ok := authEntry["auth"].(string); ok && authVal != "" { - decoded, err := base64.StdEncoding.DecodeString(authVal) - if err != nil { - return fmt.Errorf("failed to decode auth: %w", err) - } - parts := strings.SplitN(string(decoded), ":", 2) - if len(parts) != 2 { - return fmt.Errorf("invalid auth format") - } - username = parts[0] - password = parts[1] - } else { - if u, ok := authEntry["username"].(string); ok { - username = u - } - if p, ok := authEntry["password"].(string); ok { - password = p - } - } - - if username == "" || password == "" { - return fmt.Errorf("empty username or password for registry %s", registryHost) - } - - fmt.Printf("Logging in to registry %s as %s\n", registryHost, username) - - loginOpts := append(nerdctlBaseArgs(), "login", "-u", username, "-p", password) - if insecure { - loginOpts = append(loginOpts, "--insecure-registry") - } - loginOpts = append(loginOpts, registryHost) - - cmd := exec.Command("nerdctl", loginOpts...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("nerdctl login failed: %v, output: %s", err, string(output)) - } - - fmt.Printf("Login succeeded for %s\n", registryHost) - return nil -} - -func shouldUseInsecureRegistry(registryHost string) bool { - if raw := strings.TrimSpace(os.Getenv("SNAPSHOT_REGISTRY_INSECURE")); raw != "" { - value, err := strconv.ParseBool(raw) - if err == nil { - return value - } - fmt.Fprintf(os.Stderr, "WARNING: invalid SNAPSHOT_REGISTRY_INSECURE=%q, falling back to registry host heuristic\n", raw) - } - - return strings.Contains(registryHost, "local") || - strings.Contains(registryHost, "localhost") || - strings.HasPrefix(registryHost, "127.") || - strings.HasPrefix(registryHost, "10.") || - strings.HasPrefix(registryHost, "192.168.") || - isPrivate172Registry(registryHost) -} - -func isPrivate172Registry(registryHost string) bool { - host := strings.Split(registryHost, ":")[0] - parts := strings.Split(host, ".") - if len(parts) < 2 || parts[0] != "172" { - return false - } - secondOctet, err := strconv.Atoi(parts[1]) - if err != nil { - return false - } - return secondOctet >= 16 && secondOctet <= 31 -} - -// getImageDigest uses nerdctl to get the digest of the image -func getImageDigest(imageRef string) (string, error) { - args := append(nerdctlBaseArgs(), "inspect", "--format", "{{.Id}}", imageRef) - output, err := commandCombinedOutput("nerdctl", args...) - if err != nil { - return "", fmt.Errorf("nerdctl inspect failed for image %s: %w, output: %s", imageRef, err, strings.TrimSpace(string(output))) - } - digest := strings.TrimSpace(string(output)) - if digest == "" { - return "", fmt.Errorf("nerdctl inspect returned empty digest for image %s", imageRef) - } - return digest, nil } diff --git a/kubernetes/cmd/image-committer/main_test.go b/kubernetes/cmd/image-committer/main_test.go deleted file mode 100644 index df9add332..000000000 --- a/kubernetes/cmd/image-committer/main_test.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2025 Alibaba Group Holding Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestGetImageDigestReturnsErrorOnInspectFailure(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - commandCombinedOutput = func(_ string, _ ...string) ([]byte, error) { - return []byte("inspect failed"), errors.New("exit status 1") - } - - digest, err := getImageDigest("registry.example.com/test/image:snap") - - if err == nil { - t.Fatal("expected digest extraction error") - } - if digest != "" { - t.Fatalf("expected empty digest on error, got %q", digest) - } - if digest == "sha256:placeholder" { - t.Fatal("digest extraction must not return placeholder") - } -} - -func TestGetImageDigestReturnsErrorOnEmptyInspectOutput(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - commandCombinedOutput = func(_ string, _ ...string) ([]byte, error) { - return []byte(" \n"), nil - } - - digest, err := getImageDigest("registry.example.com/test/image:snap") - - if err == nil { - t.Fatal("expected empty digest error") - } - if digest != "" { - t.Fatalf("expected empty digest on error, got %q", digest) - } -} - -func TestGetImageDigestReturnsDigest(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - commandCombinedOutput = func(_ string, _ ...string) ([]byte, error) { - return []byte("sha256:abc123\n"), nil - } - - digest, err := getImageDigest("registry.example.com/test/image:snap") - - if err != nil { - t.Fatalf("expected digest extraction to succeed, got %v", err) - } - if digest != "sha256:abc123" { - t.Fatalf("unexpected digest %q", digest) - } -} - -func TestGetContainerIDByNerdctlReturnsRunningContainer(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - - calls := 0 - commandCombinedOutput = func(name string, args ...string) ([]byte, error) { - calls++ - if name != "nerdctl" { - t.Fatalf("unexpected command %q", name) - } - if calls != 1 { - t.Fatalf("expected a single nerdctl lookup, got %d", calls) - } - return []byte("container-running\n"), nil - } - - container, err := getContainerByNerdctl("pod-1", "default", "sandbox") - if err != nil { - t.Fatalf("expected running container lookup to succeed, got %v", err) - } - if container.ID != "container-running" { - t.Fatalf("unexpected container ID %q", container.ID) - } - if !container.Running { - t.Fatal("expected container to be reported as running") - } -} - -func TestGetContainerIDByNerdctlFallsBackToStoppedContainers(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - - var calls [][]string - commandCombinedOutput = func(name string, args ...string) ([]byte, error) { - if name != "nerdctl" { - t.Fatalf("unexpected command %q", name) - } - calls = append(calls, append([]string(nil), args...)) - switch len(calls) { - case 1: - return []byte("\n"), nil - case 2: - return []byte("container-stopped\n"), nil - default: - t.Fatalf("unexpected extra nerdctl lookup #%d", len(calls)) - return nil, nil - } - } - - container, err := getContainerByNerdctl("pod-1", "default", "sandbox") - if err != nil { - t.Fatalf("expected stopped container fallback to succeed, got %v", err) - } - if container.ID != "container-stopped" { - t.Fatalf("unexpected container ID %q", container.ID) - } - if container.Running { - t.Fatal("expected stopped container to be reported as stopped") - } - if len(calls) != 2 { - t.Fatalf("expected two nerdctl lookups, got %d", len(calls)) - } - if contains(calls[0], "-a") { - t.Fatalf("first lookup should only inspect running containers: %v", calls[0]) - } - if !contains(calls[1], "-a") { - t.Fatalf("second lookup should include stopped containers: %v", calls[1]) - } -} - -func TestGetContainerIDByNerdctlReturnsHelpfulErrorWhenBothLookupsAreEmpty(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - - commandCombinedOutput = func(_ string, _ ...string) ([]byte, error) { - return []byte("\n"), nil - } - - _, err := getContainerIDByNerdctl("pod-1", "default", "sandbox") - if err == nil { - t.Fatal("expected lookup failure when both running and stopped container searches are empty") - } - if got := err.Error(); got != "container 'sandbox' not found in pod default/pod-1 (nerdctl ps and nerdctl ps -a returned empty)" { - t.Fatalf("unexpected error %q", got) - } -} - -func contains(values []string, target string) bool { - for _, value := range values { - if value == target { - return true - } - } - return false -} - -func TestSyncRunningContainerFilesystemsSyncsEveryRunningContainerAndSkipsStopped(t *testing.T) { - original := commandCombinedOutput - t.Cleanup(func() { commandCombinedOutput = original }) - t.Setenv("CONTAINERD_SOCKET", "/test/containerd.sock") - t.Setenv("CONTAINERD_NAMESPACE", "test-ns") - - var calls [][]string - commandCombinedOutput = func(name string, args ...string) ([]byte, error) { - if name != "nerdctl" { - t.Fatalf("unexpected command %q", name) - } - calls = append(calls, append([]string(nil), args...)) - if contains(args, "container-main") { - return []byte("guest sync failed"), errors.New("exit status 1") - } - return nil, nil - } - - err := syncRunningContainerFilesystems( - []ContainerSpec{{Name: "main"}, {Name: "sidecar"}, {Name: "stopped"}}, - map[string]discoveredContainer{ - "main": {ID: "container-main", Running: true}, - "sidecar": {ID: "container-sidecar", Running: true}, - "stopped": {ID: "container-stopped"}, - }, - ) - - if err == nil { - t.Fatal("expected a running container sync failure to be reported") - } - if !strings.Contains(err.Error(), `container "main"`) || !strings.Contains(err.Error(), "guest sync failed") { - t.Fatalf("expected contextual sync failure, got %q", err) - } - if len(calls) != 2 { - t.Fatalf("expected every running container and no stopped containers to be synced, got %d calls", len(calls)) - } - wantMain := []string{"--address", "/test/containerd.sock", "--namespace", "test-ns", "exec", "container-main", "sync"} - wantSidecar := []string{"--address", "/test/containerd.sock", "--namespace", "test-ns", "exec", "container-sidecar", "sync"} - for i, want := range [][]string{wantMain, wantSidecar} { - if strings.Join(calls[i], "\x00") != strings.Join(want, "\x00") { - t.Fatalf("unexpected nerdctl call %d: got %v, want %v", i+1, calls[i], want) - } - } -} - -func TestWriteSnapshotResultWritesTerminationMessage(t *testing.T) { - original := terminationMessagePath - t.Cleanup(func() { terminationMessagePath = original }) - terminationMessagePath = filepath.Join(t.TempDir(), "termination.log") - - err := writeSnapshotResult( - []ContainerSpec{ - {Name: "main", URI: "registry.example.com/main:snap"}, - {Name: "sidecar", URI: "registry.example.com/sidecar:snap"}, - }, - map[string]string{ - "main": "sha256:main", - "sidecar": "sha256:sidecar", - }, - ) - if err != nil { - t.Fatalf("writeSnapshotResult failed: %v", err) - } - - data, err := os.ReadFile(terminationMessagePath) - if err != nil { - t.Fatalf("failed to read termination message: %v", err) - } - - var result snapshotResult - if err := json.Unmarshal(data, &result); err != nil { - t.Fatalf("termination message is not valid JSON: %v", err) - } - if len(result.Containers) != 2 { - t.Fatalf("expected 2 container results, got %d", len(result.Containers)) - } - if result.Containers[0].Name != "main" || result.Containers[0].Digest != "sha256:main" { - t.Fatalf("unexpected first result: %#v", result.Containers[0]) - } - if result.Containers[1].Name != "sidecar" || result.Containers[1].Digest != "sha256:sidecar" { - t.Fatalf("unexpected second result: %#v", result.Containers[1]) - } -} diff --git a/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml b/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml index 26452ecb0..7468922e8 100644 --- a/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml +++ b/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml @@ -114,8 +114,8 @@ spec: description: ContainerName is the name of the container. type: string imageDigest: - description: ImageDigest is the digest of the pushed snapshot - image. + description: ImageDigest is the config digest of the pushed + snapshot image. type: string imageUri: description: ImageURI is the snapshot image URI for this container. diff --git a/kubernetes/config/manager/manager.yaml b/kubernetes/config/manager/manager.yaml index be4598fec..26a17abd1 100644 --- a/kubernetes/config/manager/manager.yaml +++ b/kubernetes/config/manager/manager.yaml @@ -68,6 +68,7 @@ spec: - --snapshot-push-secret=registry-snapshot-push-secret - --resume-pull-secret=registry-pull-secret - --image-committer-image=image-committer:dev + - --image-committer-pod-template-file= - --commit-job-timeout=2m - --image-committer-pull-secret= image: controller:dev diff --git a/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml b/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml index f902640c1..961acb61a 100644 --- a/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml +++ b/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml @@ -31,7 +31,7 @@ spec: - name: opensandbox-bin mountPath: /opt/opensandbox - name: execd-installer - image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21 + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 command: [ "/bin/sh", "-c" ] args: - | @@ -49,7 +49,9 @@ spec: - "/bin/sh" - "-c" - | - /opt/opensandbox/task-executor -listen-addr=0.0.0.0:5758 >/tmp/task-executor.log 2>&1 + /opt/opensandbox/task-executor \ + -listen-addr=0.0.0.0:5758 \ + -log-dir=/tmp env: - name: SANDBOX_MAIN_CONTAINER value: main diff --git a/kubernetes/config/samples/sandbox_v1alpha1_pool_restart.yaml b/kubernetes/config/samples/sandbox_v1alpha1_pool_restart.yaml index 1d20415d8..fec211838 100644 --- a/kubernetes/config/samples/sandbox_v1alpha1_pool_restart.yaml +++ b/kubernetes/config/samples/sandbox_v1alpha1_pool_restart.yaml @@ -24,7 +24,9 @@ spec: - /bin/sh - -c - | - exec /opt/opensandbox/task-executor -listen-addr=0.0.0.0:5758 >/tmp/task-executor.log 2>&1 + exec /opt/opensandbox/task-executor \ + -listen-addr=0.0.0.0:5758 \ + -log-dir=/tmp env: - name: SANDBOX_MAIN_CONTAINER value: main @@ -56,7 +58,7 @@ spec: command: - /bin/sh - -c - image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21 + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 name: execd-installer volumeMounts: - mountPath: /opt/opensandbox @@ -67,4 +69,4 @@ spec: - emptyDir: {} name: sandbox-storage - emptyDir: {} - name: opensandbox-bin \ No newline at end of file + name: opensandbox-bin diff --git a/kubernetes/docs/HELM-DEPLOYMENT.md b/kubernetes/docs/HELM-DEPLOYMENT.md index 78120f946..ddcdad232 100644 --- a/kubernetes/docs/HELM-DEPLOYMENT.md +++ b/kubernetes/docs/HELM-DEPLOYMENT.md @@ -446,10 +446,10 @@ Tag naming convention: `helm/{component}/{version}` This automatically triggers the workflow to: 1. Parse the tag to extract component and version -2. Update the version in the corresponding Chart.yaml -3. Package the Helm Chart -4. Create a GitHub Release -5. Publish the .tgz package to the Release +2. Verify the tag version matches the chart `version` +3. Preserve the committed chart `appVersion` +4. Package the Helm Chart +5. Create a GitHub Release and publish the .tgz package Important versioning note: @@ -457,9 +457,9 @@ Important versioning note: `helm/{component}/{version}` tags. - The chart `appVersion` is the default image/application version used by that chart release. -- The `publish-helm-chart.yml` workflow updates `appVersion` for the published - release, but intentionally does not auto-bump the chart `version` inside - `Chart.yaml` on server release branches. +- Tag-triggered publishing preserves the committed chart `appVersion` and + verifies that the tag matches the committed chart `version`. Manual runs can + override `appVersion` independently. - If you need a specific server image release, set the image tag explicitly (for example `--set server.image.tag=v0.1.13`) or publish a new Helm chart package version for the chart itself. diff --git a/kubernetes/go.mod b/kubernetes/go.mod index 18f3948db..d4e2551d1 100644 --- a/kubernetes/go.mod +++ b/kubernetes/go.mod @@ -3,9 +3,15 @@ module github.com/alibaba/OpenSandbox/sandbox-k8s go 1.25.0 require ( + github.com/containerd/containerd v1.7.33 + github.com/containerd/errdefs v1.0.0 + github.com/distribution/reference v0.6.0 github.com/golang/mock v1.6.0 + github.com/google/go-containerregistry v0.20.6 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.1 github.com/stretchr/testify v1.11.1 k8s.io/api v0.33.0 k8s.io/apimachinery v0.33.0 @@ -18,10 +24,43 @@ require ( require github.com/cenkalti/backoff/v5 v5.0.3 // indirect require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.11.7 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/continuity v0.4.4 // indirect + github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/cyphar/filepath-securejoin v0.5.1 // indirect + github.com/docker/cli v28.2.2+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.5.1 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/signal v0.7.0 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/vbatts/tar-split v0.12.1 // indirect + go.opencensus.io v0.24.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect ) require ( @@ -65,8 +104,8 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect @@ -86,7 +125,7 @@ require ( golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.9.0 // indirect + golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.44.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect @@ -105,5 +144,5 @@ require ( sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/yaml v1.4.0 ) diff --git a/kubernetes/go.sum b/kubernetes/go.sum index 6e9cb736d..1165fa1d4 100644 --- a/kubernetes/go.sum +++ b/kubernetes/go.sum @@ -1,5 +1,15 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -10,15 +20,56 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/containerd v1.7.33 h1:iAkYGC/ifR/V+0eR4iXWHNGYUF0DF2PmGV5iz4Irj5M= +github.com/containerd/containerd v1.7.33/go.mod h1:gSbSCVjPCdkfJCjyrzz7aRC+xFlqVbatNpfHfVCYGUM= +github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= +github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= +github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.5.1 h1:eYgfMq5yryL4fbWfkLpFFy2ukSELzaJOTaUTuh+oF48= +github.com/cyphar/filepath-securejoin v0.5.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsyN9SdInc1A= +github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= @@ -48,8 +99,23 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -58,14 +124,24 @@ github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU= +github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= @@ -93,8 +169,22 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -108,12 +198,21 @@ github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= @@ -123,26 +222,33 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= +github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= @@ -174,32 +280,48 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -209,9 +331,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -226,12 +352,35 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -246,6 +395,10 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= diff --git a/kubernetes/internal/controller/pool_static_volume_test.go b/kubernetes/internal/controller/pool_static_volume_test.go new file mode 100644 index 000000000..3168104ac --- /dev/null +++ b/kubernetes/internal/controller/pool_static_volume_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1" + controllerutils "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/controller" +) + +func TestCreatePoolPodPreservesStaticPVC(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, sandboxv1alpha1.AddToScheme(scheme)) + + var createdPod *corev1.Pod + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.CreateOption) error { + pod, ok := obj.(*corev1.Pod) + require.True(t, ok) + pod.Name = "shared-workspace-pool-test" + createdPod = pod.DeepCopy() + return nil + }, + }). + Build() + + pool := &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-workspace-pool", + Namespace: "opensandbox", + UID: types.UID("pool-uid"), + }, + Spec: sandboxv1alpha1.PoolSpec{ + Template: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sandbox-container", + Image: "python:3.11", + VolumeMounts: []corev1.VolumeMount{ + {Name: "shared-workspace", MountPath: "/workspace"}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "shared-workspace", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "shared-workspace-pvc", + }, + }, + }, + }, + }, + }, + }, + } + defer PoolScaleExpectations.DeleteExpectations(controllerutils.GetControllerKey(pool)) + + r := &PoolReconciler{ + Client: fakeClient, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + } + require.NoError(t, r.createPoolPod(context.Background(), pool, "revision-1")) + require.NotNil(t, createdPod) + require.Len(t, createdPod.Spec.Volumes, 1) + require.NotNil(t, createdPod.Spec.Volumes[0].PersistentVolumeClaim) + assert.Equal(t, "shared-workspace-pvc", createdPod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName) + require.Len(t, createdPod.Spec.Containers, 1) + assert.Contains(t, createdPod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "shared-workspace", + MountPath: "/workspace", + }) +} diff --git a/kubernetes/internal/controller/recycle/restart/restart_default.go b/kubernetes/internal/controller/recycle/restart/restart_default.go index 674910375..3f23b0c71 100644 --- a/kubernetes/internal/controller/recycle/restart/restart_default.go +++ b/kubernetes/internal/controller/recycle/restart/restart_default.go @@ -39,6 +39,12 @@ import ( // container. Containers using the Restart recycle strategy must have a PID 1 process that // handles SIGTERM and exits gracefully (e.g., a real application server, not bare // "sleep"). When PID 1 exits, the kubelet restarts the container per its restartPolicy. +// +// Init-mode execd (OSEP-0018) keeps this contract: it installs a SIGTERM handler that +// forwards the signal to the workload and exits with the workload's status, so the +// in-namespace `kill 1` performed by this recycle path still restarts the container. +// When a trusted out-of-band stop channel replaces signal-driven stop (OSEP-0018 ยง3), +// this command must be reconciled with that channel instead. var DefaultRestartCommand = []string{"kill", "1"} const ( diff --git a/kubernetes/internal/controller/registry_image_deleter.go b/kubernetes/internal/controller/registry_image_deleter.go new file mode 100644 index 000000000..654255fae --- /dev/null +++ b/kubernetes/internal/controller/registry_image_deleter.go @@ -0,0 +1,186 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + corev1 "k8s.io/api/core/v1" +) + +type registryImageDeleter interface { + Delete(ctx context.Context, imageReference, imageDigest string, registrySecret *corev1.Secret, insecure bool) error +} + +type remoteRegistryImageDeleter struct{} + +func (remoteRegistryImageDeleter) Delete( + ctx context.Context, + imageReference string, + imageDigest string, + registrySecret *corev1.Secret, + insecure bool, +) error { + nameOptions := []name.Option{name.StrictValidation} + if insecure { + nameOptions = append(nameOptions, name.Insecure) + } + + ref, err := name.ParseReference(imageReference, nameOptions...) + if err != nil { + return fmt.Errorf("parse image reference %q: %w", imageReference, err) + } + + authenticator, err := registryAuthenticator(registrySecret, ref.Context().RegistryStr()) + if err != nil { + return err + } + remoteOptions := []remote.Option{remote.WithContext(ctx), remote.WithAuth(authenticator)} + + if imageDigest != "" { + digestRef := ref.Context().Digest(imageDigest) + deleteErr := remote.Delete(digestRef, remoteOptions...) + if deleteErr != nil && !isRegistryNotFound(deleteErr) { + return fmt.Errorf("delete image %q by digest %q: %w", imageReference, imageDigest, deleteErr) + } + // nerdctl can convert the pushed manifest media type, and some registries + // report a successful DELETE even when the supplied digest is not the + // digest currently referenced by the tag. Resolve the tag after every + // digest attempt and remove a different registry manifest when present. + if _, ok := ref.(name.Digest); !ok { + if err := deleteRegistryTagIfDifferent(ctx, ref, imageDigest, remoteOptions, imageReference); err != nil { + return err + } + } + return nil + } + + if _, ok := ref.(name.Digest); ok { + if err := remote.Delete(ref, remoteOptions...); err != nil && !isRegistryNotFound(err) { + return fmt.Errorf("delete image %q: %w", imageReference, err) + } + return nil + } + + return deleteRegistryTag(ctx, ref, remoteOptions, imageReference) +} + +func deleteRegistryTagIfDifferent(ctx context.Context, ref name.Reference, imageDigest string, remoteOptions []remote.Option, imageReference string) error { + descriptor, err := remote.Head(ref, remoteOptions...) + if isRegistryNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("resolve image digest for %q: %w", imageReference, err) + } + if descriptor.Digest.String() == imageDigest { + return nil + } + registryRef := ref.Context().Digest(descriptor.Digest.String()) + if err := remote.Delete(registryRef, remoteOptions...); err != nil && !isRegistryNotFound(err) { + return fmt.Errorf("delete image %q by registry digest %q: %w", imageReference, descriptor.Digest, err) + } + return nil +} + +func deleteRegistryTag(ctx context.Context, ref name.Reference, remoteOptions []remote.Option, imageReference string) error { + descriptor, err := remote.Head(ref, remoteOptions...) + if isRegistryNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("resolve image digest for %q: %w", imageReference, err) + } + + registryRef := ref.Context().Digest(descriptor.Digest.String()) + if err := remote.Delete(registryRef, remoteOptions...); err != nil && !isRegistryNotFound(err) { + return fmt.Errorf("delete image %q by registry digest %q: %w", imageReference, descriptor.Digest, err) + } + return nil +} + +func registryAuthenticator(secret *corev1.Secret, registry string) (authn.Authenticator, error) { + if secret == nil { + return authn.Anonymous, nil + } + + var auths map[string]authn.AuthConfig + var credentialHelpers map[string]string + var credentialStore string + switch { + case len(secret.Data[corev1.DockerConfigJsonKey]) > 0: + var config struct { + Auths map[string]authn.AuthConfig `json:"auths"` + CredHelpers map[string]string `json:"credHelpers"` + CredsStore string `json:"credsStore"` + } + if err := json.Unmarshal(secret.Data[corev1.DockerConfigJsonKey], &config); err != nil { + return nil, fmt.Errorf("parse registry secret %s/%s: %w", secret.Namespace, secret.Name, err) + } + auths = config.Auths + credentialHelpers = config.CredHelpers + credentialStore = config.CredsStore + case len(secret.Data[corev1.DockerConfigKey]) > 0: + if err := json.Unmarshal(secret.Data[corev1.DockerConfigKey], &auths); err != nil { + return nil, fmt.Errorf("parse registry secret %s/%s: %w", secret.Namespace, secret.Name, err) + } + default: + return nil, fmt.Errorf("registry secret %s/%s has neither %s nor %s", secret.Namespace, secret.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) + } + + for server, config := range auths { + if normalizeRegistry(server) == normalizeRegistry(registry) { + return authn.FromConfig(config), nil + } + } + for server, helper := range credentialHelpers { + if normalizeRegistry(server) == normalizeRegistry(registry) { + return nil, fmt.Errorf("registry secret %s/%s uses credential helper %q for %s; controller requires inline auths credentials", secret.Namespace, secret.Name, helper, registry) + } + } + if credentialStore != "" && len(auths) == 0 { + return nil, fmt.Errorf("registry secret %s/%s uses credential store %q; controller requires inline auths credentials", secret.Namespace, secret.Name, credentialStore) + } + return nil, fmt.Errorf("registry secret %s/%s has no credentials for %s", secret.Namespace, secret.Name, registry) +} + +func normalizeRegistry(registry string) string { + registry = strings.TrimPrefix(registry, "https://") + registry = strings.TrimPrefix(registry, "http://") + registry = strings.TrimSuffix(registry, "/") + registry = strings.TrimSuffix(registry, "/v1") + registry = strings.TrimSuffix(registry, "/v2") + if registry == "index.docker.io" { + return name.DefaultRegistry + } + return registry +} + +func isRegistryNotFound(err error) bool { + if err == nil { + return false + } + var transportErr *transport.Error + return errors.As(err, &transportErr) && transportErr.StatusCode == http.StatusNotFound +} diff --git a/kubernetes/internal/controller/registry_image_deleter_test.go b/kubernetes/internal/controller/registry_image_deleter_test.go new file mode 100644 index 000000000..a70aab9d7 --- /dev/null +++ b/kubernetes/internal/controller/registry_image_deleter_test.go @@ -0,0 +1,207 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestRemoteRegistryImageDeleter_ResolvesTagAndDeletesDigest(t *testing.T) { + const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + deletedPath := make(chan string, 1) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/v2/": + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodHead && request.URL.Path == "/v2/snapshots/test/manifests/tag": + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Content-Length", "2") + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete: + deletedPath <- request.URL.Path + w.WriteHeader(http.StatusAccepted) + default: + http.NotFound(w, request) + } + })) + defer registry.Close() + + imageReference := strings.TrimPrefix(registry.URL, "http://") + "/snapshots/test:tag" + err := (remoteRegistryImageDeleter{}).Delete(context.Background(), imageReference, "", nil, true) + require.NoError(t, err) + select { + case path := <-deletedPath: + assert.Equal(t, "/v2/snapshots/test/manifests/"+digest, path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for registry DELETE request") + } +} + +func TestRemoteRegistryImageDeleter_TreatsMissingManifestAsSuccess(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/v2/" { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, request) + })) + defer registry.Close() + + imageReference := strings.TrimPrefix(registry.URL, "http://") + "/snapshots/missing@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + err := (remoteRegistryImageDeleter{}).Delete(context.Background(), imageReference, "", nil, true) + require.NoError(t, err) +} + +func TestRemoteRegistryImageDeleter_FallsBackToTagAfterRecordedDigestMiss(t *testing.T) { + const recordedDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + const registryDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + deletedPaths := make(chan string, 2) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/v2/": + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodHead && request.URL.Path == "/v2/snapshots/test/manifests/tag": + w.Header().Set("Docker-Content-Digest", registryDigest) + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Content-Length", "2") + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasSuffix(request.URL.Path, "/"+recordedDigest): + deletedPaths <- request.URL.Path + http.NotFound(w, request) + case request.Method == http.MethodDelete && strings.HasSuffix(request.URL.Path, "/"+registryDigest): + deletedPaths <- request.URL.Path + w.WriteHeader(http.StatusAccepted) + default: + http.NotFound(w, request) + } + })) + defer registry.Close() + + imageReference := strings.TrimPrefix(registry.URL, "http://") + "/snapshots/test:tag" + err := (remoteRegistryImageDeleter{}).Delete(context.Background(), imageReference, recordedDigest, nil, true) + require.NoError(t, err) + select { + case path := <-deletedPaths: + assert.Equal(t, "/v2/snapshots/test/manifests/"+recordedDigest, path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for recorded-digest DELETE request") + } + select { + case path := <-deletedPaths: + assert.Equal(t, "/v2/snapshots/test/manifests/"+registryDigest, path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for fallback registry DELETE request") + } +} + +func TestRemoteRegistryImageDeleter_RemovesTagDigestAfterRecordedDeleteAccepted(t *testing.T) { + const recordedDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + const registryDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + deletedPaths := make(chan string, 2) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/v2/": + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodHead && request.URL.Path == "/v2/snapshots/test/manifests/tag": + w.Header().Set("Docker-Content-Digest", registryDigest) + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Content-Length", "2") + w.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasSuffix(request.URL.Path, "/"+recordedDigest): + deletedPaths <- request.URL.Path + w.WriteHeader(http.StatusAccepted) + case request.Method == http.MethodDelete && strings.HasSuffix(request.URL.Path, "/"+registryDigest): + deletedPaths <- request.URL.Path + w.WriteHeader(http.StatusAccepted) + default: + http.NotFound(w, request) + } + })) + defer registry.Close() + + imageReference := strings.TrimPrefix(registry.URL, "http://") + "/snapshots/test:tag" + err := (remoteRegistryImageDeleter{}).Delete(context.Background(), imageReference, recordedDigest, nil, true) + require.NoError(t, err) + select { + case path := <-deletedPaths: + assert.Equal(t, "/v2/snapshots/test/manifests/"+recordedDigest, path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for recorded-digest DELETE request") + } + select { + case path := <-deletedPaths: + assert.Equal(t, "/v2/snapshots/test/manifests/"+registryDigest, path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for registry-digest DELETE request") + } +} + +func TestRegistryAuthenticator_ReadsDockerConfigJSON(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "registry-secret", Namespace: "default"}, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(`{"auths":{"https://registry.example.com/v1/":{"username":"user","password":"pass"}}}`), + }, + } + + authenticator, err := registryAuthenticator(secret, "registry.example.com") + require.NoError(t, err) + config, err := authn.Authorization(context.Background(), authenticator) + require.NoError(t, err) + assert.Equal(t, "user", config.Username) + assert.Equal(t, "pass", config.Password) +} + +func TestRegistryAuthenticator_RejectsSecretWithoutMatchingRegistry(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "registry-secret", Namespace: "default"}, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(`{"auths":{"other.example.com":{"auth":"dXNlcjpwYXNz"}}}`), + }, + } + + _, err := registryAuthenticator(secret, "registry.example.com") + require.ErrorContains(t, err, "has no credentials for registry.example.com") +} + +func TestRegistryAuthenticator_AllowsAnonymousRegistry(t *testing.T) { + authenticator, err := registryAuthenticator(nil, "registry.example.com") + require.NoError(t, err) + assert.Equal(t, authn.Anonymous, authenticator) +} + +func TestRegistryAuthenticator_RejectsCredentialHelpers(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "registry-secret", Namespace: "default"}, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(`{"credHelpers":{"registry.example.com":"osxkeychain"}}`), + }, + } + + _, err := registryAuthenticator(secret, "registry.example.com") + require.ErrorContains(t, err, "requires inline auths credentials") +} diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller.go b/kubernetes/internal/controller/sandboxsnapshot_controller.go index b79769cdb..ae88c6275 100644 --- a/kubernetes/internal/controller/sandboxsnapshot_controller.go +++ b/kubernetes/internal/controller/sandboxsnapshot_controller.go @@ -19,6 +19,7 @@ import ( "time" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" @@ -51,8 +52,8 @@ const ( // ContainerdSocketPath is the default containerd socket path ContainerdSocketPath = "/var/run/containerd/containerd.sock" - // ContainerdFIFODir is shared with the host so nerdctl exec's I/O FIFOs are - // visible to the host-side containerd shim. + // ContainerdFIFODir is available to image-committer implementations that + // use containerd task exec with FIFO-backed process I/O. ContainerdFIFODir = "/run/containerd/fifo" // LabelSandboxSnapshotName is the label key for sandbox snapshot name @@ -68,10 +69,10 @@ type SandboxSnapshotReconciler struct { Scheme *runtime.Scheme Recorder record.EventRecorder - // ImageCommitterImage is the image for image-committer (uses nerdctl to commit/push container images) + // ImageCommitterImage is the image used for commit and unpause Jobs. ImageCommitterImage string - // ContainerdSocketPath is containerd socket path for image-committer (nerdctl --address) + // ContainerdSocketPath is the host containerd socket mounted into image-committer Jobs. ContainerdSocketPath string // CommitJobTimeout is the timeout for commit jobs (default: 10 minutes) @@ -80,15 +81,22 @@ type SandboxSnapshotReconciler struct { // SnapshotRegistry is the OCI registry for snapshot images (from Controller Manager startup params) SnapshotRegistry string - // SnapshotPushSecret is the K8s Secret name for pushing to registry (from Controller Manager startup params) + // SnapshotPushSecret is the K8s Secret name for pushing and deleting snapshot images (from Controller Manager startup params) SnapshotPushSecret string // ImageCommitterPullSecret is the K8s Secret name used to pull the image-committer image in commit Jobs. // Required when imageCommitterImage lives in a private registry. ImageCommitterPullSecret string + // ImageCommitterPodTemplate overlays operator-controlled commit Job Pod settings. + ImageCommitterPodTemplate *corev1.PodTemplateSpec + // SnapshotRegistryInsecure controls whether image-committer uses insecure registry mode. SnapshotRegistryInsecure bool + + // registryImageDeleter performs remote manifest cleanup. When nil, the + // reconciler uses remoteRegistryImageDeleter. + registryImageDeleter registryImageDeleter } // +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=sandboxsnapshots,verbs=get;list;watch;create;update;patch;delete diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go b/kubernetes/internal/controller/sandboxsnapshot_controller_test.go index 31ada010a..c292ca993 100644 --- a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go +++ b/kubernetes/internal/controller/sandboxsnapshot_controller_test.go @@ -16,6 +16,7 @@ package controller import ( "context" + "errors" "fmt" "testing" "time" @@ -36,6 +37,38 @@ import ( sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1" ) +type registryDeleteCall struct { + imageReference string + imageDigest string + secretName string + insecure bool +} + +type recordingRegistryImageDeleter struct { + calls []registryDeleteCall + err error +} + +func (d *recordingRegistryImageDeleter) Delete( + _ context.Context, + imageReference string, + imageDigest string, + secret *corev1.Secret, + insecure bool, +) error { + secretName := "" + if secret != nil { + secretName = secret.Name + } + d.calls = append(d.calls, registryDeleteCall{ + imageReference: imageReference, + imageDigest: imageDigest, + secretName: secretName, + insecure: insecure, + }) + return d.err +} + func newTestSnapshotReconciler(objs ...client.Object) *SandboxSnapshotReconciler { scheme := k8sruntime.NewScheme() utilruntime.Must(corev1.AddToScheme(scheme)) @@ -55,6 +88,112 @@ func newTestSnapshotReconciler(objs ...client.Object) *SandboxSnapshotReconciler } } +func TestSandboxSnapshotHandleDeletion_DeletesRegistryImagesBeforeRemovingFinalizer(t *testing.T) { + snapshot := &sandboxv1alpha1.SandboxSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-snapshot", + Namespace: "default", + Finalizers: []string{SandboxSnapshotFinalizer}, + }, + Status: sandboxv1alpha1.SandboxSnapshotStatus{ + Containers: []sandboxv1alpha1.ContainerSnapshot{ + {ContainerName: "main", ImageURI: "registry.example.com/snapshots/main:tag", ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + {ContainerName: "main-duplicate", ImageURI: "registry.example.com/snapshots/main:tag", ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + {ContainerName: "sidecar", ImageURI: "registry.example.com/snapshots/sidecar:tag"}, + }, + }, + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "registry-secret", Namespace: "default"}} + commitJob := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot-commit", Namespace: "default"}} + unpauseJob := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot-unpause", Namespace: "default"}} + deleter := &recordingRegistryImageDeleter{} + r := newTestSnapshotReconciler(snapshot, secret, commitJob, unpauseJob) + r.SnapshotPushSecret = secret.Name + r.SnapshotRegistryInsecure = true + r.registryImageDeleter = deleter + + result, err := r.handleDeletion(context.Background(), snapshot) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + assert.Empty(t, deleter.calls, "registry cleanup must wait for jobs to terminate") + + result, err = r.handleDeletion(context.Background(), snapshot) + require.NoError(t, err) + assert.Equal(t, ctrl.Result{}, result) + require.Len(t, deleter.calls, 2) + assert.Equal(t, "registry.example.com/snapshots/main:tag", deleter.calls[0].imageReference) + assert.Equal(t, "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", deleter.calls[0].imageDigest) + assert.Equal(t, "registry.example.com/snapshots/sidecar:tag", deleter.calls[1].imageReference) + assert.Equal(t, "registry-secret", deleter.calls[0].secretName) + assert.True(t, deleter.calls[0].insecure) + + updated := &sandboxv1alpha1.SandboxSnapshot{} + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, updated)) + assert.NotContains(t, updated.Finalizers, SandboxSnapshotFinalizer) +} + +func TestSandboxSnapshotHandleDeletion_WaitsForJobPods(t *testing.T) { + now := metav1.Now() + snapshot := &sandboxv1alpha1.SandboxSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot", Namespace: "default", Finalizers: []string{SandboxSnapshotFinalizer}}, + Status: sandboxv1alpha1.SandboxSnapshotStatus{Containers: []sandboxv1alpha1.ContainerSnapshot{{ContainerName: "main", ImageURI: "registry.example.com/snapshots/main:tag"}}}, + } + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot-commit", Namespace: "default", DeletionTimestamp: &now, Finalizers: []string{"foregroundDeletion"}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot-commit-pod", Namespace: "default", Labels: map[string]string{"job-name": "test-snapshot-commit"}}} + deleter := &recordingRegistryImageDeleter{} + r := newTestSnapshotReconciler(snapshot, job, pod) + r.registryImageDeleter = deleter + + result, err := r.handleDeletion(context.Background(), snapshot) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + assert.Empty(t, deleter.calls) +} + +func TestSandboxSnapshotHandleDeletion_KeepsFinalizerWhenRegistryDeleteFails(t *testing.T) { + snapshot := &sandboxv1alpha1.SandboxSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-snapshot", + Namespace: "default", + Finalizers: []string{SandboxSnapshotFinalizer}, + }, + Status: sandboxv1alpha1.SandboxSnapshotStatus{ + Containers: []sandboxv1alpha1.ContainerSnapshot{ + {ContainerName: "main", ImageURI: "registry.example.com/snapshots/main:tag"}, + }, + }, + } + deleter := &recordingRegistryImageDeleter{err: errors.New("registry unavailable")} + r := newTestSnapshotReconciler(snapshot) + r.registryImageDeleter = deleter + + _, err := r.handleDeletion(context.Background(), snapshot) + require.ErrorContains(t, err, "registry unavailable") + + updated := &sandboxv1alpha1.SandboxSnapshot{} + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, updated)) + assert.Contains(t, updated.Finalizers, SandboxSnapshotFinalizer) +} + +func TestSandboxSnapshotHandleDeletion_KeepsFinalizerWhenRegistrySecretIsMissing(t *testing.T) { + snapshot := &sandboxv1alpha1.SandboxSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "test-snapshot", Namespace: "default", Finalizers: []string{SandboxSnapshotFinalizer}}, + Status: sandboxv1alpha1.SandboxSnapshotStatus{Containers: []sandboxv1alpha1.ContainerSnapshot{{ContainerName: "main", ImageURI: "registry.example.com/snapshots/main:tag"}}}, + } + deleter := &recordingRegistryImageDeleter{} + r := newTestSnapshotReconciler(snapshot) + r.SnapshotPushSecret = "missing-registry-secret" + r.registryImageDeleter = deleter + + _, err := r.handleDeletion(context.Background(), snapshot) + require.ErrorContains(t, err, "missing-registry-secret") + assert.Empty(t, deleter.calls) + + updated := &sandboxv1alpha1.SandboxSnapshot{} + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, updated)) + assert.Contains(t, updated.Finalizers, SandboxSnapshotFinalizer) +} + func TestSandboxSnapshotHandleCommitting_SetsSucceedReadyCondition(t *testing.T) { snapshot := &sandboxv1alpha1.SandboxSnapshot{ ObjectMeta: metav1.ObjectMeta{ @@ -251,6 +390,12 @@ func TestSandboxSnapshotHandleCommitting_CreatesUnpauseJobWhenCommitJobFailed(t Name: "test-snapshot-commit", Namespace: "default", }, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: CommitJobContainerName, + Env: []corev1.EnvVar{{Name: "SOURCE_POD_UID", Value: "source-pod-uid"}}, + }}, + }}}, Status: batchv1.JobStatus{ Conditions: []batchv1.JobCondition{ { @@ -275,6 +420,8 @@ func TestSandboxSnapshotHandleCommitting_CreatesUnpauseJobWhenCommitJobFailed(t cleanupContainer := cleanupJob.Spec.Template.Spec.Containers[0] assert.Equal(t, []string{"/usr/local/bin/image-committer"}, cleanupContainer.Command) assert.Equal(t, []string{"unpause", "source-pod", "default", "main", "sidecar"}, cleanupContainer.Args) + assert.Contains(t, cleanupContainer.Env, corev1.EnvVar{Name: "SOURCE_POD_UID", Value: "source-pod-uid"}) + assert.Empty(t, cleanupJob.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, "node-a", cleanupJob.Spec.Template.Spec.NodeName) } @@ -468,7 +615,7 @@ func TestBuildCommitJob_SetsBoundedBackoffLimit(t *testing.T) { r := newTestSnapshotReconciler(snapshot) r.SnapshotPushSecret = "registry-snapshot-push-secret" - job, err := r.buildCommitJob(snapshot) + job, err := r.buildCommitJob(snapshot, "") require.NoError(t, err) require.NotNil(t, job.Spec.BackoffLimit) assert.Equal(t, DefaultCommitJobBackoffLimit, *job.Spec.BackoffLimit) @@ -495,10 +642,35 @@ func TestBuildCommitJob_ExecutesImageCommitterDirectlyWithIsolatedArgs(t *testin r := newTestSnapshotReconciler(snapshot) r.SnapshotRegistryInsecure = true + r.ImageCommitterPodTemplate = &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"identity.example/use": "true"}, + Annotations: map[string]string{"example.com/template": "enabled"}, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "snapshot-committer", + NodeName: "must-be-overridden", + RestartPolicy: corev1.RestartPolicyAlways, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptrToBool(true)}, + Tolerations: []corev1.Toleration{{Key: "snapshot", Operator: corev1.TolerationOpExists}}, + Containers: []corev1.Container{ + { + Name: CommitJobContainerName, + Image: "must-be-overridden", + Command: []string{"must-be-overridden"}, + Env: []corev1.EnvVar{ + {Name: "CUSTOM_ENV", Value: "custom"}, + {Name: "SOURCE_POD_UID", Value: "must-be-overridden"}, + }, + }, + {Name: "audit-sidecar", Image: "example.com/audit:latest"}, + }, + }, + } - job, err := r.buildCommitJob(snapshot) + job, err := r.buildCommitJob(snapshot, "pod-uid") require.NoError(t, err) - require.Len(t, job.Spec.Template.Spec.Containers, 1) + require.Len(t, job.Spec.Template.Spec.Containers, 2) container := job.Spec.Template.Spec.Containers[0] assert.Equal(t, []string{"/usr/local/bin/image-committer"}, container.Command) @@ -507,8 +679,19 @@ func TestBuildCommitJob_ExecutesImageCommitterDirectlyWithIsolatedArgs(t *testin "default", "main;echo nope:registry.example.com/test:tag", }, container.Args) + assert.Contains(t, container.Env, corev1.EnvVar{Name: "SOURCE_POD_UID", Value: "pod-uid"}) assert.Contains(t, container.Env, corev1.EnvVar{Name: "SNAPSHOT_REGISTRY_INSECURE", Value: "true"}) + assert.Equal(t, "snapshot-committer", job.Spec.Template.Spec.ServiceAccountName) + assert.Equal(t, "node-1", job.Spec.Template.Spec.NodeName) + assert.Equal(t, corev1.RestartPolicyNever, job.Spec.Template.Spec.RestartPolicy) + assert.Equal(t, map[string]string{"identity.example/use": "true"}, job.Spec.Template.Labels) + assert.Equal(t, map[string]string{"example.com/template": "enabled"}, job.Spec.Template.Annotations) + assert.Contains(t, container.Env, corev1.EnvVar{Name: "CUSTOM_ENV", Value: "custom"}) + assert.Equal(t, r.imageCommitterImage(), container.Image) + assert.Equal(t, []string{"/usr/local/bin/image-committer"}, container.Command) assert.Contains(t, container.VolumeMounts, corev1.VolumeMount{Name: "containerd-fifo", MountPath: ContainerdFIFODir}) + assert.Contains(t, job.Spec.Template.Spec.Tolerations, corev1.Toleration{Key: "snapshot", Operator: corev1.TolerationOpExists}) + assert.Equal(t, "audit-sidecar", job.Spec.Template.Spec.Containers[1].Name) var fifoVolume *corev1.Volume for i := range job.Spec.Template.Spec.Volumes { @@ -523,7 +706,14 @@ func TestBuildCommitJob_ExecutesImageCommitterDirectlyWithIsolatedArgs(t *testin require.NotNil(t, fifoVolume.HostPath.Type) assert.Equal(t, corev1.HostPathDirectoryOrCreate, *fifoVolume.HostPath.Type) + require.NotNil(t, job.Spec.Template.Spec.SecurityContext) + require.NotNil(t, job.Spec.Template.Spec.SecurityContext.RunAsNonRoot) + assert.True(t, *job.Spec.Template.Spec.SecurityContext.RunAsNonRoot) require.NotNil(t, container.SecurityContext) + require.NotNil(t, container.SecurityContext.RunAsUser) + assert.Zero(t, *container.SecurityContext.RunAsUser) + require.NotNil(t, container.SecurityContext.RunAsNonRoot) + assert.False(t, *container.SecurityContext.RunAsNonRoot) require.NotNil(t, container.SecurityContext.AllowPrivilegeEscalation) assert.False(t, *container.SecurityContext.AllowPrivilegeEscalation) require.NotNil(t, container.SecurityContext.Capabilities) diff --git a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go index eaf41596c..94a94ed22 100644 --- a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go +++ b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go @@ -95,7 +95,7 @@ func (r *SandboxSnapshotReconciler) handlePending(ctx context.Context, snapshot snapshot.Status.SourceNodeName = sourceNodeName snapshot.Status.Containers = containers - job, err := r.buildCommitJob(snapshot) + job, err := r.buildCommitJob(snapshot, string(pod.UID)) if err != nil { msg := fmt.Sprintf("failed to build commit job: %v", err) _ = r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseFailed, "BuildCommitJobFailed", msg) @@ -153,7 +153,7 @@ func (r *SandboxSnapshotReconciler) handleCommitting(ctx context.Context, snapsh message = failedCond.Message } log.Info("Commit job failed", "job", jobName, "message", message) - if err := r.ensureUnpauseJob(ctx, snapshot); err != nil { + if err := r.ensureUnpauseJob(ctx, snapshot, imageCommitterEnvValue(job, "SOURCE_POD_UID")); err != nil { log.Error(err, "Failed to create best-effort unpause job") } r.Recorder.Eventf(snapshot, corev1.EventTypeWarning, "JobFailed", "Commit job failed") @@ -174,28 +174,24 @@ func findJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.J return nil } -// handleDeletion cleans up the commit job and removes the finalizer. +// handleDeletion stops snapshot jobs, cleans up images, then removes the finalizer. func (r *SandboxSnapshotReconciler) handleDeletion(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { log := logf.FromContext(ctx) - jobName := r.getJobName(snapshot) - job := &batchv1.Job{} - if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: jobName}, job); err == nil { - if deleteErr := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); deleteErr != nil && !errors.IsNotFound(deleteErr) { - return ctrl.Result{}, deleteErr - } - log.Info("Deleted commit job", "job", jobName) + jobsPending, err := r.deleteSnapshotJobs(ctx, snapshot) + if err != nil { + return ctrl.Result{}, err + } + if jobsPending { + return ctrl.Result{RequeueAfter: time.Second}, nil } - unpauseJobName := r.getUnpauseJobName(snapshot) - unpauseJob := &batchv1.Job{} - if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: unpauseJobName}, unpauseJob); err == nil { - if deleteErr := r.Delete(ctx, unpauseJob, client.PropagationPolicy(metav1.DeletePropagationBackground)); deleteErr != nil && !errors.IsNotFound(deleteErr) { - return ctrl.Result{}, deleteErr - } - log.Info("Deleted unpause job", "job", unpauseJobName) + if err := r.deleteSnapshotImages(ctx, snapshot); err != nil { + return ctrl.Result{}, err } + log.Info("Deleted snapshot registry images") + if controllerutil.ContainsFinalizer(snapshot, SandboxSnapshotFinalizer) { if err := utils.UpdateFinalizer(r.Client, snapshot, utils.RemoveFinalizerOpType, SandboxSnapshotFinalizer); err != nil { return ctrl.Result{}, err @@ -204,6 +200,75 @@ func (r *SandboxSnapshotReconciler) handleDeletion(ctx context.Context, snapshot return ctrl.Result{}, nil } +// deleteSnapshotJobs requests foreground deletion for both jobs and waits for +// their owned Pods to disappear before registry cleanup. This closes the race +// where a commit Job can finish pushing after a tag was observed as missing. +func (r *SandboxSnapshotReconciler) deleteSnapshotJobs(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (bool, error) { + pending := false + for _, jobName := range []string{r.getJobName(snapshot), r.getUnpauseJobName(snapshot)} { + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: jobName}, job) + switch { + case err == nil: + pending = true + if job.DeletionTimestamp.IsZero() { + if deleteErr := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); deleteErr != nil && !errors.IsNotFound(deleteErr) { + return false, deleteErr + } + } + case errors.IsNotFound(err): + default: + return false, err + } + + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(snapshot.Namespace), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + if len(pods.Items) > 0 { + pending = true + } + } + return pending, nil +} + +func (r *SandboxSnapshotReconciler) deleteSnapshotImages(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) error { + if len(snapshot.Status.Containers) == 0 { + return nil + } + + var registrySecret *corev1.Secret + if r.SnapshotPushSecret != "" { + registrySecret = &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: r.SnapshotPushSecret}, registrySecret); err != nil { + return fmt.Errorf("get snapshot registry secret %s/%s: %w", snapshot.Namespace, r.SnapshotPushSecret, err) + } + } + + deleter := r.registryImageDeleter + if deleter == nil { + deleter = remoteRegistryImageDeleter{} + } + deleted := make(map[string]struct{}, len(snapshot.Status.Containers)) + for _, container := range snapshot.Status.Containers { + if container.ImageURI == "" { + continue + } + imageReference := container.ImageURI + if container.ImageDigest != "" { + imageReference += "@" + container.ImageDigest + } + if _, exists := deleted[imageReference]; exists { + continue + } + if err := deleter.Delete(ctx, container.ImageURI, container.ImageDigest, registrySecret, r.SnapshotRegistryInsecure); err != nil { + return fmt.Errorf("delete snapshot image for container %s: %w", container.ContainerName, err) + } + deleted[imageReference] = struct{}{} + } + return nil +} + // findPodForSandbox finds the running pod belonging to a BatchSandbox. func (r *SandboxSnapshotReconciler) findPodForSandbox(ctx context.Context, bs *sandboxv1alpha1.BatchSandbox, namespace string) (*corev1.Pod, error) { alloc, err := parseSandboxAllocation(bs) @@ -322,6 +387,7 @@ func (r *SandboxSnapshotReconciler) imageCommitterPullSecrets() []corev1.LocalOb func commitJobSecurityContext() *corev1.SecurityContext { return &corev1.SecurityContext{ RunAsUser: ptrToInt64(0), + RunAsNonRoot: ptrToBool(false), AllowPrivilegeEscalation: ptrToBool(false), Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{"ALL"}, @@ -329,7 +395,7 @@ func commitJobSecurityContext() *corev1.SecurityContext { } } -func (r *SandboxSnapshotReconciler) buildCommitJob(snapshot *sandboxv1alpha1.SandboxSnapshot) (*batchv1.Job, error) { +func (r *SandboxSnapshotReconciler) buildCommitJob(snapshot *sandboxv1alpha1.SandboxSnapshot, sourcePodUID string) (*batchv1.Job, error) { jobName := r.getJobName(snapshot) imageCommitterImage := r.imageCommitterImage() @@ -376,6 +442,9 @@ func (r *SandboxSnapshotReconciler) buildCommitJob(snapshot *sandboxv1alpha1.San } args := append([]string{snapshot.Status.SourcePodName, snapshot.Namespace}, containerSpecs...) env := []corev1.EnvVar{{Name: "CONTAINERD_SOCKET", Value: ContainerdSocketPath}} + if sourcePodUID != "" { + env = append(env, corev1.EnvVar{Name: "SOURCE_POD_UID", Value: sourcePodUID}) + } if r.SnapshotRegistryInsecure { env = append(env, corev1.EnvVar{Name: "SNAPSHOT_REGISTRY_INSECURE", Value: "true"}) } @@ -416,13 +485,153 @@ func (r *SandboxSnapshotReconciler) buildCommitJob(snapshot *sandboxv1alpha1.San }, } + if err := r.applyImageCommitterPodTemplate(&job.Spec.Template); err != nil { + return nil, err + } if err := ctrl.SetControllerReference(snapshot, job, r.Scheme); err != nil { return nil, fmt.Errorf("failed to set controller reference: %w", err) } return job, nil } -func (r *SandboxSnapshotReconciler) ensureUnpauseJob(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) error { +func (r *SandboxSnapshotReconciler) applyImageCommitterPodTemplate(generated *corev1.PodTemplateSpec) error { + if generated == nil { + return fmt.Errorf("generated image-committer Pod template is required") + } + + var overlay *corev1.PodTemplateSpec + if r.ImageCommitterPodTemplate != nil { + overlay = r.ImageCommitterPodTemplate.DeepCopy() + } else { + overlay = &corev1.PodTemplateSpec{} + } + + generated.Labels = mergeStringMaps(overlay.Labels, generated.Labels) + generated.Annotations = mergeStringMaps(overlay.Annotations, generated.Annotations) + + generatedContainer := generated.Spec.Containers[0] + commitContainer := corev1.Container{Name: CommitJobContainerName} + commitCount := 0 + containers := make([]corev1.Container, 0, len(overlay.Spec.Containers)+1) + for _, container := range overlay.Spec.Containers { + if container.Name != CommitJobContainerName { + containers = append(containers, container) + continue + } + commitCount++ + commitContainer = container + } + if commitCount > 1 { + return fmt.Errorf("image-committer Pod template contains multiple %q containers", CommitJobContainerName) + } + + commitContainer.Name = generatedContainer.Name + commitContainer.Image = generatedContainer.Image + commitContainer.ImagePullPolicy = generatedContainer.ImagePullPolicy + commitContainer.Command = generatedContainer.Command + commitContainer.Args = generatedContainer.Args + commitContainer.Env = mergeEnvVars(commitContainer.Env, generatedContainer.Env) + commitContainer.VolumeMounts = mergeVolumeMounts(commitContainer.VolumeMounts, generatedContainer.VolumeMounts) + commitContainer.SecurityContext = generatedContainer.SecurityContext + commitContainer.TerminationMessagePath = "/dev/termination-log" + commitContainer.TerminationMessagePolicy = corev1.TerminationMessageReadFile + containers = append([]corev1.Container{commitContainer}, containers...) + + overlay.Spec.Containers = containers + overlay.Spec.Volumes = mergeVolumes(overlay.Spec.Volumes, generated.Spec.Volumes) + overlay.Spec.ImagePullSecrets = mergeLocalObjectReferences(overlay.Spec.ImagePullSecrets, generated.Spec.ImagePullSecrets) + overlay.Spec.RestartPolicy = generated.Spec.RestartPolicy + overlay.Spec.NodeName = generated.Spec.NodeName + + generated.Spec = overlay.Spec + return nil +} + +func mergeStringMaps(maps ...map[string]string) map[string]string { + var result map[string]string + for _, values := range maps { + for key, value := range values { + if result == nil { + result = make(map[string]string) + } + result[key] = value + } + } + return result +} + +func mergeEnvVars(base, required []corev1.EnvVar) []corev1.EnvVar { + result := append([]corev1.EnvVar(nil), base...) + for _, value := range required { + replaced := false + for i := range result { + if result[i].Name == value.Name { + result[i] = value + replaced = true + break + } + } + if !replaced { + result = append(result, value) + } + } + return result +} + +func mergeVolumeMounts(base, required []corev1.VolumeMount) []corev1.VolumeMount { + result := append([]corev1.VolumeMount(nil), base...) + for _, value := range required { + replaced := false + for i := range result { + if result[i].Name == value.Name { + result[i] = value + replaced = true + break + } + } + if !replaced { + result = append(result, value) + } + } + return result +} + +func mergeVolumes(base, required []corev1.Volume) []corev1.Volume { + result := append([]corev1.Volume(nil), base...) + for _, value := range required { + replaced := false + for i := range result { + if result[i].Name == value.Name { + result[i] = value + replaced = true + break + } + } + if !replaced { + result = append(result, value) + } + } + return result +} + +func mergeLocalObjectReferences(base, required []corev1.LocalObjectReference) []corev1.LocalObjectReference { + result := append([]corev1.LocalObjectReference(nil), base...) + for _, value := range required { + found := false + for _, existing := range result { + if existing.Name == value.Name { + found = true + break + } + } + if !found { + result = append(result, value) + } + } + return result +} + +func (r *SandboxSnapshotReconciler) ensureUnpauseJob(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot, sourcePodUID string) error { if snapshot.Status.SourcePodName == "" || snapshot.Status.SourceNodeName == "" || len(snapshot.Status.Containers) == 0 { return nil } @@ -435,19 +644,23 @@ func (r *SandboxSnapshotReconciler) ensureUnpauseJob(ctx context.Context, snapsh return err } - job, err := r.buildUnpauseJob(snapshot) + job, err := r.buildUnpauseJob(snapshot, sourcePodUID) if err != nil { return err } return r.Create(ctx, job) } -func (r *SandboxSnapshotReconciler) buildUnpauseJob(snapshot *sandboxv1alpha1.SandboxSnapshot) (*batchv1.Job, error) { +func (r *SandboxSnapshotReconciler) buildUnpauseJob(snapshot *sandboxv1alpha1.SandboxSnapshot, sourcePodUID string) (*batchv1.Job, error) { var containerNames []string for _, cs := range snapshot.Status.Containers { containerNames = append(containerNames, cs.ContainerName) } args := append([]string{"unpause", snapshot.Status.SourcePodName, snapshot.Namespace}, containerNames...) + env := []corev1.EnvVar{{Name: "CONTAINERD_SOCKET", Value: ContainerdSocketPath}} + if sourcePodUID != "" { + env = append(env, corev1.EnvVar{Name: "SOURCE_POD_UID", Value: sourcePodUID}) + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -476,9 +689,7 @@ func (r *SandboxSnapshotReconciler) buildUnpauseJob(snapshot *sandboxv1alpha1.Sa VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-sock", MountPath: ContainerdSocketPath}, }, - Env: []corev1.EnvVar{ - {Name: "CONTAINERD_SOCKET", Value: ContainerdSocketPath}, - }, + Env: env, SecurityContext: commitJobSecurityContext(), }, }, @@ -589,6 +800,23 @@ func snapshotResultFromPod(pod *corev1.Pod) (*commitJobResult, bool, error) { return nil, false, nil } +func imageCommitterEnvValue(job *batchv1.Job, name string) string { + if job == nil { + return "" + } + for _, container := range job.Spec.Template.Spec.Containers { + if container.Name != CommitJobContainerName { + continue + } + for _, env := range container.Env { + if env.Name == name { + return env.Value + } + } + } + return "" +} + func (r *SandboxSnapshotReconciler) getJobName(snapshot *sandboxv1alpha1.SandboxSnapshot) string { return fmt.Sprintf("%s-commit", snapshot.Name) } diff --git a/kubernetes/internal/controller/suite_test.go b/kubernetes/internal/controller/suite_test.go index abdb549af..027de5781 100644 --- a/kubernetes/internal/controller/suite_test.go +++ b/kubernetes/internal/controller/suite_test.go @@ -29,6 +29,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -86,7 +87,8 @@ var _ = BeforeSuite(func() { Expect(cfg).NotTo(BeNil()) k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, }) Expect(err).ToNot(HaveOccurred()) By("register field index") diff --git a/kubernetes/pkg/imagecommitter/cli/cli.go b/kubernetes/pkg/imagecommitter/cli/cli.go new file mode 100644 index 000000000..dd9754499 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/cli/cli.go @@ -0,0 +1,214 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + containerd "github.com/containerd/containerd" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/imagecommitter" +) + +const ( + defaultContainerdSocket = "/run/containerd/containerd.sock" + defaultContainerdNamespace = "k8s.io" +) + +// Config supplies implementation-specific dependencies while preserving the +// common commit and unpause CLI contract. +type Config struct { + CredentialProvider imagecommitter.CredentialProvider + SourceCredentialProvider imagecommitter.CredentialProvider + TerminationMessagePath string + Output io.Writer + ErrorOutput io.Writer +} + +// Run executes a commit or unpause operation. +func Run(ctx context.Context, args []string, config Config) error { + if config.Output == nil { + config.Output = io.Discard + } + if config.ErrorOutput == nil { + config.ErrorOutput = io.Discard + } + operation, commitRequest, unpauseRequest, err := parseOperation(args) + if err != nil { + return err + } + + client, err := containerd.New( + containerdSocket(), + containerd.WithDefaultNamespace(containerdNamespace()), + ) + if err != nil { + return fmt.Errorf("connect to containerd: %w", err) + } + defer client.Close() + + runtime := imagecommitter.NewContainerdRuntime(client) + orchestrator := &imagecommitter.Orchestrator{ + Runtime: runtime, + Executor: runtime, + Output: config.Output, + ErrorOutput: config.ErrorOutput, + } + if operation == "unpause" { + return orchestrator.Unpause(ctx, unpauseRequest) + } + + // Preserve the existing best-effort preparation behavior. It remains an + // implementation detail and is not part of the executable contract. + orchestrator.PreparationCommand = []string{"sync"} + orchestrator.Builder = imagecommitter.NewContainerdImageBuilder( + client, + config.SourceCredentialProvider, + func(source string) bool { return shouldUseInsecureSourceRegistry(source, config.ErrorOutput) }, + ) + orchestrator.Pusher = imagecommitter.NewContainerdImagePusher( + client, + config.CredentialProvider, + func(target string) bool { return shouldUseInsecureRegistry(target, config.ErrorOutput) }, + ) + result, err := orchestrator.Commit(ctx, commitRequest) + if err != nil { + return err + } + if err := writeResult(config.TerminationMessagePath, result); err != nil { + return fmt.Errorf("write snapshot result: %w", err) + } + for _, container := range result.Containers { + name := strings.ToUpper(strings.ReplaceAll(container.Name, "-", "_")) + fmt.Fprintf(config.Output, "SNAPSHOT_DIGEST_%s=%s\n", name, container.Digest) + } + if len(result.Containers) > 0 { + fmt.Fprintf(config.Output, "SNAPSHOT_DIGEST=%s\n", result.Containers[0].Digest) + } + return nil +} + +func parseOperation(args []string) (string, imagecommitter.CommitRequest, imagecommitter.UnpauseRequest, error) { + podUID := strings.TrimSpace(os.Getenv("SOURCE_POD_UID")) + if len(args) > 0 && args[0] == "unpause" { + if len(args) < 4 { + return "", imagecommitter.CommitRequest{}, imagecommitter.UnpauseRequest{}, errors.New("usage: image-committer unpause [container_name...]") + } + return "unpause", imagecommitter.CommitRequest{}, imagecommitter.UnpauseRequest{ + PodName: args[1], + Namespace: args[2], + PodUID: podUID, + ContainerNames: append([]string(nil), args[3:]...), + }, nil + } + if len(args) < 3 { + return "", imagecommitter.CommitRequest{}, imagecommitter.UnpauseRequest{}, errors.New("usage: image-committer : [:...]") + } + request := imagecommitter.CommitRequest{PodName: args[0], Namespace: args[1], PodUID: podUID} + for _, raw := range args[2:] { + spec, err := parseContainerSpec(raw) + if err != nil { + return "", imagecommitter.CommitRequest{}, imagecommitter.UnpauseRequest{}, err + } + request.Containers = append(request.Containers, spec) + } + if request.PodName == "" || request.Namespace == "" { + return "", imagecommitter.CommitRequest{}, imagecommitter.UnpauseRequest{}, errors.New("pod name and namespace are required") + } + return "commit", request, imagecommitter.UnpauseRequest{}, nil +} + +func parseContainerSpec(raw string) (imagecommitter.ContainerSpec, error) { + parts := strings.SplitN(raw, ":", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return imagecommitter.ContainerSpec{}, fmt.Errorf("invalid container spec %q; expected container_name:target_image", raw) + } + return imagecommitter.ContainerSpec{Name: parts[0], Target: parts[1]}, nil +} + +func writeResult(path string, result imagecommitter.Result) error { + if path == "" { + return errors.New("termination message path is required") + } + data, err := json.Marshal(result) + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o644) +} + +func containerdSocket() string { + if value := strings.TrimSpace(os.Getenv("CONTAINERD_SOCKET")); value != "" { + return value + } + return defaultContainerdSocket +} + +func containerdNamespace() string { + if value := strings.TrimSpace(os.Getenv("CONTAINERD_NAMESPACE")); value != "" { + return value + } + return defaultContainerdNamespace +} + +func shouldUseInsecureRegistry(targetImage string, errorOutput io.Writer) bool { + return shouldUseInsecureRegistryEnv(targetImage, "SNAPSHOT_REGISTRY_INSECURE", errorOutput) +} + +func shouldUseInsecureSourceRegistry(_ string, errorOutput io.Writer) bool { + raw := strings.TrimSpace(os.Getenv("SOURCE_IMAGE_REGISTRY_INSECURE")) + if raw == "" { + return false + } + value, err := strconv.ParseBool(raw) + if err != nil { + fmt.Fprintf(errorOutput, "WARNING: invalid SOURCE_IMAGE_REGISTRY_INSECURE=%q; using secure transport\n", raw) + return false + } + return value +} + +func shouldUseInsecureRegistryEnv(imageReference, environmentVariable string, errorOutput io.Writer) bool { + if raw := strings.TrimSpace(os.Getenv(environmentVariable)); raw != "" { + value, err := strconv.ParseBool(raw) + if err == nil { + return value + } + fmt.Fprintf(errorOutput, "WARNING: invalid %s=%q; using host compatibility heuristic\n", environmentVariable, raw) + } + registryHost := strings.SplitN(imageReference, "/", 2)[0] + return strings.Contains(registryHost, "local") || + strings.HasPrefix(registryHost, "127.") || + strings.HasPrefix(registryHost, "10.") || + strings.HasPrefix(registryHost, "192.168.") || + isPrivate172Registry(registryHost) +} + +func isPrivate172Registry(registryHost string) bool { + host := strings.SplitN(registryHost, ":", 2)[0] + parts := strings.Split(host, ".") + if len(parts) < 2 || parts[0] != "172" { + return false + } + secondOctet, err := strconv.Atoi(parts[1]) + return err == nil && secondOctet >= 16 && secondOctet <= 31 +} diff --git a/kubernetes/pkg/imagecommitter/cli/cli_test.go b/kubernetes/pkg/imagecommitter/cli/cli_test.go new file mode 100644 index 000000000..185bb4c1b --- /dev/null +++ b/kubernetes/pkg/imagecommitter/cli/cli_test.go @@ -0,0 +1,150 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/imagecommitter" +) + +func TestParseCommitOperation(t *testing.T) { + t.Setenv("SOURCE_POD_UID", "pod-uid") + operation, request, _, err := parseOperation([]string{ + "pod-1", + "default", + "main:registry.example.com:5000/snapshots/main:snap", + "sidecar:registry.example.com/snapshots/sidecar:snap", + }) + if err != nil { + t.Fatalf("parseOperation failed: %v", err) + } + if operation != "commit" { + t.Fatalf("operation = %q, want commit", operation) + } + if request.PodName != "pod-1" || request.Namespace != "default" || request.PodUID != "pod-uid" { + t.Fatalf("unexpected request identity: %#v", request) + } + if len(request.Containers) != 2 { + t.Fatalf("container count = %d, want 2", len(request.Containers)) + } + if got := request.Containers[0].Target; got != "registry.example.com:5000/snapshots/main:snap" { + t.Fatalf("target = %q", got) + } +} + +func TestParseUnpauseOperation(t *testing.T) { + operation, _, request, err := parseOperation([]string{"unpause", "pod-1", "default", "main", "sidecar"}) + if err != nil { + t.Fatalf("parseOperation failed: %v", err) + } + if operation != "unpause" { + t.Fatalf("operation = %q, want unpause", operation) + } + if len(request.ContainerNames) != 2 || request.ContainerNames[1] != "sidecar" { + t.Fatalf("unexpected container names: %v", request.ContainerNames) + } +} + +func TestParseOperationRejectsInvalidInput(t *testing.T) { + for _, args := range [][]string{ + nil, + {"pod", "namespace"}, + {"pod", "namespace", "invalid"}, + {"unpause", "pod", "namespace"}, + } { + if _, _, _, err := parseOperation(args); err == nil { + t.Fatalf("parseOperation(%v) unexpectedly succeeded", args) + } + } +} + +func TestWriteResult(t *testing.T) { + terminationMessagePath := filepath.Join(t.TempDir(), "termination.log") + + want := imagecommitter.Result{Containers: []imagecommitter.ContainerResult{ + {Name: "main", Image: "registry.example.com/main:snap", Digest: "sha256:main"}, + {Name: "sidecar", Image: "registry.example.com/sidecar:snap", Digest: "sha256:sidecar"}, + }} + if err := writeResult(terminationMessagePath, want); err != nil { + t.Fatalf("writeResult failed: %v", err) + } + data, err := os.ReadFile(terminationMessagePath) + if err != nil { + t.Fatalf("read termination result: %v", err) + } + var got imagecommitter.Result + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("decode termination result: %v", err) + } + if len(got.Containers) != 2 || got.Containers[0] != want.Containers[0] || got.Containers[1] != want.Containers[1] { + t.Fatalf("unexpected result: %#v", got) + } +} + +func TestShouldUseInsecureRegistry(t *testing.T) { + t.Run("explicit false overrides heuristic", func(t *testing.T) { + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "false") + if shouldUseInsecureRegistry("registry.local/snapshot:test", os.Stderr) { + t.Fatal("explicit false should disable insecure transport") + } + }) + t.Run("private host heuristic", func(t *testing.T) { + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "") + if !shouldUseInsecureRegistry("10.0.0.2:5000/snapshot:test", os.Stderr) { + t.Fatal("private registry should use compatibility heuristic") + } + }) + t.Run("source policy is independent from snapshot target", func(t *testing.T) { + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "true") + t.Setenv("SOURCE_IMAGE_REGISTRY_INSECURE", "false") + if shouldUseInsecureSourceRegistry("registry.example.com/source:test", os.Stderr) { + t.Fatal("source registry must not inherit the snapshot target policy") + } + }) + t.Run("source defaults to secure transport", func(t *testing.T) { + t.Setenv("SOURCE_IMAGE_REGISTRY_INSECURE", "") + for _, source := range []string{ + "registry.local/source:test", + "10.0.0.2:5000/source:test", + "172.16.0.2:5000/source:test", + "192.168.0.2:5000/source:test", + } { + if shouldUseInsecureSourceRegistry(source, os.Stderr) { + t.Fatalf("source registry %q should use secure transport by default", source) + } + } + }) + t.Run("explicit source insecure", func(t *testing.T) { + t.Setenv("SOURCE_IMAGE_REGISTRY_INSECURE", "true") + if !shouldUseInsecureSourceRegistry("registry.example.com/source:test", os.Stderr) { + t.Fatal("explicit source policy should enable insecure transport") + } + }) + t.Run("invalid source policy fails closed", func(t *testing.T) { + t.Setenv("SOURCE_IMAGE_REGISTRY_INSECURE", "invalid") + var warnings bytes.Buffer + if shouldUseInsecureSourceRegistry("registry.local/source:test", &warnings) { + t.Fatal("invalid source policy should use secure transport") + } + if warnings.Len() == 0 { + t.Fatal("invalid source policy should emit a warning") + } + }) +} diff --git a/kubernetes/pkg/imagecommitter/cli/doc.go b/kubernetes/pkg/imagecommitter/cli/doc.go new file mode 100644 index 000000000..2165353f8 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/cli/doc.go @@ -0,0 +1,18 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cli implements the controller-to-committer executable contract. +// Provider-specific binaries can call Run with their CredentialProvider while +// retaining the standard commit and unpause arguments and result format. +package cli diff --git a/kubernetes/pkg/imagecommitter/doc.go b/kubernetes/pkg/imagecommitter/doc.go new file mode 100644 index 000000000..cbe7e1924 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/doc.go @@ -0,0 +1,19 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imagecommitter defines the reusable contracts and default containerd +// implementations used to commit sandbox container filesystems to OCI images. +// Cloud-specific committers can provide a CredentialProvider and reuse the +// common orchestrator, containerd runtime, image builder, and image pusher. +package imagecommitter diff --git a/kubernetes/pkg/imagecommitter/image_containerd.go b/kubernetes/pkg/imagecommitter/image_containerd.go new file mode 100644 index 000000000..5efe57459 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/image_containerd.go @@ -0,0 +1,310 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + containerd "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/rootfs" + "github.com/containerd/errdefs" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ContainerdImageBuilder assembles OCI image content from writable snapshots. +type ContainerdImageBuilder struct { + client *containerd.Client + sourceCredentials CredentialProvider + sourceInsecure InsecureRegistryFunc +} + +// NewContainerdImageBuilder creates a builder with the credential and transport +// policies used to recover missing source-image content. The provider may +// return empty credentials for registries that permit anonymous pulls. +func NewContainerdImageBuilder(client *containerd.Client, sourceCredentials CredentialProvider, sourceInsecure InsecureRegistryFunc) *ContainerdImageBuilder { + return &ContainerdImageBuilder{ + client: client, + sourceCredentials: sourceCredentials, + sourceInsecure: sourceInsecure, + } +} + +func (b *ContainerdImageBuilder) Commit(ctx context.Context, container ResolvedContainer, target string) (LocalImage, error) { + if container.Snapshotter == "" || container.SnapshotKey == "" { + return LocalImage{}, fmt.Errorf("container %s has no writable snapshot metadata", container.ID) + } + + leaseCtx, done, err := b.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(time.Hour)) + if err != nil { + return LocalImage{}, fmt.Errorf("create containerd lease: %w", err) + } + defer done(leaseCtx) + + c, err := b.client.LoadContainer(leaseCtx, container.ID) + if err != nil { + return LocalImage{}, fmt.Errorf("load container %s: %w", container.ID, err) + } + baseImage, err := c.Image(leaseCtx) + if err != nil { + return LocalImage{}, fmt.Errorf("load source image for container %s: %w", container.ID, err) + } + store := b.client.ContentStore() + if err := b.ensureBaseImageContent(leaseCtx, baseImage); err != nil { + return LocalImage{}, fmt.Errorf("recover source image content for container %s: %w", container.ID, err) + } + // Commit Jobs are pinned to the source sandbox's node and use that node's + // containerd socket, so under the supported native execution model the + // committer process platform matches the platform selected for the source + // container. Cross-architecture emulation is not a supported snapshot mode. + // ensureBaseImageContent intentionally uses the same platform matcher. + baseManifest, err := images.Manifest(leaseCtx, store, baseImage.Target(), platforms.Default()) + if err != nil { + return LocalImage{}, fmt.Errorf("read source manifest for container %s: %w", container.ID, err) + } + configData, err := content.ReadBlob(leaseCtx, store, baseManifest.Config) + if err != nil { + return LocalImage{}, fmt.Errorf("read source config for container %s: %w", container.ID, err) + } + var imageConfig ocispec.Image + if err := json.Unmarshal(configData, &imageConfig); err != nil { + return LocalImage{}, fmt.Errorf("decode source config for container %s: %w", container.ID, err) + } + + mediaTypes := commitMediaTypes(baseManifest.MediaType) + diffDesc, err := rootfs.CreateDiff( + leaseCtx, + container.SnapshotKey, + b.client.SnapshotService(container.Snapshotter), + b.client.DiffService(), + diff.WithReference(fmt.Sprintf("opensandbox-commit-%s-%d", container.ID, time.Now().UnixNano())), + diff.WithMediaType(mediaTypes.Diff), + ) + if err != nil { + return LocalImage{}, fmt.Errorf("create writable snapshot diff for container %s: %w", container.ID, err) + } + diffInfo, err := store.Info(leaseCtx, diffDesc.Digest) + if err != nil { + return LocalImage{}, fmt.Errorf("inspect diff content for container %s: %w", container.ID, err) + } + diffIDValue := diffInfo.Labels["containerd.io/uncompressed"] + if diffIDValue == "" { + return LocalImage{}, fmt.Errorf("diff for container %s has no uncompressed digest", container.ID) + } + diffID, err := digest.Parse(diffIDValue) + if err != nil { + return LocalImage{}, fmt.Errorf("parse diff ID for container %s: %w", container.ID, err) + } + // The comparer can report an OCI layer descriptor even when the source image + // uses Docker media types. The bytes are compatible; the manifest descriptor + // must use the selected image format. + diffDesc.MediaType = mediaTypes.Layer + + now := time.Now().UTC() + imageConfig.Created = &now + imageConfig.RootFS.Type = "layers" + imageConfig.RootFS.DiffIDs = append(imageConfig.RootFS.DiffIDs, diffID) + imageConfig.History = append(imageConfig.History, ocispec.History{ + Created: &now, + CreatedBy: "OpenSandbox image committer", + }) + newConfigData, err := json.Marshal(imageConfig) + if err != nil { + return LocalImage{}, fmt.Errorf("encode committed image config: %w", err) + } + configDesc := ocispec.Descriptor{ + MediaType: mediaTypes.Config, + Digest: digest.FromBytes(newConfigData), + Size: int64(len(newConfigData)), + } + if err := content.WriteBlob( + leaseCtx, + store, + "opensandbox-config-"+configDesc.Digest.String(), + bytes.NewReader(newConfigData), + configDesc, + ); err != nil { + return LocalImage{}, fmt.Errorf("write committed image config: %w", err) + } + + layers := append([]ocispec.Descriptor(nil), baseManifest.Layers...) + layers = append(layers, diffDesc) + newManifest := ocispec.Manifest{ + Versioned: baseManifest.Versioned, + MediaType: mediaTypes.Manifest, + ArtifactType: baseManifest.ArtifactType, + Config: configDesc, + Layers: layers, + Subject: baseManifest.Subject, + Annotations: baseManifest.Annotations, + } + newManifestData, err := json.Marshal(newManifest) + if err != nil { + return LocalImage{}, fmt.Errorf("encode committed image manifest: %w", err) + } + manifestDesc := ocispec.Descriptor{ + MediaType: mediaTypes.Manifest, + Digest: digest.FromBytes(newManifestData), + Size: int64(len(newManifestData)), + } + gcLabels := map[string]string{"containerd.io/gc.ref.content.0": configDesc.Digest.String()} + for i, layer := range layers { + gcLabels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = layer.Digest.String() + } + if err := content.WriteBlob( + leaseCtx, + store, + "opensandbox-manifest-"+manifestDesc.Digest.String(), + bytes.NewReader(newManifestData), + manifestDesc, + content.WithLabels(gcLabels), + ); err != nil { + return LocalImage{}, fmt.Errorf("write committed image manifest: %w", err) + } + + imageRecord := images.Image{Name: target, Target: manifestDesc, CreatedAt: now, UpdatedAt: now} + if _, err := b.client.ImageService().Update(leaseCtx, imageRecord); err != nil { + if !errdefs.IsNotFound(err) { + return LocalImage{}, fmt.Errorf("update target image %s: %w", target, err) + } + if _, err := b.client.ImageService().Create(leaseCtx, imageRecord); err != nil { + return LocalImage{}, fmt.Errorf("create target image %s: %w", target, err) + } + } + + return LocalImage{Reference: target, Target: manifestDesc, Config: configDesc}, nil +} + +func (b *ContainerdImageBuilder) ensureBaseImageContent(ctx context.Context, image containerd.Image) error { + return ensurePlatformContent( + ctx, + b.client.ContentStore(), + image.Target(), + platforms.Default(), + func(ctx context.Context) error { + return b.fetchBaseImageContent(ctx, image) + }, + ) +} + +func (b *ContainerdImageBuilder) fetchBaseImageContent(ctx context.Context, image containerd.Image) error { + sourceReference, err := referenceWithDigest(image.Name(), image.Target()) + if err != nil { + return err + } + host, err := registryHost(sourceReference) + if err != nil { + return err + } + credential := RegistryCredential{} + if b.sourceCredentials != nil { + credential, err = b.sourceCredentials.Credential(ctx, host) + if err != nil { + return fmt.Errorf("resolve source credentials for %s: %w", host, err) + } + } + insecure := b.sourceInsecure != nil && b.sourceInsecure(sourceReference) + if err := b.fetchBaseImageContentWithTransport(ctx, sourceReference, host, credential, "https", insecure); err != nil { + if !insecure || !shouldFallbackToPlainHTTP(err) { + return fmt.Errorf("fetch source image %s: %w", sourceReference, err) + } + if err := b.fetchBaseImageContentWithTransport(ctx, sourceReference, host, credential, "http", false); err != nil { + return fmt.Errorf("fetch source image %s over plain HTTP: %w", sourceReference, err) + } + } + return nil +} + +func (b *ContainerdImageBuilder) fetchBaseImageContentWithTransport( + ctx context.Context, + sourceReference string, + host string, + credential RegistryCredential, + scheme string, + skipVerify bool, +) error { + resolver := newDockerResolver(host, credential, scheme, skipVerify) + _, err := b.client.Fetch( + ctx, + sourceReference, + containerd.WithResolver(resolver), + containerd.WithPlatform(platforms.DefaultString()), + ) + return err +} + +func ensurePlatformContent( + ctx context.Context, + store content.Store, + target ocispec.Descriptor, + platform platforms.MatchComparer, + fetch func(context.Context) error, +) error { + _, _, _, missing, err := images.Check(ctx, store, target, platform) + if err != nil { + return fmt.Errorf("check source image content: %w", err) + } + if len(missing) == 0 { + return nil + } + if err := fetch(ctx); err != nil { + return fmt.Errorf("fetch %d missing source image blob(s): %w", len(missing), err) + } + _, _, _, missing, err = images.Check(ctx, store, target, platform) + if err != nil { + return fmt.Errorf("recheck source image content: %w", err) + } + if len(missing) > 0 { + return fmt.Errorf("source image still has %d missing blob(s) after fetch", len(missing)) + } + return nil +} + +type commitMediaTypeSet struct { + Manifest string + Config string + Layer string + Diff string +} + +func commitMediaTypes(baseManifestMediaType string) commitMediaTypeSet { + // containerd's diff service accepts the OCI compression media type. Docker + // schema 2 uses the same gzip bytes with a different descriptor media type, + // so request an OCI diff and relabel the resulting descriptor when writing a + // Docker manifest. + if baseManifestMediaType == images.MediaTypeDockerSchema2Manifest { + return commitMediaTypeSet{ + Manifest: images.MediaTypeDockerSchema2Manifest, + Config: images.MediaTypeDockerSchema2Config, + Layer: images.MediaTypeDockerSchema2LayerGzip, + Diff: ocispec.MediaTypeImageLayerGzip, + } + } + return commitMediaTypeSet{ + Manifest: ocispec.MediaTypeImageManifest, + Config: ocispec.MediaTypeImageConfig, + Layer: ocispec.MediaTypeImageLayerGzip, + Diff: ocispec.MediaTypeImageLayerGzip, + } +} diff --git a/kubernetes/pkg/imagecommitter/image_containerd_test.go b/kubernetes/pkg/imagecommitter/image_containerd_test.go new file mode 100644 index 000000000..8a2085bcf --- /dev/null +++ b/kubernetes/pkg/imagecommitter/image_containerd_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestCommitMediaTypesUsesOCIDiffForDockerManifest(t *testing.T) { + mediaTypes := commitMediaTypes(images.MediaTypeDockerSchema2Manifest) + if mediaTypes.Manifest != images.MediaTypeDockerSchema2Manifest { + t.Fatalf("manifest media type = %q", mediaTypes.Manifest) + } + if mediaTypes.Config != images.MediaTypeDockerSchema2Config { + t.Fatalf("config media type = %q", mediaTypes.Config) + } + if mediaTypes.Layer != images.MediaTypeDockerSchema2LayerGzip { + t.Fatalf("manifest layer media type = %q", mediaTypes.Layer) + } + if mediaTypes.Diff != ocispec.MediaTypeImageLayerGzip { + t.Fatalf("diff service media type = %q, want OCI gzip", mediaTypes.Diff) + } +} + +func TestCommitMediaTypesUsesOCIForOCIManifest(t *testing.T) { + mediaTypes := commitMediaTypes(ocispec.MediaTypeImageManifest) + if mediaTypes.Manifest != ocispec.MediaTypeImageManifest || + mediaTypes.Config != ocispec.MediaTypeImageConfig || + mediaTypes.Layer != ocispec.MediaTypeImageLayerGzip || + mediaTypes.Diff != ocispec.MediaTypeImageLayerGzip { + t.Fatalf("unexpected OCI media types: %#v", mediaTypes) + } +} + +func TestEnsurePlatformContentFetchesMissingSelectedPlatformBlob(t *testing.T) { + store, target, missingLayer := incompleteMultiPlatformImage(t) + fetches := 0 + err := ensurePlatformContent( + context.Background(), + store, + target, + platforms.Only(ocispec.Platform{OS: "linux", Architecture: "amd64"}), + func(ctx context.Context) error { + fetches++ + return writeTestBlob(ctx, store, missingLayer, []byte("selected-platform-layer")) + }, + ) + if err != nil { + t.Fatalf("ensurePlatformContent failed: %v", err) + } + if fetches != 1 { + t.Fatalf("fetches = %d, want 1", fetches) + } +} + +func TestEnsurePlatformContentSkipsFetchWhenSelectedPlatformIsComplete(t *testing.T) { + store, target, missingLayer := incompleteMultiPlatformImage(t) + if err := writeTestBlob(context.Background(), store, missingLayer, []byte("selected-platform-layer")); err != nil { + t.Fatal(err) + } + fetches := 0 + err := ensurePlatformContent( + context.Background(), + store, + target, + platforms.Only(ocispec.Platform{OS: "linux", Architecture: "amd64"}), + func(context.Context) error { + fetches++ + return nil + }, + ) + if err != nil { + t.Fatalf("ensurePlatformContent failed: %v", err) + } + if fetches != 0 { + t.Fatalf("fetches = %d, want 0", fetches) + } +} + +func TestEnsurePlatformContentReportsFetchFailure(t *testing.T) { + store, target, _ := incompleteMultiPlatformImage(t) + err := ensurePlatformContent( + context.Background(), + store, + target, + platforms.Only(ocispec.Platform{OS: "linux", Architecture: "amd64"}), + func(context.Context) error { return errors.New("registry unavailable") }, + ) + if err == nil || !strings.Contains(err.Error(), "registry unavailable") { + t.Fatalf("error = %v, want registry failure", err) + } +} + +func TestEnsurePlatformContentRejectsIncompleteFetch(t *testing.T) { + store, target, _ := incompleteMultiPlatformImage(t) + err := ensurePlatformContent( + context.Background(), + store, + target, + platforms.Only(ocispec.Platform{OS: "linux", Architecture: "amd64"}), + func(context.Context) error { return nil }, + ) + if err == nil || !strings.Contains(err.Error(), "still has 1 missing blob") { + t.Fatalf("error = %v, want incomplete fetch failure", err) + } +} + +func incompleteMultiPlatformImage(t *testing.T) (content.Store, ocispec.Descriptor, ocispec.Descriptor) { + t.Helper() + ctx := context.Background() + store, err := local.NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + configData := []byte(`{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`) + config := descriptorFor(ocispec.MediaTypeImageConfig, configData) + if err := writeTestBlob(ctx, store, config, configData); err != nil { + t.Fatal(err) + } + layerData := []byte("selected-platform-layer") + layer := descriptorFor(ocispec.MediaTypeImageLayerGzip, layerData) + manifestData, err := json.Marshal(ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispec.MediaTypeImageManifest, + Config: config, + Layers: []ocispec.Descriptor{layer}, + }) + if err != nil { + t.Fatal(err) + } + manifest := descriptorFor(ocispec.MediaTypeImageManifest, manifestData) + manifest.Platform = &ocispec.Platform{OS: "linux", Architecture: "amd64"} + if err := writeTestBlob(ctx, store, manifest, manifestData); err != nil { + t.Fatal(err) + } + foreignManifest := descriptorFor(ocispec.MediaTypeImageManifest, []byte("absent-arm64-manifest")) + foreignManifest.Platform = &ocispec.Platform{OS: "linux", Architecture: "arm64"} + indexData, err := json.Marshal(ocispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{manifest, foreignManifest}, + }) + if err != nil { + t.Fatal(err) + } + index := descriptorFor(ocispec.MediaTypeImageIndex, indexData) + if err := writeTestBlob(ctx, store, index, indexData); err != nil { + t.Fatal(err) + } + return store, index, layer +} + +func descriptorFor(mediaType string, data []byte) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: mediaType, Digest: digest.FromBytes(data), Size: int64(len(data))} +} + +func writeTestBlob(ctx context.Context, store content.Store, descriptor ocispec.Descriptor, data []byte) error { + return content.WriteBlob(ctx, store, "test-"+descriptor.Digest.String(), bytes.NewReader(data), descriptor) +} diff --git a/kubernetes/pkg/imagecommitter/orchestrator.go b/kubernetes/pkg/imagecommitter/orchestrator.go new file mode 100644 index 000000000..b14a2e9a2 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/orchestrator.go @@ -0,0 +1,219 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "context" + "errors" + "fmt" + "io" + "time" +) + +// Orchestrator applies the stable commit and unpause operation ordering. +type Orchestrator struct { + Runtime ContainerRuntime + Executor ContainerExecutor + Builder ImageBuilder + Pusher ImagePusher + PreparationCommand []string + Output io.Writer + ErrorOutput io.Writer +} + +func (o *Orchestrator) Commit(ctx context.Context, request CommitRequest) (Result, error) { + if err := o.validate(); err != nil { + return Result{}, err + } + if request.PodName == "" || request.Namespace == "" || len(request.Containers) == 0 { + return Result{}, errors.New("pod name, namespace, and at least one container are required") + } + + containers := make([]ResolvedContainer, 0, len(request.Containers)) + for _, spec := range request.Containers { + container, err := o.Runtime.Resolve(ctx, ContainerSelector{ + PodName: request.PodName, + PodNamespace: request.Namespace, + PodUID: request.PodUID, + ContainerName: spec.Name, + }) + if err != nil { + return Result{}, fmt.Errorf("resolve container %q: %w", spec.Name, err) + } + containers = append(containers, container) + o.logf("Resolved container %s as %s (%s)\n", spec.Name, container.ID, container.State) + } + + if o.Executor != nil && len(o.PreparationCommand) > 0 { + for _, container := range containers { + if container.State != TaskStateRunning { + continue + } + result, err := o.Executor.Exec(ctx, container, ExecRequest{Args: o.PreparationCommand}) + if err != nil { + o.errorf("WARNING: preparation command failed for container %s: %v\n", container.Name, err) + continue + } + if result.ExitCode != 0 { + o.errorf("WARNING: preparation command exited with code %d for container %s\n", result.ExitCode, container.Name) + } + } + } + + pauseHandles := make([]PauseHandle, 0, len(containers)) + cleanupComplete := false + defer func() { + if !cleanupComplete { + _ = o.resumeOwned(pauseHandles) + } + }() + for _, container := range containers { + if container.State == TaskStateStopped { + continue + } + handle, err := o.Runtime.Pause(ctx, container) + if err != nil { + // Preserve the current best-effort pause behavior. The image build can + // still succeed for a task that stopped between resolve and pause. + o.errorf("WARNING: could not pause container %s: %v\n", container.Name, err) + continue + } + if handle.PausedByUs { + pauseHandles = append(pauseHandles, handle) + } + } + + localImages := make([]LocalImage, len(containers)) + var buildErrors []error + for i, container := range containers { + image, err := o.Builder.Commit(ctx, container, request.Containers[i].Target) + if err != nil { + buildErrors = append(buildErrors, fmt.Errorf("commit container %q: %w", container.Name, err)) + continue + } + localImages[i] = image + o.logf("Committed container %s to %s\n", container.Name, image.Reference) + } + + resumeErr := o.resumeOwned(pauseHandles) + cleanupComplete = true + if len(buildErrors) > 0 || resumeErr != nil { + return Result{}, errors.Join(append(buildErrors, resumeErr)...) + } + + result := Result{Containers: make([]ContainerResult, 0, len(localImages))} + var pushErrors []error + for i, image := range localImages { + if image.Config.Digest == "" { + pushErrors = append(pushErrors, fmt.Errorf("push container %q: committed image has no config digest", request.Containers[i].Name)) + continue + } + descriptor, err := o.Pusher.Push(ctx, image) + if err != nil { + pushErrors = append(pushErrors, fmt.Errorf("push container %q: %w", request.Containers[i].Name, err)) + continue + } + result.Containers = append(result.Containers, ContainerResult{ + Name: request.Containers[i].Name, + Image: image.Reference, + Digest: image.Config.Digest.String(), + }) + o.logf("Pushed image %s (manifest %s, config %s)\n", image.Reference, descriptor.Digest, image.Config.Digest) + } + if len(pushErrors) > 0 { + return Result{}, errors.Join(pushErrors...) + } + return result, nil +} + +func (o *Orchestrator) Unpause(ctx context.Context, request UnpauseRequest) error { + if o.Runtime == nil { + return errors.New("container runtime is required") + } + if request.PodName == "" || request.Namespace == "" || len(request.ContainerNames) == 0 { + return errors.New("pod name, namespace, and at least one container are required") + } + + var errs []error + for _, name := range request.ContainerNames { + container, err := o.Runtime.Resolve(ctx, ContainerSelector{ + PodName: request.PodName, + PodNamespace: request.Namespace, + PodUID: request.PodUID, + ContainerName: name, + }) + if err != nil { + errs = append(errs, fmt.Errorf("resolve container %q: %w", name, err)) + continue + } + state, err := o.Runtime.Status(ctx, container) + if err != nil { + errs = append(errs, fmt.Errorf("inspect container %q: %w", name, err)) + continue + } + switch state { + case TaskStateRunning: + o.logf("Container %s is already running\n", name) + case TaskStateStopped: + o.logf("Container %s is stopped; no unpause is required\n", name) + case TaskStatePaused: + if err := o.Runtime.Resume(ctx, container); err != nil { + errs = append(errs, fmt.Errorf("unpause container %q: %w", name, err)) + continue + } + o.logf("Unpaused container %s\n", name) + default: + errs = append(errs, fmt.Errorf("container %q has unsupported state %s", name, state)) + } + } + return errors.Join(errs...) +} + +func (o *Orchestrator) resumeOwned(handles []PauseHandle) error { + var errs []error + for i := len(handles) - 1; i >= 0; i-- { + if !handles[i].PausedByUs { + continue + } + // Give every container an independent cleanup window so one stuck task + // cannot prevent attempts to resume the remaining tasks. + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + err := o.Runtime.Resume(cleanupCtx, handles[i].Container) + cancel() + if err != nil { + errs = append(errs, fmt.Errorf("resume container %s: %w", handles[i].Container.ID, err)) + } + } + return errors.Join(errs...) +} + +func (o *Orchestrator) validate() error { + if o.Runtime == nil || o.Builder == nil || o.Pusher == nil { + return errors.New("runtime, image builder, and image pusher are required") + } + return nil +} + +func (o *Orchestrator) logf(format string, args ...any) { + if o.Output != nil { + fmt.Fprintf(o.Output, format, args...) + } +} + +func (o *Orchestrator) errorf(format string, args ...any) { + if o.ErrorOutput != nil { + fmt.Fprintf(o.ErrorOutput, format, args...) + } +} diff --git a/kubernetes/pkg/imagecommitter/orchestrator_test.go b/kubernetes/pkg/imagecommitter/orchestrator_test.go new file mode 100644 index 000000000..af17092cc --- /dev/null +++ b/kubernetes/pkg/imagecommitter/orchestrator_test.go @@ -0,0 +1,223 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeRuntime struct { + states map[string]TaskState + resolveError map[string]error + operations *[]string +} + +func (f *fakeRuntime) Resolve(_ context.Context, selector ContainerSelector) (ResolvedContainer, error) { + if err := f.resolveError[selector.ContainerName]; err != nil { + return ResolvedContainer{}, err + } + state, ok := f.states[selector.ContainerName] + if !ok { + return ResolvedContainer{}, errors.New("not found") + } + *f.operations = append(*f.operations, "resolve:"+selector.ContainerName) + return ResolvedContainer{ID: selector.ContainerName + "-id", Name: selector.ContainerName, State: state}, nil +} + +func (f *fakeRuntime) Status(_ context.Context, container ResolvedContainer) (TaskState, error) { + return f.states[container.Name], nil +} + +func (f *fakeRuntime) Pause(_ context.Context, container ResolvedContainer) (PauseHandle, error) { + *f.operations = append(*f.operations, "pause:"+container.Name) + if f.states[container.Name] == TaskStateRunning { + f.states[container.Name] = TaskStatePaused + return PauseHandle{Container: container, PausedByUs: true}, nil + } + return PauseHandle{Container: container}, nil +} + +func (f *fakeRuntime) Resume(_ context.Context, container ResolvedContainer) error { + *f.operations = append(*f.operations, "resume:"+container.Name) + f.states[container.Name] = TaskStateRunning + return nil +} + +type fakeBuilder struct { + operations *[]string + fail string + panicFor string +} + +func (f fakeBuilder) Commit(_ context.Context, container ResolvedContainer, target string) (LocalImage, error) { + *f.operations = append(*f.operations, "commit:"+container.Name) + if container.Name == f.panicFor { + panic("commit panic") + } + if container.Name == f.fail { + return LocalImage{}, errors.New("commit failed") + } + return LocalImage{ + Reference: target, + Target: ocispec.Descriptor{Digest: digest.FromString("manifest:" + target)}, + Config: ocispec.Descriptor{Digest: digest.FromString("config:" + target)}, + }, nil +} + +type fakePusher struct { + operations *[]string + runtime *fakeRuntime +} + +func (f fakePusher) Push(_ context.Context, image LocalImage) (ocispec.Descriptor, error) { + for name, state := range f.runtime.states { + if state == TaskStatePaused { + return ocispec.Descriptor{}, fmt.Errorf("container %s was still paused during push", name) + } + } + *f.operations = append(*f.operations, "push:"+image.Reference) + return image.Target, nil +} + +func TestCommitResolvesBeforePauseAndResumesBeforePush(t *testing.T) { + var operations []string + runtime := &fakeRuntime{ + states: map[string]TaskState{"main": TaskStateRunning, "stopped": TaskStateStopped}, + resolveError: map[string]error{}, + operations: &operations, + } + orchestrator := &Orchestrator{ + Runtime: runtime, + Builder: fakeBuilder{operations: &operations}, + Pusher: fakePusher{operations: &operations, runtime: runtime}, + } + result, err := orchestrator.Commit(context.Background(), CommitRequest{ + PodName: "pod", Namespace: "default", + Containers: []ContainerSpec{ + {Name: "main", Target: "registry.example.com/main:snap"}, + {Name: "stopped", Target: "registry.example.com/stopped:snap"}, + }, + }) + if err != nil { + t.Fatalf("Commit failed: %v", err) + } + wantOperations := []string{ + "resolve:main", "resolve:stopped", + "pause:main", + "commit:main", "commit:stopped", + "resume:main", + "push:registry.example.com/main:snap", "push:registry.example.com/stopped:snap", + } + if !reflect.DeepEqual(operations, wantOperations) { + t.Fatalf("operations = %v, want %v", operations, wantOperations) + } + if len(result.Containers) != 2 || result.Containers[0].Name != "main" || result.Containers[1].Name != "stopped" { + t.Fatalf("unexpected result: %#v", result) + } + wantDigest := digest.FromString("config:registry.example.com/main:snap").String() + if result.Containers[0].Digest != wantDigest { + t.Fatalf("reported digest = %q, want config digest %q", result.Containers[0].Digest, wantDigest) + } + manifestDigest := digest.FromString("manifest:registry.example.com/main:snap").String() + if result.Containers[0].Digest == manifestDigest { + t.Fatalf("reported digest unexpectedly used manifest digest %q", manifestDigest) + } +} + +func TestCommitResumesOwnedContainersAfterBuildFailure(t *testing.T) { + var operations []string + runtime := &fakeRuntime{ + states: map[string]TaskState{"main": TaskStateRunning}, + resolveError: map[string]error{}, + operations: &operations, + } + orchestrator := &Orchestrator{ + Runtime: runtime, + Builder: fakeBuilder{operations: &operations, fail: "main"}, + Pusher: fakePusher{operations: &operations, runtime: runtime}, + } + _, err := orchestrator.Commit(context.Background(), CommitRequest{ + PodName: "pod", Namespace: "default", + Containers: []ContainerSpec{{Name: "main", Target: "registry.example.com/main:snap"}}, + }) + if err == nil { + t.Fatal("expected build failure") + } + if runtime.states["main"] != TaskStateRunning { + t.Fatalf("container was not resumed: %s", runtime.states["main"]) + } +} + +func TestCommitResumesOwnedContainersAfterProviderPanic(t *testing.T) { + var operations []string + runtime := &fakeRuntime{ + states: map[string]TaskState{"main": TaskStateRunning}, + resolveError: map[string]error{}, + operations: &operations, + } + orchestrator := &Orchestrator{ + Runtime: runtime, + Builder: fakeBuilder{operations: &operations, panicFor: "main"}, + Pusher: fakePusher{operations: &operations, runtime: runtime}, + } + func() { + defer func() { + if recover() == nil { + t.Fatal("expected provider panic") + } + }() + _, _ = orchestrator.Commit(context.Background(), CommitRequest{ + PodName: "pod", Namespace: "default", + Containers: []ContainerSpec{{Name: "main", Target: "registry.example.com/main:snap"}}, + }) + }() + if runtime.states["main"] != TaskStateRunning { + t.Fatalf("container was not resumed after panic: %s", runtime.states["main"]) + } +} + +func TestUnpauseIsIdempotentAndContinuesAfterErrors(t *testing.T) { + var operations []string + runtime := &fakeRuntime{ + states: map[string]TaskState{ + "paused": TaskStatePaused, + "running": TaskStateRunning, + "stopped": TaskStateStopped, + }, + resolveError: map[string]error{"missing": errors.New("not found")}, + operations: &operations, + } + orchestrator := &Orchestrator{Runtime: runtime} + err := orchestrator.Unpause(context.Background(), UnpauseRequest{ + PodName: "pod", Namespace: "default", + ContainerNames: []string{"missing", "paused", "running", "stopped"}, + }) + if err == nil { + t.Fatal("expected aggregated missing-container error") + } + if runtime.states["paused"] != TaskStateRunning { + t.Fatal("paused container was not resumed") + } + if !reflect.DeepEqual(operations, []string{"resolve:paused", "resume:paused", "resolve:running", "resolve:stopped"}) { + t.Fatalf("unexpected operations: %v", operations) + } +} diff --git a/kubernetes/pkg/imagecommitter/registry_containerd.go b/kubernetes/pkg/imagecommitter/registry_containerd.go new file mode 100644 index 000000000..2ba056e27 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/registry_containerd.go @@ -0,0 +1,237 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "syscall" + + containerd "github.com/containerd/containerd" + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" + "github.com/distribution/reference" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// DockerConfigCredentialProvider reads the Kubernetes dockerconfigjson mount. +type DockerConfigCredentialProvider struct { + Path string + ErrorOutput io.Writer +} + +type dockerConfig struct { + Auths map[string]dockerAuthEntry `json:"auths"` +} + +type dockerAuthEntry struct { + Auth string `json:"auth"` + Username string `json:"username"` + Password string `json:"password"` + IdentityToken string `json:"identitytoken"` + RegistryToken string `json:"registrytoken"` +} + +func (p DockerConfigCredentialProvider) Credential(_ context.Context, registryHost string) (RegistryCredential, error) { + if p.Path == "" { + return RegistryCredential{}, nil + } + data, err := os.ReadFile(p.Path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + p.warn("read registry credential config: %v", err) + } + return RegistryCredential{}, nil + } + var config dockerConfig + if err := json.Unmarshal(data, &config); err != nil { + p.warn("parse registry credential config: %v", err) + return RegistryCredential{}, nil + } + for configuredHost, entry := range config.Auths { + if normalizeRegistryHost(configuredHost) != normalizeRegistryHost(registryHost) { + continue + } + credential := RegistryCredential{ + Username: entry.Username, + Password: entry.Password, + AccessToken: entry.RegistryToken, + RefreshToken: entry.IdentityToken, + } + if entry.Auth != "" && credential.Username == "" && credential.Password == "" { + decoded, err := base64.StdEncoding.DecodeString(entry.Auth) + if err != nil { + p.warn("decode registry auth for %s: %v", registryHost, err) + return RegistryCredential{}, fmt.Errorf("decode registry auth for %s: %w", registryHost, err) + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + p.warn("invalid registry auth for %s", registryHost) + return RegistryCredential{}, fmt.Errorf("invalid registry auth for %s", registryHost) + } + credential.Username, credential.Password = parts[0], parts[1] + } + return credential, nil + } + p.warn("no registry credential found for %s; attempting registry access without credentials", registryHost) + return RegistryCredential{}, nil +} + +func (p DockerConfigCredentialProvider) warn(format string, args ...any) { + if p.ErrorOutput != nil { + fmt.Fprintf(p.ErrorOutput, "WARNING: "+format+"\n", args...) + } +} + +// InsecureRegistryFunc reports whether a registry reference may skip TLS +// verification and fall back to plain HTTP. +type InsecureRegistryFunc func(imageReference string) bool + +// ContainerdImagePusher pushes image content through containerd's resolver. +type ContainerdImagePusher struct { + client *containerd.Client + credentials CredentialProvider + insecure InsecureRegistryFunc +} + +func NewContainerdImagePusher(client *containerd.Client, credentials CredentialProvider, insecure InsecureRegistryFunc) *ContainerdImagePusher { + return &ContainerdImagePusher{client: client, credentials: credentials, insecure: insecure} +} + +func (p *ContainerdImagePusher) Push(ctx context.Context, image LocalImage) (ocispec.Descriptor, error) { + host, err := registryHost(image.Reference) + if err != nil { + return ocispec.Descriptor{}, err + } + credential := RegistryCredential{} + if p.credentials != nil { + credential, err = p.credentials.Credential(ctx, host) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("resolve credentials for %s: %w", host, err) + } + } + insecure := p.insecure != nil && p.insecure(image.Reference) + if err := p.push(ctx, image, host, credential, "https", insecure); err != nil { + if !insecure || !shouldFallbackToPlainHTTP(err) { + return ocispec.Descriptor{}, fmt.Errorf("push image %s: %w", image.Reference, err) + } + if err := p.push(ctx, image, host, credential, "http", false); err != nil { + return ocispec.Descriptor{}, fmt.Errorf("push image %s over plain HTTP: %w", image.Reference, err) + } + } + return image.Target, nil +} + +func (p *ContainerdImagePusher) push(ctx context.Context, image LocalImage, host string, credential RegistryCredential, scheme string, skipVerify bool) error { + resolver := newDockerResolver(host, credential, scheme, skipVerify) + return p.client.Push(ctx, image.Reference, image.Target, containerd.WithResolver(resolver)) +} + +func newDockerResolver(host string, credential RegistryCredential, scheme string, skipVerify bool) remotes.Resolver { + transport := http.DefaultTransport.(*http.Transport).Clone() + if skipVerify { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicitly configured insecure registry + } + client := &http.Client{Transport: transport} + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && (req.URL.Scheme != via[len(via)-1].URL.Scheme || req.URL.Host != via[len(via)-1].URL.Host) { + req.Header.Del("Authorization") + } + return nil + } + headers := http.Header{} + if credential.AccessToken != "" { + headers.Set("Authorization", "Bearer "+credential.AccessToken) + } + credentials := func(requestedHost string) (string, string, error) { + if normalizeRegistryHost(requestedHost) != normalizeRegistryHost(host) { + return "", "", nil + } + if credential.AccessToken != "" { + return "", "", nil + } + if credential.RefreshToken != "" { + return "", credential.RefreshToken, nil + } + return credential.Username, credential.Password, nil + } + authorizer := docker.NewDockerAuthorizer( + docker.WithAuthClient(client), + docker.WithAuthHeader(headers), + docker.WithAuthCreds(credentials), + ) + return docker.NewResolver(docker.ResolverOptions{Hosts: func(requestedHost string) ([]docker.RegistryHost, error) { + actualHost := requestedHost + if requestedHost == "docker.io" { + actualHost = "registry-1.docker.io" + } + return []docker.RegistryHost{{ + Client: client, + Authorizer: authorizer, + Host: actualHost, + Scheme: scheme, + Path: "/v2", + Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush, + Header: headers.Clone(), + }}, nil + }}) +} + +func shouldFallbackToPlainHTTP(err error) bool { + return errors.Is(err, http.ErrSchemeMismatch) || errors.Is(err, syscall.ECONNREFUSED) +} + +func registryHost(imageReference string) (string, error) { + named, err := reference.ParseNormalizedNamed(imageReference) + if err != nil { + return "", fmt.Errorf("parse image reference %q: %w", imageReference, err) + } + return reference.Domain(named), nil +} + +func referenceWithDigest(imageReference string, descriptor ocispec.Descriptor) (string, error) { + named, err := reference.ParseNormalizedNamed(imageReference) + if err != nil { + return "", fmt.Errorf("parse source image %q: %w", imageReference, err) + } + canonical, err := reference.WithDigest(reference.TrimNamed(named), descriptor.Digest) + if err != nil { + return "", fmt.Errorf("pin source image %q to digest %s: %w", imageReference, descriptor.Digest, err) + } + return canonical.String(), nil +} + +func normalizeRegistryHost(value string) string { + value = strings.TrimSpace(value) + value = strings.TrimPrefix(value, "https://") + value = strings.TrimPrefix(value, "http://") + value = strings.TrimSuffix(value, "/v1/") + value = strings.TrimSuffix(value, "/v2/") + value = strings.TrimSuffix(value, "/") + switch value { + case "index.docker.io", "registry-1.docker.io": + return "docker.io" + default: + return value + } +} diff --git a/kubernetes/pkg/imagecommitter/registry_containerd_test.go b/kubernetes/pkg/imagecommitter/registry_containerd_test.go new file mode 100644 index 000000000..5f4a33a14 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/registry_containerd_test.go @@ -0,0 +1,146 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestDockerConfigCredentialProvider(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + auth := base64.StdEncoding.EncodeToString([]byte("robot:secret")) + data := []byte(`{"auths":{"https://registry.example.com/v1/":{"auth":"` + auth + `"}}}`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + provider := DockerConfigCredentialProvider{Path: path} + credential, err := provider.Credential(context.Background(), "registry.example.com") + if err != nil { + t.Fatalf("Credential failed: %v", err) + } + if credential.Username != "robot" || credential.Password != "secret" { + t.Fatalf("unexpected credential: %#v", credential) + } + other, err := provider.Credential(context.Background(), "other.example.com") + if err != nil { + t.Fatalf("Credential for other host failed: %v", err) + } + if other != (RegistryCredential{}) { + t.Fatalf("credential leaked to another host: %#v", other) + } +} + +func TestDockerConfigCredentialProviderFallsBackToAnonymousOnInvalidConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`not-json`), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + var warnings bytes.Buffer + credential, err := (DockerConfigCredentialProvider{Path: path, ErrorOutput: &warnings}).Credential(context.Background(), "registry.example.com") + if err != nil { + t.Fatalf("invalid config should remain best effort: %v", err) + } + if credential != (RegistryCredential{}) { + t.Fatalf("invalid config returned credential: %#v", credential) + } + if warnings.Len() == 0 { + t.Fatal("expected invalid config warning") + } +} + +func TestDockerConfigCredentialProviderRejectsMalformedMatchingAuth(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + data := []byte(`{"auths":{"registry.example.com":{"auth":"not-base64"}}}`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := (DockerConfigCredentialProvider{Path: path}).Credential(context.Background(), "registry.example.com") + if err == nil { + t.Fatal("expected malformed matching auth to fail") + } + if !strings.Contains(err.Error(), "decode registry auth") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestDockerConfigCredentialProviderSupportsIdentityToken(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"auths":{"registry.example.com":{"identitytoken":"token"}}}`), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + credential, err := (DockerConfigCredentialProvider{Path: path}).Credential(context.Background(), "registry.example.com") + if err != nil { + t.Fatalf("Credential failed: %v", err) + } + if credential.RefreshToken != "token" { + t.Fatalf("refresh token = %q", credential.RefreshToken) + } +} + +func TestShouldFallbackToPlainHTTP(t *testing.T) { + if !shouldFallbackToPlainHTTP(http.ErrSchemeMismatch) { + t.Fatal("scheme mismatch should fall back to HTTP") + } + if !shouldFallbackToPlainHTTP(fmt.Errorf("connect: %w", syscall.ECONNREFUSED)) { + t.Fatal("connection refused should fall back to HTTP") + } + if shouldFallbackToPlainHTTP(errors.New("unauthorized")) { + t.Fatal("authentication errors must not fall back to HTTP") + } +} + +func TestNormalizeRegistryHostAliasesDockerHub(t *testing.T) { + for _, host := range []string{"docker.io", "registry-1.docker.io", "https://index.docker.io/v1/"} { + if got := normalizeRegistryHost(host); got != "docker.io" { + t.Fatalf("normalizeRegistryHost(%q) = %q", host, got) + } + } +} + +func TestRegistryHost(t *testing.T) { + host, err := registryHost("registry.example.com:5000/project/image:snap") + if err != nil { + t.Fatalf("registryHost failed: %v", err) + } + if host != "registry.example.com:5000" { + t.Fatalf("host = %q", host) + } +} + +func TestReferenceWithDigestReplacesMutableTag(t *testing.T) { + descriptor := ocispec.Descriptor{Digest: digest.FromString("source image")} + got, err := referenceWithDigest("registry.example.com/project/image:latest", descriptor) + if err != nil { + t.Fatalf("referenceWithDigest failed: %v", err) + } + want := "registry.example.com/project/image@" + descriptor.Digest.String() + if got != want { + t.Fatalf("reference = %q, want %q", got, want) + } +} diff --git a/kubernetes/pkg/imagecommitter/runtime_containerd.go b/kubernetes/pkg/imagecommitter/runtime_containerd.go new file mode 100644 index 000000000..d3785410e --- /dev/null +++ b/kubernetes/pkg/imagecommitter/runtime_containerd.go @@ -0,0 +1,278 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "sort" + "syscall" + "time" + + containerd "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/errdefs" +) + +// ContainerdRuntime implements runtime and exec operations through containerd. +type ContainerdRuntime struct { + client *containerd.Client +} + +func NewContainerdRuntime(client *containerd.Client) *ContainerdRuntime { + return &ContainerdRuntime{client: client} +} + +func (r *ContainerdRuntime) Resolve(ctx context.Context, selector ContainerSelector) (ResolvedContainer, error) { + containers, err := r.client.Containers(ctx) + if err != nil { + return ResolvedContainer{}, fmt.Errorf("list containerd containers: %w", err) + } + + type candidate struct { + container ResolvedContainer + createdAt time.Time + } + var candidates []candidate + for _, c := range containers { + info, err := c.Info(ctx) + if err != nil { + if errdefs.IsNotFound(err) { + // A container can be removed between list and inspect. + continue + } + return ResolvedContainer{}, fmt.Errorf("inspect container %s: %w", c.ID(), err) + } + labels := info.Labels + if labels[PodNameLabel] != selector.PodName || + labels[PodNamespaceLabel] != selector.PodNamespace || + labels[ContainerNameLabel] != selector.ContainerName { + continue + } + if selector.PodUID != "" && labels[PodUIDLabel] != selector.PodUID { + continue + } + + resolved := ResolvedContainer{ + ID: c.ID(), + Name: selector.ContainerName, + Snapshotter: info.Snapshotter, + SnapshotKey: info.SnapshotKey, + SourceImage: info.Image, + } + resolved.State, err = r.Status(ctx, resolved) + if err != nil { + return ResolvedContainer{}, err + } + candidates = append(candidates, candidate{container: resolved, createdAt: info.CreatedAt}) + } + + if len(candidates) == 0 { + return ResolvedContainer{}, fmt.Errorf("container %q not found in pod %s/%s", selector.ContainerName, selector.PodNamespace, selector.PodName) + } + + var active []candidate + for _, candidate := range candidates { + if candidate.container.State == TaskStateRunning || candidate.container.State == TaskStatePaused { + active = append(active, candidate) + } + } + if len(active) == 1 { + return active[0].container, nil + } + if len(active) > 1 { + return ResolvedContainer{}, fmt.Errorf("container %q in pod %s/%s is ambiguous: %d active matches", selector.ContainerName, selector.PodNamespace, selector.PodName, len(active)) + } + if len(candidates) > 1 { + sort.Slice(candidates, func(i, j int) bool { return candidates[i].createdAt.After(candidates[j].createdAt) }) + if candidates[0].createdAt.Equal(candidates[1].createdAt) { + return ResolvedContainer{}, fmt.Errorf("container %q in pod %s/%s is ambiguous: %d stopped matches", selector.ContainerName, selector.PodNamespace, selector.PodName, len(candidates)) + } + } + return candidates[0].container, nil +} + +func (r *ContainerdRuntime) Status(ctx context.Context, container ResolvedContainer) (TaskState, error) { + c, err := r.client.LoadContainer(ctx, container.ID) + if err != nil { + return TaskStateUnknown, fmt.Errorf("load container %s: %w", container.ID, err) + } + task, err := c.Task(ctx, nil) + if err != nil { + if errdefs.IsNotFound(err) { + return TaskStateStopped, nil + } + return TaskStateUnknown, fmt.Errorf("load task for container %s: %w", container.ID, err) + } + status, err := task.Status(ctx) + if err != nil { + return TaskStateUnknown, fmt.Errorf("get task status for container %s: %w", container.ID, err) + } + switch status.Status { + case containerd.Running, containerd.Created: + return TaskStateRunning, nil + case containerd.Paused, containerd.Pausing: + return TaskStatePaused, nil + case containerd.Stopped: + return TaskStateStopped, nil + default: + return TaskStateUnknown, nil + } +} + +func (r *ContainerdRuntime) Pause(ctx context.Context, container ResolvedContainer) (PauseHandle, error) { + state, err := r.Status(ctx, container) + if err != nil { + return PauseHandle{}, err + } + handle := PauseHandle{Container: container} + switch state { + case TaskStatePaused, TaskStateStopped: + return handle, nil + case TaskStateRunning: + default: + return PauseHandle{}, fmt.Errorf("cannot pause container %s in state %s", container.ID, state) + } + + c, err := r.client.LoadContainer(ctx, container.ID) + if err != nil { + return PauseHandle{}, fmt.Errorf("load container %s: %w", container.ID, err) + } + task, err := c.Task(ctx, nil) + if err != nil { + return PauseHandle{}, fmt.Errorf("load task for container %s: %w", container.ID, err) + } + if err := task.Pause(ctx); err != nil { + return PauseHandle{}, fmt.Errorf("pause container %s: %w", container.ID, err) + } + handle.PausedByUs = true + return handle, nil +} + +func (r *ContainerdRuntime) Resume(ctx context.Context, container ResolvedContainer) error { + state, err := r.Status(ctx, container) + if err != nil { + return err + } + if state == TaskStateRunning || state == TaskStateStopped { + return nil + } + if state != TaskStatePaused { + return fmt.Errorf("cannot resume container %s in state %s", container.ID, state) + } + + c, err := r.client.LoadContainer(ctx, container.ID) + if err != nil { + return fmt.Errorf("load container %s: %w", container.ID, err) + } + task, err := c.Task(ctx, nil) + if err != nil { + return fmt.Errorf("load task for container %s: %w", container.ID, err) + } + if err := task.Resume(ctx); err != nil { + return fmt.Errorf("resume container %s: %w", container.ID, err) + } + state, err = r.Status(ctx, container) + if err != nil { + return err + } + if state != TaskStateRunning { + return fmt.Errorf("container %s remained in state %s after resume", container.ID, state) + } + return nil +} + +func (r *ContainerdRuntime) Exec(ctx context.Context, container ResolvedContainer, request ExecRequest) (result ExecResult, retErr error) { + if len(request.Args) == 0 { + return ExecResult{}, errors.New("exec arguments are required") + } + state, err := r.Status(ctx, container) + if err != nil { + return ExecResult{}, err + } + if state != TaskStateRunning { + return ExecResult{}, fmt.Errorf("cannot exec in container %s in state %s", container.ID, state) + } + + c, err := r.client.LoadContainer(ctx, container.ID) + if err != nil { + return ExecResult{}, fmt.Errorf("load container %s: %w", container.ID, err) + } + spec, err := c.Spec(ctx) + if err != nil { + return ExecResult{}, fmt.Errorf("load container spec %s: %w", container.ID, err) + } + if spec.Process == nil { + return ExecResult{}, fmt.Errorf("container %s has no process spec", container.ID) + } + processSpec := *spec.Process + processSpec.Args = append([]string(nil), request.Args...) + processSpec.CommandLine = "" + processSpec.Terminal = false + + task, err := c.Task(ctx, nil) + if err != nil { + return ExecResult{}, fmt.Errorf("load task for container %s: %w", container.ID, err) + } + execID, err := randomID() + if err != nil { + return ExecResult{}, err + } + process, err := task.Exec(ctx, execID, &processSpec, cio.NullIO) + if err != nil { + return ExecResult{}, fmt.Errorf("create exec process for container %s: %w", container.ID, err) + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := process.Delete(cleanupCtx, containerd.WithProcessKill) + if retErr == nil && err != nil && !errdefs.IsNotFound(err) { + retErr = fmt.Errorf("delete exec process %s: %w", execID, err) + } + }() + + exitCh, err := process.Wait(ctx) + if err != nil { + return ExecResult{}, fmt.Errorf("wait for exec process %s: %w", execID, err) + } + if err := process.Start(ctx); err != nil { + return ExecResult{}, fmt.Errorf("start exec process %s: %w", execID, err) + } + + select { + case status := <-exitCh: + code, _, err := status.Result() + if err != nil { + return ExecResult{}, fmt.Errorf("wait result for exec process %s: %w", execID, err) + } + return ExecResult{ExitCode: code}, nil + case <-ctx.Done(): + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = process.Kill(cleanupCtx, syscall.SIGKILL) + return ExecResult{}, ctx.Err() + } +} + +func randomID() (string, error) { + var data [16]byte + if _, err := rand.Read(data[:]); err != nil { + return "", fmt.Errorf("generate exec ID: %w", err) + } + return "opensandbox-" + hex.EncodeToString(data[:]), nil +} diff --git a/kubernetes/pkg/imagecommitter/types.go b/kubernetes/pkg/imagecommitter/types.go new file mode 100644 index 000000000..c571ebad4 --- /dev/null +++ b/kubernetes/pkg/imagecommitter/types.go @@ -0,0 +1,154 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecommitter + +import ( + "context" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + PodNameLabel = "io.kubernetes.pod.name" + PodNamespaceLabel = "io.kubernetes.pod.namespace" + PodUIDLabel = "io.kubernetes.pod.uid" + ContainerNameLabel = "io.kubernetes.container.name" +) + +// ContainerSpec maps a source container to its target image. +type ContainerSpec struct { + Name string + Target string +} + +// ContainerSelector identifies a Kubernetes container in containerd metadata. +type ContainerSelector struct { + PodName string + PodNamespace string + PodUID string + ContainerName string +} + +// TaskState is the runtime state relevant to commit and unpause operations. +type TaskState string + +const ( + TaskStateUnknown TaskState = "unknown" + TaskStateRunning TaskState = "running" + TaskStatePaused TaskState = "paused" + TaskStateStopped TaskState = "stopped" +) + +// ResolvedContainer contains runtime metadata needed by providers. +type ResolvedContainer struct { + ID string + Name string + State TaskState + Snapshotter string + SnapshotKey string + SourceImage string +} + +// PauseHandle records whether this invocation owns a pause transition. +type PauseHandle struct { + Container ResolvedContainer + PausedByUs bool +} + +// ExecRequest describes an optional command executed in a source container. +type ExecRequest struct { + Args []string +} + +// ExecResult is the result of a successfully created exec process. +type ExecResult struct { + ExitCode uint32 +} + +// LocalImage identifies image content assembled in containerd. +type LocalImage struct { + Reference string + // Target is the manifest descriptor pushed to the registry. + Target ocispec.Descriptor + // Config is the image config descriptor reported as the snapshot image + // digest for compatibility with the previous image-committer behavior. + Config ocispec.Descriptor +} + +// RegistryCredential supports standard OCI Distribution authentication forms. +type RegistryCredential struct { + Username string + Password string + AccessToken string + RefreshToken string +} + +// ContainerResult is written to the Kubernetes termination message. +type ContainerResult struct { + Name string `json:"name"` + Image string `json:"image"` + // Digest is the image config digest, preserving the image ID semantics of + // the previous image-committer implementation. + Digest string `json:"digest"` +} + +// Result is the stable commit output contract. +type Result struct { + Containers []ContainerResult `json:"containers"` +} + +// CommitRequest is the parsed commit operation input. +type CommitRequest struct { + PodName string + Namespace string + PodUID string + Containers []ContainerSpec +} + +// UnpauseRequest is the parsed unpause operation input. +type UnpauseRequest struct { + PodName string + Namespace string + PodUID string + ContainerNames []string +} + +// ContainerRuntime abstracts container lookup and task lifecycle operations. +type ContainerRuntime interface { + Resolve(context.Context, ContainerSelector) (ResolvedContainer, error) + Status(context.Context, ResolvedContainer) (TaskState, error) + Pause(context.Context, ResolvedContainer) (PauseHandle, error) + Resume(context.Context, ResolvedContainer) error +} + +// ContainerExecutor optionally runs preparation commands in source containers. +type ContainerExecutor interface { + Exec(context.Context, ResolvedContainer, ExecRequest) (ExecResult, error) +} + +// ImageBuilder creates local image content without contacting a registry. +type ImageBuilder interface { + Commit(context.Context, ResolvedContainer, string) (LocalImage, error) +} + +// CredentialProvider resolves credentials only for the requested registry host. +type CredentialProvider interface { + Credential(context.Context, string) (RegistryCredential, error) +} + +// ImagePusher uploads local image content and returns its pushed descriptor. +type ImagePusher interface { + Push(context.Context, LocalImage) (ocispec.Descriptor, error) +} diff --git a/kubernetes/test/e2e/pause_resume_test.go b/kubernetes/test/e2e/pause_resume_test.go index ebc334538..dc3eb81e4 100644 --- a/kubernetes/test/e2e/pause_resume_test.go +++ b/kubernetes/test/e2e/pause_resume_test.go @@ -15,9 +15,10 @@ package e2e import ( - "encoding/base64" "encoding/json" "fmt" + "net" + "net/http" "os" "os/exec" "path/filepath" @@ -123,24 +124,41 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { }) AfterAll(func() { - By("cleaning up Docker Registry") - cmd := exec.Command("kubectl", "delete", "deployment", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") + By("cleaning up any remaining batchsandboxes") + cmd := exec.Command("kubectl", "delete", "batchsandboxes", "--all", "-n", pauseResumeNamespace, + "--ignore-not-found=true", "--wait=false") utils.Run(cmd) - cmd = exec.Command("kubectl", "delete", "service", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") + + By("requesting cleanup of any remaining sandboxsnapshots") + cmd = exec.Command("kubectl", "delete", "sandboxsnapshots", "--all", "-n", pauseResumeNamespace, + "--ignore-not-found=true", "--wait=false") utils.Run(cmd) + // Failure-path tests can intentionally leave snapshots whose image URI + // cannot be cleaned up. Exercise the documented operator escape hatch so + // teardown cannot block forever on their strict cleanup finalizers. + By("removing finalizers from snapshots that could not be cleaned up") + cmd = exec.Command("kubectl", "get", "sandboxsnapshots", "-n", pauseResumeNamespace, + "-o", "jsonpath={range .items[*]}{.metadata.name}{'\\n'}{end}") + remainingSnapshots, err := utils.Run(cmd) + if err == nil { + for _, snapshotName := range strings.Fields(remainingSnapshots) { + cmd = exec.Command("kubectl", "patch", "sandboxsnapshot", snapshotName, "-n", pauseResumeNamespace, + "--type=merge", "-p", `{"metadata":{"finalizers":[]}}`) + utils.Run(cmd) + } + } + By("cleaning up secrets") for _, secret := range []string{"registry-auth", "registry-snapshot-push-secret", "registry-pull-secret"} { cmd = exec.Command("kubectl", "delete", "secret", secret, "-n", pauseResumeNamespace, "--ignore-not-found=true") utils.Run(cmd) } - By("cleaning up any remaining sandboxsnapshots") - cmd = exec.Command("kubectl", "delete", "sandboxsnapshots", "--all", "-n", pauseResumeNamespace, "--ignore-not-found=true") + By("cleaning up Docker Registry") + cmd = exec.Command("kubectl", "delete", "deployment", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") utils.Run(cmd) - - By("cleaning up any remaining batchsandboxes") - cmd = exec.Command("kubectl", "delete", "batchsandboxes", "--all", "-n", pauseResumeNamespace, "--ignore-not-found=true") + cmd = exec.Command("kubectl", "delete", "service", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") utils.Run(cmd) By("undeploying the controller-manager") @@ -261,6 +279,12 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { Expect(err).NotTo(HaveOccurred()) Expect(output).To(Equal("Succeed"), "Internal pause snapshot should be ready after pause") + cmd = exec.Command("kubectl", "get", "sandboxsnapshot", sandboxName+"-pause", + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.containers[0].imageUri}") + snapshotImageURI, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(snapshotImageURI).NotTo(BeEmpty()) + // --- Step 4: Resume - patch spec.pause=false --- By("triggering resume by patching spec.pause=false") cmd = exec.Command("kubectl", "patch", "batchsandbox", sandboxName, @@ -279,10 +303,15 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { }, 2*time.Minute).Should(Succeed()) By("verifying the reserved internal SandboxSnapshot is deleted after successful resume") - cmd = exec.Command("kubectl", "get", "sandboxsnapshot", sandboxName+"-pause", - "-n", pauseResumeNamespace, "-o", "name") - output, err = utils.Run(cmd) - Expect(err).To(HaveOccurred(), "Internal pause snapshot should be deleted after successful resume") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", sandboxName+"-pause", + "-n", pauseResumeNamespace, "-o", "name") + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred(), "Internal pause snapshot should be deleted after successful resume") + }, 2*time.Minute).Should(Succeed()) + + By("verifying the deleted snapshot manifest is absent from the registry") + expectRegistryManifestMissing(snapshotImageURI) // --- Step 5: Verify rootfs data persistence --- By("getting resumed pod name") @@ -811,7 +840,7 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { utils.Run(cmd) }) - It("should set Phase=Succeed+PauseFailed when commit/push fails with invalid registry", func() { + It("should set Phase=Succeed+PauseFailed when the snapshot registry is unavailable", func() { const sandboxName = "test-pause-commit-fail" By("creating BatchSandbox with template") @@ -841,17 +870,51 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { g.Expect(output).To(Equal("Succeed")) }, 2*time.Minute).Should(Succeed()) - By("patching registry push secret to invalid value (invalid docker config)") - // Create an invalid docker config JSON (base64 encoded) - invalidConfig := `{"auths":{"docker-registry.default.svc.cluster.local:5000":{"username":"invalid","password":"wrong","auth":"aW52YWxpZDp3cm9uZw=="}}}` - encoded := base64.StdEncoding.EncodeToString([]byte(invalidConfig)) - patchData := fmt.Sprintf(`{"data":{".dockerconfigjson":"%s"}}`, encoded) - cmd = exec.Command("kubectl", "patch", "secret", "registry-snapshot-push-secret", "-n", pauseResumeNamespace, - "--type=merge", "-p", patchData) + By("making the snapshot registry unavailable") + cmd = exec.Command("kubectl", "scale", "deployment", "docker-registry", + "-n", pauseResumeNamespace, "--replicas=0") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) - By("triggering pause with invalid registry config") + By("waiting for the snapshot registry to have no available replicas") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "docker-registry", "-n", pauseResumeNamespace, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + var deployment struct { + Status struct { + AvailableReplicas int `json:"availableReplicas"` + } `json:"status"` + } + g.Expect(json.Unmarshal([]byte(output), &deployment)).To(Succeed()) + g.Expect(deployment.Status.AvailableReplicas).To(Equal(0)) + + cmd = exec.Command("kubectl", "get", "endpointslice", "-l", "kubernetes.io/service-name=docker-registry", + "-n", pauseResumeNamespace, "-o", "json") + output, err = utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + var endpointSlices struct { + Items []struct { + Endpoints []struct { + Conditions struct { + Ready *bool `json:"ready"` + } `json:"conditions"` + } `json:"endpoints"` + } `json:"items"` + } + g.Expect(json.Unmarshal([]byte(output), &endpointSlices)).To(Succeed()) + readyCount := 0 + for _, endpointSlice := range endpointSlices.Items { + for _, endpoint := range endpointSlice.Endpoints { + if endpoint.Conditions.Ready == nil || *endpoint.Conditions.Ready { + readyCount++ + } + } + } + g.Expect(readyCount).To(Equal(0)) + }, 2*time.Minute).Should(Succeed()) + + By("triggering pause while the snapshot registry is unavailable") cmd = exec.Command("kubectl", "patch", "batchsandbox", sandboxName, "-n", pauseResumeNamespace, "--type=merge", "-p", `{"spec":{"pause":true}}`) @@ -878,6 +941,53 @@ var _ = Describe("PauseResume", Ordered, Label("PauseResume"), func() { cmd = exec.Command("kubectl", "delete", "batchsandbox", sandboxName, "-n", pauseResumeNamespace, "--ignore-not-found=true") utils.Run(cmd) + By("restoring the snapshot registry") + cmd = exec.Command("kubectl", "scale", "deployment", "docker-registry", + "-n", pauseResumeNamespace, "--replicas=1") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for the snapshot registry to become available") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "docker-registry", "-n", pauseResumeNamespace, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + var deployment struct { + Status struct { + AvailableReplicas int `json:"availableReplicas"` + } `json:"status"` + } + g.Expect(json.Unmarshal([]byte(output), &deployment)).To(Succeed()) + g.Expect(deployment.Status.AvailableReplicas).To(Equal(1)) + }, 2*time.Minute).Should(Succeed()) + + By("waiting for the snapshot registry endpoint to become ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpointslice", "-l", "kubernetes.io/service-name=docker-registry", + "-n", pauseResumeNamespace, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + var endpointSlices struct { + Items []struct { + Endpoints []struct { + Conditions struct { + Ready *bool `json:"ready"` + } `json:"conditions"` + } `json:"endpoints"` + } `json:"items"` + } + g.Expect(json.Unmarshal([]byte(output), &endpointSlices)).To(Succeed()) + readyCount := 0 + for _, endpointSlice := range endpointSlices.Items { + for _, endpoint := range endpointSlice.Endpoints { + if endpoint.Conditions.Ready == nil || *endpoint.Conditions.Ready { + readyCount++ + } + } + } + g.Expect(readyCount).To(BeNumerically(">=", 1)) + }, 2*time.Minute).Should(Succeed()) + By("restoring registry push secret to valid credentials") err = createDockerRegistrySecrets(pauseResumeNamespace) Expect(err).NotTo(HaveOccurred()) @@ -1069,3 +1179,45 @@ func createDockerRegistrySecrets(namespace string) error { return nil } + +func expectRegistryManifestMissing(imageURI string) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + port := listener.Addr().(*net.TCPAddr).Port + Expect(listener.Close()).To(Succeed()) + + portForward := exec.Command("kubectl", "port-forward", "-n", pauseResumeNamespace, + "service/docker-registry", fmt.Sprintf("%d:5000", port)) + Expect(portForward.Start()).To(Succeed()) + defer func() { + _ = portForward.Process.Kill() + _ = portForward.Wait() + }() + + client := &http.Client{Timeout: 2 * time.Second} + registryURL := fmt.Sprintf("http://127.0.0.1:%d", port) + Eventually(func(g Gomega) { + request, requestErr := http.NewRequest(http.MethodGet, registryURL+"/v2/", nil) + g.Expect(requestErr).NotTo(HaveOccurred()) + request.SetBasicAuth(registryUsername, registryPassword) + response, requestErr := client.Do(request) + g.Expect(requestErr).NotTo(HaveOccurred()) + defer response.Body.Close() + g.Expect(response.StatusCode).To(Equal(http.StatusOK)) + }, 30*time.Second).Should(Succeed()) + + repositoryAndTag := strings.TrimPrefix(imageURI, registryServiceAddr+"/") + tagSeparator := strings.LastIndex(repositoryAndTag, ":") + Expect(tagSeparator).To(BeNumerically(">", 0), "snapshot image must include a tag") + manifestURL := fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, repositoryAndTag[:tagSeparator], repositoryAndTag[tagSeparator+1:]) + + for range 2 { + request, requestErr := http.NewRequest(http.MethodHead, manifestURL, nil) + Expect(requestErr).NotTo(HaveOccurred()) + request.SetBasicAuth(registryUsername, registryPassword) + response, requestErr := client.Do(request) + Expect(requestErr).NotTo(HaveOccurred()) + response.Body.Close() + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + } +} diff --git a/kubernetes/test/e2e/testdata/registry-deployment.yaml b/kubernetes/test/e2e/testdata/registry-deployment.yaml index b97044312..59d82601b 100644 --- a/kubernetes/test/e2e/testdata/registry-deployment.yaml +++ b/kubernetes/test/e2e/testdata/registry-deployment.yaml @@ -25,6 +25,8 @@ spec: value: "Registry Realm" - name: REGISTRY_AUTH_HTPASSWD_PATH value: /auth/htpasswd + - name: REGISTRY_STORAGE_DELETE_ENABLED + value: "true" volumeMounts: - name: auth mountPath: /auth @@ -48,4 +50,4 @@ spec: - port: 5000 targetPort: 5000 selector: - app: docker-registry \ No newline at end of file + app: docker-registry diff --git a/oseps/0004-secure-container-runtime.md b/oseps/0004-secure-container-runtime.md index 365eb2840..65d1ec765 100644 --- a/oseps/0004-secure-container-runtime.md +++ b/oseps/0004-secure-container-runtime.md @@ -3,8 +3,8 @@ title: Pluggable Secure Container Runtime Support authors: - "@hittyt" creation-date: 2026-02-05 -last-updated: 2026-02-09 -status: implementing +last-updated: 2026-07-27 +status: implemented --- # OSEP-0004: Pluggable Secure Container Runtime Support @@ -182,7 +182,7 @@ Extension to `~/.sandbox.toml`. A single `[secure_runtime]` section configures t ```toml [runtime] type = "docker" # or "kubernetes" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" # Secure container runtime configuration. # When enabled, ALL sandboxes on this server use the specified runtime. @@ -212,7 +212,7 @@ Example 1 โ€” gVisor on Docker: # ~/.sandbox.toml [runtime] type = "docker" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [secure_runtime] type = "gvisor" @@ -226,7 +226,7 @@ Example 2 โ€” Kata Containers (QEMU) on Kubernetes: # ~/.sandbox.toml [runtime] type = "kubernetes" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [secure_runtime] type = "kata" diff --git a/oseps/0005-client-side-sandbox-pool.md b/oseps/0005-client-side-sandbox-pool.md index fd2d63013..c4fa3a4e1 100644 --- a/oseps/0005-client-side-sandbox-pool.md +++ b/oseps/0005-client-side-sandbox-pool.md @@ -3,8 +3,8 @@ title: Client-Side Sandbox Pool authors: - "@ninan" creation-date: 2026-03-02 -last-updated: 2026-03-06 -status: implementing +last-updated: 2026-07-27 +status: implemented --- # OSEP-0005: Client-Side Sandbox Pool diff --git a/oseps/0013-isolated-execution-api.md b/oseps/0013-isolated-execution-api.md index 38861f974..0a7f5088b 100644 --- a/oseps/0013-isolated-execution-api.md +++ b/oseps/0013-isolated-execution-api.md @@ -1,7 +1,7 @@ --- title: Isolated Execution API authors: - - "@pjp" + - "@Pangjiping" creation-date: 2026-06-06 last-updated: 2026-06-23 status: implementing diff --git a/oseps/0014-multi-tenancy.md b/oseps/0014-multi-tenancy.md index 796c1cec4..8d76bfebc 100644 --- a/oseps/0014-multi-tenancy.md +++ b/oseps/0014-multi-tenancy.md @@ -3,8 +3,8 @@ title: Multi-Tenancy Support for Kubernetes Runtime authors: - "@Pangjiping" creation-date: 2026-04-29 -last-updated: 2026-05-07 -status: draft +last-updated: 2026-07-27 +status: implemented --- # OSEP-0014: Multi-Tenancy Support for Kubernetes Runtime diff --git a/oseps/0016-unified-umbrella-release-governance.md b/oseps/0016-unified-umbrella-release-governance.md index e2ff3699d..2e06b9403 100644 --- a/oseps/0016-unified-umbrella-release-governance.md +++ b/oseps/0016-unified-umbrella-release-governance.md @@ -146,7 +146,7 @@ Boolean rules that keep the eras mechanically distinguishable: 1. **Umbrella git tags** are v-less: `opensandbox/1.4.0`. 2. **Container image tags** carry a `release-` prefix: `opensandbox/execd:release-1.4.0`. Chosen because pre-umbrella - images (`opensandbox/execd:v1.0.21`) sit in the same registry + images (`opensandbox/execd:v1.0.22`) sit in the same registry repositories; the prefix eliminates any collision or ambiguity at a glance. 3. **Package-registry artifacts** use bare `X.Y.Z`. PyPI, npm, diff --git a/oseps/0018-execd-as-sandbox-init.md b/oseps/0018-execd-as-sandbox-init.md index 2f73be74f..c8e0ec00c 100644 --- a/oseps/0018-execd-as-sandbox-init.md +++ b/oseps/0018-execd-as-sandbox-init.md @@ -1,10 +1,10 @@ --- title: execd as Sandbox Init authors: - - "@pjp" + - "@Pangjiping" creation-date: 2026-07-27 -last-updated: 2026-07-27 -status: draft +last-updated: 2026-08-18 +status: implementing --- # OSEP-0018: execd as Sandbox Init @@ -34,6 +34,78 @@ status: draft - [Upgrade & Migration Strategy](#upgrade--migration-strategy) +## Implementation Status + +> Updated 2026-08-18. Status: **implementing** โ€” the phased rollout below is +> implemented on branch `feat/execd-init-mode` (Phases 1โ€“5 plus the server +> switch); remaining items are the trusted stop channel (ยง3), kernel-5.10 +> empirical validation, cross-language e2e, and the default-on rollout (see +> the Remaining work table below). R-i (server-path hardening e2e, docker +> bridge) landed in PR #1554. + +| Phase | Scope | Status | +|---|---|---| +| 1 | execd `--init` mode: single reaper (only wait4 caller), managedProcess abstraction replacing `Cmd.Wait` on every launch path, signal forwarding (TERM/HUP/USR1/USR2/WINCH), entrypoint-owned container lifecycle, subreaper fallback, `PR_SET_DUMPABLE`, `bootstrap.sh` `EXECD_INIT` exec branch, hardening endpoint reporting | โœ… implemented | +| 2 | Pre-exec floor (`[hardening] enabled`): `opensandbox-launcher` native helper (env strip โ†’ KEEPCAPS โ†’ bounding trim โ†’ no_new_privs โ†’ identity drop โ†’ ambient caps โ†’ seccomp last โ†’ execve), `[seccomp] deny` reuse with `execve` reserved, `keep_capabilities`; fail-open per layer | โœ… implemented | +| 3 | Landlock (`[landlock] enabled`): allowlist policy (system paths + `/proc/self` + device files + `/tmp`/`/run`/`allowed_writable` + extras), ABI probe v1โ€“v4 with access-bit trimming; ABI < 1 โ†’ `unsupported` | โœ… implemented | +| 4 | eBPF observation (`[ebpf] enabled`, `execd-ebpf` variant): exec/connect/privilege hooks (CO-RE), sandbox cgroup filter, rotating JSONL audit file; kernel โ‰ฅ5.10 with BTF | โœ… implemented | +| 5 | Pool taskTemplate: with `execd_run_as_init`, tasks are no longer backgrounded โ€” execd becomes the task process-tree root (subreaper + exit-code propagation); `kill 1` recycle contract confirmed compatible under current SIGTERM semantics | โœ… implemented (task level) | +| โ€” | Server switch `runtime.execd_run_as_init` injects `EXECD_INIT=1` across Docker / K8s Batch/Agent / Pool paths โ€” resolves Open Question 1 | โœ… implemented | + +**Open implementation questions โ€” resolution:** + +1. **Single enable switch** โ€” resolved: the server sets `EXECD_INIT` from the + single `runtime.execd_run_as_init` config (default `false`); `bootstrap.sh` + passes `--init` iff `EXECD_INIT` is truthy, so topology and flag stay in + lockstep by construction. +2. **External SIGTERM forwarding** โ€” implemented: SIGTERM is forwarded to the + entrypoint and execd exits with the workload's status. Distinguishing + external vs in-namespace SIGTERM (the ยง3 trusted-stop mechanism) is still + open: today an in-namespace `kill 1` also stops the sandbox (same as the + pre-OSEP bootstrap behavior) โ€” tracked as remaining work. +3. **Managed-process abstraction** โ€” implemented (reaper delivers + `WaitStatus`; per-call-site pipes/teardown are owned by the abstraction). +4. **Landlock device files** โ€” resolved: `null/zero/full/random/urandom/tty` + writable + `/dev/pts` subtree for the controlling terminal. +5. **Landlock `/proc/self` for descendants** โ€” resolved by accepting the + limitation: only the initial workload keeps `/proc/self` access; forked + descendants get EACCES on their own procfs (documented in + `docs/components/execd.md`). + +**Remaining work** (status 2026-08-18, PR #1474 + #1546 + #1554): + +| # | Item | Status / plan | +|---|---|---| +| R-a | Trusted out-of-band stop channel (ยง3, R2) | **Open โ€” biggest remaining item, separate PR.** Any SIGTERM (incl. in-namespace `kill 1`) still stops the sandbox (same as pre-OSEP behavior); `kill -9 1` is inert. Target: authenticated channel unreachable from the workload; then point the K8s Restart recycle at it instead of `kill 1` (`restart_default.go`; current SIGTERM semantics confirmed compatible, comment added) | +| R-b | Pool pod-level PID 1 | **Declined follow-up.** Pool sandboxes run execd as task-level subreaper (Phase 5): the per-child floor holds without PID 1; the lost signal shield is no regression (pool exposure equals the pre-OSEP era; task-executor owns pod reaping). Operators wanting full PID 1 configure the Pool template command manually (`bootstrap.sh` + keepalive + `EXECD_INIT=1`); server auto-injection out of scope | +| R-c | Kernel-5.10 eBPF empirical validation | Open. The 5.10โ€“5.15 exec-hook fallback (inline `filename[1024]`, `__data_loc` argv) is derived from the tracepoint definition, not boot-tested; worst case the hook degrades (fail-open). Needs a real 5.10 node with BTF + `CAP_BPF` | +| R-d | Cross-language SDK e2e | Python covered (docker-bridge + k8s nightly: PID 1, reaping, kill-9 inert, `/proc/1/environ` denial, capabilities endpoint). JS/Go/C#/Java/Kotlin init-mode e2e not written | +| R-e | `execd-ebpf` server-side selection | **Deferred.** The default image ships both binaries (`/execd` + `/execd-ebpf`); choosing which runs (`EXECD` env, `runtime.execd_binary`) is not wired into the server or the Docker/K8s distribution paths. Tracked in `components/execd/README.md` "Known issue / TODO" | +| R-f | Default-on rollout | `runtime.execd_run_as_init` and `[hardening] enabled` default `false` by design; flip after N releases of validation (owner decision), record in release notes | +| R-g | `OPENSANDBOX_ID` reserved-env override (Codex round 7) | **Deferred.** Docker env builder appends `OPENSANDBOX_ID` after user env (pre-existing pattern); harden reserved-key filtering first if a duplicate-key spoofing path is demonstrated | +| R-h | CI flake observation | PauseResume v1.32.2 (only) timed out at 900s on "commit/push fails with invalid registry" (Kubernetes CI); unrelated to this branch's recent commits โ€” re-run to confirm flake | +| R-i | Server-path hardening e2e (Python) | **Implemented (docker bridge + k8s)** โ€” `tests/python/tests/test_execd_hardening_e2e.py` + `scripts/python-execd-hardening-e2e.sh` + CI job `python-execd-hardening-e2e` (PR #1554), extended to the Kubernetes path in the execd-init k8s nightly. **Docker**: the hardened isolation TOML (`components/execd/configs/isolation.hardened.toml`) is injected into every sandbox via a config-level bind mount + `EXECD_ISOLATION_CONFIG` (`[docker] sandbox_env`); the workspace bind additionally exercises the launcher's mount expansion. **Kubernetes** (no server config change): the TOML travels in a ConfigMap (`opensandbox-e2e-execd-isolation`) mounted by the e2e `batchsandbox_template_file` (added `execd-isolation` volume + mount, `optional: true`, merged by the existing template-extras path), the test points `EXECD_ISOLATION_CONFIG` at it per request env, and the workspace PVC is mounted at `/mnt/workspace-exec` via request volumes so the Landlock bind-mount expansion is still exercised. k8s root-cause note: the e2e PVC's hostPath PV used to live under the kind node's `/tmp`, which is a **noexec tmpfs** โ€” every PVC mount was therefore non-executable (writes/reads fine, exec EACCES regardless of Landlock, pod spec and CR were always correct); the e2e harness now places the PV on the node rootfs (`/var/opensandbox-e2e`, `scripts/common/kubernetes-e2e.sh`). The entrypoint dump goes to `/workspace` (writable in both runtimes) and is read back via the SDK files API on k8s. Covers: reduced caps/seccomp/NNP + env strip on entrypoint and `/command`, Landlock (`/tmp` writable, `/etc/passwd` read-only, workspace mount write+exec; skipped when the kernel reports `unsupported`, per ยง6 fail-open), capabilities endpoint layer states, and the missing-`CAP_SETPCAP` degradation (phase 2, docker only โ€” k8s degradation still open: the k8s container ceiling caps are not tuned in the e2e) | +| R-j | eBPF JSONL audit e2e | **Open.** No container/e2e test runs the `execd-ebpf` variant with `[ebpf] enabled` and asserts exec/connect/privilege events land in the rotating audit file; only event-decoding unit tests exist โ€” the `commit_creds` privilege hook has never been validated on a real kernel (ties into R-c/R-e) | +| R-k | Python e2e signal-forwarding breadth | **Implemented** โ€” `test_application_signals_forwarded_to_entrypoint` now sends HUP/USR1/USR2/WINCH to PID 1 and asserts every trap marker fires in the entrypoint (SIGTERM graceful shutdown covered by `tests/init_container.sh`) | +| R-l | K8s init-mode e2e depth | **Partially addressed.** The k8s nightly runs the same Python file with two k8s-path adaptations: `test_entrypoint_exit_code_propagates` skips on k8s (BatchSandbox stays Pending after pod completion and does not surface the container exit code โ€” verified empirically on the nightly), and the `kill 1` pin asserts execd becomes unreachable instead of a lifecycle state transition. Still open: no Pool + `EXECD_INIT` subreaper-report case (R-b); no K8s Restart recycle (`kill 1`) against an init-mode pod โ€” the `restart_default.go` "contract compatible" comment is unverified e2e | +| R-m | Default-off assertion + sustained fork-heavy | **Open (low).** No explicit e2e pin that with init/hardening off the capabilities endpoint reports `init_mode: none` and layers `disabled`; the fork-heavy e2e loop is 20ร—5 short-lived children โ€” a long-running mix of `/command` churn + background sleepers would closer match OSEP ยงTest-Plan | +| R-n | PTY path under hardening โ€” zero coverage | **Implemented** โ€” Go integration test `TestHardeningPTYSessions` (`hardening_linux_test.go`, runs in the execd `test` CI job): StartPTY + StartPipe both launch through the launcher with the reaper active and assert the session shell reports Seccomp=2 / NoNewPrivs=1 / CapEff=0 (root) / `EXECD_ACCESS_TOKEN` stripped. Container-level `/pty` WS case dropped: the alpine execd image has no WS client, and the pty fd / `setsid` / `Setctty` survival across the launcher's `execve` is covered by the integration test | +| R-o | Isolated session (bwrap) + init-mode reaper combination | **Implemented** โ€” (1) Go integration test `TestIsolatedSessionWithInitReaper` (`isolated_session_initmode_linux_test.go`, `linux && bwrap`, run as root in the `bwrap-smoke` CI job): full bwrap lifecycle (create/run/exit-code/delete) under reaper dispatch, plus a delete racing a running workload to exercise the pre-reap barrier's PGID-reuse serialization with the reaper's WNOWAIT-observe โ†’ consume path. (2) Python e2e `TestIsolatedSessionHardeningE2E` (docker bridge, runs in the hardening e2e job's phase 1): bwrap sessions under init mode + the floor โ€” capabilities available, session workload carries bwrap's seccomp/NNP floor + credential env strip, PID-namespace isolation, state persistence, delete-while-busy teardown, hardening report intact around sessions | +| R-p | `/code` (Jupyter kernels) under init/hardening e2e | **Declined follow-up.** Not validated at e2e level; kernels inherit the reduced Jupyter entrypoint by construction, and `test_execd_init_e2e.py` covers Jupyter startup under PID 1 via the ready check. Revisit if the code-interpreter entrypoint changes | +| R-q | Custom `[seccomp] deny` + `keep_capabilities` e2e | **Open (low).** Only Go unit tests cover the reserved-`execve` rejection (`TestHardeningRejectsReservedExecve`) and the ambient-raise path (`TestHardeningKeepCapabilities`). No e2e runs a hardened sandbox with a custom deny list + `keep_capabilities`: plan a TOML variant (e.g. deny `chmod`, `keep_capabilities=["CAP_NET_RAW"]`) in the hardening e2e asserting the denied syscall fails in `/command` and the workload shows CapEff=0x2000 | +| R-r | e2e consumes the SDK `hardening` model instead of a raw HTTP probe | **Implemented** โ€” `_hardening_report` now reads `sandbox.isolation.capabilities().hardening` (`HardeningStatus` model) instead of a `/command` urllib JSON probe, pinning the spec โ†’ SDK โ†’ implementation alignment of the hardening object | +| R-s | `EXECD_INIT` โ†” TOML drift pin (init off, hardening on) | **Open (low).** The "hardening enabled but execd is not the init โ†’ layers degraded" state is only pinned in Go (`TestHardeningReportDegradesWithoutInitMode`); no Python e2e asserts the endpoint reports `init_mode: none` + degraded layers with `[hardening] enabled` but `execd_run_as_init = false`. Complements R-m's default-off pin | +| R-t | Reaper sweep backstop (lost/coalesced SIGCHLD) | **Implemented** โ€” `TestReaperSweepBackstop` (`initmode_linux_test.go`): the reaper's `signal.Notify` subscription stays registered (blocking the Go runtime's auto-reap) while the run loop is severed from it, so only the sweep ticker can reap an exiting child; asserts the child is drained within the ticker budget | +| R-u | Runtime-initiated container stop (external SIGTERM) at SDK/e2e level | **Open (low).** Only `components/execd/tests/init_container.sh` (docker stop) covers the graceful-shutdown path; no SDK-level e2e asserts the entrypoint receives SIGTERM and the sandbox exits with its status when the runtime stops the container. Adjacent to R-k's signal-forwarding breadth | + +Closed this round (2026-08-12): CI green for all execd-init jobs; `/proc/1/environ` +e2e assertion (`test_workload_cannot_read_execd_environ`); hardening report +degrades honestly when hardening is on without init topology; launcher aborts on +failed identity drop; bundled-image Jupyter log relocated under `/tmp`; +generated eBPF bindings gated behind `ebpf &&` tags; `runtime.execd_run_as_init` +documented in `server/configuration.md`. + + ## Summary This proposal makes **execd** the sandbox init (PID 1): it `fork`/`exec`s the user diff --git a/oseps/README.md b/oseps/README.md index 50c98e399..6c6646ea2 100644 --- a/oseps/README.md +++ b/oseps/README.md @@ -9,8 +9,8 @@ This is the complete list of OpenSandbox Enhancement Proposals: | [OSEP-0001](0001-fqdn-based-egress-control.md) | FQDN-based Egress Control | implemented | 2026-01-22 | | [OSEP-0002](0002-kubernetes-sigs-agent-sandbox-support.md) | kubernetes-sigs/agent-sandbox Support | implemented | 2026-01-23 | | [OSEP-0003](0003-volume-and-volumebinding-support.md) | Volume Support | implementing | 2026-02-11 | -| [OSEP-0004](0004-secure-container-runtime.md) | Pluggable Secure Container Runtime Support | implemented | 2026-02-09 | -| [OSEP-0005](0005-client-side-sandbox-pool.md) | Client-Side Sandbox Pool | implementing | 2026-03-09 | +| [OSEP-0004](0004-secure-container-runtime.md) | Pluggable Secure Container Runtime Support | implemented | 2026-07-27 | +| [OSEP-0005](0005-client-side-sandbox-pool.md) | Client-Side Sandbox Pool | implemented | 2026-07-27 | | [OSEP-0006](0006-developer-console.md) | Developer Console for Sandbox Operations | implementable | 2026-03-06 | | [OSEP-0007](0007-fast-sandbox-runtime-support.md) | Fast Sandbox Runtime Support | provisional | 2026-02-08 | | [OSEP-0008](0008-pause-resume-rootfs-snapshot.md) | Pause and Resume via Rootfs Snapshot | implementing | 2026-03-13 | @@ -19,7 +19,7 @@ This is the complete list of OpenSandbox Enhancement Proposals: | [OSEP-0011](0011-secure-access-endpoint.md) | Secure Access on GetEndpoint and Signed Endpoint | implemented | 2026-04-25 | | [OSEP-0012](0012-credential-vault.md) | Credential Vault and Credential Proxy | implementing | 2026-06-10 | | [OSEP-0013](0013-isolated-execution-api.md) | Isolated Execution API | implementing | 2026-06-23 | -| [OSEP-0014](0014-multi-tenancy.md) | Multi-Tenancy Support for Kubernetes Runtime | draft | 2026-05-07 | +| [OSEP-0014](0014-multi-tenancy.md) | Multi-Tenancy Support for Kubernetes Runtime | implemented | 2026-07-27 | | [OSEP-0015](0015-pod-snapshot.md) | Spec-Driven Pod Snapshot for Pause and Resume | draft | 2026-06-27 | | [OSEP-0016](0016-unified-umbrella-release-governance.md) | Unified Umbrella Release Governance | draft | 2026-07-21 | | [OSEP-0017](0017-resilient-sdk-transport.md) | Resilient SDK Transport | implementing | 2026-07-22 | diff --git a/sandboxes/code-interpreter/scripts/code-interpreter.sh b/sandboxes/code-interpreter/scripts/code-interpreter.sh index 1b09c9198..e6ad1b71e 100755 --- a/sandboxes/code-interpreter/scripts/code-interpreter.sh +++ b/sandboxes/code-interpreter/scripts/code-interpreter.sh @@ -167,4 +167,9 @@ pids+=($!) setup_bash & pids+=($!) -jupyter notebook --ip=127.0.0.1 --port="${JUPYTER_PORT:-44771}" --allow-root --no-browser --NotebookApp.token="${JUPYTER_TOKEN:-opensandboxcodeinterpreterjupyter}" >/opt/code-interpreter/jupyter.log +# Runtime artifacts live under /tmp so the Landlock hardening floor +# (read+exec on /opt, read+write on /tmp) keeps working out of the box. +export JUPYTER_RUNTIME_DIR=/tmp/jupyter-runtime +mkdir -p /tmp/jupyter-runtime /tmp/code-interpreter + +jupyter notebook --ip=127.0.0.1 --port="${JUPYTER_PORT:-44771}" --allow-root --no-browser --NotebookApp.token="${JUPYTER_TOKEN:-opensandboxcodeinterpreterjupyter}" >/tmp/code-interpreter/jupyter.log diff --git a/scripts/common/kubernetes-e2e.sh b/scripts/common/kubernetes-e2e.sh index 06620a1cb..20e5b6e0b 100644 --- a/scripts/common/kubernetes-e2e.sh +++ b/scripts/common/kubernetes-e2e.sh @@ -78,7 +78,10 @@ spec: persistentVolumeReclaimPolicy: Retain storageClassName: manual hostPath: - path: /tmp/${PV_NAME} + # NOT under /tmp: the kind node mounts /tmp as a noexec tmpfs, so a + # hostPath PV there is not executable (writes/reads work, exec fails with + # EACCES regardless of Landlock). /var lives on the node's rootfs. + path: /var/opensandbox-e2e/${PV_NAME} type: DirectoryOrCreate --- apiVersion: v1 @@ -188,6 +191,7 @@ configToml: | [runtime] type = "kubernetes" execd_image = "${EXECD_IMG}" + execd_run_as_init = ${E2E_EXECD_RUN_AS_INIT:-false} [egress] image = "${EGRESS_IMG}" diff --git a/scripts/python-e2e.sh b/scripts/python-e2e.sh index 9278d32ee..70ac13516 100755 --- a/scripts/python-e2e.sh +++ b/scripts/python-e2e.sh @@ -90,5 +90,7 @@ if [ "${RUN_CODE_INTERPRETER_E2E}" = "true" ]; then else uv run pytest \ --ignore=tests/test_code_interpreter_e2e.py \ - --ignore=tests/test_code_interpreter_e2e_sync.py + --ignore=tests/test_code_interpreter_e2e_sync.py \ + --ignore=tests/test_execd_init_e2e.py \ + --ignore=tests/test_execd_hardening_e2e.py fi diff --git a/scripts/python-execd-hardening-e2e.sh b/scripts/python-execd-hardening-e2e.sh new file mode 100644 index 000000000..8bde5e16c --- /dev/null +++ b/scripts/python-execd-hardening-e2e.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the server-path hardening E2E (OSEP-0018, R-i) against a local +# Docker-bridge server with runtime.execd_run_as_init = true. The hardened +# isolation TOML (hardening + landlock) is injected into every sandbox via +# a config-level bind mount + EXECD_ISOLATION_CONFIG, so the whole +# server -> sandbox -> execd path runs with the floor on. +# +# Phase 1: the floor works end to end (reduced caps/seccomp/NNP, env strip, +# landlock enforcement, capabilities endpoint). +# Phase 2: fail-open degradation โ€” same server with CAP_SETPCAP dropped from +# the container ceiling; cap_drop must report degraded while the +# rest of the floor stays active. +# +# Usage: bash scripts/python-execd-hardening-e2e.sh + +set -euxo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SERVER_PID="" + +cleanup() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +run_server() { + cd server + export OPENSANDBOX_INSECURE_SERVER=YES + uv sync + uv run python -m opensandbox_server.main > server.log 2>&1 & + SERVER_PID=$! + cd "${REPO_ROOT}" + sleep 10 +} + +stop_server() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + SERVER_PID="" + sleep 2 + fi +} + +run_pytest() { + local selector="$1" + cd "${REPO_ROOT}/tests/python" + uv sync --all-extras --refresh + uv run pytest tests/test_execd_hardening_e2e.py -v -k "${selector}" + cd "${REPO_ROOT}" +} + +# --------------------------------------------------------------------------- +# Images + host workspace + hardened TOML. +# --------------------------------------------------------------------------- +docker build -f components/execd/Dockerfile -t opensandbox/execd:local "${REPO_ROOT}" +docker pull opensandbox/code-interpreter:${TAG:-latest} + +# The workload has no CAP_DAC_OVERRIDE under the floor, so the host-side +# workspace dir must be fully accessible to the container uid. The hardened +# TOML is injected via a config-level bind mount (the server does not stage +# isolation configs from the execd image). +mkdir -p /tmp/opensandbox-e2e/workspace /tmp/opensandbox-e2e/logs +chmod 0777 /tmp/opensandbox-e2e/workspace +cp components/execd/configs/isolation.hardened.toml /tmp/opensandbox-e2e/isolation.hardened.toml +echo "-------- EXECD HARDENING E2E test logs for execd --------" > /tmp/opensandbox-e2e/logs/execd.log + +write_server_config() { + local drop_capabilities="$1" + cat < ~/.sandbox.toml +[server] +host = "127.0.0.1" +port = 8080 +api_key = "" +[log] +level = "INFO" +[runtime] +type = "docker" +execd_image = "opensandbox/execd:local" +execd_run_as_init = true +[egress] +image = "opensandbox/egress:local" +mode = "dns+nft" +[docker] +network_mode = "bridge" +# The container baseline must not mask the launcher's own floor: with +# no_new_privileges=true (the server default) or Docker's default seccomp +# profile, a launcher regression would still show NoNewPrivs=1/Seccomp=2 +# in the workload. Unset both so the observed values can only come from +# the opensandbox-launcher. +no_new_privileges = false +seccomp_profile = "unconfined" +sandbox_env = { EXECD_ISOLATION_CONFIG = "/etc/opensandbox/isolation.toml" } +sandbox_binds = [ + "/tmp/opensandbox-e2e/workspace:/workspace", + "/tmp/opensandbox-e2e/isolation.hardened.toml:/etc/opensandbox/isolation.toml", +] +drop_capabilities = ${drop_capabilities} +[storage] +allowed_host_paths = ["/tmp/opensandbox-e2e"] +EOF +} + +# --------------------------------------------------------------------------- +# Phase 1: the floor applies end to end (default ceiling keeps CAP_SETPCAP). +# --------------------------------------------------------------------------- +write_server_config '["AUDIT_WRITE", "MKNOD", "NET_ADMIN", "NET_RAW", "SYS_ADMIN", "SYS_MODULE", "SYS_PTRACE", "SYS_TIME", "SYS_TTY_CONFIG"]' +run_server +run_pytest "TestHardeningE2E or TestIsolatedSessionHardeningE2E" +stop_server + +# --------------------------------------------------------------------------- +# Phase 2: degradation โ€” CAP_SETPCAP missing from the ceiling. +# --------------------------------------------------------------------------- +write_server_config '["AUDIT_WRITE", "MKNOD", "NET_ADMIN", "NET_RAW", "SYS_ADMIN", "SYS_MODULE", "SYS_PTRACE", "SYS_TIME", "SYS_TTY_CONFIG", "SETPCAP"]' +run_server +OPENSANDBOX_HARDENING_DEGRADATION=true run_pytest "TestHardeningDegradationE2E" +stop_server + +echo "Execd hardening E2E PASSED (phase 1: floor, phase 2: degradation)" diff --git a/scripts/python-execd-init-e2e.sh b/scripts/python-execd-init-e2e.sh new file mode 100755 index 000000000..766a74493 --- /dev/null +++ b/scripts/python-execd-init-e2e.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the execd-as-init E2E (OSEP-0018) against a local Docker-bridge +# server. The server config (~/.sandbox.toml) is written by the workflow +# with runtime.execd_run_as_init = true. + +set -euxo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SERVER_PID="" + +cleanup() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +docker build -f components/execd/Dockerfile -t opensandbox/execd:local "${REPO_ROOT}" +docker pull opensandbox/code-interpreter:${TAG:-latest} + +mkdir -p /tmp/opensandbox-e2e/logs +echo "-------- EXECD INIT E2E test logs for execd --------" > /tmp/opensandbox-e2e/logs/execd.log + +cd server +export OPENSANDBOX_INSECURE_SERVER=YES +uv sync +uv run python -m opensandbox_server.main > server.log 2>&1 & +SERVER_PID=$! +cd .. + +sleep 10 + +cd sdks/sandbox/python && make generate-api +cd ../../.. + +cd tests/python +uv sync --all-extras --refresh +uv run pytest tests/test_execd_init_e2e.py -v diff --git a/scripts/python-k8s-execd-init-e2e.sh b/scripts/python-k8s-execd-init-e2e.sh new file mode 100755 index 000000000..97caea6aa --- /dev/null +++ b/scripts/python-k8s-execd-init-e2e.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the execd-as-init E2E (OSEP-0018) on a Kind cluster with the +# Kubernetes runtime and runtime.execd_run_as_init = true. + +set -euxo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common/kubernetes-e2e.sh +source "${SCRIPT_DIR}/common/kubernetes-e2e.sh" + +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +KIND_CLUSTER="${KIND_CLUSTER:-opensandbox-e2e}" +KIND_K8S_VERSION="${KIND_K8S_VERSION:-v1.30.4}" +KUBECONFIG_PATH="${KUBECONFIG_PATH:-/tmp/opensandbox-kind-kubeconfig}" +E2E_NAMESPACE="${E2E_NAMESPACE:-opensandbox-e2e}" +SERVER_NAMESPACE="${SERVER_NAMESPACE:-opensandbox-system}" +PVC_NAME="${PVC_NAME:-opensandbox-e2e-pvc-test}" +PV_NAME="${PV_NAME:-opensandbox-e2e-pv-test}" +CONTROLLER_IMG="${CONTROLLER_IMG:-opensandbox/controller:e2e-local}" +SERVER_IMG="${SERVER_IMG:-opensandbox/server:e2e-local}" +EXECD_IMG="${EXECD_IMG:-opensandbox/execd:e2e-local}" +EGRESS_IMG="${EGRESS_IMG:-opensandbox/egress:e2e-local}" +SERVER_RELEASE="${SERVER_RELEASE:-opensandbox-server}" +SERVER_VALUES_FILE="${SERVER_VALUES_FILE:-/tmp/opensandbox-server-values.yaml}" +PORT_FORWARD_LOG="${PORT_FORWARD_LOG:-/tmp/opensandbox-server-port-forward.log}" +SANDBOX_TEST_IMAGE="${SANDBOX_TEST_IMAGE:-opensandbox/code-interpreter:latest}" +LIFECYCLE_LOCAL_PORT="${LIFECYCLE_LOCAL_PORT:-8080}" + +SERVER_IMG_REPOSITORY="${SERVER_IMG%:*}" +SERVER_IMG_TAG="${SERVER_IMG##*:}" + +export E2E_EXECD_RUN_AS_INIT=true + +k8s_e2e_export_kubeconfig +k8s_e2e_setup_kind_and_controller +k8s_e2e_build_runtime_images +k8s_e2e_kind_load_runtime_images +k8s_e2e_apply_pvc_and_seed +# The hardened isolation TOML travels to sandboxes via a ConfigMap mounted by +# the e2e batchsandbox template (optional: true), and the hardening e2e points +# EXECD_ISOLATION_CONFIG at it per request. No server config needed. +kubectl create configmap opensandbox-e2e-execd-isolation \ + --namespace "${E2E_NAMESPACE}" \ + --from-file=isolation.hardened.toml="${REPO_ROOT}/components/execd/configs/isolation.hardened.toml" \ + --dry-run=client -o yaml | kubectl apply -f - +k8s_e2e_write_server_helm_values +k8s_e2e_helm_install_server + +kubectl port-forward -n "${SERVER_NAMESPACE}" svc/opensandbox-server "${LIFECYCLE_LOCAL_PORT}:80" >"${PORT_FORWARD_LOG}" 2>&1 & +PORT_FORWARD_PID=$! +trap 'kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true' EXIT + +# Capture the sandbox pod specs and BatchSandbox CRs while the tests run: the +# hardening leg has twice observed PVC mount paths showing up as noexec tmpfs +# inside the container, so the pod-spec dump is needed to see what the +# controller actually created. +( + for _ in $(seq 1 600); do + kubectl get pods -n "${E2E_NAMESPACE}" -o yaml > /tmp/opensandbox-e2e-pods.yaml 2>/dev/null || true + kubectl get batchsandboxes -n "${E2E_NAMESPACE}" -o yaml > /tmp/opensandbox-e2e-batchsandboxes.yaml 2>/dev/null || true + sleep 2 + done +) & +SPEC_WATCHER_PID=$! +trap 'kill "${SPEC_WATCHER_PID}" >/dev/null 2>&1 || true; kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true' EXIT + +k8s_e2e_wait_http_ok "http://127.0.0.1:${LIFECYCLE_LOCAL_PORT}/health" + +export OPENSANDBOX_TEST_DOMAIN="localhost:${LIFECYCLE_LOCAL_PORT}" +export OPENSANDBOX_TEST_PROTOCOL="http" +export OPENSANDBOX_TEST_API_KEY="kubernetes-e2e" +export OPENSANDBOX_SANDBOX_DEFAULT_IMAGE="${SANDBOX_TEST_IMAGE}" +export OPENSANDBOX_E2E_RUNTIME="kubernetes" +export OPENSANDBOX_TEST_USE_SERVER_PROXY="true" +export OPENSANDBOX_TEST_PVC_NAME="${PVC_NAME}" + +k8s_e2e_export_sandbox_resource_env + +cd "${REPO_ROOT}/sdks/sandbox/python" +make generate-api +cd "${REPO_ROOT}/tests/python" +uv sync --all-extras --refresh +uv run pytest tests/test_execd_init_e2e.py -v +uv run pytest tests/test_execd_hardening_e2e.py -v -k "TestHardeningE2E" diff --git a/scripts/verify-license.sh b/scripts/verify-license.sh index 954860a6a..455ad3fab 100755 --- a/scripts/verify-license.sh +++ b/scripts/verify-license.sh @@ -68,6 +68,12 @@ is_generated_to_skip() { if [[ "$file" == *"deepcopy.go" ]]; then return 0 fi + # Files emitted by code generators (e.g. bpf2go, gRPC/protobuf) carry the + # standard generated marker; regeneration would clobber any manually added + # header. + if head -n 25 "$file" 2>/dev/null | grep -qE "(Code generated by|Generated by .*)\s.* DO NOT EDIT"; then + return 0 + fi return 1 } diff --git a/sdks/code-interpreter/javascript/src/adapters/openapiError.ts b/sdks/code-interpreter/javascript/src/adapters/openapiError.ts index 128a9074f..d48770104 100644 --- a/sdks/code-interpreter/javascript/src/adapters/openapiError.ts +++ b/sdks/code-interpreter/javascript/src/adapters/openapiError.ts @@ -24,10 +24,22 @@ export function throwOnOpenApiFetchError( const status = (result.response as any).status ?? 0; const err = result.error as any; + + let rawFragment: string | undefined; + if (typeof result.error === "string") { + rawFragment = result.error; + } else if (result.error && typeof result.error === "object") { + try { + rawFragment = JSON.stringify(result.error); + } catch { + rawFragment = undefined; + } + } + const message = err?.message ?? err?.error?.message ?? - fallbackMessage; + (rawFragment && rawFragment.length > 0 ? rawFragment : fallbackMessage); const code = err?.code ?? err?.error?.code; const msg = err?.message ?? err?.error?.message ?? message; diff --git a/sdks/code-interpreter/javascript/tests/contexts.test.mjs b/sdks/code-interpreter/javascript/tests/contexts.test.mjs index 72badf74b..098e493bb 100644 --- a/sdks/code-interpreter/javascript/tests/contexts.test.mjs +++ b/sdks/code-interpreter/javascript/tests/contexts.test.mjs @@ -85,3 +85,26 @@ test("DefaultAdapterFactory exposes context CRUD and interrupt operations", asyn assert.equal(entry.headers["x-endpoint"], "endpoint"); } }); + +test("code context error message carries unstructured error body", async () => { + const factory = new DefaultAdapterFactory(); + const codes = factory.createCodes({ + sandbox: { + connectionConfig: { + headers: {}, + fetch: async () => new Response("context not found", { status: 404 }), + sseFetch: async () => new Response("", { status: 200 }), + }, + }, + execdBaseUrl: "http://sandbox.internal:3456", + endpointHeaders: {}, + }); + + await assert.rejects( + () => codes.getContext("ctx-1"), + (err) => { + assert.match(err.message, /context not found/); + return true; + }, + ); +}); diff --git a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py index 55a759652..3be86fa7a 100644 --- a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py +++ b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py @@ -312,7 +312,7 @@ async def run( f"Failed to run code. Status: {response.status_code}, Body: {error_body}" ) raise SandboxApiException( - message=f"Failed to run code. Status code: {response.status_code}", + message=f"Failed to run code. Status code: {response.status_code}, Body: {error_body}", status_code=response.status_code, request_id=extract_request_id(response.headers), ) diff --git a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py index ec2dfe7f0..f09da3b56 100644 --- a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py +++ b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py @@ -291,8 +291,9 @@ def run( with self._sse_client.stream("POST", url, json=api_request) as response: if response.status_code != 200: response.read() + error_body = response.text raise SandboxApiException( - message=f"Failed to run code. Status code: {response.status_code}", + message=f"Failed to run code. Status code: {response.status_code}, Body: {error_body}", status_code=response.status_code, request_id=extract_request_id(response.headers), ) diff --git a/sdks/code-interpreter/python/tests/test_code_service_adapter_streaming.py b/sdks/code-interpreter/python/tests/test_code_service_adapter_streaming.py index 5d5898266..5b43c6aa5 100644 --- a/sdks/code-interpreter/python/tests/test_code_service_adapter_streaming.py +++ b/sdks/code-interpreter/python/tests/test_code_service_adapter_streaming.py @@ -165,3 +165,5 @@ async def test_run_code_non_200_raises_api_exception() -> None: with pytest.raises(SandboxApiException) as ei: await adapter.run("other") assert ei.value.request_id == "req-code-123" + # The server's error body is spliced into the message so logs carry the reason. + assert "bad" in str(ei.value) diff --git a/sdks/code-interpreter/python/tests/test_code_service_adapter_sync_headers.py b/sdks/code-interpreter/python/tests/test_code_service_adapter_sync_headers.py index 8ac7dcb6f..fa7c0ddbf 100644 --- a/sdks/code-interpreter/python/tests/test_code_service_adapter_sync_headers.py +++ b/sdks/code-interpreter/python/tests/test_code_service_adapter_sync_headers.py @@ -13,12 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import pytest from opensandbox.config.connection_sync import ConnectionConfigSync +from opensandbox.exceptions import SandboxApiException from opensandbox.models.sandboxes import SandboxEndpoint from code_interpreter.sync.adapters.code_adapter import CodesAdapterSync +def test_sync_adapter_non_200_includes_error_body_in_message() -> None: + import httpx + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + headers={"x-request-id": "req-code-sync-123"}, + content=b"bad request body", + request=request, + ) + + cfg = ConnectionConfigSync(protocol="http", transport=httpx.MockTransport(handler)) + endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772) + adapter = CodesAdapterSync(endpoint, cfg) + + with pytest.raises(SandboxApiException) as ei: + adapter.run("other") + assert ei.value.request_id == "req-code-sync-123" + # The server's error body is spliced into the message so logs carry the reason. + assert "bad request body" in str(ei.value) + + def test_sync_adapter_merges_endpoint_headers_into_both_clients() -> None: cfg = ConnectionConfigSync(protocol="http", headers={"X-Base": "base"}) endpoint = SandboxEndpoint( diff --git a/sdks/mcp/sandbox/python/README.md b/sdks/mcp/sandbox/python/README.md index 3f3e89cbd..222c71447 100644 --- a/sdks/mcp/sandbox/python/README.md +++ b/sdks/mcp/sandbox/python/README.md @@ -42,6 +42,7 @@ Config fields: - `protocol`: `http` or `https` for API requests. - `request_timeout_seconds`: HTTP request timeout in seconds. - `transport`: `stdio` by default, or `streamable-http`. +- `use-server-proxy`: when present, forces the SDK client to use server proxy mode. ### Streamable HTTP diff --git a/sdks/mcp/sandbox/python/pyproject.toml b/sdks/mcp/sandbox/python/pyproject.toml index bd833e859..00933ae87 100644 --- a/sdks/mcp/sandbox/python/pyproject.toml +++ b/sdks/mcp/sandbox/python/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "opensandbox>=0.1.10,<0.2.0", "starlette>=1.3.1", "python-multipart>=0.0.31", - "cryptography>=48.0.1", + "cryptography>=50.0.0", "pydantic-settings>=2.14.2", "pyjwt>=2.13.0", ] @@ -145,7 +145,7 @@ branch = true constraint-dependencies = [ "starlette>=1.3.1", "python-multipart>=0.0.31", - "cryptography>=48.0.1", + "cryptography>=50.0.0", "pydantic-settings>=2.14.2", "pyjwt>=2.13.0", ] diff --git a/sdks/mcp/sandbox/python/src/opensandbox_mcp/__main__.py b/sdks/mcp/sandbox/python/src/opensandbox_mcp/__main__.py index 623301309..d195e44d6 100644 --- a/sdks/mcp/sandbox/python/src/opensandbox_mcp/__main__.py +++ b/sdks/mcp/sandbox/python/src/opensandbox_mcp/__main__.py @@ -54,6 +54,12 @@ def main() -> None: default=30, help="HTTP request timeout in seconds.", ) + parser.add_argument( + "--use-server-proxy", + action="store_true", + default=False, + help="Route sandbox traffic through the OpenSandbox server proxy (use when direct port access is blocked).", + ) args = parser.parse_args() config_values = {} @@ -67,6 +73,9 @@ def main() -> None: config_values["request_timeout"] = timedelta( seconds=args.request_timeout_seconds ) + if args.use_server_proxy: + config_values["use_server_proxy"] = True + connection_config = ConnectionConfig(**config_values) if config_values else None mcp = create_server(connection_config=connection_config) diff --git a/sdks/mcp/sandbox/python/uv.lock b/sdks/mcp/sandbox/python/uv.lock index 4b9a3a952..93b8a8a5d 100644 --- a/sdks/mcp/sandbox/python/uv.lock +++ b/sdks/mcp/sandbox/python/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ [manifest] constraints = [ - { name = "cryptography", specifier = ">=48.0.1" }, + { name = "cryptography", specifier = ">=50.0.0" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, { name = "pyjwt", specifier = ">=2.13.0" }, { name = "python-multipart", specifier = ">=0.0.31" }, @@ -276,59 +276,59 @@ toml = [ [[package]] name = "cryptography" -version = "49.0.0" +version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, - { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, - { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, - { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, - { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, - { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, - { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, - { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, - { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, - { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9e/db72b3ae7fc9cfad53e630e56c6ae83b9b6ff0bf3718ffb8012d20b3aabf/cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7", size = 4013892, upload-time = "2026-06-12T20:02:10.735Z" }, - { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, - { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, - { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, - { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, - { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, - { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, - { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, - { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, - { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fe/93ecac273d3738939d023612ad12cca9a3740a5345d69fda04134c43fd96/cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61", size = 3799153, upload-time = "2026-06-12T20:01:39.059Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, - { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, - { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, - { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, - { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, - { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, - { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, - { url = "https://files.pythonhosted.org/packages/63/d3/4a83af35d65e3fad632c926fad684c193ea4398569ccb0bbbc7fe8f5dc9a/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b", size = 3993685, upload-time = "2026-06-12T20:02:14.883Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a7/f9dac0ab7f80368c56993a7bf638ef9935f825c91902798481fac0898138/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838", size = 4676239, upload-time = "2026-06-12T20:02:28.793Z" }, - { url = "https://files.pythonhosted.org/packages/d7/70/2ba3769dd0ae167e2f33dfa9592d45db6ff9a61d62ca1a5b3d1bdd09068f/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5", size = 4715584, upload-time = "2026-06-12T20:01:27.495Z" }, - { url = "https://files.pythonhosted.org/packages/94/64/2923570ac1c0bd3a737aa366ac3abbbbde273042308b8cde95e2364a6e6a/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615", size = 4675885, upload-time = "2026-06-12T20:01:55.49Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f8/614dc7e051418cfe53d55173c1e24c6b0085e89996fe90508c2fdf769aef/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6", size = 4715449, upload-time = "2026-06-12T20:02:05.469Z" }, - { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3e/e54cde8c01631a5a8226ccd617eab9e57fd5cfdad90f1a9e6bb570794631/cryptography-50.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c", size = 3963170, upload-time = "2026-07-31T14:24:51.968Z" }, + { url = "https://files.pythonhosted.org/packages/01/b6/0b9e125e90f3d2dcf599a218a899cda7326a3158cfa258723f0b398b08f6/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a", size = 4692441, upload-time = "2026-07-31T14:24:53.743Z" }, + { url = "https://files.pythonhosted.org/packages/53/c9/a5151588710785a96d7bc4de27d4cd62f263bbbcb203cfe29df537eb6505/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e", size = 4699810, upload-time = "2026-07-31T14:24:55.746Z" }, + { url = "https://files.pythonhosted.org/packages/c7/1a/15b92b25eb6ce3089cd49377ae990a0f3ad485a510f968aed1f19dbdcdf2/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d", size = 4691924, upload-time = "2026-07-31T14:24:58.082Z" }, + { url = "https://files.pythonhosted.org/packages/62/15/219075012ab13e8905f3cd572204f4acb4b111df787104346b9bc0cea789/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437", size = 4699593, upload-time = "2026-07-31T14:24:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b5/c2c5fce26f0ee40d21bafe7f191d29a34b35a65ac4fe8a1191d1983612e9/cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9", size = 3813796, upload-time = "2026-07-31T14:25:02.298Z" }, ] [[package]] @@ -501,6 +501,7 @@ source = { editable = "../../../sandbox/python" } dependencies = [ { name = "attrs" }, { name = "httpx" }, + { name = "httpx-sse" }, { name = "pydantic" }, { name = "python-dateutil" }, ] @@ -509,6 +510,7 @@ dependencies = [ requires-dist = [ { name = "attrs", specifier = ">=21.3.0" }, { name = "httpx", specifier = ">=0.27.0,<1.0" }, + { name = "httpx-sse", specifier = ">=0.4.3,<0.5" }, { name = "pydantic", specifier = ">=2.4.2,<3.0" }, { name = "pyjwt", marker = "extra == 'pool-redis'", specifier = ">=2.13.0" }, { name = "python-dateutil", specifier = ">=2.8.2,<3.0" }, @@ -551,7 +553,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "cryptography", specifier = ">=48.0.1" }, + { name = "cryptography", specifier = ">=50.0.0" }, { name = "mcp", extras = ["cli"] }, { name = "opensandbox", editable = "../../../sandbox/python" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, diff --git a/sdks/package.json b/sdks/package.json index f39f26939..11128a9b0 100644 --- a/sdks/package.json +++ b/sdks/package.json @@ -16,12 +16,12 @@ "rollup@^4.0.0": "4.60.2", "picomatch@^4.0.0": "4.0.4", "brace-expansion@^1.0.0": "1.1.16", - "brace-expansion@^2.0.0": "2.1.2", + "brace-expansion@^2.0.0": "2.1.4", "brace-expansion@^5.0.0": "5.0.7", "flatted@^3.0.0": "3.4.2", - "fast-uri@^3.0.0": "3.1.4", - "js-yaml@^4.0.0": "4.3.0", - "undici": "7.28.0" + "fast-uri@^3.0.0": "3.1.5", + "js-yaml@^4.0.0": "4.3.1", + "undici": "7.29.0" } }, "devDependencies": { diff --git a/sdks/pnpm-lock.yaml b/sdks/pnpm-lock.yaml index 76ab6ffc0..64aa3e6c7 100644 --- a/sdks/pnpm-lock.yaml +++ b/sdks/pnpm-lock.yaml @@ -9,12 +9,12 @@ overrides: rollup@^4.0.0: 4.60.2 picomatch@^4.0.0: 4.0.4 brace-expansion@^1.0.0: 1.1.16 - brace-expansion@^2.0.0: 2.1.2 + brace-expansion@^2.0.0: 2.1.4 brace-expansion@^5.0.0: 5.0.7 flatted@^3.0.0: 3.4.2 - fast-uri@^3.0.0: 3.1.4 - js-yaml@^4.0.0: 4.3.0 - undici: 7.28.0 + fast-uri@^3.0.0: 3.1.5 + js-yaml@^4.0.0: 4.3.1 + undici: 7.29.0 importers: @@ -64,8 +64,8 @@ importers: specifier: ^0.14.1 version: 0.14.1 undici: - specifier: 7.28.0 - version: 7.28.0 + specifier: 7.29.0 + version: 7.29.0 devDependencies: '@eslint/js': specifier: ^9.39.4 @@ -366,66 +366,79 @@ packages: resolution: {integrity: sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.60.2': resolution: {integrity: sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.60.2': resolution: {integrity: sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.60.2': resolution: {integrity: sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.60.2': resolution: {integrity: sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.60.2': resolution: {integrity: sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.60.2': resolution: {integrity: sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.60.2': resolution: {integrity: sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.60.2': resolution: {integrity: sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.60.2': resolution: {integrity: sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.60.2': resolution: {integrity: sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.60.2': resolution: {integrity: sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.60.2': resolution: {integrity: sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.60.2': resolution: {integrity: sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==} @@ -563,8 +576,8 @@ packages: brace-expansion@1.1.16: resolution: {integrity: sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==} - brace-expansion@2.1.2: - resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==} + brace-expansion@2.1.4: + resolution: {integrity: sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==} brace-expansion@5.0.7: resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} @@ -699,8 +712,8 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} - fast-uri@3.1.4: - resolution: {integrity: sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==} + fast-uri@3.1.5: + resolution: {integrity: sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==} fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} @@ -796,8 +809,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@4.3.0: - resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} + js-yaml@4.3.1: + resolution: {integrity: sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==} hasBin: true json-buffer@3.0.1: @@ -1078,8 +1091,8 @@ packages: ufo@1.6.3: resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} engines: {node: '>=20.18.1'} uri-js@4.4.1: @@ -1224,7 +1237,7 @@ snapshots: globals: 14.0.0 ignore: 5.3.2 import-fresh: 3.3.1 - js-yaml: 4.3.0 + js-yaml: 4.3.1 minimatch: 3.1.5 strip-json-comments: 3.1.1 transitivePeerDependencies: @@ -1267,7 +1280,7 @@ snapshots: '@redocly/ajv@8.17.1': dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.4 + fast-uri: 3.1.5 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 @@ -1280,7 +1293,7 @@ snapshots: colorette: 1.4.0 https-proxy-agent: 7.0.6(supports-color@10.2.2) js-levenshtein: 1.1.6 - js-yaml: 4.3.0 + js-yaml: 4.3.1 minimatch: 5.1.8 pluralize: 8.0.0 yaml-ast-parser: 0.0.43 @@ -1491,7 +1504,7 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 - brace-expansion@2.1.2: + brace-expansion@2.1.4: dependencies: balanced-match: 1.0.2 @@ -1654,7 +1667,7 @@ snapshots: fast-levenshtein@2.0.6: {} - fast-uri@3.1.4: {} + fast-uri@3.1.5: {} fdir@6.5.0(picomatch@4.0.4): optionalDependencies: @@ -1729,7 +1742,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@4.3.0: + js-yaml@4.3.1: dependencies: argparse: 2.0.1 @@ -1776,7 +1789,7 @@ snapshots: minimatch@5.1.8: dependencies: - brace-expansion: 2.1.2 + brace-expansion: 2.1.4 mlly@1.8.0: dependencies: @@ -2006,7 +2019,7 @@ snapshots: ufo@1.6.3: {} - undici@7.28.0: {} + undici@7.29.0: {} uri-js@4.4.1: dependencies: diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Adapters/CommandsAdapter.cs b/sdks/sandbox/csharp/src/OpenSandbox/Adapters/CommandsAdapter.cs index 8a6f68e1d..a58483df6 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Adapters/CommandsAdapter.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Adapters/CommandsAdapter.cs @@ -371,7 +371,8 @@ private static SandboxApiException CreateApiException(HttpResponseMessage respon } } - var message = errorMessage ?? $"Request failed with status code {(int)response.StatusCode}"; + var message = errorMessage ?? $"Request failed with status code {(int)response.StatusCode}" + + (string.IsNullOrEmpty(content) ? string.Empty : $": {content}"); return new SandboxApiException( message: message, statusCode: (int)response.StatusCode, diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Adapters/SseParser.cs b/sdks/sandbox/csharp/src/OpenSandbox/Adapters/SseParser.cs index 6be24a3db..50b5b7516 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Adapters/SseParser.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Adapters/SseParser.cs @@ -75,7 +75,8 @@ public static async IAsyncEnumerable ParseJsonEventStreamAsync( } } - var message = errorMessage ?? fallbackErrorMessage ?? $"Stream request failed (status={(int)response.StatusCode})"; + var message = errorMessage ?? $"{fallbackErrorMessage ?? $"Stream request failed (status={(int)response.StatusCode})"}" + + (string.IsNullOrEmpty(text) ? string.Empty : $": {text}"); var sandboxErrorCode = errorCode ?? SandboxErrorCodes.UnexpectedResponse; throw new SandboxApiException( diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Internal/HttpClientWrapper.cs b/sdks/sandbox/csharp/src/OpenSandbox/Internal/HttpClientWrapper.cs index af6462533..19aa542c8 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Internal/HttpClientWrapper.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Internal/HttpClientWrapper.cs @@ -382,7 +382,8 @@ private static void ThrowApiException(HttpResponseMessage response, string conte } } - var message = errorMessage ?? $"Request failed with status code {(int)response.StatusCode}"; + var message = errorMessage ?? $"Request failed with status code {(int)response.StatusCode}" + + (string.IsNullOrEmpty(content) ? string.Empty : $": {content}"); var sandboxErrorCode = errorCode ?? SandboxErrorCodes.UnexpectedResponse; throw new SandboxApiException( diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Models/Isolated.cs b/sdks/sandbox/csharp/src/OpenSandbox/Models/Isolated.cs index 61f9ac913..6d951a884 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Models/Isolated.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Models/Isolated.cs @@ -125,7 +125,8 @@ public record IsolatedCapabilities( [property: JsonPropertyName("commit_supported")] bool CommitSupported = false, [property: JsonPropertyName("diff_supported")] bool DiffSupported = false, [property: JsonPropertyName("setpriv_available")] bool SetprivAvailable = false, - [property: JsonPropertyName("userns_available")] bool UsernsAvailable = false + [property: JsonPropertyName("userns_available")] bool UsernsAvailable = false, + [property: JsonPropertyName("hardening")] HardeningStatus? Hardening = null ) { public void Deconstruct( @@ -144,3 +145,19 @@ public void Deconstruct( diffSupported = DiffSupported; } } + +/// execd init-mode and workload-hardening state (OSEP-0018). +public record HardeningStatus( + [property: JsonPropertyName("init_mode")] string? InitMode = null, // "pid1" | "subreaper" | "none" + [property: JsonPropertyName("signal_shield")] bool SignalShield = false, + [property: JsonPropertyName("cap_drop")] HardeningLayerState? CapDrop = null, + [property: JsonPropertyName("seccomp")] HardeningLayerState? Seccomp = null, + [property: JsonPropertyName("landlock")] HardeningLayerState? Landlock = null, + [property: JsonPropertyName("ebpf")] HardeningLayerState? Ebpf = null +); + +/// Whether one hardening layer is actually enforced. +public record HardeningLayerState( + [property: JsonPropertyName("state")] string? State = null, // active | disabled | degraded | unsupported + [property: JsonPropertyName("message")] string? Message = null +); diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Sandbox.cs b/sdks/sandbox/csharp/src/OpenSandbox/Sandbox.cs index 674bdec7a..143255314 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Sandbox.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Sandbox.cs @@ -815,13 +815,8 @@ public async Task WaitUntilReadyAsync( if (DateTime.UtcNow > deadline) { var context = $"domain={ConnectionConfig.Domain}, useServerProxy={ConnectionConfig.UseServerProxy}"; - var suggestion = "If this sandbox runs in Docker bridge or remote-network mode, consider enabling useServerProxy=true."; - if (!ConnectionConfig.UseServerProxy) - { - suggestion += " You can also configure server-side [docker].host_ip for direct endpoint access."; - } throw new SandboxReadyTimeoutException( - $"Sandbox health check timed out after {options.ReadyTimeoutSeconds}s ({attempt} attempts). {errorDetail} Connection context: {context}. {suggestion}"); + $"Sandbox health check timed out after {options.ReadyTimeoutSeconds}s ({attempt} attempts). {errorDetail} Connection context: {context}."); } attempt++; diff --git a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/CommandsAdapterTests.cs b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/CommandsAdapterTests.cs index d6db30db3..8fc615bfc 100644 --- a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/CommandsAdapterTests.cs +++ b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/CommandsAdapterTests.cs @@ -441,6 +441,64 @@ await act.Should().ThrowAsync() .WithMessage("*sessionId*"); } + [Fact] + public async Task GetCommandStatusAsync_ShouldIncludeBodyInMessage_WhenBodyUnparseable() + { + var body = "{\"error\":\"cursor must be positive\"}"; + var handler = new StubHttpMessageHandler((_, _) => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + }); + var adapter = CreateAdapter(handler); + + var act = () => adapter.GetCommandStatusAsync("exec-1"); + + var ex = await act.Should().ThrowAsync(); + ex.Which.Message.Should().Contain(body); + } + + [Fact] + public async Task GetCommandStatusAsync_ShouldIncludeBodyInMessage_WhenBodyIsPlainText() + { + var body = "cursor must be positive"; + var handler = new StubHttpMessageHandler((_, _) => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent(body, Encoding.UTF8, "text/plain") + }); + }); + var adapter = CreateAdapter(handler); + + var act = () => adapter.GetCommandStatusAsync("exec-1"); + + var ex = await act.Should().ThrowAsync(); + ex.Which.Message.Should().Contain(body); + ex.Which.RawBody.Should().Be(body); + } + + [Fact] + public async Task GetBackgroundCommandLogsAsync_ShouldIncludeBodyInMessage_WhenBodyUnparseable() + { + var body = "quota exceeded for sandbox"; + var handler = new StubHttpMessageHandler((_, _) => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent(body, Encoding.UTF8, "text/plain") + }); + }); + var adapter = CreateAdapter(handler); + + var act = () => adapter.GetBackgroundCommandLogsAsync("exec-1"); + + var ex = await act.Should().ThrowAsync(); + ex.Which.Message.Should().Contain(body); + } + private static CommandsAdapter CreateAdapter(HttpMessageHandler httpHandler) { var baseUrl = "http://execd.local"; diff --git a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SandboxReadinessDiagnosticsTests.cs b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SandboxReadinessDiagnosticsTests.cs index dd42f0d00..92ec9c86a 100644 --- a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SandboxReadinessDiagnosticsTests.cs +++ b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SandboxReadinessDiagnosticsTests.cs @@ -26,7 +26,7 @@ namespace OpenSandbox.Tests; public class SandboxReadinessDiagnosticsTests { [Fact] - public async Task WaitUntilReadyAsync_WhenHealthCheckThrows_IncludesLastErrorAndConnectionContext() + public async Task WaitUntilReadyAsync_WhenHealthCheckThrows_OmitsNetworkConfigurationHints() { // Arrange var healthMock = new Mock(); @@ -52,8 +52,10 @@ await sandbox.WaitUntilReadyAsync(new WaitUntilReadyOptions ex.Which.Message.Should().Contain("Last health check error"); ex.Which.Message.Should().Contain("domain=localhost:8080"); ex.Which.Message.Should().Contain("useServerProxy=False"); - ex.Which.Message.Should().Contain("useServerProxy=true"); - ex.Which.Message.Should().Contain("[docker].host_ip"); + ex.Which.Message.Should().NotContainEquivalentOf("consider enabling useServerProxy=true"); + ex.Which.Message.Should().NotContainEquivalentOf("Docker bridge"); + ex.Which.Message.Should().NotContainEquivalentOf("remote-network"); + ex.Which.Message.Should().NotContain("[docker].host_ip"); } finally { diff --git a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SseParserTests.cs b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SseParserTests.cs index 520c7ac9f..907fe7145 100644 --- a/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SseParserTests.cs +++ b/sdks/sandbox/csharp/tests/OpenSandbox.Tests/SseParserTests.cs @@ -207,7 +207,28 @@ public async Task ParseJsonEventStreamAsync_WithErrorResponseNoJson_ShouldUseFal }); exception.StatusCode.Should().Be(500); - exception.Message.Should().Be("Custom fallback"); + // The raw body is spliced into the message so logs carry the server's reason. + exception.Message.Should().Be("Custom fallback: Internal Server Error"); + } + + [Fact] + public async Task ParseJsonEventStreamAsync_WithUnstructuredJsonBody_ShouldSpliceBodyIntoMessage() + { + // Arrange + var errorContent = @"{""error"":""invalid parameter""}"; + var response = CreateMockResponse(HttpStatusCode.BadRequest, errorContent); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in SseParser.ParseJsonEventStreamAsync(response, "Run code failed")) + { + // Should not reach here + } + }); + + exception.StatusCode.Should().Be(400); + exception.Message.Should().Be(@"Run code failed: {""error"":""invalid parameter""}"); } [Fact] diff --git a/sdks/sandbox/go/README.md b/sdks/sandbox/go/README.md index 35e49b940..c7f3f74e4 100644 --- a/sdks/sandbox/go/README.md +++ b/sdks/sandbox/go/README.md @@ -156,6 +156,15 @@ _, err = sandbox.CreateCredentialVault(ctx, opensandbox.CredentialVaultCreateReq See [Credential Vault](../../../docs/guides/credential-vault.md) for auth types, binding guidance, and Git/curl examples. +### Release idle pool sandboxes + +`ReleaseAllIdle(ctx)` preserves the original fire-and-forget behavior: it drains +idle IDs and returns after scheduling best-effort kills. Call +`ReleaseAllIdleParallel(ctx, maxWorkers)` on `*DefaultSandboxPool` to bound kill +concurrency and wait until every drained ID has received a kill attempt. +`maxWorkers` must be positive. The parallel method is intentionally not part of +the `SandboxPool` interface, so existing interface implementors remain compatible. + ## API Reference ### LifecycleClient diff --git a/sdks/sandbox/go/isolated.go b/sdks/sandbox/go/isolated.go index a3b3125b3..0a0891ad5 100644 --- a/sdks/sandbox/go/isolated.go +++ b/sdks/sandbox/go/isolated.go @@ -182,16 +182,34 @@ type isolatedBackgroundRunRequest struct { Background bool `json:"background"` } +// HardeningLayerState reports whether one hardening layer is actually +// enforced: "active" | "disabled" | "degraded" | "unsupported". +type HardeningLayerState struct { + State string `json:"state"` + Message string `json:"message,omitempty"` +} + +// HardeningStatus reports execd init-mode state (OSEP-0018). +type HardeningStatus struct { + InitMode string `json:"init_mode"` // "pid1" | "subreaper" | "none" + SignalShield bool `json:"signal_shield"` // kernel PID 1 signal shield active + CapDrop *HardeningLayerState `json:"cap_drop"` + Seccomp *HardeningLayerState `json:"seccomp"` + Landlock *HardeningLayerState `json:"landlock"` + Ebpf *HardeningLayerState `json:"ebpf"` +} + // IsolatedCapabilities reports isolation capabilities. type IsolatedCapabilities struct { - Available bool `json:"available"` - Isolator string `json:"isolator,omitempty"` - Version string `json:"version,omitempty"` - Message string `json:"message,omitempty"` - SetprivAvailable bool `json:"setpriv_available"` - UsernsAvailable bool `json:"userns_available"` - CommitSupported bool `json:"commit_supported"` - DiffSupported bool `json:"diff_supported"` + Available bool `json:"available"` + Isolator string `json:"isolator,omitempty"` + Version string `json:"version,omitempty"` + Message string `json:"message,omitempty"` + SetprivAvailable bool `json:"setpriv_available"` + UsernsAvailable bool `json:"userns_available"` + CommitSupported bool `json:"commit_supported"` + DiffSupported bool `json:"diff_supported"` + Hardening *HardeningStatus `json:"hardening,omitempty"` } // IsolatedCreate creates an isolated bash session. diff --git a/sdks/sandbox/go/pool.go b/sdks/sandbox/go/pool.go index cac75c7f1..1048ea3ca 100644 --- a/sdks/sandbox/go/pool.go +++ b/sdks/sandbox/go/pool.go @@ -16,6 +16,7 @@ package opensandbox import ( "context" + "errors" "fmt" "sync" "sync/atomic" @@ -91,6 +92,20 @@ func (p *DefaultSandboxPool) Start(ctx context.Context) error { startMaxIdle := p.config.MaxIdle p.mu.Unlock() + // Refuse to bind a retired namespace. Only a definite fence blocks startup; + // a store outage is left to the writes below to surface. + if err := p.ensureNamespaceActive(ctx); err != nil { + var destroyed *PoolDestroyedError + if errors.As(err, &destroyed) { + p.mu.Lock() + if p.lifecycleState == PoolLifecycleStarting { + p.lifecycleState = PoolLifecycleNotStarted + } + p.mu.Unlock() + return err + } + } + // Initialize state store with pool configuration. if err := p.config.StateStore.SetMaxIdle(ctx, p.config.PoolName, startMaxIdle); err != nil { p.mu.Lock() @@ -187,6 +202,17 @@ func (p *DefaultSandboxPool) syncHealthState() { func (p *DefaultSandboxPool) runReconcileTick(ctx context.Context) { p.reconMu.Lock() defer p.reconMu.Unlock() + + // A destroy fences the namespace for every peer. Stop rather than keep + // replenishing a pool that is being retired. + if err := p.ensureNamespaceActive(ctx); err != nil { + var destroyed *PoolDestroyedError + if errors.As(err, &destroyed) { + p.stopAfterNamespaceDestroyed(destroyed.State) + return + } + } + createFn := func(ctx context.Context, reason PooledSandboxCreateReason) (string, error) { return p.createOneSandbox(ctx, reason) } @@ -215,6 +241,12 @@ func (p *DefaultSandboxPool) Acquire(ctx context.Context, opts AcquireOptions) ( policy = *opts.Policy } + // A fenced namespace must not mint new sandboxes, so this has to run before the + // direct-create fallthrough below and not only on the store write paths. + if err := p.ensureNamespaceActiveForAcquire(ctx, policy); err != nil { + return nil, err + } + // Resolve minTTL. minTTL := p.config.AcquireMinRemainingTTL if opts.MinRemainingTTL > 0 { @@ -297,6 +329,12 @@ func (p *DefaultSandboxPool) Acquire(ctx context.Context, opts AcquireOptions) ( go p.killDiscardedAliveSandboxes(pendingKill) return nil, &PoolNotRunningError{PoolName: p.config.PoolName, State: currentState} } + // A destroy may have landed since the preflight check. Stop retrying rather + // than pop further idle IDs out from under the drain. + if err := p.ensureNamespaceActiveForAcquire(ctx, policy); err != nil { + go p.killDiscardedAliveSandboxes(pendingKill) + return nil, err + } continue } // Connect + readiness succeeded. From here on the sandbox is a healthy, borrowable idle: @@ -320,6 +358,15 @@ func (p *DefaultSandboxPool) Acquire(ctx context.Context, opts AcquireOptions) ( return nil, fmt.Errorf("opensandbox: pool acquire: renew after connect failed: %w", renewErr) } } + // TryTakeIdle is unfenced so the destroy manager can drain, so this ID is + // already out of the store and a destroy can no longer reach it. Re-check + // before handing it over, fail-closed: if the store cannot confirm the + // namespace is ACTIVE, kill the sandbox rather than leak it into a + // namespace that may be retired. + if err := p.ensureNamespaceActiveAfterCreate(ctx, sb, nil); err != nil { + go p.killDiscardedAliveSandboxes(pendingKill) + return nil, err + } go p.killDiscardedAliveSandboxes(pendingKill) p.config.Logger.Debug("acquire: from idle", "pool_name", p.config.PoolName, @@ -348,7 +395,7 @@ func (p *DefaultSandboxPool) Acquire(ctx context.Context, opts AcquireOptions) ( "attempted_any", attemptedAny, "loop_exhausted", loopExhausted, "last_sandbox_id", lastSandboxID) - return p.directCreate(ctx, opts) + return p.directCreate(ctx, opts, policy) } // tryTakeIdle wraps the store's take primitives, returning a nil result on a legitimate empty @@ -404,7 +451,7 @@ func (p *DefaultSandboxPool) connectIdle(ctx context.Context, sandboxID string, }) } -func (p *DefaultSandboxPool) directCreate(ctx context.Context, opts AcquireOptions) (*Sandbox, error) { +func (p *DefaultSandboxPool) directCreate(ctx context.Context, opts AcquireOptions, policy AcquirePolicy) (*Sandbox, error) { var sb *Sandbox var err error @@ -428,7 +475,15 @@ func (p *DefaultSandboxPool) directCreate(ctx context.Context, opts AcquireOptio if err != nil { return nil, err } - return p.postCreateChecks(ctx, sb, opts) + sb, err = p.postCreateChecks(ctx, sb, opts) + if err != nil { + return nil, err + } + // Re-check: a destroy may have landed while this sandbox was being created. + if err := p.ensureNamespaceActiveAfterCreate(ctx, sb, &policy); err != nil { + return nil, err + } + return sb, nil } // postCreateChecks applies renew to a freshly created sandbox. @@ -556,7 +611,7 @@ func adaptHealthCheck(userCheck func(context.Context, *Sandbox) error) func(cont } } -// ReleaseAllIdle drains all idle sandboxes and kills them. +// ReleaseAllIdle drains all idle sandboxes and schedules a best-effort kill for each one. func (p *DefaultSandboxPool) ReleaseAllIdle(ctx context.Context) (int, error) { count := 0 for { @@ -576,6 +631,63 @@ func (p *DefaultSandboxPool) ReleaseAllIdle(ctx context.Context) (int, error) { return count, nil } +// ReleaseAllIdleParallel drains all idle sandboxes and kills them with bounded +// concurrency. It blocks until every drained sandbox has received a best-effort +// kill attempt. maxWorkers must be positive. +// +// ctx only bounds the drain phase. Once an ID has been drained, its kill attempt +// uses an independent timeout and completes before this method returns, even if +// ctx is cancelled. +func (p *DefaultSandboxPool) ReleaseAllIdleParallel(ctx context.Context, maxWorkers int) (int, error) { + if maxWorkers <= 0 { + return 0, fmt.Errorf("opensandbox: pool release all idle parallel: maxWorkers must be positive, got %d", maxWorkers) + } + sandboxIDs := make([]string, 0) + var drainErr error + for { + if err := ctx.Err(); err != nil { + drainErr = err + break + } + sandboxID, err := p.config.StateStore.TryTakeIdle(ctx, p.config.PoolName) + if err != nil { + drainErr = err + break + } + if sandboxID == "" { + break + } + sandboxIDs = append(sandboxIDs, sandboxID) + } + + jobs := make(chan string) + var workers sync.WaitGroup + workerCount := len(sandboxIDs) + if workerCount > maxWorkers { + workerCount = maxWorkers + } + workers.Add(workerCount) + for i := 0; i < workerCount; i++ { + go func() { + defer workers.Done() + for sandboxID := range jobs { + if err := p.killSandbox(sandboxID); err != nil { + p.config.Logger.Warn("failed to kill sandbox (best-effort)", + "pool_name", p.config.PoolName, + "sandbox_id", sandboxID, + "error", err) + } + } + }() + } + for _, sandboxID := range sandboxIDs { + jobs <- sandboxID + } + close(jobs) + workers.Wait() + return len(sandboxIDs), drainErr +} + // Resize dynamically changes the idle target. // The new value is persisted to the state store and updated locally so that // a subsequent Start() (after stop/restart) uses the latest value. @@ -753,12 +865,121 @@ done: return nil } -const killSandboxTimeout = 30 * time.Second +// ensureNamespaceActive returns *PoolDestroyedError when a destroy has fenced or +// tombstoned this pool's namespace, and *PoolStateStoreUnavailableError when the +// state store cannot answer. +func (p *DefaultSandboxPool) ensureNamespaceActive(ctx context.Context) error { + state, err := p.config.StateStore.GetDestroyState(ctx, p.config.PoolName) + if err != nil { + var unavailable *PoolStateStoreUnavailableError + if errors.As(err, &unavailable) { + return err + } + return &PoolStateStoreUnavailableError{Operation: "GetDestroyState", Cause: err} + } + if state != PoolDestroyStateActive { + return &PoolDestroyedError{PoolName: p.config.PoolName, State: state} + } + return nil +} + +// ensureNamespaceActiveForAcquire is ensureNamespaceActive with the same +// store-outage degradation the take path already applies: policies that fall +// through to direct create treat an unreachable store as "state unknown" and +// proceed, so a full store outage does not make them less available than +// documented (OSEP-0005 error-code matrix). Fail-closed policies surface it. +func (p *DefaultSandboxPool) ensureNamespaceActiveForAcquire(ctx context.Context, policy AcquirePolicy) error { + err := p.ensureNamespaceActive(ctx) + if err == nil { + return nil + } + var unavailable *PoolStateStoreUnavailableError + if errors.As(err, &unavailable) && policyFallsThroughToDirectCreate(policy) { + p.config.Logger.Warn("acquire: state store unavailable during namespace check, "+ + "assuming ACTIVE and degrading to direct create", + "pool_name", p.config.PoolName, + "policy", policy, + "error", err) + return nil + } + return err +} + +// ensureNamespaceActiveAfterCreate re-checks the fence once the acquire path holds +// a live sandbox, so a destroy that landed mid-acquire does not leak one into a +// retired namespace. On a fence the sandbox is killed and closed. +// +// policy is non-nil only for the direct-create path, where a store outage degrades +// the same way the rest of that path does. The idle path passes nil and stays +// fail-closed: that sandbox is already out of the store, so an unconfirmed +// namespace has to be treated as retired. +func (p *DefaultSandboxPool) ensureNamespaceActiveAfterCreate(ctx context.Context, sb *Sandbox, policy *AcquirePolicy) error { + err := p.ensureNamespaceActive(ctx) + if err == nil { + return nil + } + var unavailable *PoolStateStoreUnavailableError + if errors.As(err, &unavailable) && policy != nil && policyFallsThroughToDirectCreate(*policy) { + p.config.Logger.Warn("acquire: state store unavailable during post-create namespace check, "+ + "keeping sandbox and degrading per policy", + "pool_name", p.config.PoolName, + "sandbox_id", sb.ID(), + "policy", *policy, + "error", err) + return nil + } + go p.killSandboxBestEffort(sb.ID()) + _ = sb.Close() + return err +} + +// stopAfterNamespaceDestroyed stops the pool once its namespace has been retired. +// It runs on the reconcile goroutine, so unlike Shutdown it must not wait on p.wg. +func (p *DefaultSandboxPool) stopAfterNamespaceDestroyed(state PoolDestroyState) { + p.mu.Lock() + if p.lifecycleState == PoolLifecycleStopped || p.lifecycleState == PoolLifecycleDraining { + p.mu.Unlock() + return + } + p.lifecycleState = PoolLifecycleStopped + if p.ticker != nil { + p.ticker.Stop() + } + if p.done != nil && !p.doneClosed { + close(p.done) + p.doneClosed = true + } + cancelFn := p.reconCancel + sdCh := p.shutdownDone + p.mu.Unlock() + + if cancelFn != nil { + cancelFn() + } + if sdCh != nil { + select { + case <-sdCh: + default: + close(sdCh) + } + } + p.config.Logger.Info("pool stopped: namespace destroyed", + "pool_name", p.config.PoolName, + "destroy_state", state) +} + +const ( + killSandboxTimeout = 30 * time.Second +) func (p *DefaultSandboxPool) killSandboxBestEffort(sandboxID string) { + _ = p.killSandbox(sandboxID) +} + +func (p *DefaultSandboxPool) killSandbox(sandboxID string) error { ctx, cancel := context.WithTimeout(context.Background(), killSandboxTimeout) defer cancel() - _ = p.manager.KillSandbox(ctx, sandboxID) + return p.manager.KillSandbox(ctx, sandboxID) } func (p *DefaultSandboxPool) killDiscardedAliveSandboxes(ids []string) { diff --git a/sdks/sandbox/go/pool_errors.go b/sdks/sandbox/go/pool_errors.go index b1251b222..fcb7d6900 100644 --- a/sdks/sandbox/go/pool_errors.go +++ b/sdks/sandbox/go/pool_errors.go @@ -63,3 +63,31 @@ func (e *PoolStateStoreUnavailableError) Error() string { } func (e *PoolStateStoreUnavailableError) Unwrap() error { return e.Cause } + +// PoolDestroyedError is returned when a write targets a pool namespace that a +// destroy has fenced, i.e. one that is DESTROYING or DESTROYED. +type PoolDestroyedError struct { + PoolName string + State PoolDestroyState +} + +func (e *PoolDestroyedError) Error() string { + return fmt.Sprintf("opensandbox: pool %q is %s", e.PoolName, e.State) +} + +// PoolDestroyIncompleteError is returned when a destroy could not run to +// completion. The namespace stays DESTROYING and the caller should retry. +type PoolDestroyIncompleteError struct { + PoolName string + Reason string + Cause error +} + +func (e *PoolDestroyIncompleteError) Error() string { + if e.Cause == nil { + return fmt.Sprintf("opensandbox: pool %q destroy incomplete: %s", e.PoolName, e.Reason) + } + return fmt.Sprintf("opensandbox: pool %q destroy incomplete: %s: %v", e.PoolName, e.Reason, e.Cause) +} + +func (e *PoolDestroyIncompleteError) Unwrap() error { return e.Cause } diff --git a/sdks/sandbox/go/pool_manager.go b/sdks/sandbox/go/pool_manager.go new file mode 100644 index 000000000..871a9b281 --- /dev/null +++ b/sdks/sandbox/go/pool_manager.go @@ -0,0 +1,248 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opensandbox + +import ( + "context" + crypto_rand "crypto/rand" + "errors" + "fmt" + "os" + "strings" + "time" +) + +// SandboxPoolManager performs namespace-level maintenance on a shared sandbox +// pool. It does not acquire sandboxes: it exists so an operator can retire a +// pool namespace without reconstructing the SandboxPool object that originally +// owned it. +type SandboxPoolManager struct { + stateStore PoolStateStore + manager *SandboxManager + ownerID string + logger PoolLogger +} + +// Destroy retires a pool namespace. +// +// FORCE destroy writes a shared DESTROYING fence first, which every peer sharing +// the state store observes: fenced pools cannot hold the primary lock or publish +// idle sandboxes, so replenishment stops instead of racing the destroy. The +// manager then drains the visible idle IDs and kills them best-effort, clears the +// persistent coordination state, and writes a DESTROYED tombstone so later +// callers cannot silently rebind the namespace. +// +// If the drain or the cleanup cannot finish, the namespace stays DESTROYING and +// the error is a *PoolDestroyIncompleteError; retrying Destroy is safe. Calling +// Destroy on an already-tombstoned namespace is a no-op that reports DESTROYED. +func (m *SandboxPoolManager) Destroy(ctx context.Context, poolName string, options PoolDestroyOptions) (*PoolDestroyResult, error) { + if strings.TrimSpace(poolName) == "" { + return nil, fmt.Errorf("opensandbox: pool manager: poolName must not be blank") + } + if options.Strategy != PoolDestroyForce { + return nil, fmt.Errorf("opensandbox: pool manager: only FORCE destroy strategy is supported, got %s", options.Strategy) + } + + drainTimeout := DefaultPoolDrainTimeout + if options.DrainTimeout != nil { + drainTimeout = *options.DrainTimeout + if drainTimeout < 0 { + return nil, fmt.Errorf("opensandbox: pool manager: DrainTimeout must not be negative, got %v", drainTimeout) + } + } + tombstoneTTL := DefaultPoolTombstoneTTL + if options.TombstoneTTL != nil { + tombstoneTTL = *options.TombstoneTTL + if tombstoneTTL < 0 { + return nil, fmt.Errorf("opensandbox: pool manager: TombstoneTTL must not be negative, got %v", tombstoneTTL) + } + } + + state, err := m.stateStore.GetDestroyState(ctx, poolName) + if err != nil { + return nil, err + } + if state == PoolDestroyStateDestroyed { + return alreadyDestroyedResult(poolName), nil + } + + if err := m.stateStore.BeginDestroy(ctx, poolName, m.ownerID); err != nil { + // Lost the race to a concurrent destroy that already tombstoned the + // namespace; the outcome the caller asked for already holds. + var destroyed *PoolDestroyedError + if errors.As(err, &destroyed) { + return alreadyDestroyedResult(poolName), nil + } + return nil, err + } + + drained := 0 + killed := 0 + deadline := time.Now().Add(drainTimeout) + for { + sandboxID, err := m.stateStore.TryTakeIdle(ctx, poolName) + if err != nil { + return nil, &PoolDestroyIncompleteError{ + PoolName: poolName, + Reason: fmt.Sprintf("failed to drain idle sandboxes after %d drained", drained), + Cause: err, + } + } + if sandboxID == "" { + break + } + drained++ + if err := m.manager.KillSandbox(ctx, sandboxID); err != nil { + m.logger.Warn("pool destroy failed to kill idle sandbox (best-effort)", + "pool_name", poolName, + "sandbox_id", sandboxID, + "error", err) + } else { + killed++ + } + if drainTimeout > 0 && time.Now().After(deadline) { + return nil, &PoolDestroyIncompleteError{ + PoolName: poolName, + Reason: fmt.Sprintf("drain timed out after %v with %d idle sandboxes drained", drainTimeout, drained), + } + } + } + + if err := m.stateStore.ClearPoolState(ctx, poolName); err != nil { + return nil, &PoolDestroyIncompleteError{ + PoolName: poolName, + Reason: "failed to clear persistent state", + Cause: err, + } + } + if err := m.stateStore.MarkDestroyed(ctx, poolName, m.ownerID, tombstoneTTL); err != nil { + return nil, &PoolDestroyIncompleteError{ + PoolName: poolName, + Reason: "failed to write destroyed tombstone", + Cause: err, + } + } + + m.logger.Info("pool namespace destroyed", + "pool_name", poolName, + "drained_idle_count", drained, + "killed_idle_count", killed) + + return &PoolDestroyResult{ + PoolName: poolName, + State: PoolDestroyStateDestroyed, + DrainedIdleCount: drained, + KilledIdleCount: killed, + PersistentStateCleared: true, + }, nil +} + +func alreadyDestroyedResult(poolName string) *PoolDestroyResult { + return &PoolDestroyResult{ + PoolName: poolName, + State: PoolDestroyStateDestroyed, + DrainedIdleCount: 0, + KilledIdleCount: 0, + PersistentStateCleared: false, + } +} + +// SandboxPoolManagerBuilder configures and creates a SandboxPoolManager. +type SandboxPoolManagerBuilder struct { + stateStore PoolStateStore + connectionConfig ConnectionConfig + connectionConfigSet bool + ownerID string + logger PoolLogger +} + +// NewSandboxPoolManagerBuilder creates a new builder. +func NewSandboxPoolManagerBuilder() *SandboxPoolManagerBuilder { + return &SandboxPoolManagerBuilder{} +} + +// StateStore sets the pool state store to operate on (required). It must be the +// same store the pool being retired coordinates through. +func (b *SandboxPoolManagerBuilder) StateStore(s PoolStateStore) *SandboxPoolManagerBuilder { + b.stateStore = s + return b +} + +// ConnectionConfig sets the connection configuration used to kill drained +// sandboxes (required). +func (b *SandboxPoolManagerBuilder) ConnectionConfig(c ConnectionConfig) *SandboxPoolManagerBuilder { + b.connectionConfig = c + b.connectionConfigSet = true + return b +} + +// OwnerID sets the identifier recorded alongside the fence and the tombstone. +// Defaults to a generated per-process value. +func (b *SandboxPoolManagerBuilder) OwnerID(id string) *SandboxPoolManagerBuilder { + b.ownerID = id + return b +} + +// PoolLogger sets a custom structured logger. Defaults to a no-op logger. +func (b *SandboxPoolManagerBuilder) PoolLogger(l PoolLogger) *SandboxPoolManagerBuilder { + b.logger = l + return b +} + +// Build validates configuration and creates a SandboxPoolManager. +func (b *SandboxPoolManagerBuilder) Build() (*SandboxPoolManager, error) { + if b.stateStore == nil { + return nil, fmt.Errorf("opensandbox: pool manager builder: StateStore is required") + } + if !b.connectionConfigSet { + return nil, fmt.Errorf("opensandbox: pool manager builder: ConnectionConfig is required") + } + + ownerID := b.ownerID + if ownerID == "" { + generated, err := generatePoolManagerOwnerID() + if err != nil { + return nil, err + } + ownerID = generated + } + if strings.TrimSpace(ownerID) == "" { + return nil, fmt.Errorf("opensandbox: pool manager builder: OwnerID must not be blank") + } + + logger := b.logger + if logger == nil { + logger = noopPoolLogger{} + } + + return &SandboxPoolManager{ + stateStore: b.stateStore, + manager: NewSandboxManager(b.connectionConfig), + ownerID: ownerID, + logger: logger, + }, nil +} + +func generatePoolManagerOwnerID() (string, error) { + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "unknown" + } + var randBytes [4]byte + if _, randErr := crypto_rand.Read(randBytes[:]); randErr != nil { + return "", fmt.Errorf("opensandbox: pool manager builder: failed to generate random owner ID: %w", randErr) + } + return fmt.Sprintf("pool-manager-%s-%d-%d-%x", hostname, os.Getpid(), time.Now().UnixNano(), randBytes), nil +} diff --git a/sdks/sandbox/go/pool_manager_test.go b/sdks/sandbox/go/pool_manager_test.go new file mode 100644 index 000000000..da9335cf0 --- /dev/null +++ b/sdks/sandbox/go/pool_manager_test.go @@ -0,0 +1,1020 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opensandbox + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// killRecorder is a mock lifecycle server that records DELETE calls and can be +// told to fail them or to stall before answering. +type killRecorder struct { + srv *httptest.Server + deleted atomic.Int32 + fail atomic.Bool + delay atomic.Int64 // nanoseconds +} + +func newKillRecorder(t *testing.T) *killRecorder { + t.Helper() + rec := &killRecorder{} + rec.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusNotFound) + return + } + if d := time.Duration(rec.delay.Load()); d > 0 { + time.Sleep(d) + } + rec.deleted.Add(1) + if rec.fail.Load() { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(rec.srv.Close) + return rec +} + +func newTestPoolManager(t *testing.T, store PoolStateStore, serverURL string) *SandboxPoolManager { + t.Helper() + manager, err := NewSandboxPoolManagerBuilder(). + StateStore(store). + ConnectionConfig(ConnectionConfig{Domain: serverURL, Protocol: "http"}). + OwnerID("test-pool-manager"). + Build() + if err != nil { + t.Fatalf("newTestPoolManager: Build failed: %v", err) + } + return manager +} + +func seedIdle(t *testing.T, store PoolStateStore, poolName string, n int) { + t.Helper() + ctx := context.Background() + for i := 0; i < n; i++ { + if err := store.PutIdle(ctx, poolName, fmt.Sprintf("sbx-idle-%d", i)); err != nil { + t.Fatalf("seedIdle: PutIdle failed: %v", err) + } + } +} + +func durationPtr(d time.Duration) *time.Duration { return &d } + +// ---------- Destroy Protocol Tests ---------- + +func TestSandboxPoolManager_Destroy(t *testing.T) { + tests := []struct { + name string + idleCount int + killsFail bool + options PoolDestroyOptions + wantDrained int + wantKilled int + wantDeletions int32 + }{ + { + name: "empty pool", + options: PoolDestroyOptions{}, + }, + { + name: "drains and kills every idle sandbox", + idleCount: 3, + wantDrained: 3, + wantKilled: 3, + wantDeletions: 3, + }, + { + name: "kill failures are best-effort", + idleCount: 2, + killsFail: true, + wantDrained: 2, + wantKilled: 0, + wantDeletions: 2, + }, + { + name: "zero drain timeout disables the deadline", + idleCount: 2, + options: PoolDestroyOptions{DrainTimeout: durationPtr(0)}, + wantDrained: 2, + wantKilled: 2, + wantDeletions: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + rec.fail.Store(tt.killsFail) + seedIdle(t, store, "test-pool", tt.idleCount) + + manager := newTestPoolManager(t, store, rec.srv.URL) + result, err := manager.Destroy(ctx, "test-pool", tt.options) + if err != nil { + t.Fatalf("Destroy failed: %v", err) + } + + if result.State != PoolDestroyStateDestroyed { + t.Errorf("state = %s, want DESTROYED", result.State) + } + if result.PoolName != "test-pool" { + t.Errorf("poolName = %q, want %q", result.PoolName, "test-pool") + } + if !result.PersistentStateCleared { + t.Error("PersistentStateCleared = false, want true") + } + if result.DrainedIdleCount != tt.wantDrained { + t.Errorf("DrainedIdleCount = %d, want %d", result.DrainedIdleCount, tt.wantDrained) + } + if result.KilledIdleCount != tt.wantKilled { + t.Errorf("KilledIdleCount = %d, want %d", result.KilledIdleCount, tt.wantKilled) + } + if got := rec.deleted.Load(); got != tt.wantDeletions { + t.Errorf("DELETE requests = %d, want %d", got, tt.wantDeletions) + } + + state, err := store.GetDestroyState(ctx, "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateDestroyed { + t.Errorf("store destroy state = %s, want DESTROYED", state) + } + counters, err := store.SnapshotCounters(ctx, "test-pool") + if err != nil { + t.Fatalf("SnapshotCounters failed: %v", err) + } + if counters.IdleCount != 0 { + t.Errorf("idle count after destroy = %d, want 0", counters.IdleCount) + } + }) + } +} + +// recordingPoolLogger captures warnings so tests can assert on best-effort paths. +type recordingPoolLogger struct { + mu sync.Mutex + warns []string +} + +func (l *recordingPoolLogger) Info(_ string, _ ...interface{}) {} +func (l *recordingPoolLogger) Debug(_ string, _ ...interface{}) {} + +func (l *recordingPoolLogger) Warn(msg string, _ ...interface{}) { + l.mu.Lock() + defer l.mu.Unlock() + l.warns = append(l.warns, msg) +} + +func (l *recordingPoolLogger) warnCount() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.warns) +} + +func TestSandboxPoolManager_Destroy_LogsBestEffortKillFailures(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + rec.fail.Store(true) + seedIdle(t, store, "test-pool", 2) + + logger := &recordingPoolLogger{} + manager, err := NewSandboxPoolManagerBuilder(). + StateStore(store). + ConnectionConfig(ConnectionConfig{Domain: rec.srv.URL, Protocol: "http"}). + PoolLogger(logger). + Build() + if err != nil { + t.Fatalf("Build failed: %v", err) + } + + result, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}) + if err != nil { + t.Fatalf("Destroy failed: %v", err) + } + if result.State != PoolDestroyStateDestroyed { + t.Errorf("state = %s, want DESTROYED (kill failures must not abort the destroy)", result.State) + } + if got := logger.warnCount(); got != 2 { + t.Errorf("warn count = %d, want 2 (one per failed kill)", got) + } +} + +func TestSandboxPoolManager_Destroy_IsIdempotent(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + seedIdle(t, store, "test-pool", 2) + + manager := newTestPoolManager(t, store, rec.srv.URL) + if _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}); err != nil { + t.Fatalf("first Destroy failed: %v", err) + } + + result, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}) + if err != nil { + t.Fatalf("second Destroy failed: %v", err) + } + if result.State != PoolDestroyStateDestroyed { + t.Errorf("state = %s, want DESTROYED", result.State) + } + if result.PersistentStateCleared { + t.Error("PersistentStateCleared = true on a repeat destroy, want false") + } + if result.DrainedIdleCount != 0 || result.KilledIdleCount != 0 { + t.Errorf("repeat destroy drained/killed = %d/%d, want 0/0", result.DrainedIdleCount, result.KilledIdleCount) + } + if got := rec.deleted.Load(); got != 2 { + t.Errorf("DELETE requests = %d, want 2 (the repeat destroy must not kill again)", got) + } +} + +func TestSandboxPoolManager_Destroy_DrainTimeoutLeavesFence(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + rec.delay.Store(int64(30 * time.Millisecond)) + seedIdle(t, store, "test-pool", 5) + + manager := newTestPoolManager(t, store, rec.srv.URL) + _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{ + DrainTimeout: durationPtr(10 * time.Millisecond), + }) + + var incomplete *PoolDestroyIncompleteError + if !errors.As(err, &incomplete) { + t.Fatalf("Destroy error = %v, want *PoolDestroyIncompleteError", err) + } + if incomplete.PoolName != "test-pool" { + t.Errorf("PoolName = %q, want %q", incomplete.PoolName, "test-pool") + } + + // The namespace stays fenced so a retry can finish the job. + state, err := store.GetDestroyState(ctx, "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateDestroying { + t.Errorf("state after timeout = %s, want DESTROYING", state) + } + + rec.delay.Store(0) + result, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}) + if err != nil { + t.Fatalf("retry Destroy failed: %v", err) + } + if result.State != PoolDestroyStateDestroyed { + t.Errorf("state after retry = %s, want DESTROYED", result.State) + } +} + +func TestSandboxPoolManager_Destroy_TombstoneTTLExpires(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + + manager := newTestPoolManager(t, store, rec.srv.URL) + if _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{ + TombstoneTTL: durationPtr(20 * time.Millisecond), + }); err != nil { + t.Fatalf("Destroy failed: %v", err) + } + + if err := store.PutIdle(ctx, "test-pool", "sbx-blocked"); err == nil { + t.Fatal("PutIdle succeeded while the tombstone was live, want *PoolDestroyedError") + } + + time.Sleep(40 * time.Millisecond) + + state, err := store.GetDestroyState(ctx, "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateActive { + t.Errorf("state after tombstone TTL = %s, want ACTIVE", state) + } + if err := store.PutIdle(ctx, "test-pool", "sbx-rebound"); err != nil { + t.Errorf("PutIdle after tombstone expiry failed: %v", err) + } +} + +func TestSandboxPoolManager_Destroy_ZeroTombstoneTTLNeverExpires(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + + manager := newTestPoolManager(t, store, rec.srv.URL) + if _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{ + TombstoneTTL: durationPtr(0), + }); err != nil { + t.Fatalf("Destroy failed: %v", err) + } + + time.Sleep(20 * time.Millisecond) + + state, err := store.GetDestroyState(ctx, "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateDestroyed { + t.Errorf("state = %s, want DESTROYED (a zero TTL must never expire)", state) + } +} + +func TestSandboxPoolManager_Destroy_InvalidOptions(t *testing.T) { + tests := []struct { + name string + poolName string + options PoolDestroyOptions + }{ + {name: "blank pool name", poolName: " ", options: PoolDestroyOptions{}}, + {name: "unsupported strategy", poolName: "test-pool", options: PoolDestroyOptions{Strategy: PoolDestroyStrategy(99)}}, + {name: "negative drain timeout", poolName: "test-pool", options: PoolDestroyOptions{DrainTimeout: durationPtr(-time.Second)}}, + {name: "negative tombstone TTL", poolName: "test-pool", options: PoolDestroyOptions{TombstoneTTL: durationPtr(-time.Second)}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := NewInMemoryPoolStateStore() + rec := newKillRecorder(t) + manager := newTestPoolManager(t, store, rec.srv.URL) + + if _, err := manager.Destroy(context.Background(), tt.poolName, tt.options); err == nil { + t.Fatal("Destroy succeeded, want validation error") + } + + // A rejected destroy must not have fenced anything. + state, err := store.GetDestroyState(context.Background(), "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateActive { + t.Errorf("state = %s, want ACTIVE", state) + } + }) + } +} + +// ---------- Fence Observation Tests ---------- + +func TestSandboxPoolManager_Destroy_FenceStopsLivePool(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycleSrv := newMockLifecycleServer(t, execdSrv.URL) + store := NewInMemoryPoolStateStore() + + pool := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(2).ReconcileInterval(10 * time.Millisecond) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + waitForIdleCount(t, store, "test-pool", 2) + + manager := newTestPoolManager(t, store, lifecycleSrv.URL) + if _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}); err != nil { + t.Fatalf("Destroy failed: %v", err) + } + + // The still-running pool must not replenish the destroyed namespace. + for i := 0; i < 5; i++ { + time.Sleep(20 * time.Millisecond) + counters, err := store.SnapshotCounters(ctx, "test-pool") + if err != nil { + t.Fatalf("SnapshotCounters failed: %v", err) + } + if counters.IdleCount != 0 { + t.Fatalf("idle count = %d after destroy, want 0 (fenced pool must not warm up)", counters.IdleCount) + } + } + + // The reconcile tick observes the fence and stops the pool outright. + deadline := time.Now().Add(5 * time.Second) + for { + snapshot, err := pool.Snapshot(ctx) + if err != nil { + t.Fatalf("Snapshot failed: %v", err) + } + if snapshot.LifecycleState == PoolLifecycleStopped { + break + } + if time.Now().After(deadline) { + t.Fatalf("pool state = %s after destroy, want STOPPED", snapshot.LifecycleState) + } + time.Sleep(10 * time.Millisecond) + } + + // A peer starting fresh against the tombstoned namespace must refuse to run. + peer := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(2) + }) + err := peer.Start(ctx) + var destroyed *PoolDestroyedError + if !errors.As(err, &destroyed) { + t.Fatalf("peer Start error = %v, want *PoolDestroyedError", err) + } +} + +// countingLifecycleServer is a mock lifecycle API that records how many +// sandboxes were created and killed. +type countingLifecycleServer struct { + srv *httptest.Server + created atomic.Int32 + deleted atomic.Int32 +} + +func newCountingLifecycleServer(t *testing.T, execdURL string) *countingLifecycleServer { + t.Helper() + c := &countingLifecycleServer{} + c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case r.Method == http.MethodPost && path == "/v1/sandboxes": + c.created.Add(1) + jsonResponse(w, http.StatusCreated, SandboxInfo{ + ID: fmt.Sprintf("sbx-created-%d", c.created.Load()), + Status: SandboxStatus{State: StateRunning}, + Entrypoint: []string{"tail", "-f", "/dev/null"}, + CreatedAt: time.Now().UTC(), + }) + case r.Method == http.MethodGet && strings.Contains(path, "/endpoints/"): + jsonResponse(w, http.StatusOK, Endpoint{ + Endpoint: execdURL, + Headers: map[string]string{"X-EXECD-ACCESS-TOKEN": "test-token"}, + }) + case r.Method == http.MethodGet && strings.HasPrefix(path, "/v1/sandboxes/"): + parts := strings.Split(path, "/") + jsonResponse(w, http.StatusOK, SandboxInfo{ + ID: parts[len(parts)-1], + Status: SandboxStatus{State: StateRunning}, + Entrypoint: []string{"tail", "-f", "/dev/null"}, + CreatedAt: time.Now().UTC(), + }) + case r.Method == http.MethodDelete && strings.HasPrefix(path, "/v1/sandboxes/"): + c.deleted.Add(1) + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && strings.HasSuffix(path, "/renew-expiration"): + jsonResponse(w, http.StatusOK, RenewExpirationResponse{ExpiresAt: time.Now().Add(time.Hour).UTC()}) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(c.srv.Close) + return c +} + +// scriptedDestroyStateStore overrides GetDestroyState so a test can drive the +// fence independently of the rest of the store. +type scriptedDestroyStateStore struct { + *InMemoryPoolStateStore + + // err, when set, is returned from every GetDestroyState call. + err error + // activeCalls is how many leading calls report ACTIVE before the namespace + // starts reporting DESTROYED. Ignored when err is set. + activeCalls int32 + + calls atomic.Int32 +} + +func (s *scriptedDestroyStateStore) GetDestroyState(_ context.Context, _ string) (PoolDestroyState, error) { + n := s.calls.Add(1) + if s.err != nil { + return PoolDestroyStateActive, s.err + } + if n <= s.activeCalls { + return PoolDestroyStateActive, nil + } + return PoolDestroyStateDestroyed, nil +} + +// TestSandboxPoolManager_Destroy_BlocksDirectCreateOnLivePool covers the case a +// store-level fence alone cannot: a peer that is still RUNNING when the fence +// lands would otherwise find an empty idle buffer and mint a fresh sandbox into +// the retired namespace via the direct-create fallthrough. +func TestSandboxPoolManager_Destroy_BlocksDirectCreateOnLivePool(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycle := newCountingLifecycleServer(t, execdSrv.URL) + store := NewInMemoryPoolStateStore() + + // MaxIdle 0 and a long interval keep the pool RUNNING: no reconcile tick + // fires to observe the fence and stop it. + pool := newTestPool(t, lifecycle.srv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour). + EmptyBehavior(AcquirePolicyDirectCreate) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + // Sanity check: before the destroy, direct create is the expected behavior. + sb, err := pool.Acquire(ctx, AcquireOptions{}) + if err != nil { + t.Fatalf("Acquire before destroy failed: %v", err) + } + _ = sb.Close() + if got := lifecycle.created.Load(); got != 1 { + t.Fatalf("created = %d before destroy, want 1", got) + } + + manager := newTestPoolManager(t, store, lifecycle.srv.URL) + if _, err := manager.Destroy(ctx, "test-pool", PoolDestroyOptions{}); err != nil { + t.Fatalf("Destroy failed: %v", err) + } + + snapshot, err := pool.Snapshot(ctx) + if err != nil { + t.Fatalf("Snapshot failed: %v", err) + } + if snapshot.LifecycleState != PoolLifecycleRunning { + t.Fatalf("pool state = %s, want RUNNING (the test needs a live peer)", snapshot.LifecycleState) + } + + _, err = pool.Acquire(ctx, AcquireOptions{}) + var destroyed *PoolDestroyedError + if !errors.As(err, &destroyed) { + t.Fatalf("Acquire after destroy = %v, want *PoolDestroyedError", err) + } + if got := lifecycle.created.Load(); got != 1 { + t.Errorf("created = %d after destroy, want 1 (no sandbox may be minted into a retired namespace)", got) + } +} + +// TestPool_Acquire_KillsSandboxFencedMidCreate covers a destroy that lands while +// a direct create is already in flight. +func TestPool_Acquire_KillsSandboxFencedMidCreate(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycle := newCountingLifecycleServer(t, execdSrv.URL) + + // The namespace is checked at Start and again before the acquire; both must + // see ACTIVE. The third check is the post-create one, and that is the one + // this test wants fenced. + store := &scriptedDestroyStateStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + activeCalls: 2, + } + + pool := newTestPool(t, lifecycle.srv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + _, err := pool.Acquire(ctx, AcquireOptions{}) + var destroyed *PoolDestroyedError + if !errors.As(err, &destroyed) { + t.Fatalf("Acquire = %v, want *PoolDestroyedError", err) + } + if got := lifecycle.created.Load(); got != 1 { + t.Fatalf("created = %d, want 1", got) + } + + // The orphaned sandbox is killed asynchronously. + deadline := time.Now().Add(5 * time.Second) + for lifecycle.deleted.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("sandbox created before the fence was never killed") + } + time.Sleep(10 * time.Millisecond) + } +} + +// TestPool_Acquire_KillsIdleSandboxFencedMidAcquire covers the fence landing +// between the preflight check and the idle take. TryTakeIdle is unfenced so the +// destroy manager can drain, which means the ID is already out of the store by +// then and a concurrent Destroy can no longer reach it: the acquire has to kill +// it rather than hand back a sandbox from a retired namespace. +func TestPool_Acquire_KillsIdleSandboxFencedMidAcquire(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycle := newCountingLifecycleServer(t, execdSrv.URL) + + // Start and the acquire preflight both see ACTIVE; the post-connect check + // is the third call and sees the fence. + store := &scriptedDestroyStateStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + activeCalls: 2, + } + + pool := newTestPool(t, lifecycle.srv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + if err := store.PutIdle(ctx, "test-pool", "sbx-idle-fenced"); err != nil { + t.Fatalf("PutIdle failed: %v", err) + } + + sb, err := pool.Acquire(ctx, AcquireOptions{}) + var destroyed *PoolDestroyedError + if !errors.As(err, &destroyed) { + if sb != nil { + _ = sb.Close() + } + t.Fatalf("Acquire = %v, want *PoolDestroyedError", err) + } + if got := lifecycle.created.Load(); got != 0 { + t.Errorf("created = %d, want 0 (the idle candidate must not be replaced)", got) + } + + // The idle sandbox is no longer tracked anywhere, so the acquire must kill it. + deadline := time.Now().Add(5 * time.Second) + for lifecycle.deleted.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("idle sandbox taken before the fence was never killed") + } + time.Sleep(10 * time.Millisecond) + } +} + +// TestPool_Acquire_IdlePathFenceCheckIsFailClosed pins the deliberate asymmetry +// with the direct-create path: an idle sandbox is already out of the store, so an +// unreachable store cannot be assumed ACTIVE the way direct create may. +func TestPool_Acquire_IdlePathFenceCheckIsFailClosed(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycle := newCountingLifecycleServer(t, execdSrv.URL) + + inner := NewInMemoryPoolStateStore() + if err := inner.PutIdle(ctx, "test-pool", "sbx-idle-outage"); err != nil { + t.Fatalf("PutIdle failed: %v", err) + } + + // Report ACTIVE for Start and the preflight, then fail. DIRECT_CREATE would + // degrade and keep going; the idle path must not. + store := &outageAfterNCallsStore{ + InMemoryPoolStateStore: inner, + okCalls: 2, + } + + pool := newTestPool(t, lifecycle.srv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour). + EmptyBehavior(AcquirePolicyDirectCreate) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + sb, err := pool.Acquire(ctx, AcquireOptions{}) + if err == nil { + _ = sb.Close() + t.Fatal("Acquire succeeded with an unconfirmable namespace, want an error") + } + var unavailable *PoolStateStoreUnavailableError + if !errors.As(err, &unavailable) { + t.Fatalf("Acquire = %v, want *PoolStateStoreUnavailableError", err) + } + + deadline := time.Now().Add(5 * time.Second) + for lifecycle.deleted.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("idle sandbox was never killed after the fail-closed check") + } + time.Sleep(10 * time.Millisecond) + } +} + +// outageAfterNCallsStore answers GetDestroyState normally for the first okCalls +// calls and then reports the store as unreachable. +type outageAfterNCallsStore struct { + *InMemoryPoolStateStore + + okCalls int32 + calls atomic.Int32 +} + +func (s *outageAfterNCallsStore) GetDestroyState(ctx context.Context, poolName string) (PoolDestroyState, error) { + if s.calls.Add(1) <= s.okCalls { + return s.InMemoryPoolStateStore.GetDestroyState(ctx, poolName) + } + return PoolDestroyStateActive, &PoolStateStoreUnavailableError{ + Operation: "GetDestroyState", + Cause: errors.New("redis is down"), + } +} + +// TestPool_Acquire_NamespaceCheckDegradesOnStoreOutage keeps a store outage from +// making direct-create policies less available than the OSEP-0005 matrix +// documents, while fail-closed policies still surface it. +func TestPool_Acquire_NamespaceCheckDegradesOnStoreOutage(t *testing.T) { + tests := []struct { + name string + policy AcquirePolicy + wantCreated int32 + wantErr bool + }{ + {name: "direct create degrades", policy: AcquirePolicyDirectCreate, wantCreated: 1}, + {name: "retry then create degrades", policy: AcquirePolicyRetryNextIdleThenCreate, wantCreated: 1}, + {name: "fail fast surfaces the outage", policy: AcquirePolicyFailFast, wantErr: true}, + {name: "retry next idle surfaces the outage", policy: AcquirePolicyRetryNextIdle, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycle := newCountingLifecycleServer(t, execdSrv.URL) + store := &scriptedDestroyStateStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + err: errors.New("redis is down"), + } + + pool := newTestPool(t, lifecycle.srv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour). + EmptyBehavior(tt.policy) + }) + if err := pool.Start(ctx); err != nil { + t.Fatalf("Start failed: %v", err) + } + t.Cleanup(func() { _ = pool.Shutdown(context.Background(), false) }) + + sb, err := pool.Acquire(ctx, AcquireOptions{}) + if tt.wantErr { + var unavailable *PoolStateStoreUnavailableError + if !errors.As(err, &unavailable) { + t.Fatalf("Acquire = %v, want *PoolStateStoreUnavailableError", err) + } + } else { + if err != nil { + t.Fatalf("Acquire failed: %v", err) + } + _ = sb.Close() + } + if got := lifecycle.created.Load(); got != tt.wantCreated { + t.Errorf("created = %d, want %d", got, tt.wantCreated) + } + }) + } +} + +func TestPool_Start_RefusesDestroyedNamespace(t *testing.T) { + ctx := context.Background() + execdSrv := newMockExecdServer(t) + lifecycleSrv := newMockLifecycleServer(t, execdSrv.URL) + store := &scriptedDestroyStateStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + } + + pool := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { + b.StateStore(store).MaxIdle(0).ReconcileInterval(time.Hour) + }) + + err := pool.Start(ctx) + var destroyed *PoolDestroyedError + if !errors.As(err, &destroyed) { + t.Fatalf("Start = %v, want *PoolDestroyedError", err) + } + + snapshot, err := pool.Snapshot(ctx) + if err != nil { + t.Fatalf("Snapshot failed: %v", err) + } + if snapshot.LifecycleState != PoolLifecycleNotStarted { + t.Errorf("state after refused Start = %s, want NOT_STARTED", snapshot.LifecycleState) + } +} + +func TestInMemoryPoolStateStore_FenceRejectsWrites(t *testing.T) { + ctx := context.Background() + + for _, state := range []PoolDestroyState{PoolDestroyStateDestroying, PoolDestroyStateDestroyed} { + t.Run(state.String(), func(t *testing.T) { + store := NewInMemoryPoolStateStore() + if err := store.BeginDestroy(ctx, "test-pool", "owner-1"); err != nil { + t.Fatalf("BeginDestroy failed: %v", err) + } + if state == PoolDestroyStateDestroyed { + if err := store.MarkDestroyed(ctx, "test-pool", "owner-1", time.Hour); err != nil { + t.Fatalf("MarkDestroyed failed: %v", err) + } + } + + writes := map[string]func() error{ + "PutIdle": func() error { return store.PutIdle(ctx, "test-pool", "sbx-1") }, + "SetMaxIdle": func() error { return store.SetMaxIdle(ctx, "test-pool", 5) }, + "SetIdleEntryTTL": func() error { return store.SetIdleEntryTTL(ctx, "test-pool", time.Minute) }, + } + for name, write := range writes { + var destroyed *PoolDestroyedError + if err := write(); !errors.As(err, &destroyed) { + t.Errorf("%s error = %v, want *PoolDestroyedError", name, err) + } else if destroyed.State != state { + t.Errorf("%s error state = %s, want %s", name, destroyed.State, state) + } + } + + acquired, err := store.TryAcquirePrimaryLock(ctx, "test-pool", "owner-2", time.Minute) + if err != nil { + t.Fatalf("TryAcquirePrimaryLock failed: %v", err) + } + if acquired { + t.Error("TryAcquirePrimaryLock succeeded on a fenced namespace, want false") + } + + renewed, err := store.RenewPrimaryLock(ctx, "test-pool", "owner-2", time.Minute) + if err != nil { + t.Fatalf("RenewPrimaryLock failed: %v", err) + } + if renewed { + t.Error("RenewPrimaryLock succeeded on a fenced namespace, want false") + } + }) + } +} + +func TestInMemoryPoolStateStore_BeginDestroyRejectsTombstoned(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + + if err := store.BeginDestroy(ctx, "test-pool", "owner-1"); err != nil { + t.Fatalf("BeginDestroy failed: %v", err) + } + // Re-entrant while DESTROYING, so a retrying owner can make progress. + if err := store.BeginDestroy(ctx, "test-pool", "owner-1"); err != nil { + t.Fatalf("second BeginDestroy on a DESTROYING namespace failed: %v", err) + } + + if err := store.MarkDestroyed(ctx, "test-pool", "owner-1", time.Hour); err != nil { + t.Fatalf("MarkDestroyed failed: %v", err) + } + + var destroyed *PoolDestroyedError + if err := store.BeginDestroy(ctx, "test-pool", "owner-2"); !errors.As(err, &destroyed) { + t.Fatalf("BeginDestroy on a tombstoned namespace = %v, want *PoolDestroyedError", err) + } +} + +func TestInMemoryPoolStateStore_ClearPoolStateKeepsFence(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + + if err := store.SetMaxIdle(ctx, "test-pool", 7); err != nil { + t.Fatalf("SetMaxIdle failed: %v", err) + } + if err := store.PutIdle(ctx, "test-pool", "sbx-1"); err != nil { + t.Fatalf("PutIdle failed: %v", err) + } + if err := store.BeginDestroy(ctx, "test-pool", "owner-1"); err != nil { + t.Fatalf("BeginDestroy failed: %v", err) + } + if err := store.ClearPoolState(ctx, "test-pool"); err != nil { + t.Fatalf("ClearPoolState failed: %v", err) + } + + counters, err := store.SnapshotCounters(ctx, "test-pool") + if err != nil { + t.Fatalf("SnapshotCounters failed: %v", err) + } + if counters.IdleCount != 0 { + t.Errorf("idle count = %d, want 0", counters.IdleCount) + } + maxIdle, err := store.GetMaxIdle(ctx, "test-pool") + if err != nil { + t.Fatalf("GetMaxIdle failed: %v", err) + } + if maxIdle != 0 { + t.Errorf("maxIdle = %d, want 0", maxIdle) + } + state, err := store.GetDestroyState(ctx, "test-pool") + if err != nil { + t.Fatalf("GetDestroyState failed: %v", err) + } + if state != PoolDestroyStateDestroying { + t.Errorf("state = %s, want DESTROYING (ClearPoolState must not lift the fence)", state) + } +} + +func TestInMemoryPoolStateStore_MarkDestroyedRejectsBlankOwnerAndNegativeTTL(t *testing.T) { + ctx := context.Background() + store := NewInMemoryPoolStateStore() + + if err := store.MarkDestroyed(ctx, "test-pool", "", time.Hour); err == nil { + t.Error("MarkDestroyed with a blank owner succeeded, want error") + } + if err := store.MarkDestroyed(ctx, "test-pool", "owner-1", -time.Second); err == nil { + t.Error("MarkDestroyed with a negative TTL succeeded, want error") + } + if err := store.BeginDestroy(ctx, "test-pool", ""); err == nil { + t.Error("BeginDestroy with a blank owner succeeded, want error") + } +} + +// ---------- Builder Tests ---------- + +func TestSandboxPoolManagerBuilder_Validation(t *testing.T) { + tests := []struct { + name string + build func() (*SandboxPoolManager, error) + wantErr bool + }{ + { + name: "missing state store", + build: func() (*SandboxPoolManager, error) { + return NewSandboxPoolManagerBuilder(). + ConnectionConfig(ConnectionConfig{Domain: "localhost:8080"}). + Build() + }, + wantErr: true, + }, + { + name: "missing connection config", + build: func() (*SandboxPoolManager, error) { + return NewSandboxPoolManagerBuilder(). + StateStore(NewInMemoryPoolStateStore()). + Build() + }, + wantErr: true, + }, + { + name: "blank owner ID", + build: func() (*SandboxPoolManager, error) { + return NewSandboxPoolManagerBuilder(). + StateStore(NewInMemoryPoolStateStore()). + ConnectionConfig(ConnectionConfig{Domain: "localhost:8080"}). + OwnerID(" "). + Build() + }, + wantErr: true, + }, + { + name: "defaults the owner ID", + build: func() (*SandboxPoolManager, error) { + return NewSandboxPoolManagerBuilder(). + StateStore(NewInMemoryPoolStateStore()). + ConnectionConfig(ConnectionConfig{Domain: "localhost:8080"}). + Build() + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager, err := tt.build() + if tt.wantErr { + if err == nil { + t.Fatal("Build succeeded, want error") + } + return + } + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if manager.ownerID == "" { + t.Error("ownerID is empty, want a generated value") + } + }) + } +} + +// waitForIdleCount blocks until the store reports want idle entries. +func waitForIdleCount(t *testing.T, store PoolStateStore, poolName string, want int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + counters, err := store.SnapshotCounters(context.Background(), poolName) + if err != nil { + t.Fatalf("SnapshotCounters failed: %v", err) + } + if counters.IdleCount >= want { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %d idle entries in pool %q", want, poolName) +} diff --git a/sdks/sandbox/go/pool_store.go b/sdks/sandbox/go/pool_store.go index e45f994ca..6a7ae44bf 100644 --- a/sdks/sandbox/go/pool_store.go +++ b/sdks/sandbox/go/pool_store.go @@ -71,4 +71,23 @@ type PoolStateStore interface { // SetIdleEntryTTL persists the idle entry TTL for the pool. SetIdleEntryTTL(ctx context.Context, poolName string, ttl time.Duration) error + + // GetDestroyState returns the destroy state of the pool namespace. + // An expired tombstone reads back as ACTIVE. + GetDestroyState(ctx context.Context, poolName string) (PoolDestroyState, error) + + // BeginDestroy writes the DESTROYING fence, making the namespace + // unwritable for every peer sharing this store. Returns *PoolDestroyedError + // if the namespace is already tombstoned. Re-entrant while DESTROYING. + BeginDestroy(ctx context.Context, poolName string, ownerID string) error + + // ClearPoolState wipes the pool's coordination state: idle entries, the + // primary lock, maxIdle, and the idle entry TTL. The destroy state itself + // is left in place. + ClearPoolState(ctx context.Context, poolName string) error + + // MarkDestroyed replaces the fence with a DESTROYED tombstone so later + // callers cannot silently rebind the namespace. A zero tombstoneTTL writes + // a tombstone that never expires; it must not be negative. + MarkDestroyed(ctx context.Context, poolName string, ownerID string, tombstoneTTL time.Duration) error } diff --git a/sdks/sandbox/go/pool_store_memory.go b/sdks/sandbox/go/pool_store_memory.go index 4557374a7..86661449b 100644 --- a/sdks/sandbox/go/pool_store_memory.go +++ b/sdks/sandbox/go/pool_store_memory.go @@ -42,6 +42,12 @@ type poolState struct { // configurable per-pool settings. idleTTL time.Duration maxIdle int + + // destroy fence / tombstone state. + destroyState PoolDestroyState + destroyOwnerID string + // destroyExpiresAt is zero when the current destroy state never expires. + destroyExpiresAt time.Time } // InMemoryPoolStateStore is a pure in-memory implementation of PoolStateStore. @@ -165,6 +171,9 @@ func (s *InMemoryPoolStateStore) PutIdle(_ context.Context, poolName string, san defer ps.mu.Unlock() now := time.Now() + if err := ps.rejectIfFencedLocked(poolName, now); err != nil { + return err + } if existing, exists := ps.idleMap[sandboxID]; exists { if existing.ExpiresAt.IsZero() || now.Before(existing.ExpiresAt) { return nil // still alive, idempotent no-op @@ -204,6 +213,9 @@ func (s *InMemoryPoolStateStore) TryAcquirePrimaryLock(_ context.Context, poolNa defer ps.mu.Unlock() now := time.Now() + if ps.destroyStateLocked(now) != PoolDestroyStateActive { + return false, nil + } if ps.lock.ownerID != "" && now.Before(ps.lock.expiresAt) { // Lock is held and not expired. if ps.lock.ownerID == ownerID { @@ -228,6 +240,9 @@ func (s *InMemoryPoolStateStore) RenewPrimaryLock(_ context.Context, poolName st defer ps.mu.Unlock() now := time.Now() + if ps.destroyStateLocked(now) != PoolDestroyStateActive { + return false, nil + } if ps.lock.ownerID != ownerID { return false, nil } @@ -352,6 +367,9 @@ func (s *InMemoryPoolStateStore) SetMaxIdle(_ context.Context, poolName string, ps := s.getOrCreatePool(poolName) ps.mu.Lock() defer ps.mu.Unlock() + if err := ps.rejectIfFencedLocked(poolName, time.Now()); err != nil { + return err + } ps.maxIdle = maxIdle return nil } @@ -364,10 +382,100 @@ func (s *InMemoryPoolStateStore) SetIdleEntryTTL(_ context.Context, poolName str ps := s.getOrCreatePool(poolName) ps.mu.Lock() defer ps.mu.Unlock() + if err := ps.rejectIfFencedLocked(poolName, time.Now()); err != nil { + return err + } ps.idleTTL = ttl return nil } +// GetDestroyState returns the destroy state of the pool namespace. +func (s *InMemoryPoolStateStore) GetDestroyState(_ context.Context, poolName string) (PoolDestroyState, error) { + ps := s.getOrCreatePool(poolName) + ps.mu.Lock() + defer ps.mu.Unlock() + return ps.destroyStateLocked(time.Now()), nil +} + +// BeginDestroy writes the DESTROYING fence. Returns *PoolDestroyedError if the +// namespace already carries a live tombstone. +func (s *InMemoryPoolStateStore) BeginDestroy(_ context.Context, poolName string, ownerID string) error { + if ownerID == "" { + return fmt.Errorf("opensandbox: ownerID must not be blank") + } + ps := s.getOrCreatePool(poolName) + ps.mu.Lock() + defer ps.mu.Unlock() + + if ps.destroyStateLocked(time.Now()) == PoolDestroyStateDestroyed { + return &PoolDestroyedError{PoolName: poolName, State: PoolDestroyStateDestroyed} + } + ps.destroyState = PoolDestroyStateDestroying + ps.destroyOwnerID = ownerID + ps.destroyExpiresAt = time.Time{} + return nil +} + +// ClearPoolState wipes the pool's coordination state, leaving the destroy state +// in place so the fence survives the cleanup. +func (s *InMemoryPoolStateStore) ClearPoolState(_ context.Context, poolName string) error { + ps := s.getOrCreatePool(poolName) + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.idleMap = make(map[string]*IdleEntry) + ps.idleQueue = nil + ps.lock = poolLock{} + ps.idleTTL = DefaultIdleTimeout + ps.maxIdle = 0 + return nil +} + +// MarkDestroyed writes the DESTROYED tombstone. A zero tombstoneTTL never expires. +func (s *InMemoryPoolStateStore) MarkDestroyed(_ context.Context, poolName string, ownerID string, tombstoneTTL time.Duration) error { + if ownerID == "" { + return fmt.Errorf("opensandbox: ownerID must not be blank") + } + if tombstoneTTL < 0 { + return fmt.Errorf("opensandbox: tombstoneTTL must not be negative, got %v", tombstoneTTL) + } + ps := s.getOrCreatePool(poolName) + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.destroyState = PoolDestroyStateDestroyed + ps.destroyOwnerID = ownerID + if tombstoneTTL == 0 { + ps.destroyExpiresAt = time.Time{} + } else { + ps.destroyExpiresAt = time.Now().Add(tombstoneTTL) + } + return nil +} + +// destroyStateLocked returns the current destroy state, clearing an expired +// tombstone on the way. Must be called with ps.mu held. +func (ps *poolState) destroyStateLocked(now time.Time) PoolDestroyState { + if ps.destroyState == PoolDestroyStateActive { + return PoolDestroyStateActive + } + if !ps.destroyExpiresAt.IsZero() && !now.Before(ps.destroyExpiresAt) { + ps.destroyState = PoolDestroyStateActive + ps.destroyOwnerID = "" + ps.destroyExpiresAt = time.Time{} + } + return ps.destroyState +} + +// rejectIfFencedLocked returns *PoolDestroyedError when the namespace is fenced. +// Must be called with ps.mu held. +func (ps *poolState) rejectIfFencedLocked(poolName string, now time.Time) error { + if state := ps.destroyStateLocked(now); state != PoolDestroyStateActive { + return &PoolDestroyedError{PoolName: poolName, State: state} + } + return nil +} + // compactQueueIfNeeded copies the queue to a right-sized slice when the // underlying array has grown much larger than needed. Must be called with // ps.mu held. diff --git a/sdks/sandbox/go/pool_test.go b/sdks/sandbox/go/pool_test.go index 20e2934ac..69a6c93cc 100644 --- a/sdks/sandbox/go/pool_test.go +++ b/sdks/sandbox/go/pool_test.go @@ -136,31 +136,6 @@ func newTestPool(t *testing.T, serverURL string, opts ...func(*SandboxPoolBuilde // ---------- Builder Tests ---------- -func TestPoolBuilder_Defaults(t *testing.T) { - b := NewSandboxPoolBuilder() - if b.config.ReconcileInterval != 30*time.Second { - t.Errorf("ReconcileInterval = %v, want 30s", b.config.ReconcileInterval) - } - if b.config.PrimaryLockTTL != 60*time.Second { - t.Errorf("PrimaryLockTTL = %v, want 60s", b.config.PrimaryLockTTL) - } - if b.config.DegradedThreshold != 3 { - t.Errorf("DegradedThreshold = %d, want 3", b.config.DegradedThreshold) - } - if b.config.DrainTimeout != 30*time.Second { - t.Errorf("DrainTimeout = %v, want 30s", b.config.DrainTimeout) - } - if b.config.AcquireReadyTimeout != 30*time.Second { - t.Errorf("AcquireReadyTimeout = %v, want 30s", b.config.AcquireReadyTimeout) - } - if b.config.WarmupReadyTimeout != 30*time.Second { - t.Errorf("WarmupReadyTimeout = %v, want 30s", b.config.WarmupReadyTimeout) - } - if b.config.EmptyBehavior != AcquirePolicyDirectCreate { - t.Errorf("EmptyBehavior = %v, want DIRECT_CREATE", b.config.EmptyBehavior) - } -} - func TestPoolBuilder_MissingPoolName(t *testing.T) { _, err := NewSandboxPoolBuilder(). ConnectionConfig(ConnectionConfig{Domain: "localhost:8080", Protocol: "http"}). @@ -471,6 +446,40 @@ type failingTakeStore struct { takeErr error } +type failAfterTakeStore struct { + *InMemoryPoolStateStore + successfulTakes int + takes int + err error +} + +type cancelAfterTakeStore struct { + *InMemoryPoolStateStore + successfulTakes int + takes int + cancel context.CancelFunc +} + +func (s *failAfterTakeStore) TryTakeIdle(ctx context.Context, poolName string) (string, error) { + if s.takes >= s.successfulTakes { + return "", s.err + } + s.takes++ + return s.InMemoryPoolStateStore.TryTakeIdle(ctx, poolName) +} + +func (s *cancelAfterTakeStore) TryTakeIdle(ctx context.Context, poolName string) (string, error) { + sandboxID, err := s.InMemoryPoolStateStore.TryTakeIdle(ctx, poolName) + if err != nil || sandboxID == "" { + return sandboxID, err + } + s.takes++ + if s.takes == s.successfulTakes { + s.cancel() + } + return sandboxID, nil +} + func (s *failingTakeStore) TryTakeIdle(_ context.Context, _ string) (string, error) { return "", s.takeErr } @@ -838,6 +847,178 @@ func TestPool_Shutdown_DoesNotReleaseIdle(t *testing.T) { } } +func TestPool_ReleaseAllIdle_BoundsConcurrentKills(t *testing.T) { + const maxWorkers = 50 + var active atomic.Int32 + var maxActive atomic.Int32 + var deleted atomic.Int32 + ready := make(chan struct{}) + var readyOnce sync.Once + lifecycleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusNotFound) + return + } + current := active.Add(1) + for { + observed := maxActive.Load() + if current <= observed || maxActive.CompareAndSwap(observed, current) { + break + } + } + if current == maxWorkers { + readyOnce.Do(func() { close(ready) }) + } + select { + case <-ready: + case <-time.After(2 * time.Second): + t.Error("concurrent kills did not reach configured limit") + } + active.Add(-1) + deleted.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(lifecycleSrv.Close) + storeErr := errors.New("injected store failure") + store := &failAfterTakeStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + successfulTakes: 55, + err: storeErr, + } + pool := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { + b.MaxIdle(0).StateStore(store) + }) + ctx := context.Background() + for i := 0; i < 55; i++ { + if err := pool.config.StateStore.PutIdle(ctx, "test-pool", fmt.Sprintf("idle-%d", i)); err != nil { + t.Fatal(err) + } + } + + if _, err := pool.ReleaseAllIdleParallel(ctx, 0); err == nil { + t.Fatal("ReleaseAllIdleParallel() accepted maxWorkers=0") + } + released, err := pool.ReleaseAllIdleParallel(ctx, maxWorkers) + + if !errors.Is(err, storeErr) { + t.Fatalf("ReleaseAllIdleParallel() error = %v, want injected store failure", err) + } + if released != 55 { + t.Fatalf("released = %d, want 55", released) + } + if maxActive.Load() != maxWorkers { + t.Errorf("max concurrent kills = %d, want %d", maxActive.Load(), maxWorkers) + } + if deleted.Load() != 55 { + t.Errorf("deleted = %d, want 55", deleted.Load()) + } +} + +func TestPool_ReleaseAllIdleParallel_CompletesDrainedKillsAfterContextCancellation(t *testing.T) { + const ( + totalSandboxes = 5 + drainedBeforeCancel = 3 + ) + var deleted atomic.Int32 + lifecycleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusNotFound) + return + } + deleted.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(lifecycleSrv.Close) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store := &cancelAfterTakeStore{ + InMemoryPoolStateStore: NewInMemoryPoolStateStore(), + successfulTakes: drainedBeforeCancel, + cancel: cancel, + } + pool := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { + b.MaxIdle(0).StateStore(store) + }) + for i := 0; i < totalSandboxes; i++ { + if err := store.PutIdle(ctx, "test-pool", fmt.Sprintf("idle-%d", i)); err != nil { + t.Fatal(err) + } + } + + released, err := pool.ReleaseAllIdleParallel(ctx, 2) + + if !errors.Is(err, context.Canceled) { + t.Fatalf("ReleaseAllIdleParallel() error = %v, want context.Canceled", err) + } + if released != drainedBeforeCancel { + t.Fatalf("released = %d, want %d", released, drainedBeforeCancel) + } + if got := deleted.Load(); got != drainedBeforeCancel { + t.Errorf("deleted = %d, want %d", got, drainedBeforeCancel) + } + counters, snapshotErr := store.SnapshotCounters(context.Background(), "test-pool") + if snapshotErr != nil { + t.Fatal(snapshotErr) + } + if want := totalSandboxes - drainedBeforeCancel; counters.IdleCount != want { + t.Errorf("remaining idle = %d, want %d", counters.IdleCount, want) + } +} + +func TestPool_ReleaseAllIdle_PreservesFireAndForgetBehavior(t *testing.T) { + requestStarted := make(chan struct{}) + releaseKill := make(chan struct{}) + deleted := make(chan struct{}) + lifecycleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusNotFound) + return + } + close(requestStarted) + <-releaseKill + w.WriteHeader(http.StatusNoContent) + close(deleted) + })) + t.Cleanup(lifecycleSrv.Close) + pool := newTestPool(t, lifecycleSrv.URL, func(b *SandboxPoolBuilder) { b.MaxIdle(0) }) + ctx := context.Background() + if err := pool.config.StateStore.PutIdle(ctx, "test-pool", "idle-1"); err != nil { + t.Fatal(err) + } + type result struct { + count int + err error + } + done := make(chan result, 1) + go func() { + count, err := pool.ReleaseAllIdle(ctx) + done <- result{count: count, err: err} + }() + + select { + case got := <-done: + if got.err != nil || got.count != 1 { + t.Fatalf("ReleaseAllIdle() = (%d, %v), want (1, nil)", got.count, got.err) + } + case <-time.After(2 * time.Second): + close(releaseKill) + t.Fatal("ReleaseAllIdle blocked on the kill request") + } + select { + case <-requestStarted: + case <-time.After(2 * time.Second): + close(releaseKill) + t.Fatal("kill request did not start") + } + close(releaseKill) + select { + case <-deleted: + case <-time.After(2 * time.Second): + t.Fatal("kill request did not finish") + } +} + func TestPool_Shutdown_NonGraceful_DoesNotReleaseIdle(t *testing.T) { execdSrv := newMockExecdServer(t) lifecycleSrv := newMockLifecycleServer(t, execdSrv.URL) diff --git a/sdks/sandbox/go/pool_types.go b/sdks/sandbox/go/pool_types.go index 7065be58a..ba6b57a88 100644 --- a/sdks/sandbox/go/pool_types.go +++ b/sdks/sandbox/go/pool_types.go @@ -258,3 +258,89 @@ type AcquireOptions struct { // DefaultIdleTimeout is the default TTL for idle pool entries (24 hours, per OSEP-0005). const DefaultIdleTimeout = 24 * time.Hour + +// PoolDestroyState represents the destroy lifecycle of a pool namespace as seen +// by every process sharing the same state store. +// +// - ACTIVE: the namespace is usable. +// - DESTROYING: a destroy fence is in place. Peer pools must stop replenishing +// and must not fall back to direct create. +// - DESTROYED: a tombstone is in place. Callers must not rebind the namespace +// until the tombstone expires. +type PoolDestroyState int + +const ( + PoolDestroyStateActive PoolDestroyState = iota + PoolDestroyStateDestroying + PoolDestroyStateDestroyed +) + +func (s PoolDestroyState) String() string { + switch s { + case PoolDestroyStateActive: + return "ACTIVE" + case PoolDestroyStateDestroying: + return "DESTROYING" + case PoolDestroyStateDestroyed: + return "DESTROYED" + default: + return "UNKNOWN" + } +} + +// PoolDestroyStrategy selects how a namespace is retired. Only FORCE is +// implemented; the iota order MUST stay append-only. +type PoolDestroyStrategy int + +const ( + PoolDestroyForce PoolDestroyStrategy = iota +) + +func (s PoolDestroyStrategy) String() string { + switch s { + case PoolDestroyForce: + return "FORCE" + default: + return "UNKNOWN" + } +} + +// DefaultPoolDrainTimeout bounds the idle-drain phase of a pool destroy. +const DefaultPoolDrainTimeout = 30 * time.Second + +// DefaultPoolTombstoneTTL is how long a DESTROYED tombstone survives before the +// namespace may be rebound. +const DefaultPoolTombstoneTTL = 7 * 24 * time.Hour + +// PoolDestroyOptions configures a single SandboxPoolManager.Destroy call. +// The zero value is valid and selects FORCE with all defaults. +type PoolDestroyOptions struct { + // Strategy selects the destroy algorithm. Only PoolDestroyForce is supported. + Strategy PoolDestroyStrategy + + // DrainTimeout bounds the idle-drain loop. Nil selects DefaultPoolDrainTimeout; + // an explicit zero drains without a deadline. Must not be negative. + DrainTimeout *time.Duration + + // TombstoneTTL is how long the DESTROYED tombstone survives. Nil selects + // DefaultPoolTombstoneTTL; an explicit zero writes a tombstone that never + // expires. Must not be negative. + TombstoneTTL *time.Duration +} + +// PoolDestroyResult reports what a destroy actually did. +type PoolDestroyResult struct { + PoolName string + State PoolDestroyState + + // DrainedIdleCount is how many idle entries were taken from the store. + DrainedIdleCount int + + // KilledIdleCount is how many of those sandboxes were successfully killed. + // Killing is best-effort, so this may be lower than DrainedIdleCount. + KilledIdleCount int + + // PersistentStateCleared reports whether this call cleared the coordination + // state. It is false when the namespace was already tombstoned. + PersistentStateCleared bool +} diff --git a/sdks/sandbox/go/poolredis/store.go b/sdks/sandbox/go/poolredis/store.go index bb29b21da..57a9ad647 100644 --- a/sdks/sandbox/go/poolredis/store.go +++ b/sdks/sandbox/go/poolredis/store.go @@ -56,6 +56,10 @@ end `) putIdleScript = redis.NewScript(` +local destroy_state = redis.call('GET', KEYS[3]) +if destroy_state then + return -1 +end local redis_time = redis.call('TIME') local now_ms = tonumber(redis_time[1]) * 1000 + math.floor(tonumber(redis_time[2]) / 1000) local expires_at = now_ms + tonumber(ARGV[2]) @@ -94,6 +98,10 @@ return discarded_alive `) acquireLockScript = redis.NewScript(` +local destroy_state = redis.call('GET', KEYS[2]) +if destroy_state then + return 0 +end local current = redis.call('GET', KEYS[1]) if not current then redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) @@ -106,6 +114,10 @@ return 0 `) renewLockScript = redis.NewScript(` +local destroy_state = redis.call('GET', KEYS[2]) +if destroy_state then + return 0 +end if redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('PEXPIRE', KEYS[1], ARGV[2]) return 1 @@ -138,6 +150,39 @@ for i = 1, #entries, 2 do end end return count +`) + + // setFencedValueScript writes a single pool setting, refusing the write when + // the namespace carries a destroy fence or tombstone. + setFencedValueScript = redis.NewScript(` +local destroy_state = redis.call('GET', KEYS[2]) +if destroy_state then + return -1 +end +redis.call('SET', KEYS[1], ARGV[1]) +return 1 +`) + + beginDestroyScript = redis.NewScript(` +local destroy_state = redis.call('GET', KEYS[1]) +if destroy_state == ARGV[2] then + return -1 +end +redis.call('SET', KEYS[1], ARGV[1]) +redis.call('SET', KEYS[2], ARGV[3]) +return 1 +`) + + markDestroyedScript = redis.NewScript(` +local ttl_ms = tonumber(ARGV[3]) +if ttl_ms and ttl_ms > 0 then + redis.call('SET', KEYS[1], ARGV[1], 'PX', ttl_ms) + redis.call('SET', KEYS[2], ARGV[2], 'PX', ttl_ms) +else + redis.call('SET', KEYS[1], ARGV[1]) + redis.call('SET', KEYS[2], ARGV[2]) +end +return 1 `) ) @@ -257,14 +302,14 @@ func (s *RedisPoolStateStore) PutIdle(ctx context.Context, poolName string, sand return err } - keys := []string{s.idleListKey(poolName), s.idleExpiresKey(poolName)} + keys := []string{s.idleListKey(poolName), s.idleExpiresKey(poolName), s.destroyStateKey(poolName)} argv := []interface{}{sandboxID, strconv.FormatInt(idleTTLMs, 10)} - _, err = putIdleScript.Run(ctx, s.client, keys, argv...).Result() + result, err := putIdleScript.Run(ctx, s.client, keys, argv...).Int64() if err != nil && err != redis.Nil { return &opensandbox.PoolStateStoreUnavailableError{Operation: "PutIdle", Cause: err} } - return nil + return s.destroyedErrorIfFenced(ctx, poolName, result) } // RemoveIdle atomically removes a sandbox from the idle pool. Idempotent. @@ -289,7 +334,7 @@ func (s *RedisPoolStateStore) TryAcquirePrimaryLock(ctx context.Context, poolNam ttlMs = 1 } result, err := acquireLockScript.Run(ctx, s.client, - []string{s.PrimaryLockKey(poolName)}, + []string{s.PrimaryLockKey(poolName), s.destroyStateKey(poolName)}, ownerID, strconv.FormatInt(ttlMs, 10)).Int64() if err != nil && err != redis.Nil { return false, &opensandbox.PoolStateStoreUnavailableError{Operation: "TryAcquirePrimaryLock", Cause: err} @@ -304,7 +349,7 @@ func (s *RedisPoolStateStore) RenewPrimaryLock(ctx context.Context, poolName str ttlMs = 1 } - keys := []string{s.PrimaryLockKey(poolName)} + keys := []string{s.PrimaryLockKey(poolName), s.destroyStateKey(poolName)} argv := []interface{}{ownerID, strconv.FormatInt(ttlMs, 10)} result, err := renewLockScript.Run(ctx, s.client, keys, argv...).Int64() @@ -433,11 +478,7 @@ func (s *RedisPoolStateStore) GetMaxIdle(ctx context.Context, poolName string) ( // SetMaxIdle persists the maxIdle value for the pool. func (s *RedisPoolStateStore) SetMaxIdle(ctx context.Context, poolName string, maxIdle int) error { - err := s.client.Set(ctx, s.maxIdleKey(poolName), strconv.Itoa(maxIdle), 0).Err() - if err != nil { - return &opensandbox.PoolStateStoreUnavailableError{Operation: "SetMaxIdle", Cause: err} - } - return nil + return s.setFencedValue(ctx, poolName, "SetMaxIdle", s.maxIdleKey(poolName), strconv.Itoa(maxIdle)) } // SetIdleEntryTTL persists the idle entry TTL for the pool. @@ -446,13 +487,123 @@ func (s *RedisPoolStateStore) SetIdleEntryTTL(ctx context.Context, poolName stri if ms < 1 { ms = 1 } - err := s.client.Set(ctx, s.idleTTLKey(poolName), strconv.FormatInt(ms, 10), 0).Err() + return s.setFencedValue(ctx, poolName, "SetIdleEntryTTL", s.idleTTLKey(poolName), strconv.FormatInt(ms, 10)) +} + +func (s *RedisPoolStateStore) setFencedValue(ctx context.Context, poolName string, operation string, key string, value string) error { + keys := []string{key, s.destroyStateKey(poolName)} + + result, err := setFencedValueScript.Run(ctx, s.client, keys, value).Int64() + if err != nil && err != redis.Nil { + return &opensandbox.PoolStateStoreUnavailableError{Operation: operation, Cause: err} + } + return s.destroyedErrorIfFenced(ctx, poolName, result) +} + +// GetDestroyState returns the destroy state of the pool namespace. An expired +// tombstone has already been dropped by Redis and reads back as ACTIVE. +func (s *RedisPoolStateStore) GetDestroyState(ctx context.Context, poolName string) (opensandbox.PoolDestroyState, error) { + val, err := s.client.Get(ctx, s.destroyStateKey(poolName)).Result() + if err == redis.Nil { + return opensandbox.PoolDestroyStateActive, nil + } if err != nil { - return &opensandbox.PoolStateStoreUnavailableError{Operation: "SetIdleEntryTTL", Cause: err} + return opensandbox.PoolDestroyStateActive, &opensandbox.PoolStateStoreUnavailableError{Operation: "GetDestroyState", Cause: err} + } + switch val { + case opensandbox.PoolDestroyStateDestroying.String(): + return opensandbox.PoolDestroyStateDestroying, nil + case opensandbox.PoolDestroyStateDestroyed.String(): + return opensandbox.PoolDestroyStateDestroyed, nil + default: + return opensandbox.PoolDestroyStateActive, nil + } +} + +// BeginDestroy writes the DESTROYING fence. Returns *opensandbox.PoolDestroyedError +// if the namespace already carries a live tombstone. +func (s *RedisPoolStateStore) BeginDestroy(ctx context.Context, poolName string, ownerID string) error { + if ownerID == "" { + return fmt.Errorf("opensandbox: ownerID must not be blank") + } + keys := []string{s.destroyStateKey(poolName), s.destroyOwnerKey(poolName)} + argv := []interface{}{ + opensandbox.PoolDestroyStateDestroying.String(), + opensandbox.PoolDestroyStateDestroyed.String(), + ownerID, + } + + result, err := beginDestroyScript.Run(ctx, s.client, keys, argv...).Int64() + if err != nil && err != redis.Nil { + return &opensandbox.PoolStateStoreUnavailableError{Operation: "BeginDestroy", Cause: err} + } + if result == -1 { + return &opensandbox.PoolDestroyedError{PoolName: poolName, State: opensandbox.PoolDestroyStateDestroyed} + } + return nil +} + +// ClearPoolState deletes the pool's coordination keys, leaving the destroy keys +// in place so the fence survives the cleanup. +func (s *RedisPoolStateStore) ClearPoolState(ctx context.Context, poolName string) error { + err := s.client.Del(ctx, + s.idleListKey(poolName), + s.idleExpiresKey(poolName), + s.PrimaryLockKey(poolName), + s.maxIdleKey(poolName), + s.idleTTLKey(poolName), + ).Err() + if err != nil && err != redis.Nil { + return &opensandbox.PoolStateStoreUnavailableError{Operation: "ClearPoolState", Cause: err} + } + return nil +} + +// MarkDestroyed writes the DESTROYED tombstone. A zero tombstoneTTL writes a +// tombstone that never expires. +func (s *RedisPoolStateStore) MarkDestroyed(ctx context.Context, poolName string, ownerID string, tombstoneTTL time.Duration) error { + if ownerID == "" { + return fmt.Errorf("opensandbox: ownerID must not be blank") + } + if tombstoneTTL < 0 { + return fmt.Errorf("opensandbox: tombstoneTTL must not be negative, got %v", tombstoneTTL) + } + + // A sub-millisecond TTL would round to zero and be read as "never expires", + // so clamp it to the smallest expiry Redis can represent. + ttlMs := tombstoneTTL.Milliseconds() + if tombstoneTTL > 0 && ttlMs < 1 { + ttlMs = 1 + } + + keys := []string{s.destroyStateKey(poolName), s.destroyOwnerKey(poolName)} + argv := []interface{}{ + opensandbox.PoolDestroyStateDestroyed.String(), + ownerID, + strconv.FormatInt(ttlMs, 10), + } + + _, err := markDestroyedScript.Run(ctx, s.client, keys, argv...).Result() + if err != nil && err != redis.Nil { + return &opensandbox.PoolStateStoreUnavailableError{Operation: "MarkDestroyed", Cause: err} } return nil } +// destroyedErrorIfFenced converts the -1 sentinel returned by the fenced-write +// scripts into a *opensandbox.PoolDestroyedError carrying the observed state. +func (s *RedisPoolStateStore) destroyedErrorIfFenced(ctx context.Context, poolName string, scriptResult int64) error { + if scriptResult != -1 { + return nil + } + state, err := s.GetDestroyState(ctx, poolName) + if err != nil { + // The write was refused; report that rather than the follow-up read failure. + state = opensandbox.PoolDestroyStateDestroying + } + return &opensandbox.PoolDestroyedError{PoolName: poolName, State: state} +} + // resolveIdleTTL reads the configured idle TTL from Redis (in ms). // Falls back to DefaultIdleTimeout if not set. func (s *RedisPoolStateStore) resolveIdleTTL(ctx context.Context, poolName string) (int64, error) { @@ -501,5 +652,13 @@ func (s *RedisPoolStateStore) idleTTLKey(poolName string) string { return s.poolKey(poolName, "idleTtlMillis") } +func (s *RedisPoolStateStore) destroyStateKey(poolName string) string { + return s.poolKey(poolName, "destroy:state") +} + +func (s *RedisPoolStateStore) destroyOwnerKey(poolName string) string { + return s.poolKey(poolName, "destroy:owner") +} + // Compile-time interface check. var _ opensandbox.PoolStateStore = (*RedisPoolStateStore)(nil) diff --git a/sdks/sandbox/go/poolredis/store_test.go b/sdks/sandbox/go/poolredis/store_test.go index 2ee0e16dc..94b5e900b 100644 --- a/sdks/sandbox/go/poolredis/store_test.go +++ b/sdks/sandbox/go/poolredis/store_test.go @@ -57,6 +57,8 @@ func cleanupPool(t *testing.T, store *RedisPoolStateStore, poolName string) { store.PrimaryLockKey(poolName), store.maxIdleKey(poolName), store.idleTTLKey(poolName), + store.destroyStateKey(poolName), + store.destroyOwnerKey(poolName), } store.client.Del(ctx, keys...) } @@ -498,3 +500,220 @@ func TestRedisStore_WrapsClientFailures(t *testing.T) { t.Errorf("Operation = %q, want %q", storeErr.Operation, "GetMaxIdle") } } + +// ---------- Destroy Fence And Tombstone Tests ---------- + +func TestRedisStore_GetDestroyState_DefaultsToActive(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + state, err := store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state != opensandbox.PoolDestroyStateActive { + t.Errorf("state = %s, want ACTIVE", state) + } +} + +func TestRedisStore_BeginDestroy_FencesWrites(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.BeginDestroy(ctx, poolName, "owner-1"); err != nil { + t.Fatalf("BeginDestroy error: %v", err) + } + + state, err := store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state != opensandbox.PoolDestroyStateDestroying { + t.Fatalf("state = %s, want DESTROYING", state) + } + + writes := map[string]func() error{ + "PutIdle": func() error { return store.PutIdle(ctx, poolName, "sb-fenced") }, + "SetMaxIdle": func() error { return store.SetMaxIdle(ctx, poolName, 3) }, + "SetIdleEntryTTL": func() error { return store.SetIdleEntryTTL(ctx, poolName, time.Hour) }, + } + for name, write := range writes { + var destroyed *opensandbox.PoolDestroyedError + if err := write(); !errors.As(err, &destroyed) { + t.Errorf("%s error = %v, want *PoolDestroyedError", name, err) + } else if destroyed.State != opensandbox.PoolDestroyStateDestroying { + t.Errorf("%s error state = %s, want DESTROYING", name, destroyed.State) + } + } + + acquired, err := store.TryAcquirePrimaryLock(ctx, poolName, "owner-2", time.Minute) + if err != nil { + t.Fatalf("TryAcquirePrimaryLock error: %v", err) + } + if acquired { + t.Error("TryAcquirePrimaryLock succeeded on a fenced namespace, want false") + } + + renewed, err := store.RenewPrimaryLock(ctx, poolName, "owner-2", time.Minute) + if err != nil { + t.Fatalf("RenewPrimaryLock error: %v", err) + } + if renewed { + t.Error("RenewPrimaryLock succeeded on a fenced namespace, want false") + } +} + +func TestRedisStore_BeginDestroy_RejectsTombstoned(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.BeginDestroy(ctx, poolName, "owner-1"); err != nil { + t.Fatalf("BeginDestroy error: %v", err) + } + // Re-entrant while DESTROYING so a retrying owner can make progress. + if err := store.BeginDestroy(ctx, poolName, "owner-1"); err != nil { + t.Fatalf("second BeginDestroy error: %v", err) + } + if err := store.MarkDestroyed(ctx, poolName, "owner-1", time.Hour); err != nil { + t.Fatalf("MarkDestroyed error: %v", err) + } + + var destroyed *opensandbox.PoolDestroyedError + if err := store.BeginDestroy(ctx, poolName, "owner-2"); !errors.As(err, &destroyed) { + t.Fatalf("BeginDestroy on a tombstoned namespace = %v, want *PoolDestroyedError", err) + } +} + +func TestRedisStore_ClearPoolState_KeepsFence(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.SetMaxIdle(ctx, poolName, 4); err != nil { + t.Fatalf("SetMaxIdle error: %v", err) + } + if err := store.PutIdle(ctx, poolName, "sb-1"); err != nil { + t.Fatalf("PutIdle error: %v", err) + } + if err := store.BeginDestroy(ctx, poolName, "owner-1"); err != nil { + t.Fatalf("BeginDestroy error: %v", err) + } + if err := store.ClearPoolState(ctx, poolName); err != nil { + t.Fatalf("ClearPoolState error: %v", err) + } + + counters, err := store.SnapshotCounters(ctx, poolName) + if err != nil { + t.Fatalf("SnapshotCounters error: %v", err) + } + if counters.IdleCount != 0 { + t.Errorf("idle count = %d, want 0", counters.IdleCount) + } + maxIdle, err := store.GetMaxIdle(ctx, poolName) + if err != nil { + t.Fatalf("GetMaxIdle error: %v", err) + } + if maxIdle != 0 { + t.Errorf("maxIdle = %d, want 0", maxIdle) + } + state, err := store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state != opensandbox.PoolDestroyStateDestroying { + t.Errorf("state = %s, want DESTROYING (ClearPoolState must not lift the fence)", state) + } +} + +func TestRedisStore_MarkDestroyed_TombstoneTTLExpires(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.BeginDestroy(ctx, poolName, "owner-1"); err != nil { + t.Fatalf("BeginDestroy error: %v", err) + } + if err := store.MarkDestroyed(ctx, poolName, "owner-1", 200*time.Millisecond); err != nil { + t.Fatalf("MarkDestroyed error: %v", err) + } + + state, err := store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state != opensandbox.PoolDestroyStateDestroyed { + t.Fatalf("state = %s, want DESTROYED", state) + } + + deadline := time.Now().Add(5 * time.Second) + for { + state, err = store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state == opensandbox.PoolDestroyStateActive { + break + } + if time.Now().After(deadline) { + t.Fatalf("state = %s after the tombstone TTL, want ACTIVE", state) + } + time.Sleep(20 * time.Millisecond) + } + + if err := store.PutIdle(ctx, poolName, "sb-rebound"); err != nil { + t.Errorf("PutIdle after tombstone expiry error: %v", err) + } +} + +func TestRedisStore_MarkDestroyed_ZeroTTLNeverExpires(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.MarkDestroyed(ctx, poolName, "owner-1", 0); err != nil { + t.Fatalf("MarkDestroyed error: %v", err) + } + + ttl, err := store.client.PTTL(ctx, store.destroyStateKey(poolName)).Result() + if err != nil { + t.Fatalf("PTTL error: %v", err) + } + // -1 is Redis' answer for a key that exists with no expiry. + if ttl != -1*time.Nanosecond && ttl >= 0 { + t.Errorf("tombstone PTTL = %v, want no expiry", ttl) + } + + state, err := store.GetDestroyState(ctx, poolName) + if err != nil { + t.Fatalf("GetDestroyState error: %v", err) + } + if state != opensandbox.PoolDestroyStateDestroyed { + t.Errorf("state = %s, want DESTROYED", state) + } +} + +func TestRedisStore_MarkDestroyed_RejectsInvalidInput(t *testing.T) { + store := newRedisTestStore(t) + ctx := context.Background() + poolName := "destroy-" + t.Name() + t.Cleanup(func() { cleanupPool(t, store, poolName) }) + + if err := store.MarkDestroyed(ctx, poolName, "", time.Hour); err == nil { + t.Error("MarkDestroyed with a blank owner succeeded, want error") + } + if err := store.MarkDestroyed(ctx, poolName, "owner-1", -time.Second); err == nil { + t.Error("MarkDestroyed with a negative TTL succeeded, want error") + } + if err := store.BeginDestroy(ctx, poolName, ""); err == nil { + t.Error("BeginDestroy with a blank owner succeeded, want error") + } +} diff --git a/sdks/sandbox/javascript/package.json b/sdks/sandbox/javascript/package.json index b68761a11..53788a113 100644 --- a/sdks/sandbox/javascript/package.json +++ b/sdks/sandbox/javascript/package.json @@ -48,7 +48,7 @@ }, "dependencies": { "openapi-fetch": "^0.14.1", - "undici": "^7.28.0" + "undici": "^7.29.0" }, "devDependencies": { "@eslint/js": "^9.39.4", diff --git a/sdks/sandbox/javascript/src/adapters/commandsAdapter.ts b/sdks/sandbox/javascript/src/adapters/commandsAdapter.ts index 8092f0f19..8828a4ebc 100644 --- a/sdks/sandbox/javascript/src/adapters/commandsAdapter.ts +++ b/sdks/sandbox/javascript/src/adapters/commandsAdapter.ts @@ -194,6 +194,7 @@ export class CommandsAdapter implements ExecdCommands { stream: AsyncIterable, handlers?: ExecutionHandlers, inferExitCode = false, + isBackground = false, ): Promise { const execution: CommandExecution = { logs: { stdout: [], stderr: [] }, @@ -205,6 +206,13 @@ export class CommandsAdapter implements ExecdCommands { (ev as { text?: string }).text = execution.id; } await dispatcher.dispatch(ev as any); + if (isBackground && ev.type === "execution_complete") { + // Background commands are done once execution_complete arrives; do + // not wait for the chunked terminator, which execd sends only after + // a graceful-shutdown sleep and can be lost if the connection is + // closed early (#1528). + break; + } } if (inferExitCode) { @@ -288,6 +296,7 @@ export class CommandsAdapter implements ExecdCommands { this.runStream(command, opts, signal), handlers, !opts?.background, + !!opts?.background, ); } diff --git a/sdks/sandbox/javascript/src/adapters/openapiError.ts b/sdks/sandbox/javascript/src/adapters/openapiError.ts index c476f42b6..e798178b4 100644 --- a/sdks/sandbox/javascript/src/adapters/openapiError.ts +++ b/sdks/sandbox/javascript/src/adapters/openapiError.ts @@ -24,10 +24,22 @@ export function throwOnOpenApiFetchError( const status = (result.response as any).status ?? 0; const err = result.error as any; + + let rawFragment: string | undefined; + if (typeof result.error === "string") { + rawFragment = result.error; + } else if (result.error && typeof result.error === "object") { + try { + rawFragment = JSON.stringify(result.error); + } catch { + rawFragment = undefined; + } + } + const message = err?.message ?? err?.error?.message ?? - fallbackMessage; + (rawFragment && rawFragment.length > 0 ? rawFragment : fallbackMessage); const code = err?.code ?? err?.error?.code; const msg = err?.message ?? err?.error?.message ?? message; diff --git a/sdks/sandbox/javascript/src/adapters/sse.ts b/sdks/sandbox/javascript/src/adapters/sse.ts index a4fb380db..ce25aa087 100644 --- a/sdks/sandbox/javascript/src/adapters/sse.ts +++ b/sdks/sandbox/javascript/src/adapters/sse.ts @@ -55,41 +55,51 @@ export async function* parseJsonEventStream( const decoder = new TextDecoder("utf-8"); let buf = ""; - while (true) { - const { value, done } = await reader.read(); - if (done) break; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx: number; + buf += decoder.decode(value, { stream: true }); + let idx: number; - while ((idx = buf.indexOf("\n")) >= 0) { - const rawLine = buf.slice(0, idx); - buf = buf.slice(idx + 1); + while ((idx = buf.indexOf("\n")) >= 0) { + const rawLine = buf.slice(0, idx); + buf = buf.slice(idx + 1); - const line = rawLine.trim(); - if (!line) continue; + const line = rawLine.trim(); + if (!line) continue; - // Support standard SSE "data:" prefix - if (line.startsWith(":")) continue; - if (line.startsWith("event:") || line.startsWith("id:") || line.startsWith("retry:")) continue; + // Support standard SSE "data:" prefix + if (line.startsWith(":")) continue; + if (line.startsWith("event:") || line.startsWith("id:") || line.startsWith("retry:")) continue; - const jsonLine = line.startsWith("data:") ? line.slice("data:".length).trim() : line; - if (!jsonLine) continue; + const jsonLine = line.startsWith("data:") ? line.slice("data:".length).trim() : line; + if (!jsonLine) continue; - const parsed = tryParseJson(jsonLine); - if (!parsed) continue; - yield parsed as T; + const parsed = tryParseJson(jsonLine); + if (!parsed) continue; + yield parsed as T; + } } - } - // Flush any buffered UTF-8 bytes from the decoder. - buf += decoder.decode(); + // Flush any buffered UTF-8 bytes from the decoder. + buf += decoder.decode(); - // flush last line if exists - const last = buf.trim(); - if (last) { - const jsonLine = last.startsWith("data:") ? last.slice("data:".length).trim() : last; - const parsed = tryParseJson(jsonLine); - if (parsed) yield parsed as T; + // flush last line if exists + const last = buf.trim(); + if (last) { + const jsonLine = last.startsWith("data:") ? last.slice("data:".length).trim() : last; + const parsed = tryParseJson(jsonLine); + if (parsed) yield parsed as T; + } + } finally { + // Consumers (e.g. background commands) can stop iterating once + // execution_complete arrives, without draining the rest of the + // stream. Cancel the reader on any exit path โ€” normal completion, + // early break, or thrown error โ€” so the fetch body is released + // instead of staying locked and holding the transport connection + // open (#1528, #1532). + await reader.cancel().catch(() => undefined); } } \ No newline at end of file diff --git a/sdks/sandbox/javascript/src/api/execd.ts b/sdks/sandbox/javascript/src/api/execd.ts index 7feb6c8ed..3a9fd06b7 100644 --- a/sdks/sandbox/javascript/src/api/execd.ts +++ b/sdks/sandbox/javascript/src/api/execd.ts @@ -1514,6 +1514,30 @@ export interface components { userns_available?: boolean; commit_supported?: boolean; diff_supported?: boolean; + /** @description execd init-mode and workload-hardening state (OSEP-0018): whether execd is the sandbox init and which of its controls are in effect. Not an isolation capability; reported here so operators see enforcement state in one place. */ + hardening?: { + /** + * @description How execd supervises the sandbox process tree. pid1: execd is the kernel init of the container. subreaper: execd reaps orphans but lacks the PID 1 kernel signal shield. none: init mode is off (default). + * @enum {string} + */ + init_mode?: "pid1" | "subreaper" | "none"; + /** @description Whether the kernel PID 1 signal shield protects execd from in-namespace signals (true only in init_mode pid1). */ + signal_shield?: boolean; + /** @description Capability/bounding-set reduction on user code. */ + cap_drop?: components["schemas"]["HardeningLayerState"]; + /** @description Seccomp floor installed on user code. */ + seccomp?: components["schemas"]["HardeningLayerState"]; + /** @description Landlock filesystem confinement on user code. */ + landlock?: components["schemas"]["HardeningLayerState"]; + /** @description eBPF exec/connect/privilege observation. */ + ebpf?: components["schemas"]["HardeningLayerState"]; + }; + }; + /** @description Whether one hardening layer is actually enforced. state is "active" | "disabled" (not configured) | "degraded" (configured but a prerequisite is missing) | "unsupported" (kernel/build cannot provide it). message gives the concrete reason whenever state is not active. */ + HardeningLayerState: { + /** @enum {string} */ + state?: "active" | "disabled" | "degraded" | "unsupported"; + message?: string; }; }; responses: { diff --git a/sdks/sandbox/javascript/src/models/isolated.ts b/sdks/sandbox/javascript/src/models/isolated.ts index f5958ad2e..5f00104b3 100644 --- a/sdks/sandbox/javascript/src/models/isolated.ts +++ b/sdks/sandbox/javascript/src/models/isolated.ts @@ -113,6 +113,20 @@ export interface IsolatedRunLogs { cursor: number; } +export interface HardeningLayerState { + state: "active" | "disabled" | "degraded" | "unsupported" | string; + message?: string; +} + +export interface HardeningStatus { + init_mode: "pid1" | "subreaper" | "none" | string; + signal_shield: boolean; + cap_drop?: HardeningLayerState; + seccomp?: HardeningLayerState; + landlock?: HardeningLayerState; + ebpf?: HardeningLayerState; +} + export interface IsolatedCapabilities { available: boolean; isolator?: string; @@ -122,6 +136,7 @@ export interface IsolatedCapabilities { userns_available?: boolean; commit_supported: boolean; diff_supported: boolean; + hardening?: HardeningStatus; } export interface IsolatedSessionSummary { diff --git a/sdks/sandbox/javascript/src/sandbox.ts b/sdks/sandbox/javascript/src/sandbox.ts index 906a294ba..0cc884d74 100644 --- a/sdks/sandbox/javascript/src/sandbox.ts +++ b/sdks/sandbox/javascript/src/sandbox.ts @@ -738,12 +738,7 @@ export class Sandbox { const buildTimeoutMessage = () => { const context = `domain=${this.connectionConfig.domain}, useServerProxy=${this.connectionConfig.useServerProxy}`; - let suggestion = - "If this sandbox runs in Docker bridge or remote-network mode, consider enabling useServerProxy=true."; - if (!this.connectionConfig.useServerProxy) { - suggestion += " You can also configure server-side [docker].host_ip for direct endpoint access."; - } - return `Sandbox health check timed out after ${opts.readyTimeoutSeconds}s (${attempt} attempts). ${errorDetail} Connection context: ${context}. ${suggestion}`; + return `Sandbox health check timed out after ${opts.readyTimeoutSeconds}s (${attempt} attempts). ${errorDetail} Connection context: ${context}.`; }; // Wait until execd becomes reachable and passes health check. diff --git a/sdks/sandbox/javascript/tests/commands.run.test.mjs b/sdks/sandbox/javascript/tests/commands.run.test.mjs index e05ef98d7..60727e2a2 100644 --- a/sdks/sandbox/javascript/tests/commands.run.test.mjs +++ b/sdks/sandbox/javascript/tests/commands.run.test.mjs @@ -1,7 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { CommandsAdapter } from "../dist/internal.js"; +import { CommandsAdapter, createExecdClient } from "../dist/internal.js"; +import { SandboxApiException } from "../dist/index.js"; function createAdapter(responseBody, opts = {}) { const fetchImpl = async () => @@ -73,6 +74,106 @@ test("CommandsAdapter.run keeps exitCode null when error value is empty", async assert.equal(execution.exitCode, null); }); +function createEarlyCloseStream() { + // Delivers the two SSE chunks one read() at a time, then errors on the + // next pull โ€” simulating a peer that closes the connection right after + // execution_complete, before the chunked terminator arrives (#1528). + const encoder = new TextEncoder(); + const chunks = [ + 'data: {"type":"init","text":"cmd-bg","timestamp":1}\n\n', + 'data: {"type":"execution_complete","timestamp":2,"execution_time":3}\n\n', + ].map((chunk) => encoder.encode(chunk)); + let index = 0; + return new ReadableStream({ + pull(controller) { + if (index < chunks.length) { + controller.enqueue(chunks[index]); + index += 1; + return; + } + controller.error(new Error("peer closed connection early")); + }, + }); +} + +test("CommandsAdapter.run breaks on execution_complete for background commands", async () => { + const fetchImpl = async () => + new Response(createEarlyCloseStream(), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + + const adapter = new CommandsAdapter( + {}, + { baseUrl: "http://127.0.0.1:8080", fetch: fetchImpl }, + ); + + const execution = await adapter.run("sleep 1", { background: true }); + + assert.equal(execution.id, "cmd-bg"); + assert.equal(execution.complete?.executionTimeMs, 3); + assert.equal(execution.exitCode, undefined); +}); + +test("CommandsAdapter.run still surfaces stream errors for foreground commands", async () => { + const fetchImpl = async () => + new Response(createEarlyCloseStream(), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + + const adapter = new CommandsAdapter( + {}, + { baseUrl: "http://127.0.0.1:8080", fetch: fetchImpl }, + ); + + await assert.rejects(() => adapter.run("sleep 1")); +}); + +test("CommandsAdapter.run cancels the reader when a background command breaks early", async () => { + // The stream never signals `done` or errors on its own past + // execution_complete -- it simply stops delivering data, the way a + // peer that never sends the chunked terminator would behave. If the + // completion break left the reader un-cancelled, the stream's `cancel` + // hook would never fire and the body would stay locked indefinitely. + let cancelled = false; + const encoder = new TextEncoder(); + const chunks = [ + 'data: {"type":"init","text":"cmd-cancel","timestamp":1}\n\n', + 'data: {"type":"execution_complete","timestamp":2,"execution_time":3}\n\n', + ].map((chunk) => encoder.encode(chunk)); + let index = 0; + const stream = new ReadableStream({ + pull(controller) { + if (index < chunks.length) { + controller.enqueue(chunks[index]); + index += 1; + } + // Beyond the known chunks: no-op. The underlying source never + // closes or errors the stream by itself. + }, + cancel() { + cancelled = true; + }, + }); + + const fetchImpl = async () => + new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + + const adapter = new CommandsAdapter( + {}, + { baseUrl: "http://127.0.0.1:8080", fetch: fetchImpl }, + ); + + const execution = await adapter.run("sleep 1", { background: true }); + + assert.equal(execution.id, "cmd-cancel"); + assert.equal(cancelled, true); +}); + test("CommandsAdapter.runInSession sends command and timeout fields", async () => { let requestBody; const fetchImpl = async (url, init) => { @@ -129,3 +230,26 @@ test("CommandsAdapter.runInSession infers non-zero exitCode from final error sta assert.equal(execution.complete, undefined); assert.equal(execution.exitCode, 7); }); + +test("execd client error message carries unstructured JSON error body", async () => { + const adapter = new CommandsAdapter( + createExecdClient({ + baseUrl: "http://127.0.0.1:8080", + fetch: async () => + new Response(JSON.stringify({ error: "invalid parameter" }), { + status: 400, + headers: { "content-type": "application/json" }, + }), + }), + {}, + ); + + await assert.rejects( + () => adapter.getCommandStatus("exec-1"), + (err) => { + assert.ok(err instanceof SandboxApiException); + assert.match(err.message, /invalid parameter/); + return true; + }, + ); +}); diff --git a/sdks/sandbox/javascript/tests/health.test.mjs b/sdks/sandbox/javascript/tests/health.test.mjs index 7f892cdfa..a65e9cc4f 100644 --- a/sdks/sandbox/javascript/tests/health.test.mjs +++ b/sdks/sandbox/javascript/tests/health.test.mjs @@ -40,3 +40,21 @@ test("HealthAdapter still maps ping API errors", async () => { await assert.rejects(() => health.ping(), SandboxApiException); }); + +test("execd client error message carries unstructured plain-text body", async () => { + const health = new HealthAdapter(createExecdClient({ + baseUrl: "http://execd.test", + async fetch() { + return new Response("slow down", { status: 400 }); + }, + })); + + await assert.rejects( + () => health.ping(), + (err) => { + assert.ok(err instanceof SandboxApiException); + assert.match(err.message, /slow down/); + return true; + }, + ); +}); diff --git a/sdks/sandbox/javascript/tests/sandbox.create.test.mjs b/sdks/sandbox/javascript/tests/sandbox.create.test.mjs index b5e2fee09..b85a4fc0f 100644 --- a/sdks/sandbox/javascript/tests/sandbox.create.test.mjs +++ b/sdks/sandbox/javascript/tests/sandbox.create.test.mjs @@ -532,6 +532,45 @@ test("Sandbox.create metrics failure does not change create error", async () => ); }); +test("Sandbox.create readiness timeout omits network configuration hints", async () => { + const { adapterFactory } = createAdapterFactory(); + adapterFactory.createExecdStack = () => ({ + commands: {}, + files: {}, + health: { + async ping() { + throw new Error("connect ECONNREFUSED"); + }, + }, + metrics: {}, + }); + + const connectionConfig = new ConnectionConfig({ + domain: "http://127.0.0.1:8080", + useServerProxy: false, + disableMetrics: true, + }); + + await assert.rejects( + Sandbox.create({ + adapterFactory, + connectionConfig, + image: "python:3.12", + readyTimeoutSeconds: 0.2, + healthCheckPollingInterval: 50, + }), + (err) => { + const message = String(err && err.message); + assert.match(message, /Sandbox health check timed out after/); + assert.match(message, /domain=http:\/\/127\.0\.0\.1:8080, useServerProxy=false/); + assert.match(message, /Last health check error: connect ECONNREFUSED/); + assert.doesNotMatch(message, /consider enabling useServerProxy=true/i); + assert.doesNotMatch(message, /Docker bridge|remote-network|\[docker\]\.host_ip/i); + return true; + } + ); +}); + test("Sandbox.create metrics synchronous throw does not change create error", async () => { // Regression test: previously, payload/URL/headers construction and // `connectionConfig.fetch(...)` ran outside any try/catch in the reporter. diff --git a/sdks/sandbox/kotlin/README.md b/sdks/sandbox/kotlin/README.md index ffa47b150..b89ff047a 100644 --- a/sdks/sandbox/kotlin/README.md +++ b/sdks/sandbox/kotlin/README.md @@ -290,10 +290,15 @@ Pool lifecycle semantics: - Graceful shutdown stops admitting new warmups, keeps the primary heartbeat and completion controller alive while already-admitted warmups finish, and preserves the existing behavior of allowing those warmups to enter idle before shutdown completes. +- The pool shares one OkHttp `ConnectionPool` across every sandbox it creates + (warmup, direct create, idle connect). When the `ConnectionConfig` carries no + custom pool, the pool creates one sized by `warmupConcurrency` (5-minute + keep-alive) and evicts it on `shutdown()`; a user-provided pool is never touched. > For distributed deployment, use the optional `com.alibaba.opensandbox:sandbox-pool-redis` module or provide a custom `PoolStateStore` implementation. The Redis module accepts a caller-managed Jedis client, so your application keeps ownership of Redis connection configuration and lifecycle. Nodes sharing the same pool namespace must use the same sandbox creation and warmup definition; use a new `poolName` or namespace when changing that definition. The pool renews an owned primary lock independently from warmup execution at an internal interval no greater than `primaryLockTtl / 3`, so one slow warmup does not block leader heartbeats. > In distributed mode, `resize(maxIdle)` can be called from any node. The call returns after the target is stored in the shared state store; the current primary applies replenish or shrink work during periodic reconcile. Use `resize(0)` and wait for `snapshot().idleCount == 0` when you need to drain the distributed idle buffer; `releaseAllIdle()` is only a best-effort cleanup pass. +> `releaseAllIdle()` preserves the original serial cleanup behavior. Use `releaseAllIdle(concurrency)` for bounded parallel cleanup. `concurrency` must be positive; the overload returns only after every ID drained from the store has received a best-effort kill attempt. > `SandboxPoolManager.destroy(poolName)` is a stronger administrative operation: it writes a `DESTROYING` fence, drains visible idle IDs, best-effort kills idle sandboxes, clears persistent pool state, and then writes a `DESTROYED` tombstone for the configured TTL to prevent old nodes from recreating the same pool namespace. If drain or persistent-state cleanup cannot complete, `destroy()` throws `PoolDestroyIncompleteException` and leaves the namespace fenced as `DESTROYING`; retry `destroy()` to finish cleanup. ## Configuration diff --git a/sdks/sandbox/kotlin/gradle/libs.versions.toml b/sdks/sandbox/kotlin/gradle/libs.versions.toml index 6998d867e..d363627c8 100644 --- a/sdks/sandbox/kotlin/gradle/libs.versions.toml +++ b/sdks/sandbox/kotlin/gradle/libs.versions.toml @@ -17,6 +17,7 @@ kotlin = "2.2.21" kotlinx-serialization = "1.9.0" okhttp = "4.12.0" slf4j = "2.0.9" +opentelemetry = "1.51.0" jedis = "5.2.0" junit = "5.10.1" mockk = "1.13.8" @@ -43,6 +44,11 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa # Logging slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" } + +# Tracing (API only; no-op when no OpenTelemetry SDK is on the classpath) +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +opentelemetry-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing", version.ref = "opentelemetry" } # Redis jedis = { module = "redis.clients:jedis", version.ref = "jedis" } diff --git a/sdks/sandbox/kotlin/sandbox-api/build.gradle.kts b/sdks/sandbox/kotlin/sandbox-api/build.gradle.kts index 5204130d7..177acbf90 100644 --- a/sdks/sandbox/kotlin/sandbox-api/build.gradle.kts +++ b/sdks/sandbox/kotlin/sandbox-api/build.gradle.kts @@ -43,6 +43,10 @@ fun GenerateTask.configureCommonOptions() { generatorName.set("kotlin") library.set("jvm-okhttp4") + templateDir.set( + project.projectDir.resolve("openapi-templates").absolutePath, + ) + typeMappings.set( mapOf( "object" to "kotlinx.serialization.json.JsonElement", diff --git a/sdks/sandbox/kotlin/sandbox-api/openapi-templates/api.mustache b/sdks/sandbox/kotlin/sandbox-api/openapi-templates/api.mustache new file mode 100644 index 000000000..2ad766f36 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox-api/openapi-templates/api.mustache @@ -0,0 +1,281 @@ +{{>licenseInfo}} +{{! Custom override of the OpenAPI generator "kotlin" client api.mustache (libraries/jvm-okhttp). + Keep in sync with the bundled template for the generator version pinned in gradle/libs.versions.toml. + Only change vs. upstream: ClientException message includes the response body so 4xx errors + are diagnosable, mirroring the ServerException branch. }} +package {{apiPackage}} + +import java.io.IOException +import okhttp3.Call +import okhttp3.HttpUrl + +{{#imports}}import {{import}} +{{/imports}} + +{{^multiplatform}} +{{#gson}} +import com.google.gson.annotations.SerializedName +{{/gson}} +{{#moshi}} +import com.squareup.moshi.Json +{{/moshi}} +{{#jackson}} +import com.fasterxml.jackson.annotation.JsonProperty +{{/jackson}} +{{#kotlinx_serialization}} +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +{{/kotlinx_serialization}} +{{/multiplatform}} +{{#multiplatform}} +import kotlinx.serialization.* +{{/multiplatform}} + +{{^doNotUseRxAndCoroutines}} +{{#useCoroutines}} +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +{{/useCoroutines}} +{{/doNotUseRxAndCoroutines}} +import {{packageName}}.infrastructure.ApiClient +import {{packageName}}.infrastructure.ApiResponse +import {{packageName}}.infrastructure.ClientException +import {{packageName}}.infrastructure.ClientError +import {{packageName}}.infrastructure.ServerException +import {{packageName}}.infrastructure.ServerError +import {{packageName}}.infrastructure.MultiValueMap +import {{packageName}}.infrastructure.PartConfig +import {{packageName}}.infrastructure.RequestConfig +import {{packageName}}.infrastructure.RequestMethod +import {{packageName}}.infrastructure.ResponseType +import {{packageName}}.infrastructure.Success +import {{packageName}}.infrastructure.toMultiValue + +{{#operations}} +{{#nonPublicApi}}internal {{/nonPublicApi}}{{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}class {{classname}}(basePath: kotlin.String = defaultBasePath, client: Call.Factory = ApiClient.defaultClient) : ApiClient(basePath, client) { + {{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}companion object { + @JvmStatic + {{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}val defaultBasePath: String by lazy { + System.getProperties().getProperty(ApiClient.baseUrlKey, "{{{basePath}}}") + } + } + + {{#operation}} + {{#allParams}} + {{#isEnum}} + /** + * enum for parameter {{paramName}} + */ + {{#nonPublicApi}}internal {{/nonPublicApi}}{{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}enum class {{enumName}}{{operationIdCamelCase}}({{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}val value: {{^isContainer}}{{dataType}}{{/isContainer}}{{#isContainer}}kotlin.String{{/isContainer}}) { + {{^enumUnknownDefaultCase}} + {{#allowableValues}} + {{#enumVars}} + {{^multiplatform}} + {{#moshi}} + @Json(name = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/moshi}} + {{#gson}} + @SerializedName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/gson}} + {{#jackson}} + @JsonProperty(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/jackson}} + {{#kotlinx_serialization}} + @SerialName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/kotlinx_serialization}} + {{/multiplatform}} + {{#multiplatform}} + @SerialName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/multiplatform}} + {{/enumVars}} + {{/allowableValues}} + {{/enumUnknownDefaultCase}} + {{#enumUnknownDefaultCase}} + {{#allowableValues}} + {{#enumVars}} + {{^multiplatform}} + {{#moshi}} + @Json(name = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/moshi}} + {{#gson}} + @SerializedName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/gson}} + {{#jackson}} + @JsonProperty(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/jackson}} + {{#kotlinx_serialization}} + @SerialName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/kotlinx_serialization}} + {{/multiplatform}} + {{#multiplatform}} + @SerialName(value = {{^isString}}"{{/isString}}{{{value}}}{{^isString}}"{{/isString}}) {{&name}}({{{value}}}){{^-last}},{{/-last}}{{#-last}};{{/-last}} + {{/multiplatform}} + {{/enumVars}} + {{/allowableValues}} + {{/enumUnknownDefaultCase}} + + /** + * Override [toString()] to avoid using the enum variable name as the value, and instead use + * the actual value defined in the API spec file. + * + * This solves a problem when the variable name and its value are different, and ensures that + * the client sends the correct enum values to the server always. + */ + override fun toString(): kotlin.String = "$value" + } + + {{/isEnum}} + {{/allParams}} + /** + * {{{httpMethod}}} {{#sanitizePathComment}}{{{path}}}{{/sanitizePathComment}} + * {{summary}} + * {{notes}} + {{#allParams}}* @param {{{paramName}}} {{description}}{{^required}} (optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}}{{#required}}{{#defaultValue}} (default to {{{.}}}){{/defaultValue}}{{/required}} + {{/allParams}}* @return {{#returnType}}{{{returnType}}}{{#nullableReturnType}} or null{{/nullableReturnType}}{{/returnType}}{{^returnType}}void{{/returnType}} + * @throws IllegalStateException If the request is not correctly configured + * @throws IOException Rethrows the OkHttp execute method exception + * @throws UnsupportedOperationException If the API returns an informational or redirection response + * @throws ClientException If the API returns a client error response + * @throws ServerException If the API returns a server error response + */{{#returnType}} + @Suppress("UNCHECKED_CAST"){{/returnType}} + @Throws(IllegalStateException::class, IOException::class, UnsupportedOperationException::class, ClientException::class, ServerException::class) + {{#isDeprecated}} + @Deprecated(message = "This operation is deprecated.") + {{/isDeprecated}} + {{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}{{^doNotUseRxAndCoroutines}}{{#useCoroutines}}suspend {{/useCoroutines}}{{/doNotUseRxAndCoroutines}}fun {{operationId}}({{#allParams}}{{{paramName}}}: {{#isEnum}}{{#isContainer}}kotlin.collections.List<{{enumName}}{{operationIdCamelCase}}>{{/isContainer}}{{^isContainer}}{{enumName}}{{operationIdCamelCase}}{{/isContainer}}{{/isEnum}}{{^isEnum}}{{{dataType}}}{{/isEnum}}{{#required}}{{#defaultValue}} = {{^isNumber}}{{#isEnum}}{{enumName}}{{operationIdCamelCase}}.{{enumDefaultValue}}{{/isEnum}}{{^isEnum}}{{{defaultValue}}}{{/isEnum}}{{/isNumber}}{{#isNumber}}{{{dataType}}}("{{{defaultValue}}}"){{/isNumber}}{{/defaultValue}}{{/required}}{{^required}}?{{#defaultValue}} = {{^isNumber}}{{#isEnum}}{{enumName}}{{operationIdCamelCase}}.{{enumDefaultValue}}{{/isEnum}}{{^isEnum}}{{{defaultValue}}}{{/isEnum}}{{/isNumber}}{{#isNumber}}{{{dataType}}}("{{{defaultValue}}}"){{/isNumber}}{{/defaultValue}}{{^defaultValue}} = null{{/defaultValue}}{{/required}}{{^-last}}, {{/-last}}{{/allParams}}) : {{#returnType}}{{{returnType}}}{{#nullableReturnType}}?{{/nullableReturnType}}{{/returnType}}{{^returnType}}Unit{{/returnType}}{{^doNotUseRxAndCoroutines}}{{#useCoroutines}} = withContext(Dispatchers.IO){{/useCoroutines}}{{/doNotUseRxAndCoroutines}} { + {{#isDeprecated}} + @Suppress("DEPRECATION") + {{/isDeprecated}} + val localVarResponse = {{operationId}}WithHttpInfo({{#allParams}}{{{paramName}}} = {{{paramName}}}{{^-last}}, {{/-last}}{{/allParams}}) + + return{{^doNotUseRxAndCoroutines}}{{#useCoroutines}}@withContext{{/useCoroutines}}{{/doNotUseRxAndCoroutines}} when (localVarResponse.responseType) { + ResponseType.Success -> {{#returnType}}(localVarResponse as Success<*>).data as {{{returnType}}}{{#nullableReturnType}}?{{/nullableReturnType}}{{/returnType}}{{^returnType}}Unit{{/returnType}} + ResponseType.Informational -> throw UnsupportedOperationException("Client does not support Informational responses.") + ResponseType.Redirection -> throw UnsupportedOperationException("Client does not support Redirection responses.") + ResponseType.ClientError -> { + val localVarError = localVarResponse as ClientError<*> + throw ClientException("Client error : ${localVarError.statusCode} ${localVarError.message.orEmpty()} ${localVarError.body}", localVarError.statusCode, localVarResponse) + } + ResponseType.ServerError -> { + val localVarError = localVarResponse as ServerError<*> + throw ServerException("Server error : ${localVarError.statusCode} ${localVarError.message.orEmpty()} ${localVarError.body}", localVarError.statusCode, localVarResponse) + } + } + } + + /** + * {{{httpMethod}}} {{#sanitizePathComment}}{{{path}}}{{/sanitizePathComment}} + * {{summary}} + * {{notes}} + {{#allParams}}* @param {{{paramName}}} {{description}}{{^required}} (optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}}{{#required}}{{#defaultValue}} (default to {{{.}}}){{/defaultValue}}{{/required}} + {{/allParams}}* @return ApiResponse<{{#returnType}}{{{returnType}}}?{{/returnType}}{{^returnType}}Unit?{{/returnType}}> + * @throws IllegalStateException If the request is not correctly configured + * @throws IOException Rethrows the OkHttp execute method exception + */{{#returnType}} + @Suppress("UNCHECKED_CAST"){{/returnType}} + @Throws(IllegalStateException::class, IOException::class) + {{#isDeprecated}} + @Deprecated(message = "This operation is deprecated.") + {{/isDeprecated}} + {{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}{{^doNotUseRxAndCoroutines}}{{#useCoroutines}}suspend {{/useCoroutines}}{{/doNotUseRxAndCoroutines}}fun {{operationId}}WithHttpInfo({{#allParams}}{{{paramName}}}: {{#isEnum}}{{#isContainer}}kotlin.collections.List<{{enumName}}{{operationIdCamelCase}}>{{/isContainer}}{{^isContainer}}{{enumName}}{{operationIdCamelCase}}{{/isContainer}}{{/isEnum}}{{^isEnum}}{{{dataType}}}{{/isEnum}}{{^required}}?{{/required}}{{^-last}}, {{/-last}}{{/allParams}}) : ApiResponse<{{#returnType}}{{{returnType}}}?{{/returnType}}{{^returnType}}Unit?{{/returnType}}>{{^doNotUseRxAndCoroutines}}{{#useCoroutines}} = withContext(Dispatchers.IO){{/useCoroutines}}{{/doNotUseRxAndCoroutines}} { + {{#isDeprecated}} + @Suppress("DEPRECATION") + {{/isDeprecated}} + val localVariableConfig = {{operationId}}RequestConfig({{#allParams}}{{{paramName}}} = {{{paramName}}}{{^-last}}, {{/-last}}{{/allParams}}) + + return{{^doNotUseRxAndCoroutines}}{{#useCoroutines}}@withContext{{/useCoroutines}}{{/doNotUseRxAndCoroutines}} request<{{#hasBodyParam}}{{#bodyParams}}{{{dataType}}}{{/bodyParams}}{{/hasBodyParam}}{{^hasBodyParam}}{{^hasFormParams}}Unit{{/hasFormParams}}{{#hasFormParams}}Map>{{/hasFormParams}}{{/hasBodyParam}}, {{{returnType}}}{{^returnType}}Unit{{/returnType}}>( + localVariableConfig + ) + } + + /** + * To obtain the request config of the operation {{operationId}} + * + {{#allParams}}* @param {{{paramName}}} {{description}}{{^required}} (optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}}{{#required}}{{#defaultValue}} (default to {{{.}}}){{/defaultValue}}{{/required}} + {{/allParams}}* @return RequestConfig + */ + {{#isDeprecated}} + @Deprecated(message = "This operation is deprecated.") + {{/isDeprecated}} + {{^nonPublicApi}}{{#explicitApi}}public {{/explicitApi}}{{/nonPublicApi}}fun {{operationId}}RequestConfig({{#allParams}}{{{paramName}}}: {{#isEnum}}{{#isContainer}}kotlin.collections.List<{{enumName}}{{operationIdCamelCase}}>{{/isContainer}}{{^isContainer}}{{enumName}}{{operationIdCamelCase}}{{/isContainer}}{{/isEnum}}{{^isEnum}}{{{dataType}}}{{/isEnum}}{{^required}}?{{/required}}{{^-last}}, {{/-last}}{{/allParams}}) : RequestConfig<{{#hasBodyParam}}{{#bodyParams}}{{{dataType}}}{{/bodyParams}}{{/hasBodyParam}}{{^hasBodyParam}}{{^hasFormParams}}Unit{{/hasFormParams}}{{#hasFormParams}}Map>{{/hasFormParams}}{{/hasBodyParam}}> { + val localVariableBody = {{#hasBodyParam}}{{! + }}{{#bodyParams}}{{{paramName}}}{{/bodyParams}}{{/hasBodyParam}}{{^hasBodyParam}}{{! + }}{{^hasFormParams}}null{{/hasFormParams}}{{! + }}{{#hasFormParams}}mapOf({{#formParams}} + "{{#lambda.escapeDollar}}{{{baseName}}}{{/lambda.escapeDollar}}" to PartConfig(body = {{{paramName}}}{{#isEnum}}{{^required}}?{{/required}}.value{{/isEnum}}, headers = mutableMapOf({{#contentType}}"Content-Type" to "{{contentType}}"{{/contentType}})),{{! + }}{{/formParams}}){{/hasFormParams}}{{! + }}{{/hasBodyParam}} + val localVariableQuery: MultiValueMap = {{^hasQueryParams}}mutableMapOf() +{{/hasQueryParams}}{{#hasQueryParams}}mutableMapOf>() + .apply { + {{#queryParams}} + {{^required}} + if ({{{paramName}}} != null) { + {{#isModel}} + {{#vars}} + if ({{{paramName}}}.{{name}} != null) { + put("{{#isDeepObject}}{{{paramName}}}[{{/isDeepObject}}{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}{{#isDeepObject}}]{{/isDeepObject}}", {{#isContainer}}toMultiValue({{{paramName}}}.{{name}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.{{name}}.value{{/isString}}{{^isString}}{{{paramName}}}.{{name}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.{{name}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + } + {{/vars}} + {{/isModel}} + {{^isModel}} + put("{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}", {{#isContainer}}toMultiValue({{{paramName}}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.value{{/isString}}{{^isString}}{{{paramName}}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + {{/isModel}} + } + {{/required}} + {{#required}} + {{#isNullable}} + if ({{{paramName}}} != null) { + {{#isModel}} + {{#vars}} + if ({{{paramName}}}.{{name}} != null) { + put("{{#isDeepObject}}{{{paramName}}}[{{/isDeepObject}}{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}{{#isDeepObject}}]{{/isDeepObject}}", {{#isContainer}}toMultiValue({{{paramName}}}.{{name}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.{{name}}.value{{/isString}}{{^isString}}{{{paramName}}}.{{name}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.{{name}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + } + {{/vars}} + {{/isModel}} + {{^isModel}} + put("{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}", {{#isContainer}}toMultiValue({{{paramName}}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.value{{/isString}}{{^isString}}{{{paramName}}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + {{/isModel}} + } + {{/isNullable}} + {{^isNullable}} + {{#isModel}} + {{#vars}} + if ({{{paramName}}}.{{name}} != null) { + put("{{#isDeepObject}}{{{paramName}}}[{{/isDeepObject}}{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}{{#isDeepObject}}]{{/isDeepObject}}", {{#isContainer}}toMultiValue({{{paramName}}}.{{name}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}.{{name}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.{{name}}.value{{/isString}}{{^isString}}{{{paramName}}}.{{name}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.{{name}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + } + {{/vars}} + {{/isModel}} + {{^isModel}} + put("{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}", {{#isContainer}}toMultiValue({{{paramName}}}.toList(), "{{collectionFormat}}"){{/isContainer}}{{^isContainer}}listOf({{#isDateTime}}parseDateToQueryString({{{paramName}}}){{/isDateTime}}{{#isDate}}parseDateToQueryString({{{paramName}}}){{/isDate}}{{#isEnum}}{{#isString}}{{{paramName}}}.value{{/isString}}{{^isString}}{{{paramName}}}.toString(){{/isString}}{{/isEnum}}{{^isEnum}}{{^isDateTime}}{{^isDate}}{{{paramName}}}.toString(){{/isDate}}{{/isDateTime}}{{/isEnum}}){{/isContainer}}) + {{/isModel}} + {{/isNullable}} + {{/required}} + {{/queryParams}} + } + {{/hasQueryParams}} + val localVariableHeaders: MutableMap = mutableMapOf({{#hasFormParams}}"Content-Type" to {{^consumes}}"multipart/form-data"{{/consumes}}{{#consumes.0}}"{{{mediaType}}}"{{/consumes.0}}{{/hasFormParams}}) + {{#headerParams}} + {{{paramName}}}{{^required}}?{{/required}}.apply { localVariableHeaders["{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}"] = {{#isContainer}}this.joinToString(separator = collectionDelimiter("{{collectionFormat}}")){{/isContainer}}{{^isContainer}}this.toString(){{/isContainer}} } + {{/headerParams}} + {{^hasFormParams}}{{#hasConsumes}}{{#consumes}}localVariableHeaders["Content-Type"] = "{{{mediaType}}}" + {{/consumes}}{{/hasConsumes}}{{/hasFormParams}}{{#hasProduces}}localVariableHeaders["Accept"] = "{{#produces}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/produces}}" +{{/hasProduces}} + + return RequestConfig( + method = RequestMethod.{{httpMethod}}, + path = "{{{path}}}"{{#pathParams}}.replace("{"+"{{#lambda.escapeDollar}}{{baseName}}{{/lambda.escapeDollar}}"+"}", encodeURIComponent({{#isContainer}}{{paramName}}.joinToString(","){{/isContainer}}{{^isContainer}}{{{paramName}}}{{#isEnum}}{{^required}}?{{/required}}.value{{/isEnum}}.toString(){{/isContainer}})){{/pathParams}}, + query = localVariableQuery, + headers = localVariableHeaders, + requiresAuthentication = {{#hasAuthMethods}}true{{/hasAuthMethods}}{{^hasAuthMethods}}false{{/hasAuthMethods}}, + body = localVariableBody + ) + } + + {{/operation}} + + private fun encodeURIComponent(uriComponent: kotlin.String): kotlin.String = + HttpUrl.Builder().scheme("http").host("localhost").addPathSegment(uriComponent).build().encodedPathSegments[0] +} +{{/operations}} diff --git a/sdks/sandbox/kotlin/sandbox-bom/build.gradle.kts b/sdks/sandbox/kotlin/sandbox-bom/build.gradle.kts index d00f1c41b..a8a4a8d26 100644 --- a/sdks/sandbox/kotlin/sandbox-bom/build.gradle.kts +++ b/sdks/sandbox/kotlin/sandbox-bom/build.gradle.kts @@ -29,5 +29,6 @@ dependencies { api(libs.okhttp) api(libs.okhttp.logging) api(libs.slf4j.api) + api(libs.opentelemetry.api) } } diff --git a/sdks/sandbox/kotlin/sandbox/build.gradle.kts b/sdks/sandbox/kotlin/sandbox/build.gradle.kts index 51b1c9241..1d32b6b76 100644 --- a/sdks/sandbox/kotlin/sandbox/build.gradle.kts +++ b/sdks/sandbox/kotlin/sandbox/build.gradle.kts @@ -21,10 +21,13 @@ dependencies { implementation(libs.okhttp) implementation(libs.okhttp.logging) + implementation(libs.opentelemetry.api) compileOnly(libs.bundles.serialization) testImplementation(libs.bundles.testing) testImplementation(libs.bundles.serialization) + testImplementation(libs.opentelemetry.sdk.testing) + testRuntimeOnly(libs.logback.classic) testRuntimeOnly(libs.junit.platform.launcher) } diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/HttpClientProvider.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/HttpClientProvider.kt index 8abbbbf3f..92e58bba2 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/HttpClientProvider.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/HttpClientProvider.kt @@ -19,7 +19,12 @@ package com.alibaba.opensandbox.sandbox import com.alibaba.opensandbox.sandbox.config.ConnectionConfig import com.alibaba.opensandbox.sandbox.domain.models.execd.SECURE_ACCESS_HEADER import com.alibaba.opensandbox.sandbox.transport.RetryInterceptor +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.context.Context +import io.opentelemetry.context.propagation.TextMapPropagator +import io.opentelemetry.context.propagation.TextMapSetter import okhttp3.ConnectionPool +import okhttp3.Headers import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.Response @@ -44,12 +49,21 @@ class HttpClientProvider( private val connectionPoolOwnedBySdk: Boolean = config.connectionPool == null private val baseBuilder: OkHttpClient.Builder - get() = - OkHttpClient.Builder() - .connectionPool(connectionPool) - .addInterceptor(UserAgentInterceptor(config.userAgent)) - .addInterceptor(ExtraHeadersInterceptor(config.headers)) - .addInterceptor(ClientIpInterceptor { ClientIpDetector.clientIp() }) + get() { + val builder = + OkHttpClient.Builder() + .connectionPool(connectionPool) + .addInterceptor(UserAgentInterceptor(config.userAgent)) + .addInterceptor(ExtraHeadersInterceptor(config.headers)) + .addInterceptor(ClientIpInterceptor { ClientIpDetector.clientIp() }) + if (config.enableTracing) { + // Propagate the active trace context (W3C traceparent) so the + // lifecycle server can join the same trace. No-op when there + // is no active span in the current context. + builder.addInterceptor(TraceContextInterceptor(GlobalOpenTelemetry.getPropagators().textMapPropagator)) + } + return builder + } // 1. Explicit lazy definition to allow checking initialization status private val httpClientLazy = @@ -202,6 +216,27 @@ class HttpClientProvider( } } + /** + * Injects the W3C `traceparent` / `tracestate` headers of the current + * OpenTelemetry context into every request. When no span is active the + * propagator injects nothing and the request passes through unchanged. + */ + private class TraceContextInterceptor( + private val propagators: TextMapPropagator, + ) : Interceptor { + private val setter = + TextMapSetter { carrier: Headers.Builder?, key, value -> + carrier?.set(key, value) + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val headers = request.headers.newBuilder() + propagators.inject(Context.current(), headers, setter) + return chain.proceed(request.newBuilder().headers(headers.build()).build()) + } + } + // --- Cleanup --- /** diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/Sandbox.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/Sandbox.kt index b72c9095e..93e571f14 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/Sandbox.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/Sandbox.kt @@ -715,15 +715,9 @@ class Sandbox internal constructor( } val context = "domain=${httpClientProvider.config.getDomain()}, useServerProxy=${httpClientProvider.config.useServerProxy}" - var suggestion = - "If this sandbox runs in Docker bridge or remote-network mode, consider enabling useServerProxy=true." - if (!httpClientProvider.config.useServerProxy) { - suggestion += " You can also configure server-side [docker].host_ip for direct endpoint access." - } - val finalMessage = "Sandbox health check timed out after ${timeout.seconds}s ($attempt attempts). $errorDetail " + - "Connection context: $context. $suggestion" + "Connection context: $context." logger.error(finalMessage, lastException) diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/config/ConnectionConfig.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/config/ConnectionConfig.kt index c58533cee..acae1d41e 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/config/ConnectionConfig.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/config/ConnectionConfig.kt @@ -59,6 +59,17 @@ class ConnectionConfig private constructor( * Also honored via the `OPENSANDBOX_DISABLE_METRICS=1` environment variable. */ val disableMetrics: Boolean = false, + /** + * Enable OpenTelemetry tracing for the client-side sandbox pool warmup path. + * + * Off by default. When enabled, each pool warmup creates an OpenTelemetry + * trace (`pool.warmup` root span plus per-phase spans) and the active + * trace context is propagated to lifecycle requests via the W3C + * `traceparent` header. Tracing is best-effort: without an + * OpenTelemetry SDK + exporter on the classpath, all span calls are + * no-ops and nothing is exported. + */ + val enableTracing: Boolean = false, /** * Retry policy applied to non-streaming requests. Enabled by default; pass * [RetryPolicy.disabled] to disable SDK-policy retries and fall back to @@ -88,6 +99,35 @@ class ConnectionConfig private constructor( endpointCacheSize = this.endpointCacheSize, endpointCacheDisabled = this.endpointCacheDisabled, disableMetrics = this.disableMetrics, + enableTracing = this.enableTracing, + retryPolicy = this.retryPolicy, + ) + + /** + * Creates a copy of this ConnectionConfig that uses [connectionPool] and + * marks it as SDK-managed (evicted when the owning component closes). + * + * Internal to the SDK: only [com.alibaba.opensandbox.sandbox.pool.SandboxPool] + * injects its pool-created shared pool this way and evicts it on shutdown. + * It is not public API โ€” callers cannot rely on the eviction promise + * because [HttpClientProvider] only evicts pools it created itself. + */ + internal fun copyWithConnectionPool(connectionPool: ConnectionPool): ConnectionConfig = + ConnectionConfig( + apiKey = this.apiKey, + domain = this.domain, + protocol = this.protocol, + requestTimeout = this.requestTimeout, + debug = this.debug, + userAgent = this.userAgent, + headers = this.headers, + connectionPool = connectionPool, + connectionPoolManagedByUser = false, + useServerProxy = this.useServerProxy, + endpointCacheTtl = this.endpointCacheTtl, + endpointCacheSize = this.endpointCacheSize, + endpointCacheDisabled = this.endpointCacheDisabled, + disableMetrics = this.disableMetrics, retryPolicy = this.retryPolicy, ) @@ -188,6 +228,7 @@ class ConnectionConfig private constructor( private var endpointCacheSize: Int = 1024 private var endpointCacheDisabled: Boolean = false private var disableMetrics: Boolean = false + private var enableTracing: Boolean = false private var retryPolicy: RetryPolicy = RetryPolicy() /** @@ -228,6 +269,17 @@ class ConnectionConfig private constructor( return this } + /** + * Enable OpenTelemetry tracing for the client-side sandbox pool warmup path. + * + * Off by default; pass `true` to opt in. Tracing is best-effort and no-ops + * unless an OpenTelemetry SDK + exporter is on the classpath. + */ + fun enableTracing(enable: Boolean = true): Builder { + this.enableTracing = enable + return this + } + /** * Set the API key used for authentication. * @@ -383,6 +435,7 @@ class ConnectionConfig private constructor( endpointCacheSize = endpointCacheSize, endpointCacheDisabled = endpointCacheDisabled, disableMetrics = disableMetrics, + enableTracing = enableTracing, retryPolicy = retryPolicy, ) } diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/execd/isolated/IsolatedModels.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/execd/isolated/IsolatedModels.kt index f6f2f318c..254445703 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/execd/isolated/IsolatedModels.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/execd/isolated/IsolatedModels.kt @@ -149,4 +149,23 @@ data class IsolatedCapabilities( val diffSupported: Boolean = false, val setprivAvailable: Boolean = false, val usernsAvailable: Boolean = false, + val hardening: HardeningStatus? = null, +) + +/** execd init-mode and workload-hardening state (OSEP-0018). */ +data class HardeningStatus( + // "pid1" | "subreaper" | "none" + val initMode: String? = null, + val signalShield: Boolean = false, + val capDrop: HardeningLayerState? = null, + val seccomp: HardeningLayerState? = null, + val landlock: HardeningLayerState? = null, + val ebpf: HardeningLayerState? = null, +) + +/** Whether one hardening layer is actually enforced. */ +data class HardeningLayerState( + // active | disabled | degraded | unsupported + val state: String? = null, + val message: String? = null, ) diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/pool/PoolSnapshot.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/pool/PoolSnapshot.kt index bf648eed4..526caa19b 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/pool/PoolSnapshot.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/pool/PoolSnapshot.kt @@ -24,7 +24,7 @@ package com.alibaba.opensandbox.sandbox.domain.pool * @property idleCount Number of idle sandboxes in the store. * @property maxIdle Current max idle target visible to this pool. * @property failureCount Number of consecutive reconcile failures currently tracked. - * @property backoffActive Whether reconcile create attempts are currently suppressed by backoff. + * @property backoffActive Whether reconcile create attempts are currently suppressed by degraded backoff or an active warmup rate-limit throttle. * @property lastError Last error message if pool is DEGRADED or after failure; null otherwise. * @property inFlightOperations Number of pool operations currently in flight on this node. */ diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapter.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapter.kt index 71014a096..27f25d496 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapter.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapter.kt @@ -145,7 +145,7 @@ internal class CommandsAdapter( ResponseType.ClientError -> { val localVarError = localVarResponse as ClientError<*> throw ClientException( - "Client error : ${localVarError.statusCode} ${localVarError.message.orEmpty()}", + "Client error : ${localVarError.statusCode} ${localVarError.message.orEmpty()} ${localVarError.body}", localVarError.statusCode, localVarResponse, ) diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapter.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapter.kt index 20c6ed2a6..4df37a04b 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapter.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapter.kt @@ -22,6 +22,8 @@ import com.alibaba.opensandbox.sandbox.domain.models.execd.executions.Execution import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.BindMount import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.CreateIsolatedSessionRequest import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.EnvPassthroughSpec +import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.HardeningLayerState +import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.HardeningStatus import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.IsolatedBackgroundRun import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.IsolatedCapabilities import com.alibaba.opensandbox.sandbox.domain.models.execd.isolated.IsolatedRunLogs @@ -154,8 +156,27 @@ private data class IsolatedCapabilitiesResponse( val userns_available: Boolean = false, val commit_supported: Boolean = false, val diff_supported: Boolean = false, + val hardening: HardeningStatusResponse? = null, ) +@Serializable +private data class HardeningStatusResponse( + val init_mode: String? = null, + val signal_shield: Boolean = false, + val cap_drop: HardeningLayerStateResponse? = null, + val seccomp: HardeningLayerStateResponse? = null, + val landlock: HardeningLayerStateResponse? = null, + val ebpf: HardeningLayerStateResponse? = null, +) + +@Serializable +private data class HardeningLayerStateResponse( + val state: String? = null, + val message: String? = null, +) + +private fun HardeningLayerStateResponse.toDomain(): HardeningLayerState = HardeningLayerState(state = state, message = message) + private val json = Json { ignoreUnknownKeys = true } private const val TAIL_CURSOR_HEADER = "EXECD-ISOLATED-TAIL-CURSOR" @@ -555,6 +576,17 @@ internal class IsolatedSessionsAdapter( usernsAvailable = resp.userns_available, commitSupported = resp.commit_supported, diffSupported = resp.diff_supported, + hardening = + resp.hardening?.let { + HardeningStatus( + initMode = it.init_mode, + signalShield = it.signal_shield, + capDrop = it.cap_drop?.toDomain(), + seccomp = it.seccomp?.toDomain(), + landlock = it.landlock?.toDomain(), + ebpf = it.ebpf?.toDomain(), + ) + }, ) } } catch (e: Exception) { diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitState.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitState.kt new file mode 100644 index 000000000..c9eaec2e4 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitState.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.infrastructure.pool + +import com.alibaba.opensandbox.sandbox.transport.RETRY_AFTER_CAP +import java.time.Duration +import java.time.Instant + +/** Per-run warmup throttle established by rate-limited sandbox creates. */ +internal class PoolRateLimitState( + private val defaultDelay: Duration = DEFAULT_RATE_LIMIT_DELAY, + private val maxDelay: Duration = RETRY_AFTER_CAP, +) { + init { + require(!defaultDelay.isNegative) { "defaultDelay must not be negative" } + require(!maxDelay.isNegative) { "maxDelay must not be negative" } + } + + @Volatile + private var throttleUntil: Instant? = null + + /** Extends, but never shortens, the current throttle deadline. */ + @Synchronized + fun recordRateLimit( + retryAfter: Duration?, + now: Instant = Instant.now(), + ) { + val requestedDelay = retryAfter?.takeUnless { it.isNegative || it.isZero } ?: defaultDelay + val candidate = now.plus(minOf(requestedDelay, maxDelay)) + val current = throttleUntil + if (current == null || candidate.isAfter(current)) { + throttleUntil = candidate + } + } + + fun isActive(now: Instant = Instant.now()): Boolean { + val until = throttleUntil ?: return false + return now.isBefore(until) + } + + fun remainingDelay(now: Instant = Instant.now()): Duration { + val until = throttleUntil ?: return Duration.ZERO + val remaining = Duration.between(now, until) + return if (remaining.isNegative || remaining.isZero) Duration.ZERO else remaining + } + + companion object { + internal val DEFAULT_RATE_LIMIT_DELAY: Duration = Duration.ofSeconds(10) + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolReconciler.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolReconciler.kt index f8bf0aa0c..e2975d4ac 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolReconciler.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolReconciler.kt @@ -44,6 +44,7 @@ internal object PoolReconciler { onDiscardSandbox: (String) -> Unit = {}, reconcileState: ReconcileState, warmingCount: Int, + rateLimitState: PoolRateLimitState? = null, submitWarmups: (Int) -> Unit, ): Boolean { val poolName = config.poolName @@ -60,6 +61,7 @@ internal object PoolReconciler { onDiscardSandbox = onDiscardSandbox, reconcileState = reconcileState, warmingCount = warmingCount, + rateLimitState = rateLimitState, submitWarmups = submitWarmups, ) // Do not release primary lock here; leader holds until renew fails or TTL expires. @@ -72,6 +74,7 @@ internal object PoolReconciler { onDiscardSandbox: (String) -> Unit, reconcileState: ReconcileState, warmingCount: Int, + rateLimitState: PoolRateLimitState?, submitWarmups: (Int) -> Unit, ) { val poolName = config.poolName @@ -101,17 +104,20 @@ internal object PoolReconciler { warmupConcurrency = config.warmupConcurrency, ) - if (plan.toSubmit == 0 || reconcileState.isBackoffActive(now)) { + val degradedBackoffActive = reconcileState.isBackoffActive(now) + val rateLimitActive = rateLimitState?.isActive(now) == true + if (plan.toSubmit == 0 || degradedBackoffActive || rateLimitActive) { stateStore.renewPrimaryLock(poolName, ownerId, ttl) logger.debug( "Reconcile tick: pool_name={} idle={} warming={} deficit={} available_slots={} " + - "to_submit=0 backoff={}", + "to_submit=0 backoff={} rate_limited={}", poolName, counters.idleCount, warmingCount, plan.deficit, plan.availableSlots, - reconcileState.isBackoffActive(now), + degradedBackoffActive, + rateLimitActive, ) return } diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/internal/PoolTracer.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/internal/PoolTracer.kt new file mode 100644 index 000000000..23d66f05a --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/internal/PoolTracer.kt @@ -0,0 +1,236 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.internal + +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.Tracer +import org.slf4j.MDC +import java.util.concurrent.TimeUnit + +/** + * Best-effort OpenTelemetry tracing for the client-side pool warmup path. + * + * Tracing is opt-in via [ConnectionConfig.enableTracing]. When enabled, each + * warmup task produces one trace rooted at a [WARMUP_ROOT_SPAN] span with + * per-phase child spans (create / prepare / renew / commit). The root span + * starts at task submission time so queue-waiting time is visible as the gap + * before the first child span. + * + * While a warmup trace is current, `trace_id` and `span_id` are published to + * the SLF4J [MDC] so application logs emitted by the pool (which already + * carry `pool_name` / `sandbox_id`) can be correlated back to a trace โ€” + * search logs by sandbox_id to obtain the trace_id, then open it in the + * trace backend. + * + * All span/MDC calls are best-effort and MUST NOT surface any exception to + * the caller: without an OpenTelemetry SDK on the classpath every call is a + * no-op, and MDC access can fail under unusual logging setups. + */ +internal class PoolTracer private constructor( + private val tracer: Tracer?, +) { + val enabled: Boolean + get() = tracer != null + + /** + * Starts the root span of one warmup trace, backdated to + * [submittedEpochNanos] (epoch wall-clock, see + * [WarmupTrace.endSuccess]) so the queue-wait time is part of the trace. + * Returns null when tracing is disabled; the caller then runs without + * spans. + */ + fun startWarmupRoot( + poolName: String, + ownerId: String, + runGeneration: Long, + submittedEpochNanos: Long, + ): WarmupTrace? { + val t = tracer ?: return null + val root = + t.spanBuilder(WARMUP_ROOT_SPAN) + .setAttribute(ATTR_POOL_NAME, poolName) + .setAttribute(ATTR_POOL_OWNER, ownerId) + .setAttribute(ATTR_POOL_RUN_GENERATION, runGeneration) + .setStartTimestamp(submittedEpochNanos, TimeUnit.NANOSECONDS) + .startSpan() + return WarmupTrace(root) + } + + /** + * Runs [block] under a child span of the currently-current span (the + * warmup root). No-op span when tracing is disabled. + */ + internal inline fun withPhaseSpan( + spanName: String, + crossinline block: () -> T, + ): T { + val t = tracer ?: return block() + val span = t.spanBuilder(spanName).startSpan() + val scope = span.makeCurrent() + return try { + block() + } finally { + safeClose(scope) + span.end() + } + } + + companion object { + const val WARMUP_ROOT_SPAN = "pool.warmup" + const val WARMUP_CREATE_SPAN = "pool.warmup.create" + const val WARMUP_PREPARE_SPAN = "pool.warmup.prepare" + const val WARMUP_RENEW_SPAN = "pool.warmup.renew" + const val WARMUP_COMMIT_SPAN = "pool.warmup.commit" + + const val MDC_TRACE_ID = "trace_id" + const val MDC_SPAN_ID = "span_id" + + const val ATTR_POOL_NAME = "pool.name" + const val ATTR_POOL_OWNER = "pool.owner" + const val ATTR_POOL_RUN_GENERATION = "pool.run.generation" + const val ATTR_SANDBOX_ID = "sandbox.id" + const val ATTR_SANDBOX_IMAGE = "sandbox.image" + const val ATTR_RESULT = "result" + const val ATTR_DROP_REASON = "drop.reason" + + private const val INSTRUMENTATION_NAME = "com.alibaba.opensandbox.sandbox" + + fun from(connectionConfig: ConnectionConfig): PoolTracer { + if (!connectionConfig.enableTracing) return PoolTracer(null) + return PoolTracer( + GlobalOpenTelemetry.get().tracerBuilder(INSTRUMENTATION_NAME).build(), + ) + } + } +} + +/** + * One in-flight warmup trace: the root [Span] plus the ability to run work in + * its context (with `trace_id` / `span_id` published to MDC) and to end the + * trace with outcome attributes. + */ +internal class WarmupTrace internal constructor( + private val root: Span, +) { + val traceId: String + get() = root.spanContext.traceId + + val spanId: String + get() = root.spanContext.spanId + + /** + * Runs [block] with this trace's root span current (child spans + * auto-parent to it) and `trace_id`/`span_id` in the SLF4J MDC for the + * duration of [block]. The previous thread-local MDC values are restored + * afterwards. Never throws. + */ + fun withCurrent(block: () -> T): T { + val prevTrace = safeMdcGet(PoolTracer.MDC_TRACE_ID) + val prevSpan = safeMdcGet(PoolTracer.MDC_SPAN_ID) + safeMdcPut(PoolTracer.MDC_TRACE_ID, traceId) + safeMdcPut(PoolTracer.MDC_SPAN_ID, spanId) + val scope = root.makeCurrent() + return try { + block() + } finally { + safeClose(scope) + safeMdcRestore(PoolTracer.MDC_TRACE_ID, prevTrace) + safeMdcRestore(PoolTracer.MDC_SPAN_ID, prevSpan) + } + } + + /** Ends the trace as successful, recording sandbox identity for drill-down. */ + fun endSuccess( + sandboxId: String, + image: String?, + ) { + root.setAttribute(PoolTracer.ATTR_SANDBOX_ID, sandboxId) + if (!image.isNullOrBlank()) { + root.setAttribute(PoolTracer.ATTR_SANDBOX_IMAGE, image) + } + root.setAttribute(PoolTracer.ATTR_RESULT, RESULT_SUCCESS) + root.end() + } + + /** Ends the trace as failed, recording the failure. */ + fun endFailure(error: Throwable) { + root.recordException(error) + root.setAttribute(PoolTracer.ATTR_RESULT, RESULT_FAILURE) + root.end() + } + + /** + * Ends the trace as failed because the warmup outcome could not be + * committed (stale run, primary lock lost, or putIdle failure) โ€” the + * sandbox never entered the idle pool and is scheduled for cleanup. + * [source] matches the pool's cleanup-source values, e.g. + * `warmup-lock-lost`. + */ + fun endDropped(source: String) { + root.setAttribute(PoolTracer.ATTR_RESULT, RESULT_FAILURE) + root.setAttribute(PoolTracer.ATTR_DROP_REASON, source) + root.end() + } + + private companion object { + const val RESULT_SUCCESS = "success" + const val RESULT_FAILURE = "failure" + } +} + +private fun safeMdcGet(key: String): String? = + try { + MDC.get(key) + } catch (_: Throwable) { + null + } + +private fun safeMdcPut( + key: String, + value: String, +) { + try { + MDC.put(key, value) + } catch (_: Throwable) { + // best-effort + } +} + +private fun safeMdcRestore( + key: String, + previous: String?, +) { + try { + if (previous == null) { + MDC.remove(key) + } else { + MDC.put(key, previous) + } + } catch (_: Throwable) { + // best-effort + } +} + +private fun safeClose(scope: io.opentelemetry.context.Scope) { + try { + scope.close() + } catch (_: Throwable) { + // best-effort + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt index a2d581dd6..c896b3dc2 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt @@ -24,6 +24,7 @@ import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolDestroyedException import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolEmptyException import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolNotRunningException import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolStateStoreUnavailableException +import com.alibaba.opensandbox.sandbox.domain.exceptions.SandboxRateLimitException import com.alibaba.opensandbox.sandbox.domain.pool.AcquirePolicy import com.alibaba.opensandbox.sandbox.domain.pool.IdleEntry import com.alibaba.opensandbox.sandbox.domain.pool.PoolConfig @@ -36,9 +37,13 @@ import com.alibaba.opensandbox.sandbox.domain.pool.PoolStateStore import com.alibaba.opensandbox.sandbox.domain.pool.PooledSandboxCreateContext import com.alibaba.opensandbox.sandbox.domain.pool.PooledSandboxCreator import com.alibaba.opensandbox.sandbox.domain.pool.SandboxPreparer +import com.alibaba.opensandbox.sandbox.infrastructure.pool.PoolRateLimitState import com.alibaba.opensandbox.sandbox.infrastructure.pool.PoolReconciler import com.alibaba.opensandbox.sandbox.infrastructure.pool.ReconcileState +import com.alibaba.opensandbox.sandbox.internal.PoolTracer +import com.alibaba.opensandbox.sandbox.internal.WarmupTrace import com.alibaba.opensandbox.sandbox.internal.isCausedByInterruption +import okhttp3.ConnectionPool import org.slf4j.LoggerFactory import java.time.Duration import java.util.concurrent.ConcurrentHashMap @@ -92,7 +97,7 @@ import kotlin.concurrent.withLock class SandboxPool internal constructor( config: PoolConfig, private val sandboxManagerFactory: (ConnectionConfig) -> SandboxManager, - private val idleSandboxConnector: (String) -> Sandbox, + idleSandboxConnector: ((String) -> Sandbox)?, ) { internal constructor( config: PoolConfig, @@ -102,7 +107,7 @@ class SandboxPool internal constructor( ) : this( config = config, sandboxManagerFactory = sandboxManagerFactory, - idleSandboxConnector = defaultIdleSandboxConnector(config), + idleSandboxConnector = null, ) private val logger = LoggerFactory.getLogger(SandboxPool::class.java) @@ -113,6 +118,48 @@ class SandboxPool internal constructor( private val creationSpec: PoolCreationSpec = config.creationSpec private val sandboxCreator: PooledSandboxCreator? = config.sandboxCreator private val reconcileState = ReconcileState(config.degradedThreshold) + private val poolTracer = PoolTracer.from(config.connectionConfig) + + /** + * A pool-wide shared OkHttp connection pool, created by the pool when the + * user's [ConnectionConfig] does not carry one. Sized from + * [PoolConfig.warmupConcurrency] so concurrent warmup creates reuse + * connections instead of each opening fresh TCP connections โ€” at high + * concurrency the per-sandbox connection churn otherwise causes + * connection resets and retry amplification. Pool-managed: evicted when + * the pool closes. Null when the user supplied their own pool. + */ + private val sharedConnectionPool: ConnectionPool? = + if (config.connectionConfig.connectionPool == null) { + ConnectionPool( + maxIdleConnections = maxOf(config.warmupConcurrency, 1), + keepAliveDuration = DEFAULT_SHARED_POOL_KEEPALIVE_MINUTES, + timeUnit = TimeUnit.MINUTES, + ) + } else { + null + } + + /** + * The [ConnectionConfig] used for every sandbox the pool creates + * (warmup, direct create, idle connect). When [sharedConnectionPool] was + * created it is injected here so all sandbox HTTP clients reuse it. The + * pool's internal manager client deliberately keeps + * [ConnectionConfig.copyWithoutConnectionPool] semantics and is not part + * of this sharing. + */ + private val poolConnectionConfig: ConnectionConfig = + sharedConnectionPool?.let { connectionConfig.copyWithConnectionPool(it) } ?: connectionConfig + + /** + * The default idle-sandbox connector, resolved after [poolConnectionConfig] + * so acquired sandboxes share the pool's connection pool. + */ + private val idleSandboxConnector: (String) -> Sandbox = + idleSandboxConnector ?: defaultIdleSandboxConnector(config, poolConnectionConfig) + + /** Exposed for tests: the pool-created shared connection pool, or null when user-provided. */ + internal fun sharedConnectionPoolForTests(): ConnectionPool? = sharedConnectionPool @Volatile private var currentMaxIdle: Int = config.maxIdle @@ -484,6 +531,100 @@ class SandboxPool internal constructor( return count } + /** + * Takes all idle sandbox IDs from the store and terminates them with bounded concurrency. + * This method blocks until every ID taken from the store has received a best-effort kill attempt. + * + * @param concurrency Maximum number of concurrent kill requests. Must be positive. + * @return Number of idle sandboxes taken from the store. + */ + fun releaseAllIdle(concurrency: Int): Int { + require(concurrency > 0) { "concurrency must be positive" } + val poolName = config.poolName + val sandboxIds = mutableListOf() + var drainFailure: Exception? = null + var temporaryManager: SandboxManager? = null + try { + while (true) { + val sandboxId = + try { + stateStore.tryTakeIdle(poolName) + } catch (e: Exception) { + drainFailure = e + break + } ?: break + sandboxIds.add(sandboxId) + } + + if (sandboxIds.isNotEmpty()) { + val manager = + sandboxManager ?: try { + createSandboxManager().also { temporaryManager = it } + } catch (e: Exception) { + logger.warn( + "releaseAllIdle(concurrency): failed to create sandbox manager; " + + "draining idle ids without remote kill: " + + "pool_name={} error={}", + poolName, + e.message, + ) + null + } + if (manager != null) { + val threadIndex = AtomicInteger() + val executor = + Executors.newFixedThreadPool(minOf(concurrency, sandboxIds.size)) { runnable -> + Thread( + runnable, + "sandbox-pool-release-$poolName-${threadIndex.incrementAndGet()}", + ).apply { isDaemon = true } + } + try { + sandboxIds.forEach { sandboxId -> + executor.submit { + try { + manager.killSandbox(sandboxId) + } catch (e: Exception) { + logger.warn( + "releaseAllIdle(concurrency): failed to kill sandbox (best-effort): " + + "pool_name={} sandbox_id={} error={}", + poolName, + sandboxId, + e.message, + ) + } + } + } + } finally { + executor.shutdown() + var interrupted = false + while (!executor.isTerminated) { + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) + } catch (_: InterruptedException) { + interrupted = true + } + } + if (interrupted) { + Thread.currentThread().interrupt() + } + } + } + } + } finally { + temporaryManager?.close() + } + drainFailure?.let { throw it } + if (sandboxIds.isNotEmpty()) { + logger.info( + "releaseAllIdle(concurrency): released {} idle sandbox(es): pool_name={}", + sandboxIds.size, + poolName, + ) + } + return sandboxIds.size + } + /** * Returns a point-in-time snapshot of pool state for observability. */ @@ -504,7 +645,7 @@ class SandboxPool internal constructor( idleCount = counters.idleCount, maxIdle = resolveMaxIdle(), failureCount = reconcileState.failureCount, - backoffActive = reconcileState.isBackoffActive(), + backoffActive = reconcileState.isBackoffActive() || currentRun?.rateLimitState?.isActive() == true, lastError = reconcileState.lastError, inFlightOperations = currentRun?.inFlightOperations?.get() ?: 0, ) @@ -717,7 +858,10 @@ class SandboxPool internal constructor( private fun complete() { if (completed.compareAndSet(false, true)) { - run?.let { endOperation(it) } + run?.let { + endOperation(it) + requestReconcile(it) + } } } } @@ -820,6 +964,7 @@ class SandboxPool internal constructor( onDiscardSandbox = { sandboxId -> killSandboxBestEffort(sandboxId) }, reconcileState = reconcileState, warmingCount = run.warmingCount.get(), + rateLimitState = run.rateLimitState, submitWarmups = { count -> submitWarmups(run, count) }, ), ) @@ -862,8 +1007,20 @@ class SandboxPool internal constructor( } val exec = run.scheduler if (!run.reconcileQueued.compareAndSet(false, true)) return - try { - exec.execute { + // Completion-driven ticks are a latency optimization, not the correctness loop (the + // periodic tick is). Coalesce them into a minimum interval so bursts of fast + // completions cannot drive reconcile ticks โ€” each of which costs several state-store + // round-trips โ€” at unbounded frequency. The floor advances from the later of the last + // request and the previous floor so it applies between tick executions, not merely + // between requests: a completion that lands right after a scheduled tick must still + // wait out the full window. + val nowNanos = System.nanoTime() + val nextAllowedNanos = run.nextCompletionReconcileAtNanos + run.nextCompletionReconcileAtNanos = + maxOf(nextAllowedNanos, nowNanos) + TimeUnit.MILLISECONDS.toNanos(COMPLETION_RECONCILE_MIN_INTERVAL_MS) + val delayMs = TimeUnit.NANOSECONDS.toMillis(nextAllowedNanos - nowNanos).coerceAtLeast(0L) + val tick = + Runnable { run.reconcileQueued.set(false) try { runReconcileTick(run) @@ -871,6 +1028,12 @@ class SandboxPool internal constructor( logger.error("Pool completion-driven reconcile failed: pool_name={}", config.poolName, t) } } + try { + if (delayMs == 0L) { + exec.execute(tick) + } else { + exec.schedule(tick, delayMs, TimeUnit.MILLISECONDS) + } } catch (e: Exception) { run.reconcileQueued.set(false) if (lifecycleState.get() == LifecycleState.RUNNING) { @@ -883,6 +1046,57 @@ class SandboxPool internal constructor( } } + private fun scheduleRateLimitReconcile(run: RunContext) { + if (!isCurrentRun(run) || lifecycleState.get() != LifecycleState.RUNNING) return + synchronized(run.rateLimitScheduleLock) { + if (!isCurrentRun(run) || lifecycleState.get() != LifecycleState.RUNNING) return + run.rateLimitReconcileTask?.cancel(false) + val sequence = ++run.rateLimitReconcileSequence + val delayNanos = run.rateLimitState.remainingDelay().toNanos() + try { + run.rateLimitReconcileTask = + run.scheduler.schedule( + { onRateLimitReconcileDue(run, sequence) }, + delayNanos, + TimeUnit.NANOSECONDS, + ) + } catch (e: Exception) { + run.rateLimitReconcileTask = null + if (lifecycleState.get() == LifecycleState.RUNNING) { + logger.debug( + "Pool rate-limit reconcile submit rejected: pool_name={} error={}", + config.poolName, + e.message, + ) + } + } + } + } + + private fun onRateLimitReconcileDue( + run: RunContext, + sequence: Long, + ) { + synchronized(run.rateLimitScheduleLock) { + if (sequence != run.rateLimitReconcileSequence) return + run.rateLimitReconcileTask = null + } + if (!isCurrentRun(run) || lifecycleState.get() != LifecycleState.RUNNING) return + if (run.rateLimitState.isActive()) { + scheduleRateLimitReconcile(run) + } else { + requestReconcile(run) + } + } + + private fun cancelRateLimitReconcile(run: RunContext) { + synchronized(run.rateLimitScheduleLock) { + run.rateLimitReconcileSequence++ + run.rateLimitReconcileTask?.cancel(false) + run.rateLimitReconcileTask = null + } + } + private fun submitWarmups( run: RunContext, count: Int, @@ -894,7 +1108,7 @@ class SandboxPool internal constructor( ) { return } - val task = TrackedWarmupTask(run) + val task = TrackedWarmupTask(run, submittedEpochNanos()) try { run.warmupExecutor.execute(task) } catch (e: Exception) { @@ -909,8 +1123,16 @@ class SandboxPool internal constructor( } } + /** + * Epoch-based submission timestamp (OpenTelemetry start timestamps are + * wall-clock, not monotonic). Used to backdate the warmup root span so the + * queue-wait window is part of the trace. + */ + private fun submittedEpochNanos(): Long = System.currentTimeMillis() * 1_000_000L + private inner class TrackedWarmupTask( private val run: RunContext, + private val submittedEpochNanos: Long, ) : Runnable { private val completed = AtomicBoolean(false) @@ -920,21 +1142,45 @@ class SandboxPool internal constructor( } override fun run() { - val outcome = - try { - WarmupOutcome.Success(createOneSandbox()) - } catch (failure: Throwable) { - WarmupOutcome.Failure(failure) + // Backdate the root span to task submission so queue-wait time + // (submit -> run) is visible inside the trace. + val trace = + poolTracer.startWarmupRoot( + poolName = config.poolName, + ownerId = config.ownerId, + runGeneration = run.generation, + submittedEpochNanos = submittedEpochNanos, + ) + val outcome: WarmupOutcome + if (trace == null) { + outcome = captureOutcome() + } else { + outcome = trace.withCurrent { captureOutcome() } + if (outcome is WarmupOutcome.Failure) { + trace.endFailure(outcome.error) } - dispatchCompletion(outcome) + } + // Keep the trace open on success: the commit phase (scheduler + // thread) ends it after the sandbox is put idle. + dispatchCompletion(outcome, if (outcome is WarmupOutcome.Success) trace else null) } + private fun captureOutcome(): WarmupOutcome = + try { + WarmupOutcome.Success(createOneSandbox()) + } catch (failure: Throwable) { + WarmupOutcome.Failure(failure) + } + fun completeIfDropped() { - complete(WarmupOutcome.Cancelled) + complete(WarmupOutcome.Cancelled, null) } - private fun dispatchCompletion(outcome: WarmupOutcome) { - val completion = TrackedWarmupCompletionTask(run, this, outcome) + private fun dispatchCompletion( + outcome: WarmupOutcome, + trace: WarmupTrace?, + ) { + val completion = TrackedWarmupCompletionTask(run, this, outcome, trace) run.pendingWarmupCompletions.add(completion) try { run.scheduler.execute(completion) @@ -948,14 +1194,24 @@ class SandboxPool internal constructor( } } - fun complete(outcome: WarmupOutcome) { + fun complete( + outcome: WarmupOutcome, + trace: WarmupTrace?, + ) { if (!completed.compareAndSet(false, true)) return try { - handleWarmupOutcome(run, outcome) + handleWarmupOutcome(run, outcome, trace) } finally { run.warmingCount.decrementAndGet() endOperation(run) - requestReconcile(run) + // Only successful completions trigger an immediate reconcile. A failed warmup + // frees its slot but must not cause an immediate retry: fast-failing creates + // would otherwise form a self-sustaining reconcile/create loop that amplifies + // state-store load far beyond the periodic tick. Retries are driven by the + // periodic tick, which the backoff window already paces. + if (outcome is WarmupOutcome.Success) { + requestReconcile(run) + } } } } @@ -972,10 +1228,11 @@ class SandboxPool internal constructor( private val run: RunContext, private val warmupTask: TrackedWarmupTask, private val outcome: WarmupOutcome, + private val trace: WarmupTrace?, ) : Runnable { override fun run() { try { - warmupTask.complete(outcome) + warmupTask.complete(outcome, trace) } finally { run.pendingWarmupCompletions.remove(this) } @@ -1005,12 +1262,25 @@ class SandboxPool internal constructor( private fun handleWarmupOutcome( run: RunContext, outcome: WarmupOutcome, + trace: WarmupTrace?, ) { when (outcome) { - is WarmupOutcome.Success -> commitWarmupSandbox(run, outcome.sandboxId) + is WarmupOutcome.Success -> commitWarmupSandbox(run, outcome.sandboxId, trace) is WarmupOutcome.Failure -> { if (isCurrentRun(run) && lifecycleState.get() == LifecycleState.RUNNING) { - reconcileState.recordAsyncFailure(outcome.error.message) + val error = outcome.error + if (error is SandboxRateLimitException) { + run.rateLimitState.recordRateLimit(error.retryAfter) + scheduleRateLimitReconcile(run) + logger.debug( + "Pool warmup rate limited: pool_name={} retry_after_ms={} throttle_remaining_ms={}", + config.poolName, + error.retryAfter?.toMillis(), + run.rateLimitState.remainingDelay().toMillis(), + ) + } else { + reconcileState.recordAsyncFailure(error.message) + } } } WarmupOutcome.Cancelled -> Unit @@ -1020,57 +1290,77 @@ class SandboxPool internal constructor( private fun commitWarmupSandbox( run: RunContext, sandboxId: String, + trace: WarmupTrace?, ) { var cleanupSource: String? = null - run.commitLock.lock() - try { - val state = lifecycleState.get() - if (!isCurrentRun(run) || (state != LifecycleState.RUNNING && state != LifecycleState.DRAINING)) { - cleanupSource = "warmup-stale-run" - } else { - try { - ensurePoolNamespaceActive() - if (!stateStore.renewPrimaryLock(config.poolName, config.ownerId, config.primaryLockTtl)) { - run.primaryOwned.set(false) + val commit: () -> Unit = { + run.commitLock.lock() + try { + val state = lifecycleState.get() + if (!isCurrentRun(run) || (state != LifecycleState.RUNNING && state != LifecycleState.DRAINING)) { + cleanupSource = "warmup-stale-run" + } else { + try { + ensurePoolNamespaceActive() + if (!stateStore.renewPrimaryLock(config.poolName, config.ownerId, config.primaryLockTtl)) { + run.primaryOwned.set(false) + logger.warn( + "Pool lost primary lock before putIdle; dropping warmup sandbox: " + + "pool_name={} sandbox_id={} run={}", + config.poolName, + sandboxId, + run.generation, + ) + cleanupSource = "warmup-lock-lost" + } else { + stateStore.putIdle(config.poolName, sandboxId) + reconcileState.recordSuccess() + logger.debug( + "Pool warmup sandbox entered idle: pool_name={} sandbox_id={} run={}", + config.poolName, + sandboxId, + run.generation, + ) + } + } catch (e: Exception) { + if (isCurrentRun(run) && lifecycleState.get() == LifecycleState.RUNNING) { + reconcileState.recordAsyncFailure(e.message) + } + try { + stateStore.removeIdle(config.poolName, sandboxId) + } catch (_: Exception) { + // best-effort remove before remote cleanup + } + cleanupSource = "warmup-commit-failed" logger.warn( - "Pool lost primary lock before putIdle; dropping warmup sandbox: " + - "pool_name={} sandbox_id={} run={}", - config.poolName, - sandboxId, - run.generation, - ) - cleanupSource = "warmup-lock-lost" - } else { - stateStore.putIdle(config.poolName, sandboxId) - reconcileState.recordSuccess() - logger.debug( - "Pool warmup sandbox entered idle: pool_name={} sandbox_id={} run={}", + "Pool warmup commit failed; dropped sandbox: pool_name={} sandbox_id={} run={} error={}", config.poolName, sandboxId, run.generation, + e.message, ) } - } catch (e: Exception) { - if (isCurrentRun(run) && lifecycleState.get() == LifecycleState.RUNNING) { - reconcileState.recordAsyncFailure(e.message) - } - try { - stateStore.removeIdle(config.poolName, sandboxId) - } catch (_: Exception) { - // best-effort remove before remote cleanup - } - cleanupSource = "warmup-commit-failed" - logger.warn( - "Pool warmup commit failed; dropped sandbox: pool_name={} sandbox_id={} run={} error={}", - config.poolName, - sandboxId, - run.generation, - e.message, - ) } + } finally { + run.commitLock.unlock() + } + } + if (trace == null) { + commit() + } else { + // Runs on the scheduler thread; re-attach the warmup trace's + // context (and MDC trace_id/span_id) so the commit span and its + // log lines belong to the same trace as the create phases. + trace.withCurrent { + poolTracer.withPhaseSpan(PoolTracer.WARMUP_COMMIT_SPAN) { commit() } + } + if (cleanupSource != null) { + // The sandbox never entered the idle pool (stale run, lock + // lost, or commit failure); do not report a success. + trace.endDropped(cleanupSource) + } else { + trace.endSuccess(sandboxId, creationSpec.imageSpec.image) } - } finally { - run.commitLock.unlock() } cleanupSource?.let { source -> scheduleKillDiscardedAlive( @@ -1090,17 +1380,23 @@ class SandboxPool internal constructor( */ private fun createOneSandbox(): String { return try { - val sandbox = buildWarmupSandbox() + // Phase spans auto-parent to the warmup root via the trace context + // made current by the warmup task; they no-op when tracing is off. + val sandbox = poolTracer.withPhaseSpan(PoolTracer.WARMUP_CREATE_SPAN) { buildWarmupSandbox() } var failure: Throwable? = null try { - config.warmupSandboxPreparer?.prepare(sandbox) + poolTracer.withPhaseSpan(PoolTracer.WARMUP_PREPARE_SPAN) { + config.warmupSandboxPreparer?.prepare(sandbox) + } // The server-side TTL has been ticking since sandbox creation; readiness // wait and `warmupSandboxPreparer` can both consume meaningful time (think // initialization scripts). Renew right before handing the id back to the // reconciler so the store's stamped expiry (now + idleTimeout) actually matches // what the server will honor โ€” otherwise `acquireMinRemainingTtl` overestimates // remaining TTL by the warmup duration. - sandbox.renew(config.idleTimeout) + poolTracer.withPhaseSpan(PoolTracer.WARMUP_RENEW_SPAN) { + sandbox.renew(config.idleTimeout) + } sandbox.id } catch (t: Throwable) { failure = t @@ -1174,7 +1470,7 @@ class SandboxPool internal constructor( .readyTimeout(config.warmupReadyTimeout) .healthCheckPollingInterval(config.warmupHealthCheckPollingInterval) .skipHealthCheck(config.warmupSkipHealthCheck) - .connectionConfig(connectionConfig), + .connectionConfig(poolConnectionConfig), ) config.warmupHealthCheck?.let { builder.healthCheck(it) } return builder.build() @@ -1219,7 +1515,7 @@ class SandboxPool internal constructor( .readyTimeout(config.acquireReadyTimeout) .healthCheckPollingInterval(config.acquireHealthCheckPollingInterval) .skipHealthCheck(config.acquireSkipHealthCheck) - .connectionConfig(connectionConfig), + .connectionConfig(poolConnectionConfig), ) config.acquireHealthCheck?.let { builder.healthCheck(it) } val sandbox = builder.build() @@ -1342,7 +1638,7 @@ class SandboxPool internal constructor( healthCheckPollingInterval = healthCheckPollingInterval, skipHealthCheck = skipHealthCheck, healthCheck = customHealthCheck, - connectionConfig = connectionConfig, + connectionConfig = poolConnectionConfig, ) return creator.create(context) } @@ -1566,6 +1862,13 @@ class SandboxPool internal constructor( logger.warn("Error closing pool SandboxManager", e) } sandboxManager = null + // Evict the pool-created shared pool so its idle connections are + // released on shutdown. A user-provided pool is never touched here. + try { + sharedConnectionPool?.evictAll() + } catch (e: Exception) { + logger.warn("Error evicting pool shared connection pool", e) + } } private fun isCurrentRun(run: RunContext): Boolean = currentRun === run && run.active.get() @@ -1581,6 +1884,7 @@ class SandboxPool internal constructor( } finally { run.commitLock.unlock() } + cancelRateLimitReconcile(run) } /** @@ -1600,6 +1904,15 @@ class SandboxPool internal constructor( val warmingCount = AtomicInteger(0) val warmupSubmissionsOpen = AtomicBoolean(true) val reconcileQueued = AtomicBoolean(false) + val rateLimitState = PoolRateLimitState() + val rateLimitScheduleLock = Any() + + @Volatile + var rateLimitReconcileTask: ScheduledFuture<*>? = null + var rateLimitReconcileSequence: Long = 0 + + @Volatile + var nextCompletionReconcileAtNanos: Long = 0 val primaryOwned = AtomicBoolean(false) val inFlightOperations = AtomicInteger(0) val inFlightLock = ReentrantLock() @@ -1627,17 +1940,26 @@ class SandboxPool internal constructor( } companion object { + /** Minimum spacing between completion-driven reconcile ticks (see [requestReconcile]). */ + private const val COMPLETION_RECONCILE_MIN_INTERVAL_MS = 500L + + /** Keep-alive of the pool-created shared connection pool. */ + private const val DEFAULT_SHARED_POOL_KEEPALIVE_MINUTES = 5L + @JvmStatic fun builder(): Builder = Builder() - private fun defaultIdleSandboxConnector(config: PoolConfig): (String) -> Sandbox = + private fun defaultIdleSandboxConnector( + config: PoolConfig, + connectionConfig: ConnectionConfig, + ): (String) -> Sandbox = { sandboxId -> Sandbox.connector() .sandboxId(sandboxId) .connectTimeout(config.acquireReadyTimeout) .healthCheckPollingInterval(config.acquireHealthCheckPollingInterval) .skipHealthCheck(config.acquireSkipHealthCheck) - .connectionConfig(config.connectionConfig) + .connectionConfig(connectionConfig) .run { config.acquireHealthCheck?.let { healthCheck(it) } ?: this }.connect() diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/SandboxTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/SandboxTest.kt index cc9597fe8..ef469b37b 100644 --- a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/SandboxTest.kt +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/SandboxTest.kt @@ -356,7 +356,7 @@ class SandboxTest { } @Test - fun `checkReady timeout should include connection context and bridge hint`() { + fun `checkReady timeout should include diagnostics without network configuration hints`() { every { healthService.ping(sandboxId) } throws RuntimeException("connect ECONNREFUSED") val ex = @@ -365,27 +365,10 @@ class SandboxTest { } assertTrue(ex.message!!.contains("Connection context: domain=localhost:8080, useServerProxy=false")) - assertTrue(ex.message!!.contains("useServerProxy=true")) - assertTrue(ex.message!!.contains("[docker].host_ip")) - assertTrue(ex.message!!.contains("Last error: connect ECONNREFUSED")) - } - - @Test - fun `checkReady timeout should omit host_ip hint when server proxy is enabled`() { - val proxyEnabledConfig = - ConnectionConfig.builder() - .domain("localhost:8080") - .useServerProxy(true) - .build() - every { httpClientProvider.config } returns proxyEnabledConfig - every { healthService.ping(sandboxId) } returns false - - val ex = - assertThrows(SandboxReadyTimeoutException::class.java) { - sandbox.checkReady(Duration.ofMillis(100), Duration.ofMillis(10)) - } - - assertTrue(ex.message!!.contains("useServerProxy=true")) + assertFalse(ex.message!!.contains("consider enabling useServerProxy=true", ignoreCase = true)) + assertFalse(ex.message!!.contains("Docker bridge", ignoreCase = true)) + assertFalse(ex.message!!.contains("remote-network", ignoreCase = true)) assertFalse(ex.message!!.contains("[docker].host_ip")) + assertTrue(ex.message!!.contains("Last error: connect ECONNREFUSED")) } } diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapterTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapterTest.kt index fe9d76f1f..3975609ef 100644 --- a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapterTest.kt +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/CommandsAdapterTest.kt @@ -17,6 +17,8 @@ package com.alibaba.opensandbox.sandbox.infrastructure.adapters.service import com.alibaba.opensandbox.sandbox.HttpClientProvider +import com.alibaba.opensandbox.sandbox.api.execd.CommandApi +import com.alibaba.opensandbox.sandbox.api.execd.infrastructure.ClientException import com.alibaba.opensandbox.sandbox.config.ConnectionConfig import com.alibaba.opensandbox.sandbox.domain.exceptions.InvalidArgumentException import com.alibaba.opensandbox.sandbox.domain.exceptions.SandboxApiException @@ -318,6 +320,45 @@ class CommandsAdapterTest { assertTrue(ex.isRetryable) } + @Test + fun `getBackgroundCommandLogs should include response body in client error`() { + val responseBody = """{"code":"INVALID_ARGUMENT","message":"cursor must be positive"}""" + mockWebServer.enqueue( + MockResponse() + .setResponseCode(400) + .setBody(responseBody), + ) + + val ex = + assertThrows(SandboxApiException::class.java) { + commandsAdapter.getBackgroundCommandLogs("exec-1") + } + + assertEquals(400, ex.statusCode) + assertTrue(ex.message!!.contains(responseBody)) + assertEquals(responseBody, ex.responseBody) + } + + @Test + fun `generated client error message should include response body`() { + val responseBody = """{"code":"QUOTA_EXCEEDED","message":"sandbox quota exceeded"}""" + mockWebServer.enqueue( + MockResponse() + .setResponseCode(400) + .setBody(responseBody), + ) + + val api = + CommandApi( + "http://${mockWebServer.hostName}:${mockWebServer.port}", + httpClientProvider.httpClient, + ) + + val ex = assertThrows(ClientException::class.java) { api.getCommandStatus("exec-1") } + + assertTrue(ex.message!!.contains(responseBody)) + } + @Test fun `createSession should use generated api and return session id`() { mockWebServer.enqueue( diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapterTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapterTest.kt index fd1b991d6..fa7c6f506 100644 --- a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapterTest.kt +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/service/IsolatedSessionsAdapterTest.kt @@ -171,6 +171,38 @@ class IsolatedSessionsAdapterTest { assertEquals(false, capabilities.usernsAvailable) } + @Test + fun `capabilities parses hardening status`() { + mockWebServer.enqueue( + MockResponse() + .setBody( + """ + { + "available": true, + "hardening": { + "init_mode": "pid1", + "signal_shield": true, + "cap_drop": {"state": "active"}, + "seccomp": {"state": "active"}, + "landlock": {"state": "unsupported", "message": "kernel ABI < 1"}, + "ebpf": {"state": "disabled"} + } + } + """.trimIndent(), + ), + ) + val capabilities = adapter.capabilities() + + val hardening = capabilities.hardening + assertEquals("pid1", hardening?.initMode) + assertEquals(true, hardening?.signalShield) + assertEquals("active", hardening?.capDrop?.state) + assertEquals("active", hardening?.seccomp?.state) + assertEquals("unsupported", hardening?.landlock?.state) + assertEquals("kernel ABI < 1", hardening?.landlock?.message) + assertEquals("disabled", hardening?.ebpf?.state) + } + @Test fun `create serializes uid and gid above Int MaxValue`() { // Spec declares uid/gid as uint32; values above Int.MAX_VALUE must not fail. diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitStateTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitStateTest.kt new file mode 100644 index 000000000..6744be290 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolRateLimitStateTest.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.infrastructure.pool + +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import com.alibaba.opensandbox.sandbox.domain.pool.PoolConfig +import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Duration +import java.time.Instant +import java.util.concurrent.atomic.AtomicInteger + +class PoolRateLimitStateTest { + private val now: Instant = Instant.parse("2026-08-14T00:00:00Z") + + @Test + fun `zero or non-positive retry after falls back to default delay`() { + val state = PoolRateLimitState() + + state.recordRateLimit(retryAfter = Duration.ZERO, now = now) + + assertTrue(state.isActive(now.plusSeconds(9))) + assertFalse(state.isActive(now.plusSeconds(10))) + } + + @Test + fun `missing retry after uses bounded default delay`() { + val state = PoolRateLimitState() + + state.recordRateLimit(retryAfter = null, now = now) + + assertTrue(state.isActive(now.plusSeconds(9))) + assertFalse(state.isActive(now.plusSeconds(10))) + } + + @Test + fun `retry after is capped at transport ceiling`() { + val state = PoolRateLimitState() + + state.recordRateLimit(retryAfter = Duration.ofMinutes(5), now = now) + + assertTrue(state.isActive(now.plusSeconds(59))) + assertFalse(state.isActive(now.plusSeconds(60))) + } + + @Test + fun `concurrent rate limits only extend throttle deadline`() { + val state = PoolRateLimitState() + + state.recordRateLimit(retryAfter = Duration.ofSeconds(30), now = now) + state.recordRateLimit(retryAfter = Duration.ofSeconds(5), now = now.plusSeconds(1)) + + assertEquals(Duration.ofSeconds(1), state.remainingDelay(now.plusSeconds(29))) + state.recordRateLimit(retryAfter = Duration.ofSeconds(60), now = now.plusSeconds(1)) + assertTrue(state.isActive(now.plusSeconds(60))) + assertFalse(state.isActive(now.plusSeconds(61))) + } + + @Test + fun `rate limit suppresses warmups without blocking excess idle shrink`() { + val stateStore = InMemoryPoolStateStore() + val poolName = "rate-limited-shrink" + stateStore.putIdle(poolName, "idle-1") + stateStore.putIdle(poolName, "idle-2") + val config = + PoolConfig.builder() + .poolName(poolName) + .ownerId("owner-1") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(stateStore) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .build() + val rateLimitState = PoolRateLimitState() + rateLimitState.recordRateLimit(Duration.ofSeconds(30)) + val discarded = mutableListOf() + val submitted = AtomicInteger(0) + + PoolReconciler.runReconcileTick( + config = config, + stateStore = stateStore, + onDiscardSandbox = { discarded += it }, + reconcileState = ReconcileState(degradedThreshold = 3), + warmingCount = 0, + rateLimitState = rateLimitState, + submitWarmups = { submitted.addAndGet(it) }, + ) + + assertEquals(1, discarded.size) + assertEquals(0, submitted.get()) + assertEquals(1, stateStore.snapshotCounters(poolName).idleCount) + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupTracingTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupTracingTest.kt new file mode 100644 index 000000000..a239486bd --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupTracingTest.kt @@ -0,0 +1,385 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.pool + +import com.alibaba.opensandbox.sandbox.HttpClientProvider +import com.alibaba.opensandbox.sandbox.Sandbox +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import com.alibaba.opensandbox.sandbox.domain.pool.PooledSandboxCreator +import com.alibaba.opensandbox.sandbox.domain.pool.SandboxPreparer +import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore +import com.alibaba.opensandbox.sandbox.internal.PoolTracer +import io.mockk.every +import io.mockk.mockk +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator +import io.opentelemetry.context.propagation.ContextPropagators +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import okhttp3.Request +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.slf4j.MDC +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class PoolWarmupTracingTest { + private var exporter: InMemorySpanExporter? = null + private var openTelemetry: OpenTelemetrySdk? = null + + @AfterEach + fun tearDown() { + openTelemetry?.close() + openTelemetry = null + exporter = null + GlobalOpenTelemetry.resetForTest() + } + + private fun installSdkTracerProvider(): InMemorySpanExporter { + val spanExporter = InMemorySpanExporter.create() + val sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build() + val otel = + OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + // Default propagators are noop; set W3C so traceparent injection + // (HttpClientProvider) can be asserted. + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .build() + GlobalOpenTelemetry.set(otel) + exporter = spanExporter + openTelemetry = otel + return spanExporter + } + + @Test + fun `warmup emits a full span tree with drill-down attributes when tracing enabled`() { + val spanExporter = installSdkTracerProvider() + val capturedTraceId = AtomicReference() + val capturedSpanId = AtomicReference() + + val store = InMemoryPoolStateStore() + val pool = + SandboxPool.builder() + .poolName("trace-pool") + .ownerId("trace-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().enableTracing().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + mockk(relaxed = true).also { sandbox -> + every { sandbox.id } returns "warmup-trace-1" + } + }, + ).warmupSkipHealthCheck() + .warmupSandboxPreparer( + SandboxPreparer { + capturedTraceId.set(MDC.get(PoolTracer.MDC_TRACE_ID)) + capturedSpanId.set(MDC.get(PoolTracer.MDC_SPAN_ID)) + }, + ).reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofSeconds(2)) + .build() + + pool.start() + try { + awaitCondition { store.snapshotCounters("trace-pool").idleCount == 1 } + val spans = spanExporter.finishedSpanItems + val root = spans.single { it.name == PoolTracer.WARMUP_ROOT_SPAN } + assertEquals("trace-pool", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_POOL_NAME)]) + assertEquals("trace-owner", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_POOL_OWNER)]) + assertEquals(1L, root.attributes[AttributeKey.longKey(PoolTracer.ATTR_POOL_RUN_GENERATION)]) + assertEquals("warmup-trace-1", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_SANDBOX_ID)]) + assertEquals("ubuntu:22.04", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_SANDBOX_IMAGE)]) + assertEquals("success", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_RESULT)]) + + // MDC must expose the same trace while warmup code runs. + assertEquals(root.traceId, capturedTraceId.get()) + assertEquals(root.spanId, capturedSpanId.get()) + + val names = spans.map { it.name }.toSet() + assertTrue(names.contains(PoolTracer.WARMUP_CREATE_SPAN)) + assertTrue(names.contains(PoolTracer.WARMUP_PREPARE_SPAN)) + assertTrue(names.contains(PoolTracer.WARMUP_RENEW_SPAN)) + assertTrue(names.contains(PoolTracer.WARMUP_COMMIT_SPAN)) + + // All spans share one trace; phase spans are sequential siblings + // under the root (each phase duration stands alone for drill-down), + // and commit re-attaches to the root on the scheduler thread. + spans.forEach { span -> + assertEquals(root.traceId, span.traceId, "all spans must share the warmup trace id") + } + val create = spans.single { it.name == PoolTracer.WARMUP_CREATE_SPAN } + val prepare = spans.single { it.name == PoolTracer.WARMUP_PREPARE_SPAN } + val renew = spans.single { it.name == PoolTracer.WARMUP_RENEW_SPAN } + val commit = spans.single { it.name == PoolTracer.WARMUP_COMMIT_SPAN } + assertEquals(root.spanId, create.parentSpanId) + assertEquals(root.spanId, prepare.parentSpanId) + assertEquals(root.spanId, renew.parentSpanId) + assertEquals(root.spanId, commit.parentSpanId) + + // Root span is backdated to submission, so the trace covers the + // queue wait before the create phase. + assertTrue(root.startEpochNanos <= create.startEpochNanos) + // Root start must be epoch wall-clock (not monotonic nanoTime): + // within a minute of test start, and not some arbitrary boot-relative value. + val testStartEpochNanos = System.currentTimeMillis() * 1_000_000L + assertTrue( + root.startEpochNanos <= testStartEpochNanos, + "root start must be in the past relative to test start", + ) + assertTrue( + root.startEpochNanos >= testStartEpochNanos - Duration.ofMinutes(1).toNanos(), + "root start must be epoch wall-clock near test start", + ) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `failed warmup emits a failure root span without a commit span`() { + val spanExporter = installSdkTracerProvider() + val store = InMemoryPoolStateStore() + val pool = + SandboxPool.builder() + .poolName("trace-fail-pool") + .ownerId("trace-fail-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().enableTracing().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + throw RuntimeException("create boom") + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofMillis(200)) + .build() + + pool.start() + try { + awaitCondition { pool.snapshot().failureCount >= 1 } + val spans = spanExporter.finishedSpanItems + val root = spans.single { it.name == PoolTracer.WARMUP_ROOT_SPAN } + assertEquals("failure", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_RESULT)]) + assertEquals( + 1, + root.events.count { it.name == "exception" }, + "failure must be recorded on the root span", + ) + assertTrue(spans.none { it.name == PoolTracer.WARMUP_COMMIT_SPAN }) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `dropped warmup commit is traced as a failure`() { + val spanExporter = installSdkTracerProvider() + val store = LockLossOnCommitPoolStateStore() + val pool = + SandboxPool.builder() + .poolName("trace-drop-pool") + .ownerId("trace-drop-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().enableTracing().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + mockk(relaxed = true).also { sandbox -> + every { sandbox.id } returns "warmup-dropped-1" + } + }, + ).warmupSkipHealthCheck() + .warmupSandboxPreparer( + SandboxPreparer { + store.failRenewPrimaryLock = true + }, + ).reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofSeconds(2)) + .build() + + pool.start() + try { + awaitCondition { spanExporter.finishedSpanItems.any { it.name == PoolTracer.WARMUP_ROOT_SPAN } } + val spans = spanExporter.finishedSpanItems + val root = spans.single { it.name == PoolTracer.WARMUP_ROOT_SPAN } + assertEquals("failure", root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_RESULT)]) + assertEquals( + "warmup-lock-lost", + root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_DROP_REASON)], + ) + assertNull(root.attributes[AttributeKey.stringKey(PoolTracer.ATTR_SANDBOX_ID)]) + assertTrue( + spans.any { it.name == PoolTracer.WARMUP_COMMIT_SPAN }, + "commit phase must still be traced", + ) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `warmup emits no spans when tracing is disabled`() { + val spanExporter = installSdkTracerProvider() + val store = InMemoryPoolStateStore() + val pool = + SandboxPool.builder() + .poolName("no-trace-pool") + .ownerId("no-trace-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + mockk(relaxed = true).also { sandbox -> + every { sandbox.id } returns "no-trace-1" + } + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofSeconds(2)) + .build() + + pool.start() + try { + awaitCondition { store.snapshotCounters("no-trace-pool").idleCount == 1 } + assertTrue( + spanExporter.finishedSpanItems.isEmpty(), + "no spans may be emitted when enableTracing is false", + ) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `requests inject traceparent only when tracing is enabled and a span is current`() { + val spanExporter = installSdkTracerProvider() + val server = MockWebServer() + try { + server.enqueue(MockResponse().setResponseCode(204)) + server.enqueue(MockResponse().setResponseCode(204)) + server.enqueue(MockResponse().setResponseCode(204)) + val tracer = GlobalOpenTelemetry.get().tracerBuilder("test").build() + val span = tracer.spanBuilder("test-span").startSpan() + span.makeCurrent().use { + HttpClientProvider( + ConnectionConfig.builder() + .domain(server.url("/").toString().removeSuffix("/")) + .enableTracing() + .build(), + ).use { provider -> + provider.httpClient.newCall(Request.Builder().url(server.url("/tracing")).build()).execute() + .use { } + val recorded = server.takeRequest(1, TimeUnit.SECONDS)!! + val traceparent = recorded.getHeader("traceparent") + assertNotNull(traceparent, "traceparent must be injected for an active span") + assertTrue(traceparent!!.startsWith("00-"), "traceparent must be W3C v00 format") + assertTrue(traceparent.contains(span.spanContext.traceId), "traceparent must carry the trace id") + } + } + span.end() + + // No active span -> no injection, even with tracing enabled. + HttpClientProvider( + ConnectionConfig.builder() + .domain(server.url("/").toString().removeSuffix("/")) + .enableTracing() + .build(), + ).use { provider -> + provider.httpClient.newCall(Request.Builder().url(server.url("/no-span")).build()).execute().use { } + val recorded = server.takeRequest(1, TimeUnit.SECONDS)!! + assertNull(recorded.getHeader("traceparent")) + } + + // Tracing disabled -> no injection even with an active span. + val disabledSpan = tracer.spanBuilder("disabled-span").startSpan() + disabledSpan.makeCurrent().use { + HttpClientProvider( + ConnectionConfig.builder() + .domain(server.url("/").toString().removeSuffix("/")) + .build(), + ).use { provider -> + provider.httpClient.newCall(Request.Builder().url(server.url("/disabled")).build()).execute() + .use { } + val recorded = server.takeRequest(1, TimeUnit.SECONDS)!! + assertNull(recorded.getHeader("traceparent")) + } + } + disabledSpan.end() + assertTrue(spanExporter.finishedSpanItems.isNotEmpty()) + } finally { + server.shutdown() + } + } + + private fun awaitCondition( + timeout: Duration = Duration.ofSeconds(5), + condition: () -> Boolean, + ) { + val deadline = System.nanoTime() + timeout.toNanos() + while (System.nanoTime() < deadline) { + if (condition()) return + Thread.sleep(20) + } + throw AssertionError("condition not met within $timeout") + } + + /** + * In-memory store whose primary-lock renewal starts failing once a warmup + * preparer sets [failRenewPrimaryLock], so the commit path drops the + * warmed sandbox with `warmup-lock-lost`. + */ + private class LockLossOnCommitPoolStateStore( + private val delegate: InMemoryPoolStateStore = InMemoryPoolStateStore(), + ) : com.alibaba.opensandbox.sandbox.domain.pool.PoolStateStore by delegate { + @Volatile + var failRenewPrimaryLock: Boolean = false + + override fun renewPrimaryLock( + poolName: String, + ownerId: String, + ttl: Duration, + ): Boolean { + if (failRenewPrimaryLock) return false + return delegate.renewPrimaryLock(poolName, ownerId, ttl) + } + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolRateLimitTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolRateLimitTest.kt new file mode 100644 index 000000000..b1c01c663 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolRateLimitTest.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2025 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.pool + +import com.alibaba.opensandbox.sandbox.Sandbox +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import com.alibaba.opensandbox.sandbox.domain.exceptions.SandboxRateLimitException +import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import com.alibaba.opensandbox.sandbox.domain.pool.PoolState +import com.alibaba.opensandbox.sandbox.domain.pool.PooledSandboxCreator +import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +class SandboxPoolRateLimitTest { + @Test + fun `rate limited warmup honors retry after without degrading pool`() { + val attempts = AtomicInteger(0) + val firstAttemptAt = AtomicLong(0) + val secondAttemptAt = AtomicLong(0) + val sandbox = mockk(relaxed = true) + every { sandbox.id } returns "rate-limit-recovery" + + val pool = + SandboxPool.builder() + .poolName("rate-limited-pool") + .ownerId("rate-limited-owner") + .maxIdle(1) + .warmupConcurrency(1) + .degradedThreshold(1) + .stateStore(InMemoryPoolStateStore()) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + when (attempts.incrementAndGet()) { + 1 -> { + firstAttemptAt.set(System.nanoTime()) + throw SandboxRateLimitException( + message = "rate limited", + retryAfter = Duration.ofSeconds(1), + ) + } + else -> { + secondAttemptAt.compareAndSet(0, System.nanoTime()) + sandbox + } + } + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .build() + + pool.start() + try { + awaitCondition { pool.snapshot().backoffActive } + + val throttled = pool.snapshot() + assertEquals(PoolState.HEALTHY, throttled.state) + assertEquals(0, throttled.failureCount) + assertFalse( + awaitCondition(timeout = Duration.ofMillis(250)) { attempts.get() > 1 }, + "warmup must not retry before Retry-After expires", + ) + + assertTrue(awaitCondition { pool.snapshot().idleCount == 1 }) + val retryDelay = Duration.ofNanos(secondAttemptAt.get() - firstAttemptAt.get()) + assertTrue(retryDelay >= Duration.ofMillis(800), "warmup retried too early: $retryDelay") + assertEquals(2, attempts.get()) + assertFalse(pool.snapshot().backoffActive) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `pool restart clears rate limit throttle from previous run`() { + val attempts = AtomicInteger(0) + val sandbox = mockk(relaxed = true) + every { sandbox.id } returns "restart-rate-limit-recovery" + val pool = + SandboxPool.builder() + .poolName("restart-rate-limited-pool") + .ownerId("restart-rate-limited-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(InMemoryPoolStateStore()) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + if (attempts.incrementAndGet() == 1) { + throw SandboxRateLimitException( + message = "rate limited", + retryAfter = Duration.ofSeconds(30), + ) + } + sandbox + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .build() + + pool.start() + assertTrue(awaitCondition { pool.snapshot().backoffActive }) + pool.shutdown(graceful = false) + + pool.start() + try { + assertFalse(pool.snapshot().backoffActive) + assertTrue(awaitCondition { pool.snapshot().idleCount == 1 }) + assertEquals(2, attempts.get()) + } finally { + pool.shutdown(graceful = false) + } + } + + private fun awaitCondition( + timeout: Duration = Duration.ofSeconds(5), + condition: () -> Boolean, + ): Boolean { + val deadline = System.nanoTime() + timeout.toNanos() + while (System.nanoTime() < deadline) { + if (condition()) return true + TimeUnit.MILLISECONDS.sleep(10) + } + return condition() + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolSharedConnectionTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolSharedConnectionTest.kt new file mode 100644 index 000000000..59b488564 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolSharedConnectionTest.kt @@ -0,0 +1,187 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.pool + +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore +import okhttp3.ConnectionPool +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +/** + * Verifies the pool-level shared connection pool: created by default, + * reused by warmup and acquire paths, evicted on shutdown, and never created + * when the user provides their own pool. + */ +class SandboxPoolSharedConnectionTest { + private lateinit var lifecycle: MockWebServer + private lateinit var execd: MockWebServer + private val sandboxSeq = AtomicInteger() + + @BeforeEach + fun setUp() { + lifecycle = MockWebServer() + execd = MockWebServer() + lifecycle.start() + execd.start() + lifecycle.dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + val path = request.path.orEmpty() + return when { + request.method == "POST" && path == "/v1/sandboxes" -> + MockResponse().setResponseCode(201).setBody( + """{"id":"sbx-${sandboxSeq.incrementAndGet()}","status":{"state":"Running"},""" + + """"createdAt":"2026-01-01T00:00:00Z","entrypoint":["tail"]}""", + ) + request.method == "GET" && path.contains("/endpoints/") -> + MockResponse().setResponseCode(200).setBody( + """{"endpoint":"${execd.hostName}:${execd.port}","headers":{"X-EXECD-ACCESS-TOKEN":"t"}}""", + ) + request.method == "POST" && path.endsWith("/renew-expiration") -> + MockResponse().setResponseCode(200).setBody("""{"expiresAt":"2026-12-31T00:00:00Z"}""") + request.method == "DELETE" -> MockResponse().setResponseCode(204) + else -> MockResponse().setResponseCode(404) + } + } + } + execd.dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse = + MockResponse().setResponseCode(200).setBody("""{"status":"ok"}""") + } + } + + @AfterEach + fun tearDown() { + lifecycle.shutdown() + execd.shutdown() + } + + private fun connectionConfig(userPool: ConnectionPool? = null): ConnectionConfig { + val builder = + ConnectionConfig.builder() + .domain(lifecycle.hostName + ":" + lifecycle.port) + .protocol("http") + .requestTimeout(Duration.ofSeconds(10)) + .disableMetrics() + userPool?.let { builder.connectionPool(it) } + return builder.build() + } + + private fun buildPool( + userPool: ConnectionPool? = null, + maxIdle: Int = 4, + warmupConcurrency: Int = 2, + ): SandboxPool = + SandboxPool.builder() + .poolName("shared-conn-test") + .ownerId("owner") + .maxIdle(maxIdle) + .warmupConcurrency(warmupConcurrency) + .stateStore(InMemoryPoolStateStore()) + .connectionConfig(connectionConfig(userPool)) + .creationSpec( + PoolCreationSpec.builder() + .image("test:latest") + .entrypoint("tail", "-f", "/dev/null") + .build(), + ) + .reconcileInterval(Duration.ofMillis(100)) + .idleTimeout(Duration.ofMinutes(30)) + .acquireSkipHealthCheck(false) + .build() + + private fun waitForIdle( + pool: SandboxPool, + target: Int, + timeoutMs: Long = 15000, + ) { + val deadline = System.currentTimeMillis() + timeoutMs + while (pool.snapshot().idleCount < target) { + assertTrue(System.currentTimeMillis() < deadline, "idle never reached $target") + Thread.sleep(50) + } + } + + @Test + fun `pool creates a default shared pool when none provided`() { + val pool = buildPool(warmupConcurrency = 5) + assertNotNull(pool.sharedConnectionPoolForTests(), "expected a pool-created shared connection pool") + pool.shutdown(graceful = false) + } + + @Test + fun `user provided pool is used as-is and no default pool is created`() { + val userPool = ConnectionPool(1, 1, TimeUnit.MINUTES) + val pool = buildPool(userPool = userPool) + assertNull(pool.sharedConnectionPoolForTests(), "user pool must not be shadowed by a default pool") + pool.shutdown(graceful = false) + } + + @Test + fun `warmup and acquire reuse the shared pool instead of fresh connections`() { + val pool = buildPool(maxIdle = 4, warmupConcurrency = 2) + val shared = pool.sharedConnectionPoolForTests()!! + pool.start() + try { + waitForIdle(pool, 4) + // 4 sandboxes were created over HTTP; with a shared pool the + // connections stay in the pool instead of being opened fresh per + // sandbox. The bound is loose to stay robust on all platforms. + assertTrue(shared.idleConnectionCount() > 0, "shared pool should hold idle connections after warmup") + assertTrue( + shared.connectionCount() <= 6, + "expected connection reuse, got ${shared.connectionCount()} connections", + ) + + // Acquire reuses the same shared pool. + val sb = pool.acquire(Duration.ofMinutes(10)) + assertTrue(sb.id.startsWith("sbx-")) + sb.kill() + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `pool-owned shared pool is evicted on shutdown`() { + val pool = buildPool(maxIdle = 2, warmupConcurrency = 1) + val shared = pool.sharedConnectionPoolForTests()!! + pool.start() + try { + waitForIdle(pool, 2) + assertTrue(shared.idleConnectionCount() > 0) + } finally { + pool.shutdown(graceful = false) + } + assertEquals(0, shared.idleConnectionCount(), "pool-owned connections must be evicted on shutdown") + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolTest.kt b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolTest.kt index 7b6f3c160..953b45a52 100644 --- a/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolTest.kt +++ b/sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPoolTest.kt @@ -203,6 +203,89 @@ class SandboxPoolTest { } } + @Test + fun `failed warmup does not trigger an immediate completion-driven reconcile`() { + val store = CountingPoolStateStore() + val created = AtomicInteger(0) + val pool = + SandboxPool.builder() + .poolName("failure-no-retrigger-pool") + .ownerId("failure-no-retrigger-owner") + .maxIdle(2) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + created.incrementAndGet() + throw RuntimeException("fast create failure") + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofMillis(200)) + .build() + + pool.start() + try { + awaitCondition { pool.snapshot().failureCount >= 1 } + // Fast-failing warmups previously queued a new reconcile tick on every failure, + // causing unbounded create/retry churn within a single reconcileInterval. + Thread.sleep(800) + assertEquals(1, created.get(), "failed warmup must not be retried before the periodic tick") + assertTrue( + store.reconcileTicks.get() <= 2, + "failed warmup must not drive extra reconcile ticks, got=${store.reconcileTicks.get()}", + ) + } finally { + pool.shutdown(graceful = false) + } + } + + @Test + fun `completion-driven reconcile ticks are rate limited during fast completions`() { + val store = CountingPoolStateStore() + val created = AtomicInteger(0) + val pool = + SandboxPool.builder() + .poolName("tick-rate-limit-pool") + .ownerId("tick-rate-limit-owner") + .maxIdle(4) + .warmupConcurrency(2) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + if (created.getAndIncrement() % 2 == 0) { + throw RuntimeException("fast create failure") + } + mockk(relaxed = true).also { sandbox -> + every { sandbox.id } returns "alternating-${created.get()}" + } + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofMillis(200)) + .build() + + pool.start() + try { + // A mixed burst of fast success/failure completions keeps the pool in deficit and + // previously drove one reconcile tick per completion round (~tens per second), + // amplified by several state-store round-trips per tick. The minimum-interval + // coalescing window must cap the tick rate regardless of outcome mix. + Thread.sleep(1100) + assertTrue( + store.reconcileTicks.get() <= 5, + "completion-driven ticks must be rate limited, got=${store.reconcileTicks.get()}", + ) + assertTrue(created.get() >= 3, "burst must still make progress, created=${created.get()}") + } finally { + pool.shutdown(graceful = false) + } + } + @Test fun `primary heartbeat continues while warmup is blocked`() { val store = HeartbeatRecordingStore() @@ -934,6 +1017,61 @@ class SandboxPoolTest { } } + @Test + fun `stale acquire cleanup triggers replenish before periodic reconcile`() { + val store = CountingPoolStateStore() + val manager = mockk(relaxed = true) + val created = AtomicInteger(0) + val killed = CountDownLatch(1) + every { manager.killSandbox("warmup-1") } answers { killed.countDown() } + + val config = + PoolConfig.builder() + .poolName("cleanup-reconcile-pool") + .ownerId("cleanup-reconcile-owner") + .maxIdle(1) + .warmupConcurrency(1) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .sandboxCreator( + PooledSandboxCreator { + val index = created.incrementAndGet() + mockk(relaxed = true).also { sandbox -> + every { sandbox.id } returns "warmup-$index" + } + }, + ).warmupSkipHealthCheck() + .reconcileInterval(Duration.ofSeconds(30)) + .drainTimeout(Duration.ofSeconds(2)) + .build() + val pool = + SandboxPool( + config = config, + sandboxManagerFactory = { manager }, + idleSandboxConnector = { throw RuntimeException("stale sandbox") }, + ) + + pool.start() + try { + awaitCondition { + store.snapshotCounters("cleanup-reconcile-pool").idleCount == 1 && + store.reconcileTicks.get() >= 2 + } + + assertThrows(PoolAcquireFailedException::class.java) { + pool.acquire(policy = AcquirePolicy.FAIL_FAST) + } + + assertTrue(killed.await(5, TimeUnit.SECONDS)) + awaitCondition { + created.get() == 2 && store.snapshotCounters("cleanup-reconcile-pool").idleCount == 1 + } + } finally { + pool.shutdown(graceful = false) + } + } + @Test fun `acquire with RETRY_NEXT_IDLE and empty idle throws PoolEmptyException`() { val pool = buildPool() @@ -952,11 +1090,12 @@ class SandboxPoolTest { @Test fun `acquire with RETRY_NEXT_IDLE and all stale idle drains up to maxAcquireRetries and throws`() { val store = InMemoryPoolStateStore() + val connectAttempts = AtomicInteger(0) // maxIdle=0 keeps the reconcile loop from creating fresh sandboxes against the (missing) // server; we drive idle membership manually via putIdle so the test only exercises the // acquire retry loop. - val pool = - SandboxPool.builder() + val config = + PoolConfig.builder() .poolName("test-pool") .ownerId("test-owner") .maxIdle(0) @@ -967,7 +1106,21 @@ class SandboxPoolTest { .reconcileInterval(Duration.ofSeconds(30)) .maxAcquireRetries(3) .build() - // 5 stale IDs in idle; retry policy should try 3, leave 2 behind. + val pool = + SandboxPool( + config = config, + sandboxManagerFactory = { cfg -> + SandboxManager.builder().connectionConfig(cfg).build() + }, + idleSandboxConnector = { sandboxId -> + connectAttempts.incrementAndGet() + throw RuntimeException("stale sandbox $sandboxId") + }, + ) + // 5 stale IDs in idle; retry policy should try exactly 3. The leftover two are excess + // under maxIdle=0 and the completion-driven reconcile may remove them before the + // assertion, so the retry budget is verified via connector attempts instead of the + // residual idle count. repeat(5) { store.putIdle("test-pool", "stale-id-$it") } pool.start() @@ -975,7 +1128,7 @@ class SandboxPoolTest { assertThrows(PoolAcquireFailedException::class.java) { pool.acquire(policy = AcquirePolicy.RETRY_NEXT_IDLE) } - assertEquals(2, store.snapshotCounters("test-pool").idleCount) + assertEquals(3, connectAttempts.get()) } finally { pool.shutdown(graceful = false) } @@ -1305,14 +1458,12 @@ class SandboxPoolTest { } @Test - fun `warmup Error cleans sandbox and releases rolling slot`() { + fun `warmup Error cleans sandbox and releases rolling slot without immediate replacement`() { val store = InMemoryPoolStateStore() val manager = mockk(relaxed = true) val firstSandbox = mockk(relaxed = true) - val replacementSandbox = mockk(relaxed = true) val created = AtomicInteger(0) every { firstSandbox.id } returns "warmup-error" - every { replacementSandbox.id } returns "warmup-replacement" val config = PoolConfig.builder() @@ -1325,7 +1476,8 @@ class SandboxPoolTest { .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) .sandboxCreator( PooledSandboxCreator { - if (created.incrementAndGet() == 1) firstSandbox else replacementSandbox + created.incrementAndGet() + firstSandbox }, ).warmupSkipHealthCheck() .warmupSandboxPreparer( @@ -1342,15 +1494,15 @@ class SandboxPoolTest { pool.start() try { awaitCondition { - store.snapshotCounters("warmup-error-pool").idleCount == 1 && + pool.snapshot().failureCount >= 1 && pool.snapshot().inFlightOperations == 0 && currentRunWarming(pool).get() == 0 } - assertEquals(2, created.get()) + assertEquals(1, created.get(), "failed warmup must not be retried before the periodic tick") + assertEquals(0, store.snapshotCounters("warmup-error-pool").idleCount) verify(exactly = 1) { firstSandbox.kill() } verify(exactly = 1) { firstSandbox.close() } - assertEquals("warmup-replacement", pool.snapshotIdleEntries().single().sandboxId) } finally { pool.releaseAllIdle() pool.shutdown(graceful = false) @@ -1599,6 +1751,106 @@ class SandboxPoolTest { verify(exactly = 1) { temporaryManager.close() } } + @Test + fun `releaseAllIdle bounds kills and cleans up before store failure`() { + val delegate = InMemoryPoolStateStore() + repeat(55) { delegate.putIdle("test-pool", "id-$it") } + val store = + object : PoolStateStore by delegate { + var takes = 0 + + override fun tryTakeIdle(poolName: String): String? { + if (takes == 55) throw RuntimeException("injected store failure") + takes++ + return delegate.tryTakeIdle(poolName) + } + } + val active = AtomicInteger() + val maxActive = AtomicInteger() + val ready = CountDownLatch(50) + val killed = AtomicInteger() + val temporaryManager = mockk() + every { temporaryManager.killSandbox(any()) } answers { + val current = active.incrementAndGet() + maxActive.updateAndGet { maxOf(it, current) } + ready.countDown() + assertTrue(ready.await(2, TimeUnit.SECONDS)) + killed.incrementAndGet() + active.decrementAndGet() + if (firstArg() == "id-0") throw RuntimeException("injected kill failure") + } + every { temporaryManager.close() } just runs + val pool = + SandboxPool( + config = + PoolConfig.builder() + .poolName("test-pool") + .ownerId("test-owner") + .maxIdle(0) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .build(), + sandboxManagerFactory = { temporaryManager }, + ) + + assertThrows(IllegalArgumentException::class.java) { pool.releaseAllIdle(0) } + val failure = assertThrows(RuntimeException::class.java) { pool.releaseAllIdle(50) } + + assertEquals("injected store failure", failure.message) + assertEquals(50, maxActive.get()) + assertEquals(55, killed.get()) + assertEquals(0, store.snapshotCounters("test-pool").idleCount) + verify(exactly = 1) { temporaryManager.close() } + } + + @Test + fun `releaseAllIdle waits for kills before closing manager when caller is interrupted`() { + val store = InMemoryPoolStateStore() + store.putIdle("test-pool", "id-1") + val killStarted = CountDownLatch(1) + val releaseKill = CountDownLatch(1) + val temporaryManager = mockk() + every { temporaryManager.killSandbox("id-1") } answers { + killStarted.countDown() + releaseKill.await() + } + every { temporaryManager.close() } just runs + val pool = + SandboxPool( + config = + PoolConfig.builder() + .poolName("test-pool") + .ownerId("test-owner") + .maxIdle(0) + .stateStore(store) + .connectionConfig(ConnectionConfig.builder().build()) + .creationSpec(PoolCreationSpec.builder().image("ubuntu:22.04").build()) + .build(), + sandboxManagerFactory = { temporaryManager }, + ) + val released = AtomicInteger() + val interruptRestored = AtomicBoolean() + val caller = + Thread { + released.set(pool.releaseAllIdle(1)) + interruptRestored.set(Thread.currentThread().isInterrupted) + } + + caller.start() + assertTrue(killStarted.await(2, TimeUnit.SECONDS)) + caller.interrupt() + Thread.sleep(20) + assertTrue(caller.isAlive) + verify(exactly = 0) { temporaryManager.close() } + releaseKill.countDown() + caller.join(2_000) + + assertEquals(1, released.get()) + assertTrue(interruptRestored.get()) + verify(exactly = 1) { temporaryManager.close() } + } + @Test fun `releaseAllIdle drains store even when temporary sandbox manager creation fails`() { val store = InMemoryPoolStateStore() @@ -1991,6 +2243,22 @@ class SandboxPoolTest { ) } + private class CountingPoolStateStore( + private val delegate: InMemoryPoolStateStore = InMemoryPoolStateStore(), + ) : PoolStateStore by delegate { + /** Number of reconcile ticks, proxied by the primary-lock acquisition that opens each tick. */ + val reconcileTicks = AtomicInteger(0) + + override fun tryAcquirePrimaryLock( + poolName: String, + ownerId: String, + ttl: Duration, + ): Boolean { + reconcileTicks.incrementAndGet() + return delegate.tryAcquirePrimaryLock(poolName, ownerId, ttl) + } + } + private class BlockingTakePoolStateStore( private val delegate: InMemoryPoolStateStore = InMemoryPoolStateStore(), ) : PoolStateStore by delegate { diff --git a/sdks/sandbox/python/README.md b/sdks/sandbox/python/README.md index 73db649a5..6826efaf8 100644 --- a/sdks/sandbox/python/README.md +++ b/sdks/sandbox/python/README.md @@ -229,6 +229,10 @@ Notes: idle buffer. `release_all_idle()` is only a best-effort cleanup pass in distributed mode because another primary may put new idle sandboxes concurrently unless the shared target has already been reduced. +- `release_all_idle()` preserves the original serial cleanup behavior. Use + `release_all_idle_parallel(max_workers=50)` for bounded parallel cleanup. The + parallel method requires a positive worker count and returns only after every ID + drained from the store has received a best-effort kill attempt. - Configure `primary_lock_ttl` greater than `warmup_ready_timeout` plus expected warmup preparer time and buffer. - Redis outages are surfaced as pool state store errors. The pool fails closed; it diff --git a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py index 040c0ab49..ba54a380e 100644 --- a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py +++ b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py @@ -214,6 +214,7 @@ async def _execute_streaming_request( handlers: ExecutionHandlers | None, infer_exit_code: bool, failure_message: str, + is_background: bool = False, ) -> Execution: execution = Execution( id=None, @@ -237,6 +238,12 @@ async def _execute_streaming_request( if event_node is None: continue await dispatcher.dispatch(event_node) + if is_background and event_node.type == "execution_complete": + # Background commands are done once execution_complete + # arrives; do not wait for the chunked terminator, which + # execd sends only after a graceful-shutdown sleep and can + # be lost if the connection is closed early (#1528). + break if infer_exit_code: execution.exit_code = _infer_foreground_exit_code(execution) @@ -268,6 +275,7 @@ async def run( handlers=handlers, infer_exit_code=not opts.background, failure_message="Failed to run command", + is_background=opts.background, ) except Exception as e: diff --git a/sdks/sandbox/python/src/opensandbox/adapters/converter/exception_converter.py b/sdks/sandbox/python/src/opensandbox/adapters/converter/exception_converter.py index b2eee13a2..a2a19fc5b 100644 --- a/sdks/sandbox/python/src/opensandbox/adapters/converter/exception_converter.py +++ b/sdks/sandbox/python/src/opensandbox/adapters/converter/exception_converter.py @@ -216,11 +216,29 @@ def _build_api_exception( retry_after: timedelta | None = None, ) -> SandboxApiException: """Build a Sandbox(ApiException|RateLimitException) from raw fields.""" + from opensandbox.adapters.converter.response_handler import ( + _raw_body_message_fragment, + ) + sandbox_error = _parse_error_body(content) if content else None + message = f"API error: HTTP {status_code}" + if sandbox_error is not None and sandbox_error.code != SandboxError.UNEXPECTED_RESPONSE: + if sandbox_error.message: + message = f"{message}: {sandbox_error.message}" + else: + # Unstructured body: splice the raw response body (truncated) into the + # message so logs carry the server's own explanation instead of only + # "API error: HTTP 400". The full body stays on ``response_body``. + # Structured codes with an empty message are preserved on the error. + raw_fragment = _raw_body_message_fragment(content) + if raw_fragment: + message = f"{message}: {raw_fragment}" + if sandbox_error is None or sandbox_error.code == SandboxError.UNEXPECTED_RESPONSE: + sandbox_error = SandboxError(SandboxError.UNEXPECTED_RESPONSE, raw_fragment) is_retryable = status_code in _RETRYABLE_STATUS_CODES if status_code == HTTPStatus.TOO_MANY_REQUESTS: return SandboxRateLimitException( - message=f"API error: HTTP {status_code}", + message=message, status_code=status_code, cause=cause, error=sandbox_error, @@ -230,7 +248,7 @@ def _build_api_exception( is_retryable=is_retryable, ) return SandboxApiException( - message=f"API error: HTTP {status_code}", + message=message, status_code=status_code, cause=cause, error=sandbox_error, diff --git a/sdks/sandbox/python/src/opensandbox/adapters/endpoint_cache.py b/sdks/sandbox/python/src/opensandbox/adapters/endpoint_cache.py index b037e92ce..e9dffdd5a 100644 --- a/sdks/sandbox/python/src/opensandbox/adapters/endpoint_cache.py +++ b/sdks/sandbox/python/src/opensandbox/adapters/endpoint_cache.py @@ -125,7 +125,8 @@ def get_or_fetch( raise finally: with self._lock: - self._inflight.pop(key, None) + if self._inflight.get(key) is inf: + self._inflight.pop(key) inf.event.set() @@ -214,4 +215,5 @@ async def get_or_fetch( future.exception() raise finally: - self._inflight.pop(key, None) + if self._inflight.get(key) is future: + self._inflight.pop(key) diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/__init__.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/__init__.py index 69dc795bc..d08d9360f 100644 --- a/sdks/sandbox/python/src/opensandbox/api/execd/models/__init__.py +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/__init__.py @@ -18,6 +18,8 @@ from .bind_mount import BindMount from .capabilities_response import CapabilitiesResponse +from .capabilities_response_hardening import CapabilitiesResponseHardening +from .capabilities_response_hardening_init_mode import CapabilitiesResponseHardeningInitMode from .chmod_files_body import ChmodFilesBody from .code_context import CodeContext from .code_context_request import CodeContextRequest @@ -34,6 +36,8 @@ from .file_info_type import FileInfoType from .file_metadata import FileMetadata from .get_files_info_response_200 import GetFilesInfoResponse200 +from .hardening_layer_state import HardeningLayerState +from .hardening_layer_state_state import HardeningLayerStateState from .isolated_background_run_response import IsolatedBackgroundRunResponse from .isolated_chmod_files_body import IsolatedChmodFilesBody from .isolated_create_session_response import IsolatedCreateSessionResponse @@ -75,6 +79,8 @@ __all__ = ( "BindMount", "CapabilitiesResponse", + "CapabilitiesResponseHardening", + "CapabilitiesResponseHardeningInitMode", "ChmodFilesBody", "CodeContext", "CodeContextRequest", @@ -91,6 +97,8 @@ "FileInfoType", "FileMetadata", "GetFilesInfoResponse200", + "HardeningLayerState", + "HardeningLayerStateState", "IsolatedBackgroundRunResponse", "IsolatedChmodFilesBody", "IsolatedCreateSessionResponse", diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response.py index 0fac8c0bf..bc45b18a7 100644 --- a/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response.py +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response.py @@ -17,13 +17,17 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from attrs import define as _attrs_define from attrs import field as _attrs_field from ..types import UNSET, Unset +if TYPE_CHECKING: + from ..models.capabilities_response_hardening import CapabilitiesResponseHardening + + T = TypeVar("T", bound="CapabilitiesResponse") @@ -41,6 +45,9 @@ class CapabilitiesResponse: userns_available (bool | Unset): Whether sessions using uid_mode userns can be created commit_supported (bool | Unset): diff_supported (bool | Unset): + hardening (CapabilitiesResponseHardening | Unset): execd init-mode and workload-hardening state (OSEP-0018): + whether execd is the sandbox init and which of its controls are in effect. Not an isolation capability; reported + here so operators see enforcement state in one place. """ available: bool | Unset = UNSET @@ -51,6 +58,7 @@ class CapabilitiesResponse: userns_available: bool | Unset = UNSET commit_supported: bool | Unset = UNSET diff_supported: bool | Unset = UNSET + hardening: CapabilitiesResponseHardening | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -70,6 +78,10 @@ def to_dict(self) -> dict[str, Any]: diff_supported = self.diff_supported + hardening: dict[str, Any] | Unset = UNSET + if not isinstance(self.hardening, Unset): + hardening = self.hardening.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -89,11 +101,15 @@ def to_dict(self) -> dict[str, Any]: field_dict["commit_supported"] = commit_supported if diff_supported is not UNSET: field_dict["diff_supported"] = diff_supported + if hardening is not UNSET: + field_dict["hardening"] = hardening return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.capabilities_response_hardening import CapabilitiesResponseHardening + d = dict(src_dict) available = d.pop("available", UNSET) @@ -111,6 +127,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: diff_supported = d.pop("diff_supported", UNSET) + _hardening = d.pop("hardening", UNSET) + hardening: CapabilitiesResponseHardening | Unset + if isinstance(_hardening, Unset): + hardening = UNSET + else: + hardening = CapabilitiesResponseHardening.from_dict(_hardening) + capabilities_response = cls( available=available, isolator=isolator, @@ -120,6 +143,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: userns_available=userns_available, commit_supported=commit_supported, diff_supported=diff_supported, + hardening=hardening, ) capabilities_response.additional_properties = d diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening.py new file mode 100644 index 000000000..53374623a --- /dev/null +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening.py @@ -0,0 +1,177 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.capabilities_response_hardening_init_mode import CapabilitiesResponseHardeningInitMode +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.hardening_layer_state import HardeningLayerState + + +T = TypeVar("T", bound="CapabilitiesResponseHardening") + + +@_attrs_define +class CapabilitiesResponseHardening: + """execd init-mode and workload-hardening state (OSEP-0018): whether execd is the sandbox init and which of its + controls are in effect. Not an isolation capability; reported here so operators see enforcement state in one place. + + Attributes: + init_mode (CapabilitiesResponseHardeningInitMode | Unset): How execd supervises the sandbox process tree. pid1: + execd is the kernel init of the container. subreaper: execd reaps orphans but lacks the PID 1 kernel signal + shield. none: init mode is off (default). + signal_shield (bool | Unset): Whether the kernel PID 1 signal shield protects execd from in-namespace signals + (true only in init_mode pid1). + cap_drop (HardeningLayerState | Unset): Whether one hardening layer is actually enforced. state is "active" | + "disabled" (not configured) | "degraded" (configured but a prerequisite is missing) | "unsupported" + (kernel/build cannot provide it). message gives the concrete reason whenever state is not active. + seccomp (HardeningLayerState | Unset): Whether one hardening layer is actually enforced. state is "active" | + "disabled" (not configured) | "degraded" (configured but a prerequisite is missing) | "unsupported" + (kernel/build cannot provide it). message gives the concrete reason whenever state is not active. + landlock (HardeningLayerState | Unset): Whether one hardening layer is actually enforced. state is "active" | + "disabled" (not configured) | "degraded" (configured but a prerequisite is missing) | "unsupported" + (kernel/build cannot provide it). message gives the concrete reason whenever state is not active. + ebpf (HardeningLayerState | Unset): Whether one hardening layer is actually enforced. state is "active" | + "disabled" (not configured) | "degraded" (configured but a prerequisite is missing) | "unsupported" + (kernel/build cannot provide it). message gives the concrete reason whenever state is not active. + """ + + init_mode: CapabilitiesResponseHardeningInitMode | Unset = UNSET + signal_shield: bool | Unset = UNSET + cap_drop: HardeningLayerState | Unset = UNSET + seccomp: HardeningLayerState | Unset = UNSET + landlock: HardeningLayerState | Unset = UNSET + ebpf: HardeningLayerState | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + init_mode: str | Unset = UNSET + if not isinstance(self.init_mode, Unset): + init_mode = self.init_mode.value + + signal_shield = self.signal_shield + + cap_drop: dict[str, Any] | Unset = UNSET + if not isinstance(self.cap_drop, Unset): + cap_drop = self.cap_drop.to_dict() + + seccomp: dict[str, Any] | Unset = UNSET + if not isinstance(self.seccomp, Unset): + seccomp = self.seccomp.to_dict() + + landlock: dict[str, Any] | Unset = UNSET + if not isinstance(self.landlock, Unset): + landlock = self.landlock.to_dict() + + ebpf: dict[str, Any] | Unset = UNSET + if not isinstance(self.ebpf, Unset): + ebpf = self.ebpf.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if init_mode is not UNSET: + field_dict["init_mode"] = init_mode + if signal_shield is not UNSET: + field_dict["signal_shield"] = signal_shield + if cap_drop is not UNSET: + field_dict["cap_drop"] = cap_drop + if seccomp is not UNSET: + field_dict["seccomp"] = seccomp + if landlock is not UNSET: + field_dict["landlock"] = landlock + if ebpf is not UNSET: + field_dict["ebpf"] = ebpf + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.hardening_layer_state import HardeningLayerState + + d = dict(src_dict) + _init_mode = d.pop("init_mode", UNSET) + init_mode: CapabilitiesResponseHardeningInitMode | Unset + if isinstance(_init_mode, Unset): + init_mode = UNSET + else: + init_mode = CapabilitiesResponseHardeningInitMode(_init_mode) + + signal_shield = d.pop("signal_shield", UNSET) + + _cap_drop = d.pop("cap_drop", UNSET) + cap_drop: HardeningLayerState | Unset + if isinstance(_cap_drop, Unset): + cap_drop = UNSET + else: + cap_drop = HardeningLayerState.from_dict(_cap_drop) + + _seccomp = d.pop("seccomp", UNSET) + seccomp: HardeningLayerState | Unset + if isinstance(_seccomp, Unset): + seccomp = UNSET + else: + seccomp = HardeningLayerState.from_dict(_seccomp) + + _landlock = d.pop("landlock", UNSET) + landlock: HardeningLayerState | Unset + if isinstance(_landlock, Unset): + landlock = UNSET + else: + landlock = HardeningLayerState.from_dict(_landlock) + + _ebpf = d.pop("ebpf", UNSET) + ebpf: HardeningLayerState | Unset + if isinstance(_ebpf, Unset): + ebpf = UNSET + else: + ebpf = HardeningLayerState.from_dict(_ebpf) + + capabilities_response_hardening = cls( + init_mode=init_mode, + signal_shield=signal_shield, + cap_drop=cap_drop, + seccomp=seccomp, + landlock=landlock, + ebpf=ebpf, + ) + + capabilities_response_hardening.additional_properties = d + return capabilities_response_hardening + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening_init_mode.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening_init_mode.py new file mode 100644 index 000000000..29ba4de5c --- /dev/null +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/capabilities_response_hardening_init_mode.py @@ -0,0 +1,26 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum + + +class CapabilitiesResponseHardeningInitMode(str, Enum): + NONE = "none" + PID1 = "pid1" + SUBREAPER = "subreaper" + + def __str__(self) -> str: + return str(self.value) diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state.py new file mode 100644 index 000000000..24f4886d4 --- /dev/null +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state.py @@ -0,0 +1,97 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.hardening_layer_state_state import HardeningLayerStateState +from ..types import UNSET, Unset + +T = TypeVar("T", bound="HardeningLayerState") + + +@_attrs_define +class HardeningLayerState: + """Whether one hardening layer is actually enforced. state is "active" | "disabled" (not configured) | "degraded" + (configured but a prerequisite is missing) | "unsupported" (kernel/build cannot provide it). message gives the + concrete reason whenever state is not active. + + Attributes: + state (HardeningLayerStateState | Unset): + message (str | Unset): + """ + + state: HardeningLayerStateState | Unset = UNSET + message: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + state: str | Unset = UNSET + if not isinstance(self.state, Unset): + state = self.state.value + + message = self.message + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if state is not UNSET: + field_dict["state"] = state + if message is not UNSET: + field_dict["message"] = message + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + _state = d.pop("state", UNSET) + state: HardeningLayerStateState | Unset + if isinstance(_state, Unset): + state = UNSET + else: + state = HardeningLayerStateState(_state) + + message = d.pop("message", UNSET) + + hardening_layer_state = cls( + state=state, + message=message, + ) + + hardening_layer_state.additional_properties = d + return hardening_layer_state + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state_state.py b/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state_state.py new file mode 100644 index 000000000..0611ccc0e --- /dev/null +++ b/sdks/sandbox/python/src/opensandbox/api/execd/models/hardening_layer_state_state.py @@ -0,0 +1,27 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum + + +class HardeningLayerStateState(str, Enum): + ACTIVE = "active" + DEGRADED = "degraded" + DISABLED = "disabled" + UNSUPPORTED = "unsupported" + + def __str__(self) -> str: + return str(self.value) diff --git a/sdks/sandbox/python/src/opensandbox/models/isolated.py b/sdks/sandbox/python/src/opensandbox/models/isolated.py index 47dbdbce5..e33fe57a8 100644 --- a/sdks/sandbox/python/src/opensandbox/models/isolated.py +++ b/sdks/sandbox/python/src/opensandbox/models/isolated.py @@ -323,6 +323,45 @@ class IsolatedRunLogs(BaseModel): model_config = ConfigDict(populate_by_name=True) +class HardeningLayerState(BaseModel): + """Whether one hardening layer is actually enforced (OSEP-0018).""" + + state: str | None = Field( + default=None, + description="active | disabled (not configured) | degraded | unsupported", + ) + message: str | None = Field( + default=None, + description="Concrete reason whenever state is not active", + ) + + +class HardeningStatus(BaseModel): + """execd init-mode and workload-hardening state (OSEP-0018).""" + + init_mode: str | None = Field( + default=None, description='"pid1" | "subreaper" | "none"' + ) + signal_shield: bool = Field( + default=False, + description="Whether the kernel PID 1 signal shield is active", + ) + cap_drop: HardeningLayerState | None = Field( + default=None, description="Capability/bounding-set reduction on user code" + ) + seccomp: HardeningLayerState | None = Field( + default=None, description="Seccomp floor installed on user code" + ) + landlock: HardeningLayerState | None = Field( + default=None, description="Landlock filesystem confinement" + ) + ebpf: HardeningLayerState | None = Field( + default=None, description="eBPF exec/connect/privilege observation" + ) + + model_config = ConfigDict(populate_by_name=True) + + class IsolatedCapabilities(BaseModel): """Isolator capabilities reported by execd.""" @@ -355,5 +394,9 @@ class IsolatedCapabilities(BaseModel): diff_supported: bool = Field( default=False, description="Whether diff is supported" ) + hardening: HardeningStatus | None = Field( + default=None, + description="execd init-mode and workload-hardening state (OSEP-0018)", + ) model_config = ConfigDict(populate_by_name=True) diff --git a/sdks/sandbox/python/src/opensandbox/pool_async.py b/sdks/sandbox/python/src/opensandbox/pool_async.py index 8f3da6894..0a2dcf185 100644 --- a/sdks/sandbox/python/src/opensandbox/pool_async.py +++ b/sdks/sandbox/python/src/opensandbox/pool_async.py @@ -58,6 +58,7 @@ logger = logging.getLogger(__name__) _WARMUP_TERMINATION_TIMEOUT_SECONDS = 5.0 +_RELEASE_ALL_IDLE_CONCURRENCY = 50 class SandboxPoolAsync: @@ -388,6 +389,86 @@ async def release_all_idle(self) -> int: await temporary_manager.close() return count + async def release_all_idle_parallel( + self, max_workers: int = _RELEASE_ALL_IDLE_CONCURRENCY + ) -> int: + if max_workers <= 0: + raise ValueError("max_workers must be positive") + + cleanup_task = asyncio.create_task( + self._release_all_idle_parallel(max_workers) + ) + cancellation: asyncio.CancelledError | None = None + cleanup_failure: BaseException | None = None + while not cleanup_task.done(): + try: + await asyncio.shield(cleanup_task) + except asyncio.CancelledError as exc: + cancellation = cancellation or exc + except BaseException as exc: + cleanup_failure = exc + + if cancellation is not None: + if cleanup_failure is None and cleanup_task.done(): + try: + cleanup_failure = cleanup_task.exception() + except asyncio.CancelledError: + pass + if cleanup_failure is not None: + raise cancellation from cleanup_failure + raise cancellation + if cleanup_failure is not None: + raise cleanup_failure + return cleanup_task.result() + + async def _release_all_idle_parallel(self, max_workers: int) -> int: + pool_name = self._config.pool_name + sandbox_ids: list[str] = [] + drain_error: Exception | None = None + temporary_manager: SandboxManager | None = None + try: + while True: + try: + sandbox_id = await self._state_store.try_take_idle(pool_name) + except Exception as exc: + drain_error = exc + break + if sandbox_id is None: + break + sandbox_ids.append(sandbox_id) + + if sandbox_ids: + manager = self._sandbox_manager + if manager is None: + try: + manager = await self._create_sandbox_manager() + temporary_manager = manager + except Exception as exc: + logger.warning( + f"release_all_idle_parallel: failed to create sandbox manager; draining idle ids without remote kill: pool_name={pool_name} error={exc}" + ) + + semaphore = asyncio.Semaphore(max_workers) + + async def kill(sandbox_id: str) -> None: + if manager is None: + return + async with semaphore: + try: + await manager.kill_sandbox(sandbox_id) + except Exception as exc: + logger.warning( + f"release_all_idle_parallel: failed to kill sandbox: pool_name={pool_name} sandbox_id={sandbox_id} error={exc}" + ) + + await asyncio.gather(*(kill(sandbox_id) for sandbox_id in sandbox_ids)) + finally: + if temporary_manager is not None: + await temporary_manager.close() + if drain_error is not None: + raise drain_error + return len(sandbox_ids) + async def snapshot(self) -> PoolSnapshot: lifecycle_state = self._lifecycle_state if lifecycle_state in ( diff --git a/sdks/sandbox/python/src/opensandbox/sandbox.py b/sdks/sandbox/python/src/opensandbox/sandbox.py index 7b3fd440d..dbbca38ed 100644 --- a/sdks/sandbox/python/src/opensandbox/sandbox.py +++ b/sdks/sandbox/python/src/opensandbox/sandbox.py @@ -482,22 +482,9 @@ async def check_ready( f"ConnectionConfig(domain={self.connection_config.get_domain()}, " f"use_server_proxy={self.connection_config.use_server_proxy})" ) - if self.connection_config.use_server_proxy: - hint = ( - "Hint: server proxy mode is enabled. Check server-to-sandbox connectivity " - "and server API key/auth configuration." - ) - else: - hint = ( - "Hint: direct sandbox endpoint access is enabled. If the SDK cannot directly " - "reach sandbox network/ports, set ConnectionConfig(use_server_proxy=True). " - "For Docker bridge deployments where server runs in a container, also configure " - "server [docker].host_ip to a host-reachable address." - ) - final_message = ( f"Sandbox health check timed out after {timeout.total_seconds()}s " - f"({attempt} attempts). {error_detail}. {connection_detail}. {hint}" + f"({attempt} attempts). {error_detail}. {connection_detail}." ) logger.error(final_message) @@ -603,11 +590,13 @@ async def create( ) sandbox_id = response.id - execd_endpoint = await sandbox_service.get_sandbox_endpoint( - response.id, DEFAULT_EXECD_PORT, config.use_server_proxy - ) - egress_endpoint = await sandbox_service.get_sandbox_endpoint( - response.id, DEFAULT_EGRESS_PORT, config.use_server_proxy + execd_endpoint, egress_endpoint = await asyncio.gather( + sandbox_service.get_sandbox_endpoint( + response.id, DEFAULT_EXECD_PORT, config.use_server_proxy + ), + sandbox_service.get_sandbox_endpoint( + response.id, DEFAULT_EGRESS_PORT, config.use_server_proxy + ), ) sandbox = cls( @@ -715,11 +704,13 @@ async def connect( try: sandbox_service = factory.create_sandbox_service() - execd_endpoint = await sandbox_service.get_sandbox_endpoint( - sandbox_id, DEFAULT_EXECD_PORT, config.use_server_proxy - ) - egress_endpoint = await sandbox_service.get_sandbox_endpoint( - sandbox_id, DEFAULT_EGRESS_PORT, config.use_server_proxy + execd_endpoint, egress_endpoint = await asyncio.gather( + sandbox_service.get_sandbox_endpoint( + sandbox_id, DEFAULT_EXECD_PORT, config.use_server_proxy + ), + sandbox_service.get_sandbox_endpoint( + sandbox_id, DEFAULT_EGRESS_PORT, config.use_server_proxy + ), ) sandbox = cls( @@ -795,11 +786,13 @@ async def resume( sandbox_service = factory.create_sandbox_service() await sandbox_service.resume_sandbox(sandbox_id) - execd_endpoint = await sandbox_service.get_sandbox_endpoint( - sandbox_id, DEFAULT_EXECD_PORT, config.use_server_proxy - ) - egress_endpoint = await sandbox_service.get_sandbox_endpoint( - sandbox_id, DEFAULT_EGRESS_PORT, config.use_server_proxy + execd_endpoint, egress_endpoint = await asyncio.gather( + sandbox_service.get_sandbox_endpoint( + sandbox_id, DEFAULT_EXECD_PORT, config.use_server_proxy + ), + sandbox_service.get_sandbox_endpoint( + sandbox_id, DEFAULT_EGRESS_PORT, config.use_server_proxy + ), ) sandbox = cls( diff --git a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py index 6681bb0c2..e47d79a79 100644 --- a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py +++ b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py @@ -190,6 +190,7 @@ def _execute_streaming_request( handlers: ExecutionHandlersSync | None, infer_exit_code: bool, failure_message: str, + is_background: bool = False, ) -> Execution: execution = Execution(id=None, execution_count=None, result=[], error=None) dispatcher = ExecutionEventDispatcherSync(execution, handlers) @@ -204,6 +205,12 @@ def _execute_streaming_request( if event_node is None: continue dispatcher.dispatch(event_node) + if is_background and event_node.type == "execution_complete": + # Background commands are done once execution_complete + # arrives; do not wait for the chunked terminator, which + # execd sends only after a graceful-shutdown sleep and can + # be lost if the connection is closed early (#1528). + break if infer_exit_code: execution.exit_code = _infer_foreground_exit_code(execution) @@ -230,6 +237,7 @@ def run( handlers=handlers, infer_exit_code=not opts.background, failure_message="Failed to run command", + is_background=opts.background, ) except Exception as e: diff --git a/sdks/sandbox/python/src/opensandbox/sync/pool.py b/sdks/sandbox/python/src/opensandbox/sync/pool.py index 25d8a4755..33c973e8f 100644 --- a/sdks/sandbox/python/src/opensandbox/sync/pool.py +++ b/sdks/sandbox/python/src/opensandbox/sync/pool.py @@ -57,6 +57,7 @@ logger = logging.getLogger(__name__) _WARMUP_TERMINATION_TIMEOUT_SECONDS = 5.0 +_RELEASE_ALL_IDLE_CONCURRENCY = 50 class SandboxPoolSync: @@ -403,6 +404,59 @@ def release_all_idle(self) -> int: temporary_manager.close() return count + def release_all_idle_parallel( + self, max_workers: int = _RELEASE_ALL_IDLE_CONCURRENCY + ) -> int: + if max_workers <= 0: + raise ValueError("max_workers must be positive") + pool_name = self._config.pool_name + sandbox_ids: list[str] = [] + drain_error: Exception | None = None + temporary_manager: SandboxManagerSync | None = None + try: + while True: + try: + sandbox_id = self._state_store.try_take_idle(pool_name) + except Exception as exc: + drain_error = exc + break + if sandbox_id is None: + break + sandbox_ids.append(sandbox_id) + + if sandbox_ids: + manager = self._sandbox_manager + if manager is None: + try: + manager = self._create_sandbox_manager() + temporary_manager = manager + except Exception as exc: + logger.warning( + f"release_all_idle_parallel: failed to create sandbox manager; draining idle ids without remote kill: pool_name={pool_name} error={exc}" + ) + + def kill(sandbox_id: str) -> None: + if manager is None: + return + try: + manager.kill_sandbox(sandbox_id) + except Exception as exc: + logger.warning( + f"release_all_idle_parallel: failed to kill sandbox: pool_name={pool_name} sandbox_id={sandbox_id} error={exc}" + ) + + with ThreadPoolExecutor( + max_workers=min(max_workers, len(sandbox_ids)), + thread_name_prefix="sandbox-pool-release", + ) as executor: + list(executor.map(kill, sandbox_ids)) + finally: + if temporary_manager is not None: + temporary_manager.close() + if drain_error is not None: + raise drain_error + return len(sandbox_ids) + def snapshot(self) -> PoolSnapshot: lifecycle_state = self._lifecycle_state if lifecycle_state in ( diff --git a/sdks/sandbox/python/src/opensandbox/sync/sandbox.py b/sdks/sandbox/python/src/opensandbox/sync/sandbox.py index e75b25680..5d9ce65ac 100644 --- a/sdks/sandbox/python/src/opensandbox/sync/sandbox.py +++ b/sdks/sandbox/python/src/opensandbox/sync/sandbox.py @@ -469,21 +469,9 @@ def check_ready(self, timeout: timedelta, polling_interval: timedelta) -> None: f"ConnectionConfig(domain={self.connection_config.get_domain()}, " f"use_server_proxy={self.connection_config.use_server_proxy})" ) - if self.connection_config.use_server_proxy: - hint = ( - "Hint: server proxy mode is enabled. Check server-to-sandbox connectivity " - "and server API key/auth configuration." - ) - else: - hint = ( - "Hint: direct sandbox endpoint access is enabled. If the SDK cannot directly " - "reach sandbox network/ports, set ConnectionConfigSync(use_server_proxy=True). " - "For Docker bridge deployments where server runs in a container, also configure " - "server [docker].host_ip to a host-reachable address." - ) final_message = ( f"Sandbox health check timed out after {timeout.total_seconds()}s " - f"({attempt} attempts). {error_detail}. {connection_detail}. {hint}" + f"({attempt} attempts). {error_detail}. {connection_detail}." ) logger.error(final_message) raise SandboxReadyTimeoutException(final_message) diff --git a/sdks/sandbox/python/tests/test_command_service_adapter_streaming.py b/sdks/sandbox/python/tests/test_command_service_adapter_streaming.py index a5b0fa8a3..368dd9c17 100644 --- a/sdks/sandbox/python/tests/test_command_service_adapter_streaming.py +++ b/sdks/sandbox/python/tests/test_command_service_adapter_streaming.py @@ -23,7 +23,12 @@ from opensandbox.adapters.command_adapter import CommandsAdapter from opensandbox.config import ConnectionConfig -from opensandbox.exceptions import InvalidArgumentException, SandboxApiException +from opensandbox.exceptions import ( + InvalidArgumentException, + SandboxApiException, + SandboxConnectionException, +) +from opensandbox.models.execd import RunCommandOpts from opensandbox.models.sandboxes import SandboxEndpoint _UNICODE_SEPARATORS = "before\u0085middle\u2028middle\u2029after" @@ -256,3 +261,75 @@ async def test_run_in_session_non_zero_exit_updates_exit_code() -> None: assert execution.error.value == "7" assert execution.complete is None assert execution.exit_code == 7 + + +class _EarlyCloseAfterCompleteStream(httpx.AsyncByteStream): + """Yields SSE bytes then simulates the connection closing before the + chunked terminator arrives (regression case for #1528).""" + + def __init__(self, sse: bytes) -> None: + self._sse = sse + + async def __aiter__(self): + yield self._sse + raise httpx.RemoteProtocolError( + "peer closed connection without sending complete message body " + "(incomplete chunked read)" + ) + + +class _EarlyCloseTransport(httpx.AsyncBaseTransport): + """Transport whose SSE response body closes early right after the + ``execution_complete`` event, before the chunked terminator is sent.""" + + def __init__(self, sse: bytes) -> None: + self._sse = sse + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + stream=_EarlyCloseAfterCompleteStream(self._sse), + request=request, + ) + + +_EARLY_CLOSE_SSE = ( + b'data: {"type":"init","text":"exec-bg","timestamp":1}\n\n' + b'data: {"type":"execution_complete","timestamp":2,"execution_time":3}\n\n' +) + + +@pytest.mark.asyncio +async def test_run_background_command_breaks_on_complete_before_terminator() -> None: + """Background commands must not wait for the chunked terminator: once + ``execution_complete`` arrives, the SDK should stop reading the stream + even if the connection is closed early (#1528).""" + cfg = ConnectionConfig( + protocol="http", transport=_EarlyCloseTransport(_EARLY_CLOSE_SSE) + ) + endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772) + adapter = CommandsAdapter(cfg, endpoint) + + execution = await adapter.run("sleep 1", opts=RunCommandOpts(background=True)) + + assert execution.id == "exec-bg" + assert execution.complete is not None + assert execution.complete.execution_time_in_millis == 3 + # Background executions do not synthesize an exit code from the stream. + assert execution.exit_code is None + + +@pytest.mark.asyncio +async def test_run_foreground_command_still_waits_for_terminator() -> None: + """Foreground commands must keep waiting for the stream terminator + after ``execution_complete`` โ€” an early close is still surfaced as an + error, proving the background early-break did not change this path.""" + cfg = ConnectionConfig( + protocol="http", transport=_EarlyCloseTransport(_EARLY_CLOSE_SSE) + ) + endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772) + adapter = CommandsAdapter(cfg, endpoint) + + with pytest.raises(SandboxConnectionException): + await adapter.run("sleep 1", opts=RunCommandOpts(background=False)) diff --git a/sdks/sandbox/python/tests/test_converters_and_error_handling.py b/sdks/sandbox/python/tests/test_converters_and_error_handling.py index 334f14217..4048afd2a 100644 --- a/sdks/sandbox/python/tests/test_converters_and_error_handling.py +++ b/sdks/sandbox/python/tests/test_converters_and_error_handling.py @@ -45,6 +45,7 @@ InvalidArgumentException, SandboxApiException, SandboxInternalException, + SandboxRateLimitException, ) from opensandbox.models.execd import RunCommandOpts from opensandbox.models.sandboxes import ( @@ -383,6 +384,65 @@ def test_exception_converter_maps_generated_unexpected_status_to_api_exception() assert converted.error.code == "X" +def test_exception_converter_splices_unstructured_body_into_message() -> None: + body = b'{"error": "invalid parameter"}' # JSON without code/message envelope + err = UnexpectedStatus(400, body) + + converted = ExceptionConverter.to_sandbox_exception(err) + + assert isinstance(converted, SandboxApiException) + assert converted.status_code == 400 + # The raw body is spliced into str()/message so logs show the server reason. + assert '{"error": "invalid parameter"}' in str(converted) + # Full body still available on the exception field. + assert converted.response_body == body + + +def test_exception_converter_splices_plain_text_body_into_message() -> None: + body = b"cursor must be positive" + err = UnexpectedStatus(400, body) + + converted = ExceptionConverter.to_sandbox_exception(err) + + assert isinstance(converted, SandboxApiException) + assert "cursor must be positive" in str(converted) + assert converted.response_body == body + + +def test_exception_converter_maps_unstructured_429_body_into_message() -> None: + body = b"quota exhausted for tenant foo" + err = UnexpectedStatus(429, body) + + converted = ExceptionConverter.to_sandbox_exception(err) + + assert isinstance(converted, SandboxRateLimitException) + assert "quota exhausted for tenant foo" in str(converted) + assert converted.response_body == body + + +def test_exception_converter_truncates_long_unstructured_body_in_message() -> None: + body = b"x" * 2000 + err = UnexpectedStatus(502, body) + + converted = ExceptionConverter.to_sandbox_exception(err) + + assert isinstance(converted, SandboxApiException) + assert converted.response_body == body + assert "โ€ฆ" in str(converted) + assert len(str(converted)) < 1500 + + +def test_exception_converter_preserves_structured_code_without_message() -> None: + err = UnexpectedStatus(429, b'{"code":"QUOTA"}') + + converted = ExceptionConverter.to_sandbox_exception(err) + + assert isinstance(converted, SandboxRateLimitException) + # Structured code is preserved even when the body has no message field. + assert converted.error is not None + assert converted.error.code == "QUOTA" + + def test_exception_converter_maps_httpx_status_error_to_api_exception() -> None: request = Request("GET", "https://example.test") response = Response( diff --git a/sdks/sandbox/python/tests/test_endpoint_cache.py b/sdks/sandbox/python/tests/test_endpoint_cache.py index 55af56a50..1329381aa 100644 --- a/sdks/sandbox/python/tests/test_endpoint_cache.py +++ b/sdks/sandbox/python/tests/test_endpoint_cache.py @@ -107,6 +107,72 @@ def fetch(): assert result.endpoint == "cached" assert fetch_count[0] == 0 + def test_invalidate_does_not_remove_replacement_inflight(self): + c = EndpointCache(maxsize=10, ttl=60.0) + key = ("sb-1", 8080, False) + first_started = threading.Event() + release_first = threading.Event() + second_started = threading.Event() + release_second = threading.Event() + fetch_count = 0 + fetch_count_lock = threading.Lock() + first_result = [] + second_result = [] + + def fetch(): + nonlocal fetch_count + with fetch_count_lock: + fetch_count += 1 + call = fetch_count + if call == 1: + first_started.set() + assert release_first.wait(timeout=2) + return _ep("first") + if call == 2: + second_started.set() + assert release_second.wait(timeout=2) + return _ep("second") + raise AssertionError("unexpected duplicate fetch") + + first_thread = threading.Thread( + target=lambda: first_result.append(c.get_or_fetch(key, fetch)) + ) + second_thread = None + try: + first_thread.start() + assert first_started.wait(timeout=2) + with c._lock: + first_inflight = c._inflight[key] + + c.invalidate("sb-1") + second_thread = threading.Thread( + target=lambda: second_result.append(c.get_or_fetch(key, fetch)) + ) + second_thread.start() + assert second_started.wait(timeout=2) + with c._lock: + second_inflight = c._inflight[key] + assert second_inflight is not first_inflight + + release_first.set() + first_thread.join(timeout=2) + assert not first_thread.is_alive() + with c._lock: + assert c._inflight.get(key) is second_inflight + + release_second.set() + second_thread.join(timeout=2) + assert not second_thread.is_alive() + assert [result.endpoint for result in first_result] == ["first"] + assert [result.endpoint for result in second_result] == ["second"] + assert fetch_count == 2 + finally: + release_first.set() + release_second.set() + first_thread.join(timeout=2) + if second_thread is not None: + second_thread.join(timeout=2) + class TestAsyncEndpointCache: @pytest.mark.asyncio @@ -185,3 +251,44 @@ async def fetch(): # Cache should not be populated on error assert c.get(key) is None assert not [r for r in caplog.records if r.levelname == "ERROR"] + + @pytest.mark.asyncio + async def test_invalidate_does_not_remove_replacement_inflight(self): + c = AsyncEndpointCache(maxsize=10, ttl=60.0) + key = ("sb-1", 8080, False) + first_started = asyncio.Event() + release_first = asyncio.Event() + second_started = asyncio.Event() + release_second = asyncio.Event() + fetch_count = 0 + + async def fetch(): + nonlocal fetch_count + fetch_count += 1 + if fetch_count == 1: + first_started.set() + await release_first.wait() + return _ep("first") + if fetch_count == 2: + second_started.set() + await release_second.wait() + return _ep("second") + raise AssertionError("unexpected duplicate fetch") + + first_task = asyncio.create_task(c.get_or_fetch(key, fetch)) + await asyncio.wait_for(first_started.wait(), timeout=2) + first_inflight = c._inflight[key] + + c.invalidate("sb-1") + second_task = asyncio.create_task(c.get_or_fetch(key, fetch)) + await asyncio.wait_for(second_started.wait(), timeout=2) + second_inflight = c._inflight[key] + assert second_inflight is not first_inflight + + release_first.set() + assert await first_task == _ep("first") + assert c._inflight.get(key) is second_inflight + + release_second.set() + assert await second_task == _ep("second") + assert fetch_count == 2 diff --git a/sdks/sandbox/python/tests/test_pool_async.py b/sdks/sandbox/python/tests/test_pool_async.py index 0528bd21e..2643d03d4 100644 --- a/sdks/sandbox/python/tests/test_pool_async.py +++ b/sdks/sandbox/python/tests/test_pool_async.py @@ -55,6 +55,134 @@ async def test_async_acquire_fail_fast_empty_raises_pool_empty() -> None: await pool.shutdown(False) +@pytest.mark.asyncio +async def test_release_all_idle_preserves_serial_behavior() -> None: + store = InMemoryAsyncPoolStateStore() + for index in range(3): + await store.put_idle("pool", f"idle-{index}") + + class TrackingManager(FakeAsyncManager): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + + async def kill_sandbox(self, sandbox_id: str) -> None: + self.active += 1 + self.max_active = max(self.max_active, self.active) + await asyncio.sleep(0) + self.killed.append(sandbox_id) + self.active -= 1 + + manager = TrackingManager() + pool = _create_pool(max_idle=0, store=store, manager=manager) + + released = await pool.release_all_idle() + + assert released == 3 + assert manager.max_active == 1 + assert len(manager.killed) == 3 + assert manager.closed + + +@pytest.mark.asyncio +async def test_release_all_idle_parallel_rejects_nonpositive_workers() -> None: + pool = _create_pool(max_idle=0) + + with pytest.raises(ValueError, match="max_workers must be positive"): + await pool.release_all_idle_parallel(0) + + +@pytest.mark.asyncio +async def test_release_all_idle_bounds_kills_and_cleans_up_before_store_failure() -> ( + None +): + class FailingStore(InMemoryAsyncPoolStateStore): + def __init__(self) -> None: + super().__init__() + self.takes = 0 + + async def try_take_idle(self, pool_name: str) -> str | None: + if self.takes == 55: + raise RuntimeError("injected store failure") + self.takes += 1 + return await super().try_take_idle(pool_name) + + store = FailingStore() + for index in range(55): + await store.put_idle("pool", f"idle-{index}") + + class ConcurrentManager(FakeAsyncManager): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + self.ready = asyncio.Event() + + async def kill_sandbox(self, sandbox_id: str) -> None: + self.active += 1 + self.max_active = max(self.max_active, self.active) + if self.active == 50: + self.ready.set() + await self.ready.wait() + self.killed.append(sandbox_id) + self.active -= 1 + if sandbox_id == "idle-0": + raise RuntimeError("injected kill failure") + + manager = ConcurrentManager() + pool = _create_pool(max_idle=0, store=store, manager=manager) + + with pytest.raises(RuntimeError, match="injected store failure"): + await asyncio.wait_for(pool.release_all_idle_parallel(), timeout=2) + + assert manager.max_active == 50 + assert len(manager.killed) == 55 + assert (await store.snapshot_counters("pool")).idle_count == 0 + assert manager.closed + + +@pytest.mark.asyncio +async def test_release_all_idle_parallel_finishes_kills_before_cancellation() -> None: + store = InMemoryAsyncPoolStateStore() + for index in range(55): + await store.put_idle("pool", f"idle-{index}") + + class BlockingManager(FakeAsyncManager): + def __init__(self) -> None: + super().__init__() + self.started = 0 + self.first_batch_started = asyncio.Event() + self.release_kills = asyncio.Event() + + async def kill_sandbox(self, sandbox_id: str) -> None: + self.started += 1 + if self.started == 50: + self.first_batch_started.set() + await self.release_kills.wait() + self.killed.append(sandbox_id) + + manager = BlockingManager() + pool = _create_pool(max_idle=0, store=store, manager=manager) + release_task = asyncio.create_task(pool.release_all_idle_parallel()) + await asyncio.wait_for(manager.first_batch_started.wait(), timeout=2) + + try: + release_task.cancel() + await asyncio.sleep(0) + assert not release_task.done() + manager.release_kills.set() + with pytest.raises(asyncio.CancelledError): + await release_task + finally: + manager.release_kills.set() + await asyncio.gather(release_task, return_exceptions=True) + + assert len(manager.killed) == 55 + assert (await store.snapshot_counters("pool")).idle_count == 0 + assert manager.closed + + @pytest.mark.asyncio async def test_async_reconcile_batch_failures_only_advance_backoff_once() -> None: store = InMemoryAsyncPoolStateStore() diff --git a/sdks/sandbox/python/tests/test_pool_sync.py b/sdks/sandbox/python/tests/test_pool_sync.py index ce5de45e4..b9066b23f 100644 --- a/sdks/sandbox/python/tests/test_pool_sync.py +++ b/sdks/sandbox/python/tests/test_pool_sync.py @@ -109,6 +109,91 @@ def test_acquire_fail_fast_empty_raises_pool_empty() -> None: pool.shutdown(False) +def test_release_all_idle_bounds_kills_and_cleans_up_before_store_failure() -> None: + class FailingStore(InMemoryPoolStateStore): + def __init__(self) -> None: + super().__init__() + self.takes = 0 + + def try_take_idle(self, pool_name: str) -> str | None: + if self.takes == 55: + raise RuntimeError("injected store failure") + self.takes += 1 + return super().try_take_idle(pool_name) + + store = FailingStore() + for index in range(55): + store.put_idle("pool", f"idle-{index}") + + class ConcurrentManager(FakeManager): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + self.lock = threading.Lock() + self.ready = threading.Event() + + def kill_sandbox(self, sandbox_id: str) -> None: + with self.lock: + self.active += 1 + self.max_active = max(self.max_active, self.active) + if self.active == 50: + self.ready.set() + assert self.ready.wait(timeout=2) + with self.lock: + self.killed.append(sandbox_id) + self.active -= 1 + if sandbox_id == "idle-0": + raise RuntimeError("injected kill failure") + + manager = ConcurrentManager() + pool = _create_pool(max_idle=0, store=store, manager=manager) + + with pytest.raises(RuntimeError, match="injected store failure"): + pool.release_all_idle_parallel() + + assert manager.max_active == 50 + assert len(manager.killed) == 55 + assert store.snapshot_counters("pool").idle_count == 0 + assert manager.closed + + +def test_release_all_idle_preserves_serial_behavior() -> None: + store = InMemoryPoolStateStore() + for index in range(3): + store.put_idle("pool", f"idle-{index}") + + class TrackingManager(FakeManager): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + + def kill_sandbox(self, sandbox_id: str) -> None: + self.active += 1 + self.max_active = max(self.max_active, self.active) + time.sleep(0.001) + self.killed.append(sandbox_id) + self.active -= 1 + + manager = TrackingManager() + pool = _create_pool(max_idle=0, store=store, manager=manager) + + released = pool.release_all_idle() + + assert released == 3 + assert manager.max_active == 1 + assert len(manager.killed) == 3 + assert manager.closed + + +def test_release_all_idle_parallel_rejects_nonpositive_workers() -> None: + pool = _create_pool(max_idle=0) + + with pytest.raises(ValueError, match="max_workers must be positive"): + pool.release_all_idle_parallel(0) + + def test_acquire_fail_fast_stale_idle_raises_and_kills_candidate() -> None: store = InMemoryPoolStateStore() store.put_idle("pool", "stale-1") diff --git a/sdks/sandbox/python/tests/test_sandbox_business_logic.py b/sdks/sandbox/python/tests/test_sandbox_business_logic.py index 5f5296355..2dd3a4e11 100644 --- a/sdks/sandbox/python/tests/test_sandbox_business_logic.py +++ b/sdks/sandbox/python/tests/test_sandbox_business_logic.py @@ -23,7 +23,10 @@ from opensandbox.config import ConnectionConfig from opensandbox.constants import DEFAULT_EGRESS_PORT, DEFAULT_EXECD_PORT -from opensandbox.exceptions import SandboxReadyTimeoutException +from opensandbox.exceptions import ( + SandboxInternalException, + SandboxReadyTimeoutException, +) from opensandbox.models.diagnostics import DiagnosticContent from opensandbox.models.sandboxes import NetworkPolicy, NetworkRule, SandboxEndpoint from opensandbox.sandbox import Sandbox @@ -172,7 +175,7 @@ async def _always_false(_: Sandbox) -> bool: @pytest.mark.asyncio -async def test_check_ready_timeout_message_includes_troubleshooting_hints() -> None: +async def test_check_ready_timeout_message_omits_network_configuration_hints() -> None: async def _always_false(_: Sandbox) -> bool: return False @@ -188,7 +191,9 @@ async def _always_false(_: Sandbox) -> bool: message = str(exc_info.value) assert "ConnectionConfig(domain=10.0.0.1:8080, use_server_proxy=False)" in message - assert "ConnectionConfig(use_server_proxy=True)" in message + assert "set connectionconfig(use_server_proxy=true)" not in message.lower() + assert "direct sandbox endpoint access" not in message + assert "[docker].host_ip" not in message @pytest.mark.asyncio @@ -348,6 +353,200 @@ async def _healthy(_sbx: Sandbox) -> bool: ] +class _GatedEndpointServiceStub: + """get_sandbox_endpoint that blocks the execd request until released. + + Detects serial endpoint resolution: when the two endpoint requests are + awaited sequentially, the egress request cannot start while the execd + request is still blocked. + """ + + def __init__(self) -> None: + self.execd_entered = asyncio.Event() + self.egress_entered = asyncio.Event() + self.release = asyncio.Event() + + async def get_sandbox_endpoint( + self, _sandbox_id, port: int, _use_server_proxy: bool = False + ) -> SandboxEndpoint: + if port == DEFAULT_EXECD_PORT: + self.execd_entered.set() + await self.release.wait() + else: + self.egress_entered.set() + return SandboxEndpoint(endpoint=f"sbx.internal:{port}") + + +async def _assert_parallel_endpoint_resolution( + gate: _GatedEndpointServiceStub, op +) -> None: + task = asyncio.create_task(op()) + await asyncio.wait_for(gate.execd_entered.wait(), timeout=1) + try: + # Fails if the egress endpoint is requested only after the execd + # endpoint request has completed. + await asyncio.wait_for(gate.egress_entered.wait(), timeout=1) + finally: + gate.release.set() + await task + + +@pytest.mark.parametrize("flow", ["create", "connect", "resume"]) +@pytest.mark.asyncio +async def test_sandbox_resolves_endpoints_in_parallel( + monkeypatch: pytest.MonkeyPatch, flow: str +) -> None: + gate = _GatedEndpointServiceStub() + + class _CreateResponse: + id = "sbx-1" + + class _SandboxServiceStub: + async def create_sandbox(self, *_args, **_kwargs): + return _CreateResponse() + + async def resume_sandbox(self, _sandbox_id: str) -> None: + return None + + async def kill_sandbox(self, _sandbox_id: str) -> None: + return None + + async def get_sandbox_endpoint( + self, sandbox_id, port: int, use_server_proxy: bool = False + ) -> SandboxEndpoint: + return await gate.get_sandbox_endpoint(sandbox_id, port, use_server_proxy) + + class _FactoryStub: + def __init__(self, _connection_config: ConnectionConfig) -> None: + pass + + def create_sandbox_service(self): + return sandbox_service + + def create_filesystem_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_command_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_health_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_metrics_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_egress_service(self, _endpoint: SandboxEndpoint): + return _EgressServiceStub() + + def create_diagnostics_service(self): + return _DiagnosticsServiceStub() + + def create_isolated_session_service(self, endpoint: SandboxEndpoint): + return _Noop() + + sandbox_service = _SandboxServiceStub() + monkeypatch.setattr("opensandbox.sandbox.AdapterFactory", _FactoryStub) + + async def _op() -> Sandbox: + if flow == "create": + return await Sandbox.create( + "python:3.11", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + if flow == "connect": + return await Sandbox.connect( + "sbx-1", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + return await Sandbox.resume( + "sbx-1", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + + await _assert_parallel_endpoint_resolution(gate, _op) + + +@pytest.mark.parametrize("flow", ["create", "connect", "resume"]) +@pytest.mark.parametrize("failing_port", [DEFAULT_EXECD_PORT, DEFAULT_EGRESS_PORT]) +@pytest.mark.asyncio +async def test_sandbox_errors_when_either_endpoint_resolution_fails( + monkeypatch: pytest.MonkeyPatch, flow: str, failing_port: int +) -> None: + class _CreateResponse: + id = "sbx-1" + + class _SandboxServiceStub: + async def create_sandbox(self, *_args, **_kwargs): + return _CreateResponse() + + async def resume_sandbox(self, _sandbox_id: str) -> None: + return None + + async def kill_sandbox(self, _sandbox_id: str) -> None: + return None + + async def get_sandbox_endpoint( + self, _sandbox_id, port: int, _use_server_proxy: bool = False + ) -> SandboxEndpoint: + if port == failing_port: + raise RuntimeError(f"endpoint resolution failed for port {port}") + return SandboxEndpoint(endpoint=f"sbx.internal:{port}") + + class _FactoryStub: + def __init__(self, _connection_config: ConnectionConfig) -> None: + pass + + def create_sandbox_service(self): + return sandbox_service + + def create_filesystem_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_command_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_health_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_metrics_service(self, _endpoint: SandboxEndpoint): + return _Noop() + + def create_egress_service(self, _endpoint: SandboxEndpoint): + return _EgressServiceStub() + + def create_diagnostics_service(self): + return _DiagnosticsServiceStub() + + def create_isolated_session_service(self, endpoint: SandboxEndpoint): + return _Noop() + + sandbox_service = _SandboxServiceStub() + monkeypatch.setattr("opensandbox.sandbox.AdapterFactory", _FactoryStub) + + with pytest.raises(SandboxInternalException): + if flow == "create": + await Sandbox.create( + "python:3.11", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + elif flow == "connect": + await Sandbox.connect( + "sbx-1", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + else: + await Sandbox.resume( + "sbx-1", + skip_health_check=True, + connection_config=ConnectionConfig(), + ) + + @pytest.mark.asyncio async def test_create_cancellation_cleans_up_created_sandbox( monkeypatch: pytest.MonkeyPatch, diff --git a/sdks/sandbox/python/tests/test_sandbox_sync_business_logic.py b/sdks/sandbox/python/tests/test_sandbox_sync_business_logic.py index 9c2ab4b9a..8d19ed231 100644 --- a/sdks/sandbox/python/tests/test_sandbox_sync_business_logic.py +++ b/sdks/sandbox/python/tests/test_sandbox_sync_business_logic.py @@ -84,7 +84,7 @@ def get_events(self, sandbox_id: str, scope: str) -> DiagnosticContent: ) -def test_sync_check_ready_timeout_message_includes_troubleshooting_hints() -> None: +def test_sync_check_ready_timeout_message_omits_network_configuration_hints() -> None: def _always_false(_: SandboxSync) -> bool: return False @@ -109,7 +109,9 @@ def _always_false(_: SandboxSync) -> bool: message = str(exc_info.value) assert "ConnectionConfig(domain=10.0.0.2:8080, use_server_proxy=False)" in message - assert "ConnectionConfigSync(use_server_proxy=True)" in message + assert "set connectionconfigsync(use_server_proxy=true)" not in message.lower() + assert "direct sandbox endpoint access" not in message + assert "[docker].host_ip" not in message def test_sync_get_egress_policy_uses_injected_egress_service() -> None: diff --git a/sdks/sandbox/python/tests/test_sync_command_service_adapter_streaming.py b/sdks/sandbox/python/tests/test_sync_command_service_adapter_streaming.py index 9361993dc..ef119950a 100644 --- a/sdks/sandbox/python/tests/test_sync_command_service_adapter_streaming.py +++ b/sdks/sandbox/python/tests/test_sync_command_service_adapter_streaming.py @@ -19,8 +19,11 @@ from datetime import timedelta import httpx +import pytest from opensandbox.config.connection_sync import ConnectionConfigSync +from opensandbox.exceptions import SandboxConnectionException +from opensandbox.models.execd import RunCommandOpts from opensandbox.models.sandboxes import SandboxEndpoint from opensandbox.sync.adapters.command_adapter import CommandsAdapterSync @@ -202,3 +205,73 @@ def test_sync_run_in_session_non_zero_exit_updates_exit_code() -> None: assert execution.error.value == "7" assert execution.complete is None assert execution.exit_code == 7 + + +class _EarlyCloseAfterCompleteStream(httpx.SyncByteStream): + """Byte stream that yields the SSE body then fails, simulating a peer + that closes the connection before sending the chunked terminator.""" + + def __init__(self, sse: bytes) -> None: + self._sse = sse + + def __iter__(self): + yield self._sse + raise httpx.RemoteProtocolError( + "peer closed connection without sending complete message body " + "(incomplete chunked read)" + ) + + +class _EarlyCloseTransport(httpx.BaseTransport): + """Transport whose SSE response body closes early right after the + ``execution_complete`` event, before the chunked terminator is sent.""" + + def __init__(self, sse: bytes) -> None: + self._sse = sse + + def handle_request(self, request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + stream=_EarlyCloseAfterCompleteStream(self._sse), + request=request, + ) + + +_EARLY_CLOSE_SSE = ( + b'data: {"type":"init","text":"exec-bg","timestamp":1}\n\n' + b'data: {"type":"execution_complete","timestamp":2,"execution_time":3}\n\n' +) + + +def test_sync_run_background_command_breaks_on_complete_before_terminator() -> None: + """Background commands must not wait for the chunked terminator: once + ``execution_complete`` arrives, the SDK should stop reading the stream + even if the connection is closed early (#1528).""" + cfg = ConnectionConfigSync( + protocol="http", transport=_EarlyCloseTransport(_EARLY_CLOSE_SSE) + ) + endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772) + adapter = CommandsAdapterSync(cfg, endpoint) + + execution = adapter.run("sleep 1", opts=RunCommandOpts(background=True)) + + assert execution.id == "exec-bg" + assert execution.complete is not None + assert execution.complete.execution_time_in_millis == 3 + # Background executions do not synthesize an exit code from the stream. + assert execution.exit_code is None + + +def test_sync_run_foreground_command_still_waits_for_terminator() -> None: + """Foreground commands must keep waiting for the stream terminator + after ``execution_complete`` โ€” an early close is still surfaced as an + error, proving the background early-break did not change this path.""" + cfg = ConnectionConfigSync( + protocol="http", transport=_EarlyCloseTransport(_EARLY_CLOSE_SSE) + ) + endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772) + adapter = CommandsAdapterSync(cfg, endpoint) + + with pytest.raises(SandboxConnectionException): + adapter.run("sleep 1", opts=RunCommandOpts(background=False)) diff --git a/server/DEVELOPMENT.md b/server/DEVELOPMENT.md index 4d2f1941f..d3dac1fb7 100644 --- a/server/DEVELOPMENT.md +++ b/server/DEVELOPMENT.md @@ -23,7 +23,7 @@ level = "DEBUG" [runtime] type = "docker" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [docker] network_mode = "bridge" diff --git a/server/configuration.md b/server/configuration.md index c175979f5..dfd864a8c 100644 --- a/server/configuration.md +++ b/server/configuration.md @@ -54,7 +54,7 @@ Example files in this repository: | `[store]` | No | Server-managed persistent metadata backend | | `[secure_runtime]` | No | gVisor / Kata / Firecracker | | `[renew_intent]` | No | Auto-renew on access | -| `[otel]` | No | OTLP export for ingested SDK metrics | +| `[otel]` | No | OTLP export for Server HTTP and ingested SDK metrics | --- @@ -96,6 +96,7 @@ Example files in this repository: |-----|------|---------|-------------| | `type` | string | โ€” | **`docker`** or **`kubernetes`**. Selects which runtime implementation loads. | | `execd_image` | string | โ€” | OCI image containing the **execd** binary used to bootstrap command/file access inside the sandbox. | +| `execd_run_as_init` | boolean | `false` | Run **execd as the sandbox init** (OSEP-0018): sets `EXECD_INIT` in the sandbox environment so `bootstrap.sh` `exec`s into `execd --init` and execd becomes PID 1 โ€” reaping children, owning the container lifecycle, and exposing the hardening floor. Defaults to `false` (classic background-and-wait topology); intended to be flipped on after validation in production. | --- @@ -111,6 +112,8 @@ Example files in this repository: | `no_new_privileges` | boolean | `true` | Sets `no-new-privileges` to block privilege escalation. | | `seccomp_profile` | string \| omitted | `null` | Seccomp profile name or **absolute path**; empty uses Docker default seccomp. | | `pids_limit` | integer \| null | `4096` | Max PIDs per sandbox container; set to **`null`** to disable the limit. | +| `sandbox_env` | table | `{}` | Environment variables injected into **every** sandbox container; keys from a creation request override same-named keys. Docker-runtime counterpart of the Kubernetes pod template (e.g. `NODE_EXTRA_CA_CERTS` to trust a private CA, together with `sandbox_binds`). | +| `sandbox_binds` | string[] | `[]` | Host bind mounts applied to **every** sandbox container, Docker `-v` syntax (`host:container[:mode]`); prepended to binds derived from a request's `volumes`. | | `port_range_min` | integer | `40000` | Lower bound of the host port range used by bridge-mode sandbox port allocation. Must be less than `port_range_max`. Each sandbox needs 2โ€“3 host ports (2 without egress, 3 with egress sidecar). Narrow this range to match your firewall policy โ€” e.g., 100 concurrent sandboxes โ‰ˆ 300 ports. | | `port_range_max` | integer | `60000` | Upper bound of the host port range. Range must span โ‰ฅ 100 ports for reliable allocation. | @@ -204,6 +207,7 @@ Configures the **egress sidecar** image and enforcement mode. The server only at | `image` | string \| omitted | `null` | OCI image for the egress sidecar. **Required in config** when clients send **`networkPolicy`** (create request). | | `mode` | string | `"dns"` | Passed to the sidecar as `OPENSANDBOX_EGRESS_MODE`. Values: **`dns`** โ€” DNS-proxy-based enforcement (CIDR/static IP rules **not** enforced); **`dns+nft`** โ€” adds nftables where available so **CIDR/IP** rules can be enforced. | | `disable_ipv6` | bool | `true` | IPv6 egress is incomplete (especially on Kubernetes). **Default on**; set `false` only when you want IPv6 left up in the netns. Details in [IPv6 and egress](#ipv6-and-egress) below. | +| `readiness_timeout_seconds` | float | `30.0` | **Docker only.** Maximum time to wait for the egress sidecar health endpoint to become ready. Must be greater than `0`. | ### IPv6 and egress @@ -213,6 +217,7 @@ OpenSandbox egress does **not** treat IPv6 as a first-class, fully covered path - `egress.image` must be set when using `networkPolicy`. - Outbound policy requires **`docker.network_mode = "bridge"`**; `networkPolicy` is rejected for incompatible network modes. +- Increase `egress.readiness_timeout_seconds` when the sidecar needs more than 30 seconds to become ready in the deployment environment. **Kubernetes notes:** @@ -300,7 +305,7 @@ Per-sandbox enablement uses create request extensions (see OSEP-0009 and `exampl ## `[otel]` -Optional OpenTelemetry metrics export for SDK-reported sandbox creation latency (`POST /v1/metrics/events`). Off by default; the ingestion endpoint still accepts events and records them as noop. +Optional OpenTelemetry metrics export for Server HTTP requests and SDK-reported sandbox creation latency (`POST /v1/metrics/events`). Off by default; the HTTP middleware and ingestion endpoint remain active but record as noops. | Key | Type | Default | Description | |-----|------|---------|-------------| @@ -309,6 +314,15 @@ Optional OpenTelemetry metrics export for SDK-reported sandbox creation latency | `service_name` | string | `"opensandbox-server"` | `service.name` resource attribute. | | `export_interval_millis` | integer | `60000` | Periodic export interval (โ‰ฅ 1000). | +Exported metrics: + +| Metric | Type | Unit | Attributes | Description | +|-----|------|------|------------|-------------| +| `server.http.request.duration` | Histogram | `ms` | `http_method`, `http_route`, `http_status_code` | Server HTTP request latency. Histogram count provides request volume. | +| `opensandbox.sandbox.create.duration` | Histogram | `ms` | `sdk.language`, `sdk.version`, `success` | SDK-reported creation latency from create start until ready or failure. | + +The HTTP metric uses the matched route template rather than the raw request path. Requests that do not reach a matched route, including early authentication failures and unmatched URLs, use `http_route=unknown`. Standard HTTP methods are recorded in uppercase, while extension methods use `http_method=OTHER` to keep attribute cardinality bounded. The metric never includes sandbox IDs, tenant IDs, API keys, request or response bodies, query strings, or other unbounded request data. + --- ## Environment variables (outside TOML) diff --git a/server/docker-compose.example.yaml b/server/docker-compose.example.yaml index 7e18956e8..c1743426a 100644 --- a/server/docker-compose.example.yaml +++ b/server/docker-compose.example.yaml @@ -10,12 +10,13 @@ configs: [runtime] type = "docker" - # execd_image = "opensandbox/execd:v1.0.21" - execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21" + # execd_image = "opensandbox/execd:v1.0.22" + execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" [egress] - image = "opensandbox/egress:v1.1.5" - # image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5" + image = "opensandbox/egress:v1.1.6" + # image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" + readiness_timeout_seconds = 30.0 [docker] network_mode = "bridge" @@ -64,4 +65,4 @@ services: networks: opensandbox-net: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/server/opensandbox_server/api/devops.py b/server/opensandbox_server/api/devops.py index 27f6b766a..eda1b3832 100644 --- a/server/opensandbox_server/api/devops.py +++ b/server/opensandbox_server/api/devops.py @@ -15,10 +15,9 @@ """ API routes for OpenSandbox DevOps diagnostics. -Requests that include `scope` target the stable Diagnostics API. The open-source -server has not implemented that API yet, so these requests return a uniform -not-implemented error. Requests without `scope` preserve the deprecated DevOps -plain-text behavior for legacy humans, agents, and CLI clients. +Requests that include ``scope`` target the stable Diagnostics API and return an +inline JSON descriptor. Requests without ``scope`` preserve the deprecated +DevOps plain-text behavior for legacy humans, agents, and CLI clients. """ import logging @@ -28,19 +27,29 @@ from fastapi.responses import JSONResponse, PlainTextResponse from opensandbox_server.api.lifecycle import sandbox_service +from opensandbox_server.services.diagnostics import DiagnosticResult logger = logging.getLogger(__name__) router = APIRouter(tags=["DevOps"]) -def _diagnostics_not_implemented_response() -> JSONResponse: - return JSONResponse( - status_code=status.HTTP_501_NOT_IMPLEMENTED, - content={ - "code": "DIAGNOSTICS_NOT_IMPLEMENTED", - "message": "The stable Diagnostics API is not implemented by this OpenSandbox server.", - }, - ) +def _diagnostic_inline_response( + result: DiagnosticResult, +) -> JSONResponse: + """Build a Diagnostics API descriptor for inline plain-text content.""" + payload: dict[str, object] = { + "sandboxId": result.sandbox_id, + "kind": result.kind, + "scope": result.scope, + "delivery": "inline", + "contentType": "text/plain; charset=utf-8", + "content": result.content, + "contentLength": len(result.content.encode("utf-8")), + "truncated": result.truncated, + } + if result.warnings: + payload["warnings"] = list(result.warnings) + return JSONResponse(status_code=status.HTTP_200_OK, content=payload) def _deprecated_plain_text_response(content: str) -> PlainTextResponse: @@ -56,13 +65,10 @@ def _deprecated_plain_text_response(content: str) -> PlainTextResponse: status_code=status.HTTP_200_OK, responses={ 200: { - "description": "Deprecated plain-text logs when scope is omitted", - "content": {"text/plain": {}}, - }, - 501: { - "description": "Stable Diagnostics API is not implemented by this server", - "content": {"application/json": {}}, + "description": "Stable JSON descriptor, or deprecated text when scope is omitted", + "content": {"application/json": {}, "text/plain": {}}, }, + 400: {"description": "Unsupported diagnostics scope"}, 404: {"description": "Sandbox not found"}, }, ) @@ -96,10 +102,9 @@ def get_sandbox_logs( ) -> JSONResponse | PlainTextResponse: """Retrieve diagnostic logs for a sandbox.""" if scope is not None: - return _diagnostics_not_implemented_response() - text = sandbox_service.get_sandbox_logs( - sandbox_id, tail=tail, since=since, container=container - ) + result = sandbox_service.get_sandbox_log_diagnostics(sandbox_id, scope) + return _diagnostic_inline_response(result) + text = sandbox_service.get_sandbox_logs(sandbox_id, tail=tail, since=since, container=container) return _deprecated_plain_text_response(text) @@ -125,13 +130,10 @@ def get_sandbox_inspect(sandbox_id: str) -> PlainTextResponse: status_code=status.HTTP_200_OK, responses={ 200: { - "description": "Deprecated plain-text events when scope is omitted", - "content": {"text/plain": {}}, - }, - 501: { - "description": "Stable Diagnostics API is not implemented by this server", - "content": {"application/json": {}}, + "description": "Stable JSON descriptor, or deprecated text when scope is omitted", + "content": {"application/json": {}, "text/plain": {}}, }, + 400: {"description": "Unsupported diagnostics scope"}, 404: {"description": "Sandbox not found"}, }, ) @@ -151,7 +153,8 @@ def get_sandbox_events( ) -> JSONResponse | PlainTextResponse: """Retrieve diagnostic events for a sandbox.""" if scope is not None: - return _diagnostics_not_implemented_response() + result = sandbox_service.get_sandbox_event_diagnostics(sandbox_id, scope) + return _diagnostic_inline_response(result) text = sandbox_service.get_sandbox_events(sandbox_id, limit=limit) return _deprecated_plain_text_response(text) diff --git a/server/opensandbox_server/api/lifecycle.py b/server/opensandbox_server/api/lifecycle.py index fe152765d..846622dce 100644 --- a/server/opensandbox_server/api/lifecycle.py +++ b/server/opensandbox_server/api/lifecycle.py @@ -45,7 +45,10 @@ Snapshot, SnapshotFilter, ) -from opensandbox_server.services.constants import SandboxErrorCodes +from opensandbox_server.services.constants import ( + OPEN_SANDBOX_INGRESS_HEADER, + SandboxErrorCodes, +) from opensandbox_server.services.factory import create_sandbox_service from opensandbox_server.services.snapshot_service import create_snapshot_service @@ -580,5 +583,11 @@ def get_sandbox_endpoint( base_url = str(request.base_url).rstrip("/") + mount_prefix base_url = base_url.replace("https://", "").replace("http://", "") endpoint.endpoint = f"{base_url}/sandboxes/{sandbox_id}/proxy/{port}" + if endpoint.headers: + endpoint.headers = { + key: value + for key, value in endpoint.headers.items() + if key.lower() != OPEN_SANDBOX_INGRESS_HEADER.lower() + } or None return endpoint diff --git a/server/opensandbox_server/api/proxy.py b/server/opensandbox_server/api/proxy.py index caf121293..4aaf8ad7d 100644 --- a/server/opensandbox_server/api/proxy.py +++ b/server/opensandbox_server/api/proxy.py @@ -20,6 +20,7 @@ import logging from collections.abc import AsyncIterator, Mapping from typing import Optional +from urllib.parse import urlsplit import anyio import httpx @@ -27,6 +28,7 @@ from fastapi import APIRouter, Request, WebSocket, status from fastapi.exceptions import HTTPException from fastapi.responses import StreamingResponse +from starlette.types import Receive, Scope, Send from starlette.websockets import WebSocketDisconnect from websockets.asyncio.client import ClientConnection from websockets.typing import Origin @@ -52,6 +54,12 @@ "upgrade", } +# Uvicorn adds this to client-facing responses. Forwarding the backend value as +# well would produce a duplicate field on the wire. +SERVER_GENERATED_RESPONSE_HEADERS = { + "server", +} + # Headers that shouldn't be forwarded to untrusted/internal backends SENSITIVE_HEADERS = { "authorization", @@ -163,6 +171,30 @@ def _set_forwarded_headers( headers["X-Forwarded-For"] = request.client.host +def _rewrite_proxy_location( + location: str, + request: Request, + sandbox_id: str, + port: int, +) -> str: + """Keep root-relative redirects inside the current sandbox proxy route.""" + if not location.startswith("/") or location.startswith("//"): + return location + + proxy_suffix = f"/sandboxes/{sandbox_id}/proxy/{port}" + eip = (lifecycle.get_config().server.eip or "").strip().rstrip("/") + if eip: + external_url = eip if "://" in eip else f"//{eip}" + external_prefix = urlsplit(external_url).path.rstrip("/") + return f"{external_prefix}{proxy_suffix}{location}" + + proxy_start = request.url.path.find(proxy_suffix) + if proxy_start < 0: + return location + proxy_path = request.url.path[: proxy_start + len(proxy_suffix)] + return f"{proxy_path}{location}" + + def _schedule_proxy_renew(request: Request | WebSocket, sandbox_id: str) -> None: proxy_renew = getattr(request.app.state, "proxy_renew_coordinator", None) if proxy_renew is not None: @@ -207,19 +239,43 @@ async def _authenticate_websocket_tenant(websocket: WebSocket) -> bool: return True +async def _close_backend_response(resp: httpx.Response) -> None: + """Return a streamed backend response to httpx's pool, even during cancellation.""" + with anyio.CancelScope(shield=True): + await resp.aclose() + + async def _stream_backend_response(resp: httpx.Response) -> AsyncIterator[bytes]: - """ - Yield backend body chunks without httpx content decoding and always close the response. + """Yield raw backend chunks so content-encoding still matches the body bytes.""" + async for chunk in resp.aiter_raw(): + yield chunk + + +class _ProxyStreamingResponse(StreamingResponse): + """Streaming response that owns and always releases its httpx response.""" + + def __init__( + self, + resp: httpx.Response, + *, + status_code: int, + headers: Mapping[str, str], + ) -> None: + self._backend_response = resp + super().__init__( + content=_stream_backend_response(resp), + status_code=status_code, + headers=headers, + ) - httpx requires ``await resp.aclose()`` for ``stream=True`` responses so connections - return to the pool; Starlette's StreamingResponse does not do this automatically. - Use ``aiter_raw`` so forwarded ``content-encoding`` headers still match the body bytes. - """ - try: - async for chunk in resp.aiter_raw(): - yield chunk - finally: - await resp.aclose() + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + try: + await super().__call__(scope, receive, send) + finally: + # The body iterator may never start if the downstream disconnects + # while Starlette sends response headers. Keep ownership here so + # that connection is still returned to the shared httpx pool. + await _close_backend_response(self._backend_response) def _verify_secure_access(endpoint: Endpoint, caller_headers: Mapping[str, str]) -> None: @@ -295,25 +351,36 @@ async def _proxy_http_request( resp = await client.send(req, stream=True) - hop_by_hop = set(HOP_BY_HOP_HEADERS) - connection_header = resp.headers.get("connection") - if connection_header: - hop_by_hop.update( - header.strip().lower() - for header in connection_header.split(",") - if header.strip() - ) - response_headers = { - key: value - for key, value in resp.headers.items() - if key.lower() not in hop_by_hop - } + try: + hop_by_hop = set(HOP_BY_HOP_HEADERS) + connection_header = resp.headers.get("connection") + if connection_header: + hop_by_hop.update( + header.strip().lower() + for header in connection_header.split(",") + if header.strip() + ) + response_header_exclusions = hop_by_hop | SERVER_GENERATED_RESPONSE_HEADERS + response_headers = { + key: ( + _rewrite_proxy_location(value, request, sandbox_id, port) + if key.lower() == "location" + else value + ) + for key, value in resp.headers.items() + if key.lower() not in response_header_exclusions + } - return StreamingResponse( - content=_stream_backend_response(resp), - status_code=resp.status_code, - headers=response_headers, - ) + return _ProxyStreamingResponse( + resp, + status_code=resp.status_code, + headers=response_headers, + ) + except BaseException: + # Until ownership passes to _ProxyStreamingResponse, any failure + # after client.send() must release the acquired pool connection. + await _close_backend_response(resp) + raise except httpx.ConnectError as e: raise HTTPException( status_code=502, diff --git a/server/opensandbox_server/cli.py b/server/opensandbox_server/cli.py index 41f874320..6c3d77a41 100644 --- a/server/opensandbox_server/cli.py +++ b/server/opensandbox_server/cli.py @@ -305,6 +305,7 @@ def main() -> None: backlog=server_cfg.backlog, loop=server_cfg.loop, http=server_cfg.http, + date_header=False, timeout_graceful_shutdown=server_cfg.timeout_graceful_shutdown, ) diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.py index 717bc3e7e..2ceb7e887 100644 --- a/server/opensandbox_server/config.py +++ b/server/opensandbox_server/config.py @@ -147,12 +147,12 @@ def require_dsn_when_redis_enabled(self) -> "RenewIntentRedisConfig": class OtelConfig(BaseModel): - """Optional OpenTelemetry export for ingested SDK metrics.""" + """Optional OpenTelemetry export for Server and ingested SDK metrics.""" enabled: bool = Field( default=False, description=( - "Enable OTLP metrics export. When false, SDK events are accepted but recorded as noop." + "Enable OTLP metrics export. When false, Server and SDK metrics are noops." ), ) endpoint: Optional[str] = Field( @@ -751,6 +751,14 @@ class EgressConfig(BaseModel): "(e.g. IPv4-only CNI or experimenting with IPv6 egress despite gaps)." ), ) + readiness_timeout_seconds: float = Field( + default=30.0, + gt=0, + description=( + "Maximum time in seconds to wait for the egress sidecar health endpoint " + "to become ready in Docker runtime." + ), + ) class RuntimeConfig(BaseModel): @@ -765,6 +773,17 @@ class RuntimeConfig(BaseModel): description="Container image that contains the execd binary for sandbox initialization.", min_length=1, ) + execd_run_as_init: bool = Field( + default=False, + description=( + "Run execd as the sandbox init (OSEP-0018): sets EXECD_INIT in the " + "sandbox environment so bootstrap.sh execs into execd (--init) and " + "execd becomes PID 1, reaping children and owning the container " + "lifecycle. Defaults to false (classic background-and-wait " + "topology); intended to be flipped on after a few releases once " + "the init mode is validated in production." + ), + ) class SecureRuntimeConfig(BaseModel): @@ -898,6 +917,23 @@ class DockerConfig(BaseModel): ge=1, description="Maximum number of processes allowed per sandbox container. Set to null to disable the limit.", ) + sandbox_env: dict[str, str] = Field( + default_factory=dict, + description=( + "Environment variables injected into every sandbox container. Keys from a sandbox " + "creation request override same-named keys. Docker-runtime counterpart of the " + "Kubernetes pod template: useful for fleet-wide settings such as trusting a private " + "CA (e.g. NODE_EXTRA_CA_CERTS) together with sandbox_binds." + ), + ) + sandbox_binds: list[str] = Field( + default_factory=list, + description=( + "Host bind mounts applied to every sandbox container, in Docker -v syntax " + "(host_path:container_path[:mode]). Prepended to the binds derived from a request's " + "volumes. Useful for mounting a private CA certificate into all sandboxes." + ), + ) @model_validator(mode="after") def validate_port_range(self) -> "DockerConfig": diff --git a/server/opensandbox_server/examples/e2e.batchsandbox-template.yaml b/server/opensandbox_server/examples/e2e.batchsandbox-template.yaml index e1879b54f..76bddfb54 100644 --- a/server/opensandbox_server/examples/e2e.batchsandbox-template.yaml +++ b/server/opensandbox_server/examples/e2e.batchsandbox-template.yaml @@ -3,6 +3,11 @@ # # Faster Pod teardown in Kind/CI: skip the default 30s graceful termination window. # Do not use for real workloads where graceful shutdown matters. +# +# The execd-isolation ConfigMap carries the hardened isolation TOML for the +# OSEP-0018 hardening e2e (created by scripts/python-k8s-execd-init-e2e.sh). +# It is optional: e2e runs that do not create the ConfigMap get an empty +# mount, and execd only reads it when EXECD_ISOLATION_CONFIG points at it. # Metadata template (will be merged with runtime-generated metadata) metadata: @@ -15,3 +20,14 @@ spec: restartPolicy: Never tolerations: - operator: "Exists" + volumes: + - name: execd-isolation + configMap: + name: opensandbox-e2e-execd-isolation + optional: true + containers: + - name: sandbox + volumeMounts: + - name: execd-isolation + mountPath: /etc/opensandbox/execd-isolation + readOnly: true diff --git a/server/opensandbox_server/examples/example.config.k8s.toml b/server/opensandbox_server/examples/example.config.k8s.toml index 5f3f4d075..c95563548 100644 --- a/server/opensandbox_server/examples/example.config.k8s.toml +++ b/server/opensandbox_server/examples/example.config.k8s.toml @@ -32,7 +32,7 @@ level = "INFO" [runtime] type = "kubernetes" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [storage] # Allowlist of host path prefixes permitted for bind mounts. @@ -75,7 +75,7 @@ batchsandbox_template_file = "~/batchsandbox-template.yaml" mode = "direct" [egress] -image = "opensandbox/egress:v1.1.5" +image = "opensandbox/egress:v1.1.6" mode = "dns" # Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). # disable_ipv6 = false diff --git a/server/opensandbox_server/examples/example.config.k8s.zh.toml b/server/opensandbox_server/examples/example.config.k8s.zh.toml index 1327e4d56..c326649df 100644 --- a/server/opensandbox_server/examples/example.config.k8s.zh.toml +++ b/server/opensandbox_server/examples/example.config.k8s.zh.toml @@ -32,7 +32,7 @@ level = "INFO" [runtime] type = "kubernetes" -execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21" +execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" [storage] # ๅ…่ฎธ่ฟ›่กŒ bind mount ็š„ๅฎฟไธปๆœบ่ทฏๅพ„ๅ‰็ผ€็™ฝๅๅ•ใ€‚ @@ -76,7 +76,7 @@ batchsandbox_template_file = "~/batchsandbox-template.yaml" mode = "direct" [egress] -image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5" +image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" mode = "dns" # Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). # disable_ipv6 = false diff --git a/server/opensandbox_server/examples/example.config.toml b/server/opensandbox_server/examples/example.config.toml index 11482431d..245a16bbf 100644 --- a/server/opensandbox_server/examples/example.config.toml +++ b/server/opensandbox_server/examples/example.config.toml @@ -32,7 +32,7 @@ level = "INFO" [runtime] type = "docker" -execd_image = "opensandbox/execd:v1.0.21" +execd_image = "opensandbox/execd:v1.0.22" [storage] # Allowlist of host path prefixes permitted for bind mounts. @@ -61,6 +61,12 @@ no_new_privileges = true apparmor_profile = "" # Limit process count to reduce host impact from fork bombs; set to null to disable pids_limit = 4096 +# Optional: environment variables injected into every sandbox container +# (request env overrides same-named keys). Pair with sandbox_binds to make +# every sandbox trust a private CA: +# sandbox_env = { NODE_EXTRA_CA_CERTS = "/etc/ssl/private-ca/root-ca.crt" } +# Optional: host bind mounts applied to every sandbox container (docker -v syntax) +# sandbox_binds = ["/opt/certs/root-ca.crt:/etc/ssl/private-ca/root-ca.crt:ro"] # Seccomp profile: empty string uses Docker default; set to an absolute path for a custom profile seccomp_profile = "" @@ -68,8 +74,9 @@ seccomp_profile = "" mode = "direct" [egress] -image = "opensandbox/egress:v1.1.5" +image = "opensandbox/egress:v1.1.6" mode = "dns" +readiness_timeout_seconds = 30.0 # Renew-on-access. Off by default โ€” see server/README.md. [renew_intent] diff --git a/server/opensandbox_server/examples/example.config.zh.toml b/server/opensandbox_server/examples/example.config.zh.toml index ac2c20c8c..80d7001e5 100644 --- a/server/opensandbox_server/examples/example.config.zh.toml +++ b/server/opensandbox_server/examples/example.config.zh.toml @@ -32,7 +32,7 @@ level = "INFO" [runtime] type = "docker" -execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21" +execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" [storage] allowed_host_paths = [] @@ -59,6 +59,11 @@ no_new_privileges = true apparmor_profile = "" # Limit process count to reduce host impact from fork bombs; set to null to disable pids_limit = 4096 +# ๅฏ้€‰:ๆณจๅ…ฅๅˆฐๆฏไธชๆฒ™็ฎฑๅฎนๅ™จ็š„็Žฏๅขƒๅ˜้‡(ๅˆ›ๅปบ่ฏทๆฑ‚ไธญ็š„ๅŒๅ้”ฎไผ˜ๅ…ˆ)ใ€‚ +# ไธŽ sandbox_binds ๆญ้…ๅฏ่ฎฉๆ‰€ๆœ‰ๆฒ™็ฎฑไฟกไปป็งๆœ‰ CA: +# sandbox_env = { NODE_EXTRA_CA_CERTS = "/etc/ssl/private-ca/root-ca.crt" } +# ๅฏ้€‰:ๅบ”็”จๅˆฐๆฏไธชๆฒ™็ฎฑๅฎนๅ™จ็š„ๅฎฟไธปๆœบ bind ๆŒ‚่ฝฝ(docker -v ่ฏญๆณ•) +# sandbox_binds = ["/opt/certs/root-ca.crt:/etc/ssl/private-ca/root-ca.crt:ro"] # Seccomp profile: empty string uses Docker default; set to an absolute path for a custom profile seccomp_profile = "" @@ -66,8 +71,9 @@ seccomp_profile = "" mode = "direct" [egress] -image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5" +image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" mode = "dns" +readiness_timeout_seconds = 30.0 # ๆŒ‰่ฎฟ้—ฎ็ปญๆœŸใ€‚้ป˜่ฎคๅ…ณ้—ญ โ€” ่ง server/README_zh.mdใ€‚ [renew_intent] diff --git a/server/opensandbox_server/integrations/otel/__init__.py b/server/opensandbox_server/integrations/otel/__init__.py index 8a4b455d1..c61041aeb 100644 --- a/server/opensandbox_server/integrations/otel/__init__.py +++ b/server/opensandbox_server/integrations/otel/__init__.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Optional OpenTelemetry integration for SDK metrics ingestion.""" +"""Optional OpenTelemetry integration for Server and SDK metrics.""" from opensandbox_server.integrations.otel.metrics import ( + record_http_request_duration, record_sandbox_create_duration, setup_otel_metrics, shutdown_otel_metrics, ) __all__ = [ + "record_http_request_duration", "record_sandbox_create_duration", "setup_otel_metrics", "shutdown_otel_metrics", diff --git a/server/opensandbox_server/integrations/otel/metrics.py b/server/opensandbox_server/integrations/otel/metrics.py index d24b60a54..ea6ca9520 100644 --- a/server/opensandbox_server/integrations/otel/metrics.py +++ b/server/opensandbox_server/integrations/otel/metrics.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""OpenTelemetry metrics helpers for SDK lifecycle telemetry.""" +"""OpenTelemetry metrics helpers for Server and SDK lifecycle telemetry.""" from __future__ import annotations @@ -46,9 +46,34 @@ 30000.0, 60000.0, ) +_HTTP_REQUEST_DURATION_HISTOGRAM_NAME = "server.http.request.duration" +_HTTP_REQUEST_DURATION_UNIT = "ms" +_HTTP_REQUEST_DURATION_DESCRIPTION = ( + "Server HTTP request duration by method, route template, and status code" +) +_HTTP_REQUEST_METHODS = frozenset( + {"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE"} +) +_HTTP_REQUEST_DURATION_BOUNDARIES = ( + 1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1000.0, + 2500.0, + 5000.0, + 10000.0, + 30000.0, + 60000.0, +) _meter_provider: Optional[MeterProvider] = None _create_duration_histogram = None +_http_request_duration_histogram = None def _histogram_from_provider(provider: MeterProvider): @@ -59,15 +84,24 @@ def _histogram_from_provider(provider: MeterProvider): ) +def _http_request_histogram_from_provider(provider: MeterProvider): + return provider.get_meter("opensandbox.server").create_histogram( + name=_HTTP_REQUEST_DURATION_HISTOGRAM_NAME, + unit=_HTTP_REQUEST_DURATION_UNIT, + description=_HTTP_REQUEST_DURATION_DESCRIPTION, + ) + + def setup_otel_metrics(config: OtelConfig) -> None: """Configure OTEL metrics export when enabled; otherwise keep recording as noop.""" - global _meter_provider, _create_duration_histogram + global _meter_provider, _create_duration_histogram, _http_request_duration_histogram # Disabled: do not attach instruments to any global provider (may already export). if not config.enabled: _create_duration_histogram = None + _http_request_duration_histogram = None logger.info( - "OpenTelemetry metrics export disabled; SDK events are accepted but not exported" + "OpenTelemetry metrics export disabled; Server and SDK metrics are noops" ) return @@ -97,7 +131,13 @@ def setup_otel_metrics(config: OtelConfig) -> None: aggregation=ExplicitBucketHistogramAggregation( boundaries=list(_CREATE_DURATION_BOUNDARIES) ), - ) + ), + View( + instrument_name=_HTTP_REQUEST_DURATION_HISTOGRAM_NAME, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=list(_HTTP_REQUEST_DURATION_BOUNDARIES) + ), + ), ] provider = MeterProvider( resource=resource, @@ -109,7 +149,7 @@ def setup_otel_metrics(config: OtelConfig) -> None: if isinstance(current, MeterProvider): logger.warning( "A global MeterProvider is already installed; opensandbox will export " - "create-latency metrics via its own provider and will not replace the global one" + "metrics via its own provider and will not replace the global one" ) else: metrics.set_meter_provider(provider) @@ -118,6 +158,7 @@ def setup_otel_metrics(config: OtelConfig) -> None: # even when set_meter_provider() cannot override a preexisting global provider. _meter_provider = provider _create_duration_histogram = _histogram_from_provider(provider) + _http_request_duration_histogram = _http_request_histogram_from_provider(provider) logger.info( "OpenTelemetry metrics enabled (service=%s, endpoint=%s)", config.service_name, @@ -127,10 +168,11 @@ def setup_otel_metrics(config: OtelConfig) -> None: def shutdown_otel_metrics() -> None: """Flush and shut down the configured MeterProvider if any.""" - global _meter_provider, _create_duration_histogram + global _meter_provider, _create_duration_histogram, _http_request_duration_histogram provider = _meter_provider _meter_provider = None _create_duration_histogram = None + _http_request_duration_histogram = None if provider is None: return try: @@ -164,3 +206,30 @@ def record_sandbox_create_duration( ) except Exception: logger.exception("Failed to record sandbox create duration metric") + + +def record_http_request_duration( + *, + duration_ms: float, + method: str, + route: str, + status_code: int, +) -> None: + """Record a low-cardinality Server HTTP duration sample. Never raises.""" + hist = _http_request_duration_histogram + if hist is None: + return + normalized_method = method.upper() + if normalized_method not in _HTTP_REQUEST_METHODS: + normalized_method = "OTHER" + try: + hist.record( + duration_ms, + attributes={ + "http_method": normalized_method, + "http_route": route or "unknown", + "http_status_code": status_code, + }, + ) + except Exception: + logger.exception("Failed to record Server HTTP request duration metric") diff --git a/server/opensandbox_server/main.py b/server/opensandbox_server/main.py index 6b278fb5f..ad3eab615 100644 --- a/server/opensandbox_server/main.py +++ b/server/opensandbox_server/main.py @@ -29,12 +29,17 @@ from fastapi.exceptions import HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from starlette.types import ASGIApp from opensandbox_server.config import load_config from opensandbox_server.integrations.renew_intent import start_renew_intent_consumer from opensandbox_server.logging_config import configure_logging from opensandbox_server.startup_guard import api_key_confirm -from opensandbox_server.tenants import validate_tenant_config, TenantProvider +from opensandbox_server.tenants import ( + validate_tenant_config, + validate_tenant_namespaces_on_startup, + TenantProvider, +) # The deployed package version, resolved at runtime from installed metadata. # Exposed via GET /version (not /openapi.json). Mirrors @@ -90,6 +95,8 @@ def _build_tenant_provider(config) -> TenantProvider | None: from opensandbox_server.integrations.otel import setup_otel_metrics, shutdown_otel_metrics # noqa: E402 from opensandbox_server.integrations.renew_intent.proxy_renew import ProxyRenewCoordinator # noqa: E402 from opensandbox_server.middleware.auth import AuthMiddleware # noqa: E402 +from opensandbox_server.middleware.date_header import DateHeaderMiddleware # noqa: E402 +from opensandbox_server.middleware.http_metrics import HttpMetricsMiddleware # noqa: E402 from opensandbox_server.middleware.request_id import RequestIdMiddleware # noqa: E402 from opensandbox_server.services.extension_service import require_extension_service # noqa: E402 from opensandbox_server.services.runtime_resolver import ( # noqa: E402 @@ -98,6 +105,14 @@ def _build_tenant_provider(config) -> TenantProvider | None: logger = logging.getLogger(__name__) + +class _DateHeaderFastAPI(FastAPI): + """Keep Date handling outside Starlette's server error middleware.""" + + def build_middleware_stack(self) -> ASGIApp: + return DateHeaderMiddleware(super().build_middleware_stack()) + + @asynccontextmanager async def lifespan(app: FastAPI): if tenant_provider is None: @@ -111,6 +126,20 @@ async def lifespan(app: FastAPI): tenant_provider.start() sandbox_service.set_tenant_provider(tenant_provider) + # OSEP-0014: startup MUST validate all tenant namespaces exist and + # are accessible before serving traffic (fail-fast). Multi-tenancy is + # Kubernetes-only, which validate_tenant_config() already enforces. + # Providers that cannot enumerate tenants (HTTP) skip with a warning + # instead of silently validating an empty set. + try: + from opensandbox_server.services.k8s.client import K8sClient + + core_v1_api = K8sClient(app_config.kubernetes).get_core_v1_api() + validate_tenant_namespaces_on_startup(tenant_provider, core_v1_api) + except Exception as exc: + logger.error("Tenant namespace validation failed: %s", exc) + os._exit(1) + from anyio.to_thread import current_default_thread_limiter current_default_thread_limiter().total_tokens = app_config.server.thread_pool_size @@ -173,7 +202,7 @@ async def lifespan(app: FastAPI): # Initialize FastAPI application -app = FastAPI( +app = _DateHeaderFastAPI( title="OpenSandbox Lifecycle API", version=API_CONTRACT_VERSION, description="The Sandbox Lifecycle API coordinates how untrusted workloads are created, " @@ -187,7 +216,8 @@ async def lifespan(app: FastAPI): app.state.config = app_config app.state.tenant_provider = tenant_provider -# Middleware run in reverse order of addition: last added = first to run (outermost). +# User middleware run in reverse order of addition: last added = first to run. +# DateHeaderMiddleware wraps the complete stack, including ServerErrorMiddleware. # Add auth and CORS first so they run after RequestIdMiddleware. app.add_middleware(AuthMiddleware, config=app_config, tenant_provider=tenant_provider) app.add_middleware( @@ -197,9 +227,12 @@ async def lifespan(app: FastAPI): allow_methods=["*"], allow_headers=["*"], ) -# RequestIdMiddleware last = outermost: runs first, so every response (including -# 401 from AuthMiddleware) gets X-Request-ID and logs have request_id in context. +# RequestIdMiddleware wraps auth and CORS so every response (including 401 from +# AuthMiddleware) gets X-Request-ID and logs have request_id in context. app.add_middleware(RequestIdMiddleware) +# HttpMetricsMiddleware is the outermost user middleware so auth failures and +# other early responses are included. Unmatched routes use the bounded "unknown" label. +app.add_middleware(HttpMetricsMiddleware) # Include API routes at root and versioned prefix. # IMPORTANT: non-proxy routers MUST be registered before proxy_router @@ -278,5 +311,6 @@ async def version_info(): timeout_keep_alive=app_config.server.timeout_keep_alive, loop=app_config.server.loop, http=app_config.server.http, + date_header=False, timeout_graceful_shutdown=app_config.server.timeout_graceful_shutdown, ) diff --git a/server/opensandbox_server/middleware/date_header.py b/server/opensandbox_server/middleware/date_header.py new file mode 100644 index 000000000..ca7534dd3 --- /dev/null +++ b/server/opensandbox_server/middleware/date_header.py @@ -0,0 +1,43 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ensure HTTP responses have exactly one application-managed Date header.""" + +from email.utils import formatdate + +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +DATE_HEADER = b"date" + + +class DateHeaderMiddleware: + """Add a current HTTP Date only when the application did not provide one.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_date(message: Message) -> None: + if message["type"] == "http.response.start": + headers = list(message.get("headers", [])) + if not any(name.lower() == DATE_HEADER for name, _ in headers): + headers.append((DATE_HEADER, formatdate(usegmt=True).encode("ascii"))) + message = {**message, "headers": headers} + await send(message) + + await self.app(scope, receive, send_with_date) diff --git a/server/opensandbox_server/middleware/http_metrics.py b/server/opensandbox_server/middleware/http_metrics.py new file mode 100644 index 000000000..ce71c8a65 --- /dev/null +++ b/server/opensandbox_server/middleware/http_metrics.py @@ -0,0 +1,86 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ASGI middleware for low-cardinality Server HTTP request metrics.""" + +import logging +from time import perf_counter + +from starlette.routing import Match, Router +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from opensandbox_server.integrations.otel import record_http_request_duration + +logger = logging.getLogger(__name__) + + +def _matched_route_path(scope: Scope) -> str: + route_path = getattr(scope.get("route"), "path", None) + if route_path: + return route_path + + router = scope.get("router") + if not isinstance(router, Router): + return "unknown" + + partial_path = None + for registered_route in router.routes: + match, _ = registered_route.matches(scope) + registered_path = getattr(registered_route, "path", None) + if not registered_path: + continue + if match == Match.FULL: + return registered_path + if match == Match.PARTIAL and partial_path is None: + partial_path = registered_path + + return partial_path or "unknown" + + +class HttpMetricsMiddleware: + """Record request duration without exposing raw paths or request data.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + started_at = perf_counter() + status_code = 500 + + async def send_wrapper(message: Message) -> None: + nonlocal status_code + if message["type"] == "http.response.start": + status_code = message["status"] + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception: + status_code = 500 + raise + finally: + try: + route = _matched_route_path(scope) + record_http_request_duration( + duration_ms=(perf_counter() - started_at) * 1000.0, + method=scope.get("method", "unknown"), + route=route, + status_code=status_code, + ) + except Exception: + logger.exception("Failed to record Server HTTP request metric") diff --git a/server/opensandbox_server/services/diagnostics.py b/server/opensandbox_server/services/diagnostics.py new file mode 100644 index 000000000..4a94489e8 --- /dev/null +++ b/server/opensandbox_server/services/diagnostics.py @@ -0,0 +1,85 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runtime-neutral results and helpers for stable sandbox diagnostics.""" + +from dataclasses import dataclass +from typing import Literal + +from fastapi import HTTPException, status + +DiagnosticKind = Literal["logs", "events"] + + +@dataclass(frozen=True, slots=True) +class DiagnosticResult: + """Stable diagnostic content collected by a runtime service.""" + + sandbox_id: str + kind: DiagnosticKind + scope: str + content: str + truncated: bool = False + warnings: tuple[str, ...] = () + + +def limit_diagnostic_lines( + content: str, + limit: int, + *, + keep_tail: bool, +) -> tuple[str, bool]: + """Apply a line limit after collecting one extra line. + + Args: + content: Diagnostic text to bound. + limit: Maximum number of lines to return. + keep_tail: Keep the newest trailing lines when true; otherwise keep the + leading lines. + + Returns: + The bounded content and whether truncation occurred. + """ + lines = content.splitlines(keepends=True) + if len(lines) <= limit: + return content, False + bounded_lines = lines[-limit:] if keep_tail else lines[:limit] + return "".join(bounded_lines), True + + +def unsupported_scope_error( + kind: DiagnosticKind, + scope: str, + supported: tuple[str, ...], +) -> HTTPException: + """Build the stable error for a scope unsupported by a runtime. + + Args: + kind: Diagnostic payload kind. + scope: Scope requested by the caller. + supported: Scopes implemented by the runtime. + + Returns: + An HTTP 400 exception matching the diagnostics error contract. + """ + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported {kind} diagnostics scope {scope!r}. " + f"Supported scopes: {', '.join(supported)}." + ), + }, + ) diff --git a/server/opensandbox_server/services/docker/container_ops.py b/server/opensandbox_server/services/docker/container_ops.py index d8118cecb..f0f85f411 100644 --- a/server/opensandbox_server/services/docker/container_ops.py +++ b/server/opensandbox_server/services/docker/container_ops.py @@ -307,12 +307,16 @@ def _build_labels_and_env( apply_access_renew_extend_seconds_to_mapping(labels, request.extensions) apply_extensions_to_mapping(labels, request.extensions) - env_dict = request.env or {} + # Config-level defaults apply to every sandbox; request keys win. + env_dict = {**(self.app_config.docker.sandbox_env or {}), **(request.env or {})} environment = [] for key, value in env_dict.items(): if value is None: continue environment.append(f"{key}={value}") + if self.app_config and self.app_config.runtime.execd_run_as_init: + environment.append("EXECD_INIT=1") + environment.append(f"OPENSANDBOX_ID={sandbox_id}") return labels, environment def _resolve_image_auth( diff --git a/server/opensandbox_server/services/docker/docker_diagnostics.py b/server/opensandbox_server/services/docker/docker_diagnostics.py index 9f32159ba..3da2cc370 100644 --- a/server/opensandbox_server/services/docker/docker_diagnostics.py +++ b/server/opensandbox_server/services/docker/docker_diagnostics.py @@ -24,6 +24,21 @@ import re import time +from docker.errors import DockerException +from fastapi import HTTPException, status + +from opensandbox_server.services.constants import SandboxErrorCodes +from opensandbox_server.services.diagnostics import ( + DiagnosticResult, + limit_diagnostic_lines, + unsupported_scope_error, +) + +_SUPPORTED_LOG_SCOPES = ("container", "all") +_SUPPORTED_EVENT_SCOPES = ("runtime", "all") +_STABLE_LOG_LINE_LIMIT = 100 +_STABLE_EVENT_LINE_LIMIT = 50 + def _parse_since_to_timestamp(since: str) -> int: """Parse a human-readable duration string (e.g. '10m', '1h') into a Unix timestamp. @@ -44,6 +59,72 @@ def _parse_since_to_timestamp(since: str) -> int: class DockerDiagnosticsMixin: """Mixin that implements diagnostics methods for the Docker backend.""" + def get_sandbox_log_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable log diagnostics using Docker capabilities.""" + normalized_scope = scope.strip().lower() + if normalized_scope not in _SUPPORTED_LOG_SCOPES: + raise unsupported_scope_error("logs", scope, _SUPPORTED_LOG_SCOPES) + + content = self.get_sandbox_logs( + sandbox_id, + tail=_STABLE_LOG_LINE_LIMIT + 1, + since=None, + container=None, + ) + content, truncated = limit_diagnostic_lines( + content, + _STABLE_LOG_LINE_LIMIT, + keep_tail=True, + ) + warnings: tuple[str, ...] = () + if normalized_scope == "all": + warnings = ( + "The current backend only contributes sandbox container logs to the all scope.", + ) + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="logs", + scope=normalized_scope, + content=content, + truncated=truncated, + warnings=warnings, + ) + + def get_sandbox_event_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable event diagnostics using Docker capabilities.""" + normalized_scope = scope.strip().lower() + if normalized_scope not in _SUPPORTED_EVENT_SCOPES: + raise unsupported_scope_error("events", scope, _SUPPORTED_EVENT_SCOPES) + + content = self.get_sandbox_events( + sandbox_id, + limit=_STABLE_EVENT_LINE_LIMIT + 1, + ) + content, truncated = limit_diagnostic_lines( + content, + _STABLE_EVENT_LINE_LIMIT, + keep_tail=False, + ) + warnings: tuple[str, ...] = () + if normalized_scope == "all": + warnings = ("The current backend only contributes runtime events to the all scope.",) + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="events", + scope=normalized_scope, + content=content, + truncated=truncated, + warnings=warnings, + ) + def get_sandbox_logs( self, sandbox_id: str, @@ -59,7 +140,16 @@ def get_sandbox_logs( kwargs: dict = {"tail": tail, "timestamps": True} if since: kwargs["since"] = _parse_since_to_timestamp(since) - output = docker_container.logs(**kwargs) + try: + output = docker_container.logs(**kwargs) + except DockerException as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.CONTAINER_QUERY_FAILED, + "message": f"Failed to read logs for sandbox {sandbox_id}: {exc}", + }, + ) from exc if isinstance(output, bytes): output = output.decode("utf-8", errors="replace") return output or "(no logs)" diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.py index ab359e2bd..6e0fda1bd 100644 --- a/server/opensandbox_server/services/docker/docker_service.py +++ b/server/opensandbox_server/services/docker/docker_service.py @@ -153,6 +153,7 @@ def __init__(self, config: Optional[AppConfig] = None): self._bootstrap_script_cache: Dict[str, bytes] = {} self._bwrap_archive_cache: Dict[str, bytes] = {} self._session_gate_archive_cache: Dict[str, bytes] = {} + self._launcher_archive_cache: Dict[str, bytes] = {} self._windows_profile_cache: Dict[str, bytes] = {} self._daemon_platform: Optional[PlatformSpec] = None self._metadata_store = DockerMetadataStore() @@ -868,8 +869,11 @@ def _provision_sandbox( volume_binds.append( f"{runtime_volume_name}:{OPENSANDBOX_RUNTIME_MOUNT_PATH}:rw" ) - if volume_binds: - host_config_kwargs["binds"] = volume_binds + # Config-level binds (docker.sandbox_binds) apply to every sandbox + # and come first. + all_binds = list(self.app_config.docker.sandbox_binds or []) + (volume_binds or []) + if all_binds: + host_config_kwargs["binds"] = all_binds if requested_windows_profile: host_config_kwargs = apply_windows_runtime_host_config_defaults( host_config_kwargs, diff --git a/server/opensandbox_server/services/docker/networking.py b/server/opensandbox_server/services/docker/networking.py index 7509c788a..46ebf8df2 100644 --- a/server/opensandbox_server/services/docker/networking.py +++ b/server/opensandbox_server/services/docker/networking.py @@ -511,6 +511,7 @@ def build_sidecar_host_config(*, include_ipv6_sysctls: bool) -> Any: sandbox_id, egress_api_host_port, egress_token, + timeout_seconds=self.app_config.egress.readiness_timeout_seconds, ) return sidecar_container except Exception as exc: @@ -549,7 +550,7 @@ def _wait_for_egress_sidecar_ready( sandbox_id: str, host_port: int, egress_token: str, - timeout_seconds: float = 30.0, + timeout_seconds: float, ) -> None: deadline = time.monotonic() + timeout_seconds url = f"http://{self._resolve_proxy_host()}:{host_port}/healthz" diff --git a/server/opensandbox_server/services/docker/runtime.py b/server/opensandbox_server/services/docker/runtime.py index 0a85247c8..de90682aa 100644 --- a/server/opensandbox_server/services/docker/runtime.py +++ b/server/opensandbox_server/services/docker/runtime.py @@ -44,6 +44,8 @@ BOOTSTRAP_PATH = posixpath.join(OPENSANDBOX_DIR, "bootstrap.sh") SESSION_GATE_SOURCE_PATH = "/usr/local/libexec/opensandbox-session-gate" SESSION_GATE_INSTALL_PATH = posixpath.join(OPENSANDBOX_DIR, "opensandbox-session-gate") +LAUNCHER_SOURCE_PATH = "/usr/local/libexec/opensandbox-launcher" +LAUNCHER_INSTALL_PATH = posixpath.join(OPENSANDBOX_DIR, "opensandbox-launcher") DEFAULT_EXECD_ENVS_PATH = posixpath.join(OPENSANDBOX_DIR, ".env") @@ -145,6 +147,24 @@ def _fetch_execd_archive(self, platform: Optional[PlatformSpec] = None) -> bytes "session workload gate not found in execd image โ€” " "gated isolated-session lifecycle will be unavailable" ) + # Cache the hardening launcher (best-effort; older images do + # not contain it, which degrades [hardening] to unavailable). + if cache_key not in self._launcher_archive_cache: + try: + with self._docker_operation( + "execd cache read launcher", "execd-cache" + ): + launcher_stream, _ = container.get_archive( + LAUNCHER_SOURCE_PATH + ) + self._launcher_archive_cache[cache_key] = b"".join( + launcher_stream + ) + except DockerNotFound: + logger.warning( + "hardening launcher not found in execd image โ€” " + "[hardening] will degrade to unavailable" + ) except DockerException as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -319,6 +339,42 @@ def _copy_session_gate_to_container( }, ) from exc + def _copy_launcher_to_container( + self, + container, + sandbox_id: str, + platform: Optional[PlatformSpec] = None, + ) -> None: + """Copy the hardening launcher into its managed runtime path. + + Best-effort for backward compatibility with older execd images; when + it is missing, [hardening] degrades to unavailable at runtime. + """ + cache_key = self._normalize_platform_key(platform) + archive = self._launcher_archive_cache.get(cache_key) + if archive is None: + logger.warning( + "hardening launcher archive not cached for %s โ€” " + "[hardening] will be unavailable", + cache_key, + ) + return + + try: + with self._docker_operation("copy launcher to sandbox", sandbox_id): + container.put_archive(path=OPENSANDBOX_DIR, data=archive) + except DockerException as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.EXECD_DISTRIBUTION_FAILED, + "message": ( + "Failed to copy hardening launcher into sandbox " + f"at {LAUNCHER_INSTALL_PATH}: {str(exc)}" + ), + }, + ) from exc + def _prepare_sandbox_runtime( self, container, @@ -330,3 +386,4 @@ def _prepare_sandbox_runtime( self._install_bootstrap_script(container, sandbox_id, platform) self._copy_bwrap_to_container(container, sandbox_id, platform) self._copy_session_gate_to_container(container, sandbox_id, platform) + self._copy_launcher_to_container(container, sandbox_id, platform) diff --git a/server/opensandbox_server/services/fleets/__init__.py b/server/opensandbox_server/services/fleets/__init__.py new file mode 100644 index 000000000..d5e97d779 --- /dev/null +++ b/server/opensandbox_server/services/fleets/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/server/opensandbox_server/services/fleets/create_mapping.py b/server/opensandbox_server/services/fleets/create_mapping.py new file mode 100644 index 000000000..001486950 --- /dev/null +++ b/server/opensandbox_server/services/fleets/create_mapping.py @@ -0,0 +1,275 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Map OpenSandbox CreateSandboxRequest into fast-sandbox FastPath v2 CreateRequest. + +The fleets backend accepts a strict subset of the public create contract. +Unsupported fields are rejected with a clear error instead of being silently +ignored (see OSEP-0007 "Simplified Create"). +""" + +from __future__ import annotations + +import decimal +import re +from datetime import datetime, timezone +from typing import Optional + +from opensandbox_server.api.schema import CreateSandboxRequest +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2 as pb2, +) + +#: fleets-reserved FastPath metadata key persisting the renew-on-access +#: extension value. Stripped from public metadata and list filters. +#: fast-sandbox persists metadata as labels under `metadata.sandbox.fast.io/`, +#: so the key must be a DNS1123 label (lowercase alphanumeric + hyphens). +RENEW_EXTEND_SECONDS_METADATA_KEY = "renew-extend-seconds" + +#: Public extensions keys accepted by the fleets backend. +SUPPORTED_EXTENSION_KEYS = frozenset( + {"poolRef", "access.renew.extend.seconds"} +) + +#: FastPath CreateRequest has no nullable timeout; fleets requires an explicit one. +ERROR_TIMEOUT_REQUIRED = ( + "timeout is required on fleets: fast-sandbox persists an absolute " + "expires_at in the first Create write and has no non-expiring sandboxes." +) + + +class UnsupportedFieldError(ValueError): + """A CreateSandboxRequest field cannot be honored by the fleets backend.""" + + def __init__(self, field: str, reason: str): + super().__init__(f"{field}: {reason}") + self.field = field + self.reason = reason + + +def map_create_request( + request: CreateSandboxRequest, + sandbox_id: str, + namespace: str, + *, + default_pool_ref: str = "default-pool", + now: Optional[datetime] = None, + expires_at_unix_seconds: Optional[int] = None, + pool_resources: Optional[dict] = None, +) -> pb2.CreateRequest: + """Map the accepted CreateSandboxRequest subset to a FastPath v2 CreateRequest. + + Raises UnsupportedFieldError for any field the shared-Fastlet model cannot + honor. The caller supplies the OpenSandbox sandbox_id, which becomes the + idempotency key (request_id) and the Sandbox CRD name. + + Idempotency: FastPath treats expiry as part of the persisted initial + intent, so a transport retry of the same sandbox_id must reuse the first + absolute expiry. Callers can pass the previously normalized + ``expires_at_unix_seconds``; when omitted it is derived from ``now``. + + Pool compatibility: ``pool_resources`` is the selected SandboxPool profile + (e.g. {"cpu": "500m", "memory": "512Mi", "pids": "256"}). When provided, + request ``resource_limits`` must match the pool for every key the pool + defines and must not declare keys the pool does not define; FastPath has + no per-sandbox resource field, so incompatible limits are rejected rather + than silently ignored. + """ + _reject_unsupported_fields(request) + + if pool_resources is not None: + _validate_resource_limits(request, pool_resources) + + image = request.image + if image is None or not image.uri.strip(): + # A fast-sandbox SandboxPool defines Infra Components and resources, + # not the workload image, so fleets rejects image-less requests even + # when extensions.poolRef is set. + raise UnsupportedFieldError("image", "a non-empty image.uri is required on fleets") + + if image.auth is not None: + raise UnsupportedFieldError( + "image.auth", "private-registry credentials are not carried to fast-sandbox" + ) + + if request.timeout is None: + raise UnsupportedFieldError("timeout", ERROR_TIMEOUT_REQUIRED) + + if request.entrypoint is None: + raise UnsupportedFieldError("entrypoint", "entrypoint is required when image is provided") + + if expires_at_unix_seconds is not None: + expires_at = expires_at_unix_seconds + else: + now = now or datetime.now(timezone.utc) + expires_at = int(now.timestamp()) + request.timeout + + create = pb2.CreateRequest( + request_id=sandbox_id, + namespace=namespace, + image=image.uri, + command=list(request.entrypoint), + expires_at_unix_seconds=expires_at, + ) + + if request.env: + for key, value in request.env.items(): + if value is None: + raise UnsupportedFieldError( + "env", + f"null value for environment variable {key!r} is not supported " + "(FastPath v2 uses map)", + ) + create.envs[key] = value + + if request.metadata: + create.metadata.update(request.metadata) + + extensions = request.extensions or {} + # Normalize before forwarding: a whitespace-only poolRef must not select + # an invalid pool, and a padded name must not reach FastPath as-is. + pool_ref = (extensions.get("poolRef") or "").strip() + create.pool_ref = pool_ref or default_pool_ref + + renew_value = extensions.get("access.renew.extend.seconds") + if renew_value is not None: + create.metadata[RENEW_EXTEND_SECONDS_METADATA_KEY] = renew_value + + # fast-sandbox persists metadata as labels (metadata.sandbox.fast.io/) + # and validates every entry; reject incompatible keys/values here so users + # get a clear error instead of a confusing gRPC rejection. + _validate_metadata(create.metadata) + + return create + + +def _validate_resource_limits( + request: CreateSandboxRequest, pool_resources: dict +) -> None: + if request.resource_limits is None: + return + limits = request.resource_limits.root + for key, value in limits.items(): + if key not in pool_resources: + raise UnsupportedFieldError( + "resourceLimits", + f"{key!r} is not defined by the selected SandboxPool; " + "resources are fixed by SandboxPool.spec.sandboxResources", + ) + if not _quantities_equal(pool_resources[key], value): + raise UnsupportedFieldError( + "resourceLimits", + f"{key!r}={value!r} does not match the SandboxPool profile " + f"{key!r}={pool_resources[key]!r}; resources are fixed per pool", + ) + + +#: Kubernetes quantity decimal-power suffixes (m is handled separately). +_DECIMAL_QUANTITY_SUFFIXES = {"k": 3, "M": 6, "G": 9, "T": 12, "P": 15, "E": 18} +#: Kubernetes quantity binary-power suffixes. +_BINARY_QUANTITY_SUFFIXES = {"Ki": 10, "Mi": 20, "Gi": 30, "Ti": 40, "Pi": 50, "Ei": 60} + +_DNS_LABEL_PATTERN = r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" +_LABEL_VALUE_PATTERN = r"^[A-Za-z0-9]([-_.A-Za-z0-9]*[A-Za-z0-9])?$" +_MAX_LABEL_LENGTH = 63 + + +def _validate_metadata(metadata: dict) -> None: + """Reject metadata entries fast-sandbox cannot persist as labels.""" + for key, value in metadata.items(): + if ( + len(key) > _MAX_LABEL_LENGTH + or not re.fullmatch(_DNS_LABEL_PATTERN, key) + ): + raise UnsupportedFieldError( + "metadata", + f"metadata key {key!r} must be a DNS label (lowercase alphanumeric " + "and hyphens, max 63 chars): fast-sandbox persists metadata as labels", + ) + if ( + len(value) > _MAX_LABEL_LENGTH + or not re.fullmatch(_LABEL_VALUE_PATTERN, value) + ): + raise UnsupportedFieldError( + "metadata", + f"metadata value for {key!r} must be a Kubernetes label value " + "(alphanumeric, '-', '_', '.', max 63 chars)", + ) + + +def _quantities_equal(a: str, b: str) -> bool: + """Compare Kubernetes resource quantities canonically ("0.5" == "500m", "1Gi" == "1024Mi").""" + try: + return _canonical_quantity(a) == _canonical_quantity(b) + except Exception: + # Fall back to the raw comparison for unparseable values so the + # rejection message stays accurate. + return a == b + + +def _canonical_quantity(value: str) -> decimal.Decimal: + value = value.strip() + if value.endswith("m"): + return decimal.Decimal(value[:-1]) / 1000 + for suffix, exponent in _BINARY_QUANTITY_SUFFIXES.items(): + if value.endswith(suffix): + return decimal.Decimal(value[: -len(suffix)]) * (decimal.Decimal(2) ** exponent) + for suffix, exponent in _DECIMAL_QUANTITY_SUFFIXES.items(): + if value.endswith(suffix): + return decimal.Decimal(value[: -len(suffix)]) * (decimal.Decimal(10) ** exponent) + return decimal.Decimal(value) + + +def _reject_unsupported_fields(request: CreateSandboxRequest) -> None: + """Reject pod-identity-dependent fields that have no shared-Fastlet meaning.""" + if request.snapshot_id: + raise UnsupportedFieldError("snapshotId", "snapshots are not supported on fleets") + if request.platform is not None: + raise UnsupportedFieldError( + "platform", "scheduling is per Fastlet pool, not per sandbox" + ) + if request.resource_requests is not None: + raise UnsupportedFieldError( + "resourceRequests", + "resources are fixed by SandboxPool.spec.sandboxResources", + ) + if request.credential_proxy is not None and request.credential_proxy.enabled: + raise UnsupportedFieldError( + "credentialProxy", + "credential proxy rides the per-pod egress sidecar, which does not " + "exist in the shared-Fastlet model", + ) + if request.volumes: + raise UnsupportedFieldError( + "volumes", "Fastlet child containers cannot receive dynamic mounts" + ) + if request.network_policy is not None: + raise UnsupportedFieldError( + "networkPolicy", + "per-sandbox egress enforcement is deferred to phase 1b; not supported in phase 1a", + ) + if request.secure_access: + raise UnsupportedFieldError( + "secureAccess", + "secure access on fleets is deferred to phase 1b; not supported in phase 1a", + ) + for key in (request.extensions or {}): + if key not in SUPPORTED_EXTENSION_KEYS: + raise UnsupportedFieldError( + f"extensions[{key!r}]", + "extension keys are rejected unless explicitly supported by fleets", + ) diff --git a/server/opensandbox_server/services/fleets/fastpath_client.py b/server/opensandbox_server/services/fleets/fastpath_client.py new file mode 100644 index 000000000..29483fe0a --- /dev/null +++ b/server/opensandbox_server/services/fleets/fastpath_client.py @@ -0,0 +1,310 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""FastPath gRPC client for the fast-sandbox Fast-Path Server (FastPath v2). + +Wraps the generated `fastpath.v2.FastPathService` stubs with async semantics, +typed helpers, and normalized error handling. OpenSandbox must not infer +NotFound from error strings: only gRPC `codes.NotFound` maps to the public +HTTP 404 contract. + +Note: upstream fast-sandbox `GetSandbox` (at `aac0c2c` and later) returns the +raw Kubernetes Get error instead of passing it through `grpcKubernetesError`. +Until that upstream fix lands, a missing Sandbox CRD surfaces as an unknown +status code; the fleets adapter must treat only `codes.NotFound` as 404. +""" + +from __future__ import annotations + +from typing import Optional + +import grpc +from grpc import aio + +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2 as fastpath_pb2, +) +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2_grpc as fastpath_pb2_grpc, +) + +DEFAULT_FASTPATH_ENDPOINT = "fast-sandbox-fastpath.opensandbox.svc:9090" + + +class FastPathError(Exception): + """Base error for fast-sandbox FastPath communication failures.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + self.message = message + + def __str__(self) -> str: # pragma: no cover - trivial formatting + return f"FastPathError(code={self.code}, message={self.message})" + + +class FastPathNotFound(FastPathError): + """The referenced fast-sandbox resource does not exist (gRPC NotFound).""" + + +class FastPathUnavailable(FastPathError): + """The FastPath server is unreachable or the request timed out.""" + + +class FastPathInvalidArgument(FastPathError): + """The FastPath server rejected the request (gRPC InvalidArgument).""" + + +class FastPathConflict(FastPathError): + """The FastPath request conflicts with existing durable state.""" + + +class FastPathClient: + """Async gRPC client for the fast-sandbox FastPathService v2 API.""" + + def __init__( + self, + endpoint: str = DEFAULT_FASTPATH_ENDPOINT, + timeout_seconds: float = 30.0, + ) -> None: + self._endpoint = endpoint + self._timeout_seconds = timeout_seconds + self._channel: Optional[aio.Channel] = None + self._stub: Optional[fastpath_pb2_grpc.FastPathServiceStub] = None + + async def __aenter__(self) -> "FastPathClient": + await self.connect() + return self + + async def __aexit__(self, *exc_info) -> None: + await self.close() + + async def connect(self) -> None: + """Open the gRPC channel to the FastPath endpoint.""" + if self._channel is None: + self._channel = aio.insecure_channel(self._endpoint) + self._stub = fastpath_pb2_grpc.FastPathServiceStub(self._channel) + + async def close(self) -> None: + """Close the gRPC channel if open.""" + if self._channel is not None: + await self._channel.close() + self._channel = None + self._stub = None + + # -- lifecycle --------------------------------------------------------- + + async def create_sandbox( + self, request: fastpath_pb2.CreateRequest + ) -> fastpath_pb2.SandboxInfo: + """Create a sandbox through FastPath v2 (CRD-first, idempotent by request_id).""" + return await self._call( + lambda: self._require_stub().CreateSandbox(request, timeout=self._timeout_seconds) + ) + + async def get_sandbox( + self, namespace: str, sandbox_name: str + ) -> fastpath_pb2.SandboxInfo: + """Get a sandbox; raises FastPathNotFound on gRPC NotFound.""" + request = fastpath_pb2.GetRequest(namespace=namespace, sandbox_name=sandbox_name) + return await self._call(lambda: self._require_stub().GetSandbox(request, timeout=self._timeout_seconds)) + + async def delete_sandbox(self, namespace: str, sandbox_name: str) -> None: + """Submit an async (finalizer-driven) sandbox deletion.""" + request = fastpath_pb2.DeleteRequest( + namespace=namespace, sandbox_name=sandbox_name + ) + await self._call(lambda: self._require_stub().DeleteSandbox(request, timeout=self._timeout_seconds)) + + async def list_sandboxes( + self, + namespace: str, + metadata: Optional[dict] = None, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + ) -> fastpath_pb2.ListResponse: + """List sandboxes in a namespace; metadata acts as an AND-filter.""" + request = fastpath_pb2.ListRequest(namespace=namespace) + if metadata: + request.metadata.update(metadata) + if page_size is not None: + request.page_size = page_size + if page_token: + request.page_token = page_token + return await self._call(lambda: self._require_stub().ListSandboxes(request, timeout=self._timeout_seconds)) + + async def update_expiration( + self, namespace: str, sandbox_name: str, expires_at_unix_seconds: int + ) -> fastpath_pb2.SandboxInfo: + """Persist an absolute expiry on the Sandbox CRD.""" + request = fastpath_pb2.UpdateRequest( + namespace=namespace, sandbox_name=sandbox_name + ) + request.expires_at_unix_seconds = expires_at_unix_seconds + response = await self._call(lambda: self._require_stub().UpdateSandbox(request, timeout=self._timeout_seconds)) + return response.sandbox + + async def update_metadata( + self, + namespace: str, + sandbox_name: str, + upsert: Optional[dict] = None, + delete_keys: Optional[list[str]] = None, + ) -> fastpath_pb2.SandboxInfo: + """Update metadata: upsert entries and delete keys in one call.""" + request = fastpath_pb2.UpdateRequest( + namespace=namespace, sandbox_name=sandbox_name + ) + if upsert: + request.metadata_upsert.update(upsert) + if delete_keys: + request.metadata_delete_keys.extend(delete_keys) + response = await self._call(lambda: self._require_stub().UpdateSandbox(request, timeout=self._timeout_seconds)) + return response.sandbox + + async def get_sandbox_diagnostics( + self, namespace: str, sandbox_name: str, limit: int = 50 + ) -> fastpath_pb2.SandboxDiagnosticsResponse: + """Return lifecycle diagnostics (events only, not process output).""" + request = fastpath_pb2.SandboxDiagnosticsRequest( + namespace=namespace, sandbox_name=sandbox_name, limit=limit + ) + return await self._call( + lambda: self._require_stub().GetSandboxDiagnostics(request, timeout=self._timeout_seconds) + ) + + # -- readiness / endpoints -------------------------------------------- + + async def wait_sandbox_ready( + self, + reference: fastpath_pb2.SandboxReference, + *, + data_plane: bool = False, + component_name: Optional[str] = None, + wait_timeout_millis: int = 30000, + ) -> fastpath_pb2.SandboxInfo: + """Wait on the assigned Fastlet for runtime or data-plane readiness.""" + request = fastpath_pb2.WaitSandboxReadyRequest( + sandbox=reference, + wait_timeout_millis=wait_timeout_millis, + ) + if component_name is not None: + request.component_name = component_name + else: + request.data_plane = data_plane + return await self._call( + lambda: self._require_stub().WaitSandboxReady( + request, timeout=self._rpc_timeout(wait_timeout_millis) + ) + ) + + async def resolve_endpoint( + self, + reference: fastpath_pb2.SandboxReference, + target: fastpath_pb2.EndpointTarget, + *, + access_mode: fastpath_pb2.EndpointAccessMode = ( + fastpath_pb2.CENTRAL_PROXY + ), + wait_until_ready: bool = False, + wait_timeout_millis: int = 30000, + ) -> fastpath_pb2.ResolveEndpointResponse: + """Resolve an authenticated proxy route for a component or raw port.""" + request = fastpath_pb2.ResolveEndpointRequest( + sandbox=reference, + target=target, + access_mode=access_mode, + wait_until_ready=wait_until_ready, + wait_timeout_millis=wait_timeout_millis, + ) + deadline = ( + self._rpc_timeout(wait_timeout_millis) + if wait_until_ready + else self._timeout_seconds + ) + return await self._call( + lambda: self._require_stub().ResolveEndpoint(request, timeout=deadline) + ) + + # -- pools ------------------------------------------------------------- + + async def get_pool( + self, namespace: str, pool_name: str + ) -> fastpath_pb2.PoolInfo: + """Get a SandboxPool; raises FastPathNotFound when absent.""" + request = fastpath_pb2.GetPoolRequest(namespace=namespace, pool_name=pool_name) + return await self._call(lambda: self._require_stub().GetPool(request, timeout=self._timeout_seconds)) + + async def list_pools(self, namespace: str) -> fastpath_pb2.ListPoolsResponse: + """List SandboxPools in a namespace.""" + request = fastpath_pb2.ListPoolsRequest(namespace=namespace) + return await self._call(lambda: self._require_stub().ListPools(request, timeout=self._timeout_seconds)) + + # -- internals --------------------------------------------------------- + + def _rpc_timeout(self, server_wait_millis: int) -> float: + """gRPC deadline must exceed the server-side readiness wait.""" + return max(self._timeout_seconds, server_wait_millis / 1000 + 5.0) + + def _require_stub(self) -> fastpath_pb2_grpc.FastPathServiceStub: + if self._stub is None: + raise FastPathUnavailable("channel-not-open", "FastPath client is not connected") + return self._stub + + async def _call(self, call): + try: + return await call() + except grpc.aio.AioRpcError as exc: + raise _to_fastpath_error(exc) from exc + + +def _to_fastpath_error(exc: grpc.aio.AioRpcError) -> FastPathError: + """Normalize a gRPC status to a typed FastPathError, without string matching.""" + code = exc.code() + details = exc.details() or "" + if code == grpc.StatusCode.NOT_FOUND: + return FastPathNotFound(code.name, details) + if code == grpc.StatusCode.INVALID_ARGUMENT: + return FastPathInvalidArgument(code.name, details) + if code in (grpc.StatusCode.ALREADY_EXISTS, grpc.StatusCode.ABORTED): + return FastPathConflict(code.name, details) + if code in ( + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.CANCELLED, + ): + return FastPathUnavailable(code.name, details) + return FastPathError(code.name, details) + + +def namespaced_reference(namespace: str, sandbox_name: str) -> fastpath_pb2.SandboxReference: + """Build a SandboxReference by namespaced name (no UID cache required).""" + return fastpath_pb2.SandboxReference( + namespaced_name=fastpath_pb2.NamespacedName( + namespace=namespace, name=sandbox_name + ) + ) + + +def component_target(component_name: str) -> fastpath_pb2.EndpointTarget: + """Build an EndpointTarget for a named Pool Infra Component (e.g. execd).""" + return fastpath_pb2.EndpointTarget(component_name=component_name) + + +def port_target(port: int) -> fastpath_pb2.EndpointTarget: + """Build an EndpointTarget for a raw user port.""" + return fastpath_pb2.EndpointTarget(port=port) diff --git a/server/opensandbox_server/services/fleets/generated/__init__.py b/server/opensandbox_server/services/fleets/generated/__init__.py new file mode 100644 index 000000000..01d0b919f --- /dev/null +++ b/server/opensandbox_server/services/fleets/generated/__init__.py @@ -0,0 +1,16 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/server/opensandbox_server/services/fleets/generated/fastpath_pb2.py b/server/opensandbox_server/services/fleets/generated/fastpath_pb2.py new file mode 100644 index 000000000..0dbb360bb --- /dev/null +++ b/server/opensandbox_server/services/fleets/generated/fastpath_pb2.py @@ -0,0 +1,120 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: fastpath.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'fastpath.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x66\x61stpath.proto\x12\x0b\x66\x61stpath.v2\"1\n\x0eNamespacedName\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"n\n\x10SandboxReference\x12\x15\n\x0bsandbox_uid\x18\x01 \x01(\tH\x00\x12\x36\n\x0fnamespaced_name\x18\x02 \x01(\x0b\x32\x1b.fastpath.v2.NamespacedNameH\x00\x42\x0b\n\treference\"\xa6\x01\n\rComponentInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05state\x18\x02 \x01(\t\x12\x10\n\x08protocol\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\x12!\n\x19observed_route_generation\x18\x05 \x01(\x03\x12$\n\x1clast_transition_unix_seconds\x18\x06 \x01(\x03\x12\x0f\n\x07message\x18\x07 \x01(\t\"\xec\x04\n\x0bSandboxInfo\x12\x13\n\x0bsandbox_uid\x18\x01 \x01(\t\x12\x14\n\x0csandbox_name\x18\x02 \x01(\t\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x15\n\rruntime_state\x18\x04 \x01(\t\x12\x18\n\x10\x64\x61ta_plane_state\x18\x05 \x01(\t\x12\x1a\n\x12user_process_state\x18\x06 \x01(\t\x12\x13\n\x0b\x66\x61stlet_pod\x18\x07 \x01(\t\x12\x1f\n\x17\x63reated_at_unix_seconds\x18\x08 \x01(\x03\x12\r\n\x05image\x18\t \x01(\t\x12\x10\n\x08pool_ref\x18\n \x01(\t\x12\x38\n\x08metadata\x18\x0b \x03(\x0b\x32&.fastpath.v2.SandboxInfo.MetadataEntry\x12\x1f\n\x17\x65xpires_at_unix_seconds\x18\x0c \x01(\x03\x12\x32\n\x0e\x66\x61ilure_policy\x18\r \x01(\x0e\x32\x1a.fastpath.v2.FailurePolicy\x12 \n\x18recovery_timeout_seconds\x18\x0e \x01(\x05\x12\x1a\n\x12\x61ssignment_attempt\x18\x0f \x01(\x03\x12\x1b\n\x13instance_generation\x18\x10 \x01(\x03\x12\x18\n\x10route_generation\x18\x11 \x01(\x03\x12\x16\n\x0einfra_revision\x18\x12 \x01(\t\x12.\n\ncomponents\x18\x13 \x03(\x0b\x32\x1a.fastpath.v2.ComponentInfo\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd0\x03\n\rCreateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12\x10\n\x08pool_ref\x18\x04 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x05 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x06 \x03(\t\x12\x32\n\x04\x65nvs\x18\x07 \x03(\x0b\x32$.fastpath.v2.CreateRequest.EnvsEntry\x12\x13\n\x0bworking_dir\x18\x08 \x01(\t\x12\x1f\n\x17\x65xpires_at_unix_seconds\x18\t \x01(\x03\x12:\n\x08metadata\x18\n \x03(\x0b\x32(.fastpath.v2.CreateRequest.MetadataEntry\x12\x32\n\x0e\x66\x61ilure_policy\x18\x0b \x01(\x0e\x32\x1a.fastpath.v2.FailurePolicy\x12 \n\x18recovery_timeout_seconds\x18\x0c \x01(\x05\x1a+\n\tEnvsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"5\n\nGetRequest\x12\x14\n\x0csandbox_name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"\xb2\x01\n\x0bListRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x08metadata\x18\x02 \x03(\x0b\x32&.fastpath.v2.ListRequest.MetadataEntry\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"P\n\x0cListResponse\x12\'\n\x05items\x18\x01 \x03(\x0b\x32\x18.fastpath.v2.SandboxInfo\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"8\n\rDeleteRequest\x12\x14\n\x0csandbox_name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\"!\n\x0e\x44\x65leteResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\"\xf7\x02\n\rUpdateRequest\x12\x14\n\x0csandbox_name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12!\n\x17\x65xpires_at_unix_seconds\x18\x03 \x01(\x03H\x00\x12\x18\n\x0ereset_revision\x18\x04 \x01(\tH\x00\x12\x34\n\x0e\x66\x61ilure_policy\x18\x05 \x01(\x0e\x32\x1a.fastpath.v2.FailurePolicyH\x00\x12\"\n\x18recovery_timeout_seconds\x18\x06 \x01(\x05H\x00\x12G\n\x0fmetadata_upsert\x18\x07 \x03(\x0b\x32..fastpath.v2.UpdateRequest.MetadataUpsertEntry\x12\x1c\n\x14metadata_delete_keys\x18\x08 \x03(\t\x1a\x35\n\x13MetadataUpsertEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06update\"]\n\x0eUpdateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12)\n\x07sandbox\x18\x03 \x01(\x0b\x32\x18.fastpath.v2.SandboxInfo\"S\n\x19SandboxDiagnosticsRequest\x12\x14\n\x0csandbox_name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\"t\n\x16SandboxDiagnosticEvent\x12\x1b\n\x13timestamp_unix_nano\x18\x01 \x01(\x03\x12\r\n\x05level\x18\x02 \x01(\t\x12\x0e\n\x06source\x18\x03 \x01(\t\x12\r\n\x05phase\x18\x04 \x01(\t\x12\x0f\n\x07message\x18\x05 \x01(\t\"\x81\x02\n\x1aSandboxDiagnosticsResponse\x12)\n\x07sandbox\x18\x01 \x01(\x0b\x32\x18.fastpath.v2.SandboxInfo\x12\x18\n\x10\x61ssignment_state\x18\x02 \x01(\t\x12\x1b\n\x13runtime_instance_id\x18\x03 \x01(\t\x12\x1a\n\x12\x61ssignment_attempt\x18\x04 \x01(\x03\x12\x19\n\x11\x66\x61stlet_reachable\x18\x05 \x01(\x08\x12\x15\n\rfastlet_error\x18\x06 \x01(\t\x12\x33\n\x06\x65vents\x18\x07 \x03(\x0b\x32#.fastpath.v2.SandboxDiagnosticEvent\"\xa0\x01\n\x17WaitSandboxReadyRequest\x12.\n\x07sandbox\x18\x01 \x01(\x0b\x32\x1d.fastpath.v2.SandboxReference\x12\x14\n\ndata_plane\x18\x02 \x01(\x08H\x00\x12\x18\n\x0e\x63omponent_name\x18\x03 \x01(\tH\x00\x12\x1b\n\x13wait_timeout_millis\x18\x04 \x01(\x05\x42\x08\n\x06target\"D\n\x0e\x45ndpointTarget\x12\x18\n\x0e\x63omponent_name\x18\x01 \x01(\tH\x00\x12\x0e\n\x04port\x18\x02 \x01(\rH\x00\x42\x08\n\x06target\"\xe2\x01\n\x16ResolveEndpointRequest\x12.\n\x07sandbox\x18\x01 \x01(\x0b\x32\x1d.fastpath.v2.SandboxReference\x12+\n\x06target\x18\x02 \x01(\x0b\x32\x1b.fastpath.v2.EndpointTarget\x12\x34\n\x0b\x61\x63\x63\x65ss_mode\x18\x03 \x01(\x0e\x32\x1f.fastpath.v2.EndpointAccessMode\x12\x18\n\x10wait_until_ready\x18\x04 \x01(\x08\x12\x1b\n\x13wait_timeout_millis\x18\x05 \x01(\x05\"\xfc\x02\n\x17ResolveEndpointResponse\x12\x13\n\x0bsandbox_uid\x18\x01 \x01(\t\x12+\n\x06target\x18\x02 \x01(\x0b\x32\x1b.fastpath.v2.EndpointTarget\x12\x16\n\x0e\x63omponent_name\x18\x03 \x01(\t\x12\x10\n\x08protocol\x18\x04 \x01(\t\x12\x15\n\rresolved_port\x18\x05 \x01(\r\x12\x16\n\x0eproxy_endpoint\x18\x06 \x01(\t\x12S\n\x10required_headers\x18\x07 \x03(\x0b\x32\x39.fastpath.v2.ResolveEndpointResponse.RequiredHeadersEntry\x12\x18\n\x10route_generation\x18\x08 \x01(\x03\x12\x1f\n\x17\x65xpires_at_unix_seconds\x18\t \x01(\x03\x1a\x36\n\x14RequiredHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"X\n\x13\x43omponentCapability\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08protocol\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x13\n\x0bhealth_kind\x18\x04 \x01(\t\"\xb5\x01\n\rWarmImageInfo\x12\r\n\x05image\x18\x01 \x01(\t\x12\x18\n\x10\x64\x65sired_fastlets\x18\x02 \x01(\x05\x12\x17\n\x0f\x63\x61\x63hed_fastlets\x18\x03 \x01(\x05\x12\x18\n\x10pulling_fastlets\x18\x04 \x01(\x05\x12\x17\n\x0f\x66\x61iled_fastlets\x18\x05 \x01(\x05\x12\x1b\n\x13observed_generation\x18\x06 \x01(\x03\x12\x12\n\nlast_error\x18\x07 \x01(\t\"o\n\x0cRegistryInfo\x12\x19\n\x11target_generation\x18\x01 \x01(\x03\x12\x18\n\x10\x61pplied_fastlets\x18\x02 \x01(\x05\x12\x16\n\x0etotal_fastlets\x18\x03 \x01(\x05\x12\x12\n\nlast_error\x18\x04 \x01(\t\"\xac\x03\n\x08PoolInfo\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07runtime\x18\x03 \x01(\t\x12\x13\n\x0bsandbox_cpu\x18\x04 \x01(\t\x12\x16\n\x0esandbox_memory\x18\x05 \x01(\t\x12\x14\n\x0csandbox_pids\x18\x06 \x01(\x03\x12\x1d\n\x15max_sandboxes_per_pod\x18\x07 \x01(\x05\x12\x16\n\x0etotal_fastlets\x18\x08 \x01(\x05\x12\x16\n\x0eready_fastlets\x18\t \x01(\x05\x12\x15\n\ridle_fastlets\x18\n \x01(\x05\x12\x16\n\x0einfra_revision\x18\x0b \x01(\t\x12\x19\n\x11prepared_fastlets\x18\x0c \x01(\x05\x12\x34\n\ncomponents\x18\r \x03(\x0b\x32 .fastpath.v2.ComponentCapability\x12+\n\x08registry\x18\x0e \x01(\x0b\x32\x19.fastpath.v2.RegistryInfo\x12/\n\x0bwarm_images\x18\x0f \x03(\x0b\x32\x1a.fastpath.v2.WarmImageInfo\"6\n\x0eGetPoolRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tpool_name\x18\x02 \x01(\t\"%\n\x10ListPoolsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\"9\n\x11ListPoolsResponse\x12$\n\x05items\x18\x01 \x03(\x0b\x32\x15.fastpath.v2.PoolInfo*.\n\rFailurePolicy\x12\n\n\x06MANUAL\x10\x00\x12\x11\n\rAUTO_RECREATE\x10\x01*A\n\x12\x45ndpointAccessMode\x12\x11\n\rCENTRAL_PROXY\x10\x00\x12\x18\n\x14\x44IRECT_FASTLET_PROXY\x10\x01\x32\x9a\x06\n\x0f\x46\x61stPathService\x12\x45\n\rCreateSandbox\x12\x1a.fastpath.v2.CreateRequest\x1a\x18.fastpath.v2.SandboxInfo\x12H\n\rDeleteSandbox\x12\x1a.fastpath.v2.DeleteRequest\x1a\x1b.fastpath.v2.DeleteResponse\x12H\n\rUpdateSandbox\x12\x1a.fastpath.v2.UpdateRequest\x1a\x1b.fastpath.v2.UpdateResponse\x12\x44\n\rListSandboxes\x12\x18.fastpath.v2.ListRequest\x1a\x19.fastpath.v2.ListResponse\x12?\n\nGetSandbox\x12\x17.fastpath.v2.GetRequest\x1a\x18.fastpath.v2.SandboxInfo\x12h\n\x15GetSandboxDiagnostics\x12&.fastpath.v2.SandboxDiagnosticsRequest\x1a\'.fastpath.v2.SandboxDiagnosticsResponse\x12R\n\x10WaitSandboxReady\x12$.fastpath.v2.WaitSandboxReadyRequest\x1a\x18.fastpath.v2.SandboxInfo\x12\\\n\x0fResolveEndpoint\x12#.fastpath.v2.ResolveEndpointRequest\x1a$.fastpath.v2.ResolveEndpointResponse\x12=\n\x07GetPool\x12\x1b.fastpath.v2.GetPoolRequest\x1a\x15.fastpath.v2.PoolInfo\x12J\n\tListPools\x12\x1d.fastpath.v2.ListPoolsRequest\x1a\x1e.fastpath.v2.ListPoolsResponseB&Z$fast-sandbox/api/proto/v2;fastpathv2b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fastpath_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z$fast-sandbox/api/proto/v2;fastpathv2' + _globals['_SANDBOXINFO_METADATAENTRY']._loaded_options = None + _globals['_SANDBOXINFO_METADATAENTRY']._serialized_options = b'8\001' + _globals['_CREATEREQUEST_ENVSENTRY']._loaded_options = None + _globals['_CREATEREQUEST_ENVSENTRY']._serialized_options = b'8\001' + _globals['_CREATEREQUEST_METADATAENTRY']._loaded_options = None + _globals['_CREATEREQUEST_METADATAENTRY']._serialized_options = b'8\001' + _globals['_LISTREQUEST_METADATAENTRY']._loaded_options = None + _globals['_LISTREQUEST_METADATAENTRY']._serialized_options = b'8\001' + _globals['_UPDATEREQUEST_METADATAUPSERTENTRY']._loaded_options = None + _globals['_UPDATEREQUEST_METADATAUPSERTENTRY']._serialized_options = b'8\001' + _globals['_RESOLVEENDPOINTRESPONSE_REQUIREDHEADERSENTRY']._loaded_options = None + _globals['_RESOLVEENDPOINTRESPONSE_REQUIREDHEADERSENTRY']._serialized_options = b'8\001' + _globals['_FAILUREPOLICY']._serialized_start=4617 + _globals['_FAILUREPOLICY']._serialized_end=4663 + _globals['_ENDPOINTACCESSMODE']._serialized_start=4665 + _globals['_ENDPOINTACCESSMODE']._serialized_end=4730 + _globals['_NAMESPACEDNAME']._serialized_start=31 + _globals['_NAMESPACEDNAME']._serialized_end=80 + _globals['_SANDBOXREFERENCE']._serialized_start=82 + _globals['_SANDBOXREFERENCE']._serialized_end=192 + _globals['_COMPONENTINFO']._serialized_start=195 + _globals['_COMPONENTINFO']._serialized_end=361 + _globals['_SANDBOXINFO']._serialized_start=364 + _globals['_SANDBOXINFO']._serialized_end=984 + _globals['_SANDBOXINFO_METADATAENTRY']._serialized_start=937 + _globals['_SANDBOXINFO_METADATAENTRY']._serialized_end=984 + _globals['_CREATEREQUEST']._serialized_start=987 + _globals['_CREATEREQUEST']._serialized_end=1451 + _globals['_CREATEREQUEST_ENVSENTRY']._serialized_start=1359 + _globals['_CREATEREQUEST_ENVSENTRY']._serialized_end=1402 + _globals['_CREATEREQUEST_METADATAENTRY']._serialized_start=937 + _globals['_CREATEREQUEST_METADATAENTRY']._serialized_end=984 + _globals['_GETREQUEST']._serialized_start=1453 + _globals['_GETREQUEST']._serialized_end=1506 + _globals['_LISTREQUEST']._serialized_start=1509 + _globals['_LISTREQUEST']._serialized_end=1687 + _globals['_LISTREQUEST_METADATAENTRY']._serialized_start=937 + _globals['_LISTREQUEST_METADATAENTRY']._serialized_end=984 + _globals['_LISTRESPONSE']._serialized_start=1689 + _globals['_LISTRESPONSE']._serialized_end=1769 + _globals['_DELETEREQUEST']._serialized_start=1771 + _globals['_DELETEREQUEST']._serialized_end=1827 + _globals['_DELETERESPONSE']._serialized_start=1829 + _globals['_DELETERESPONSE']._serialized_end=1862 + _globals['_UPDATEREQUEST']._serialized_start=1865 + _globals['_UPDATEREQUEST']._serialized_end=2240 + _globals['_UPDATEREQUEST_METADATAUPSERTENTRY']._serialized_start=2177 + _globals['_UPDATEREQUEST_METADATAUPSERTENTRY']._serialized_end=2230 + _globals['_UPDATERESPONSE']._serialized_start=2242 + _globals['_UPDATERESPONSE']._serialized_end=2335 + _globals['_SANDBOXDIAGNOSTICSREQUEST']._serialized_start=2337 + _globals['_SANDBOXDIAGNOSTICSREQUEST']._serialized_end=2420 + _globals['_SANDBOXDIAGNOSTICEVENT']._serialized_start=2422 + _globals['_SANDBOXDIAGNOSTICEVENT']._serialized_end=2538 + _globals['_SANDBOXDIAGNOSTICSRESPONSE']._serialized_start=2541 + _globals['_SANDBOXDIAGNOSTICSRESPONSE']._serialized_end=2798 + _globals['_WAITSANDBOXREADYREQUEST']._serialized_start=2801 + _globals['_WAITSANDBOXREADYREQUEST']._serialized_end=2961 + _globals['_ENDPOINTTARGET']._serialized_start=2963 + _globals['_ENDPOINTTARGET']._serialized_end=3031 + _globals['_RESOLVEENDPOINTREQUEST']._serialized_start=3034 + _globals['_RESOLVEENDPOINTREQUEST']._serialized_end=3260 + _globals['_RESOLVEENDPOINTRESPONSE']._serialized_start=3263 + _globals['_RESOLVEENDPOINTRESPONSE']._serialized_end=3643 + _globals['_RESOLVEENDPOINTRESPONSE_REQUIREDHEADERSENTRY']._serialized_start=3589 + _globals['_RESOLVEENDPOINTRESPONSE_REQUIREDHEADERSENTRY']._serialized_end=3643 + _globals['_COMPONENTCAPABILITY']._serialized_start=3645 + _globals['_COMPONENTCAPABILITY']._serialized_end=3733 + _globals['_WARMIMAGEINFO']._serialized_start=3736 + _globals['_WARMIMAGEINFO']._serialized_end=3917 + _globals['_REGISTRYINFO']._serialized_start=3919 + _globals['_REGISTRYINFO']._serialized_end=4030 + _globals['_POOLINFO']._serialized_start=4033 + _globals['_POOLINFO']._serialized_end=4461 + _globals['_GETPOOLREQUEST']._serialized_start=4463 + _globals['_GETPOOLREQUEST']._serialized_end=4517 + _globals['_LISTPOOLSREQUEST']._serialized_start=4519 + _globals['_LISTPOOLSREQUEST']._serialized_end=4556 + _globals['_LISTPOOLSRESPONSE']._serialized_start=4558 + _globals['_LISTPOOLSRESPONSE']._serialized_end=4615 + _globals['_FASTPATHSERVICE']._serialized_start=4733 + _globals['_FASTPATHSERVICE']._serialized_end=5527 +# @@protoc_insertion_point(module_scope) diff --git a/server/opensandbox_server/services/fleets/generated/fastpath_pb2_grpc.py b/server/opensandbox_server/services/fleets/generated/fastpath_pb2_grpc.py new file mode 100644 index 000000000..63bd6e0ed --- /dev/null +++ b/server/opensandbox_server/services/fleets/generated/fastpath_pb2_grpc.py @@ -0,0 +1,490 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +from . import fastpath_pb2 as fastpath__pb2 + +GRPC_GENERATED_VERSION = '1.83.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in fastpath_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class FastPathServiceStub: + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CreateSandbox = channel.unary_unary( + '/fastpath.v2.FastPathService/CreateSandbox', + request_serializer=fastpath__pb2.CreateRequest.SerializeToString, + response_deserializer=fastpath__pb2.SandboxInfo.FromString, + _registered_method=True) + self.DeleteSandbox = channel.unary_unary( + '/fastpath.v2.FastPathService/DeleteSandbox', + request_serializer=fastpath__pb2.DeleteRequest.SerializeToString, + response_deserializer=fastpath__pb2.DeleteResponse.FromString, + _registered_method=True) + self.UpdateSandbox = channel.unary_unary( + '/fastpath.v2.FastPathService/UpdateSandbox', + request_serializer=fastpath__pb2.UpdateRequest.SerializeToString, + response_deserializer=fastpath__pb2.UpdateResponse.FromString, + _registered_method=True) + self.ListSandboxes = channel.unary_unary( + '/fastpath.v2.FastPathService/ListSandboxes', + request_serializer=fastpath__pb2.ListRequest.SerializeToString, + response_deserializer=fastpath__pb2.ListResponse.FromString, + _registered_method=True) + self.GetSandbox = channel.unary_unary( + '/fastpath.v2.FastPathService/GetSandbox', + request_serializer=fastpath__pb2.GetRequest.SerializeToString, + response_deserializer=fastpath__pb2.SandboxInfo.FromString, + _registered_method=True) + self.GetSandboxDiagnostics = channel.unary_unary( + '/fastpath.v2.FastPathService/GetSandboxDiagnostics', + request_serializer=fastpath__pb2.SandboxDiagnosticsRequest.SerializeToString, + response_deserializer=fastpath__pb2.SandboxDiagnosticsResponse.FromString, + _registered_method=True) + self.WaitSandboxReady = channel.unary_unary( + '/fastpath.v2.FastPathService/WaitSandboxReady', + request_serializer=fastpath__pb2.WaitSandboxReadyRequest.SerializeToString, + response_deserializer=fastpath__pb2.SandboxInfo.FromString, + _registered_method=True) + self.ResolveEndpoint = channel.unary_unary( + '/fastpath.v2.FastPathService/ResolveEndpoint', + request_serializer=fastpath__pb2.ResolveEndpointRequest.SerializeToString, + response_deserializer=fastpath__pb2.ResolveEndpointResponse.FromString, + _registered_method=True) + self.GetPool = channel.unary_unary( + '/fastpath.v2.FastPathService/GetPool', + request_serializer=fastpath__pb2.GetPoolRequest.SerializeToString, + response_deserializer=fastpath__pb2.PoolInfo.FromString, + _registered_method=True) + self.ListPools = channel.unary_unary( + '/fastpath.v2.FastPathService/ListPools', + request_serializer=fastpath__pb2.ListPoolsRequest.SerializeToString, + response_deserializer=fastpath__pb2.ListPoolsResponse.FromString, + _registered_method=True) + + +class FastPathServiceServicer: + """Missing associated documentation comment in .proto file.""" + + def CreateSandbox(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteSandbox(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateSandbox(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListSandboxes(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetSandbox(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetSandboxDiagnostics(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def WaitSandboxReady(self, request, context): + """WaitSandboxReady waits on the assigned Fastlet, never on CRD status + propagation. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ResolveEndpoint(self, request, context): + """ResolveEndpoint resolves one named component or raw user port. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetPool(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListPools(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_FastPathServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CreateSandbox': grpc.unary_unary_rpc_method_handler( + servicer.CreateSandbox, + request_deserializer=fastpath__pb2.CreateRequest.FromString, + response_serializer=fastpath__pb2.SandboxInfo.SerializeToString, + ), + 'DeleteSandbox': grpc.unary_unary_rpc_method_handler( + servicer.DeleteSandbox, + request_deserializer=fastpath__pb2.DeleteRequest.FromString, + response_serializer=fastpath__pb2.DeleteResponse.SerializeToString, + ), + 'UpdateSandbox': grpc.unary_unary_rpc_method_handler( + servicer.UpdateSandbox, + request_deserializer=fastpath__pb2.UpdateRequest.FromString, + response_serializer=fastpath__pb2.UpdateResponse.SerializeToString, + ), + 'ListSandboxes': grpc.unary_unary_rpc_method_handler( + servicer.ListSandboxes, + request_deserializer=fastpath__pb2.ListRequest.FromString, + response_serializer=fastpath__pb2.ListResponse.SerializeToString, + ), + 'GetSandbox': grpc.unary_unary_rpc_method_handler( + servicer.GetSandbox, + request_deserializer=fastpath__pb2.GetRequest.FromString, + response_serializer=fastpath__pb2.SandboxInfo.SerializeToString, + ), + 'GetSandboxDiagnostics': grpc.unary_unary_rpc_method_handler( + servicer.GetSandboxDiagnostics, + request_deserializer=fastpath__pb2.SandboxDiagnosticsRequest.FromString, + response_serializer=fastpath__pb2.SandboxDiagnosticsResponse.SerializeToString, + ), + 'WaitSandboxReady': grpc.unary_unary_rpc_method_handler( + servicer.WaitSandboxReady, + request_deserializer=fastpath__pb2.WaitSandboxReadyRequest.FromString, + response_serializer=fastpath__pb2.SandboxInfo.SerializeToString, + ), + 'ResolveEndpoint': grpc.unary_unary_rpc_method_handler( + servicer.ResolveEndpoint, + request_deserializer=fastpath__pb2.ResolveEndpointRequest.FromString, + response_serializer=fastpath__pb2.ResolveEndpointResponse.SerializeToString, + ), + 'GetPool': grpc.unary_unary_rpc_method_handler( + servicer.GetPool, + request_deserializer=fastpath__pb2.GetPoolRequest.FromString, + response_serializer=fastpath__pb2.PoolInfo.SerializeToString, + ), + 'ListPools': grpc.unary_unary_rpc_method_handler( + servicer.ListPools, + request_deserializer=fastpath__pb2.ListPoolsRequest.FromString, + response_serializer=fastpath__pb2.ListPoolsResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'fastpath.v2.FastPathService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('fastpath.v2.FastPathService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class FastPathService: + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def CreateSandbox(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/CreateSandbox', + fastpath__pb2.CreateRequest.SerializeToString, + fastpath__pb2.SandboxInfo.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeleteSandbox(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/DeleteSandbox', + fastpath__pb2.DeleteRequest.SerializeToString, + fastpath__pb2.DeleteResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def UpdateSandbox(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/UpdateSandbox', + fastpath__pb2.UpdateRequest.SerializeToString, + fastpath__pb2.UpdateResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListSandboxes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/ListSandboxes', + fastpath__pb2.ListRequest.SerializeToString, + fastpath__pb2.ListResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetSandbox(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/GetSandbox', + fastpath__pb2.GetRequest.SerializeToString, + fastpath__pb2.SandboxInfo.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetSandboxDiagnostics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/GetSandboxDiagnostics', + fastpath__pb2.SandboxDiagnosticsRequest.SerializeToString, + fastpath__pb2.SandboxDiagnosticsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def WaitSandboxReady(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/WaitSandboxReady', + fastpath__pb2.WaitSandboxReadyRequest.SerializeToString, + fastpath__pb2.SandboxInfo.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ResolveEndpoint(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/ResolveEndpoint', + fastpath__pb2.ResolveEndpointRequest.SerializeToString, + fastpath__pb2.ResolveEndpointResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetPool(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/GetPool', + fastpath__pb2.GetPoolRequest.SerializeToString, + fastpath__pb2.PoolInfo.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListPools(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/fastpath.v2.FastPathService/ListPools', + fastpath__pb2.ListPoolsRequest.SerializeToString, + fastpath__pb2.ListPoolsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/server/opensandbox_server/services/fleets/status_mapping.py b/server/opensandbox_server/services/fleets/status_mapping.py new file mode 100644 index 000000000..f1e6a1620 --- /dev/null +++ b/server/opensandbox_server/services/fleets/status_mapping.py @@ -0,0 +1,106 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Map fast-sandbox Sandbox status into the OpenSandbox Sandbox model. + +fast-sandbox splits RuntimeReady (runtime up) from DataPlaneReady (routes and +Infra Components published). OpenSandbox reports Running only when both are +Ready, matching the "endpoint usable" expectation. + +On expiry the reconciler keeps the Sandbox CRD with runtimeState=Stopped and a +RuntimeReady=False, reason=Expired Condition; that retained object maps to +Terminated. An actually missing CRD surfaces as gRPC NotFound at the client +layer and maps to HTTP 404 (no synthetic Sandbox object). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +from opensandbox_server.api.schema import ImageSpec, Sandbox, SandboxStatus +from opensandbox_server.services.fleets.create_mapping import ( + RENEW_EXTEND_SECONDS_METADATA_KEY, +) +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2 as pb2, +) + +#: FastPath metadata keys owned by the fleets backend, hidden from public reads. +RESERVED_METADATA_KEYS = frozenset({RENEW_EXTEND_SECONDS_METADATA_KEY}) + + +def map_state(info: pb2.SandboxInfo) -> str: + """Map a fast-sandbox SandboxInfo to the OpenSandbox lifecycle state.""" + runtime_state = info.runtime_state or "" + data_plane_state = info.data_plane_state or "" + + if runtime_state == "Ready": + if data_plane_state == "Ready": + return "Running" + # Runtime is up but routes/Infra are not published yet. + return "Pending" + if runtime_state in ("Pending", "Creating"): + return "Pending" + if runtime_state == "Draining": + return "Stopping" + if runtime_state == "Stopped": + return "Terminated" + if runtime_state in ("Failed", "Unavailable"): + return "Failed" + return "Pending" + + +def map_reason(info: pb2.SandboxInfo) -> Optional[str]: + """Best-effort machine-readable reason for the mapped state. + + FastPath v2 SandboxInfo does not carry Conditions, so an Expired reason + cannot be confirmed for a retained Stopped object; the reason is left + unset rather than inventing a termination cause. Only states that are + self-describing (Failed) report a reason. + """ + if info.runtime_state == "Failed": + return "Failed" + return None + + +def map_sandbox(info: pb2.SandboxInfo) -> Sandbox: + """Build the public Sandbox model from a FastPath SandboxInfo.""" + metadata = None + if info.metadata: + metadata = { + key: value + for key, value in info.metadata.items() + if key not in RESERVED_METADATA_KEYS + } + if not metadata: + metadata = None + + return Sandbox( + id=info.sandbox_name, + image=ImageSpec(uri=info.image) if info.image else None, + status=SandboxStatus(state=map_state(info), reason=map_reason(info)), + metadata=metadata, + expiresAt=_to_datetime(info.expires_at_unix_seconds), + createdAt=_to_datetime(info.created_at_unix_seconds), + ) + + +def _to_datetime(unix_seconds: int) -> Optional[datetime]: + if unix_seconds <= 0: + return None + return datetime.fromtimestamp(unix_seconds, tz=timezone.utc) diff --git a/server/opensandbox_server/services/helpers.py b/server/opensandbox_server/services/helpers.py index 6491f7d04..8bf60c555 100644 --- a/server/opensandbox_server/services/helpers.py +++ b/server/opensandbox_server/services/helpers.py @@ -22,6 +22,7 @@ from __future__ import annotations import logging +import math import re from datetime import datetime, timezone from typing import Dict, Optional @@ -92,10 +93,21 @@ def parse_nano_cpus(value: Optional[str]) -> Optional[int]: except ValueError: logger.warning("Invalid CPU limit format '%s'; ignoring.", value) return None + if not math.isfinite(cpus): + logger.warning("CPU limit must be finite. Got '%s'. Ignoring.", value) + return None if cpus <= 0: logger.warning("CPU limit must be positive. Got '%s'. Ignoring.", value) return None - return int(cpus * 1_000_000_000) + nano_cpus = cpus * 1_000_000_000 + if not math.isfinite(nano_cpus): + logger.warning("CPU limit is too large. Got '%s'. Ignoring.", value) + return None + nano_cpus = int(nano_cpus) + if nano_cpus > (1 << 63) - 1: + logger.warning("CPU limit is too large. Got '%s'. Ignoring.", value) + return None + return nano_cpus def parse_gpu_request(value: Optional[str]) -> Optional[int]: diff --git a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py index 4dfd1d855..921982499 100644 --- a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py +++ b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py @@ -94,6 +94,7 @@ def __init__( ) self.ingress_config = app_config.ingress if app_config else None self.execd_init_resources = k8s_config.execd_init_resources if k8s_config else None + self.execd_run_as_init = bool(app_config and app_config.runtime.execd_run_as_init) self.resolver = SecureRuntimeResolver(app_config) if app_config else None self.runtime_class = ( @@ -273,6 +274,9 @@ def _build_pod_spec( disable_ipv6_for_egress=disable_ipv6_for_egress, ) main_env = dict(env) + main_env["OPENSANDBOX_ID"] = sandbox_id + if self.execd_run_as_init: + main_env["EXECD_INIT"] = "1" if credential_proxy_enabled: main_env[OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT] = "true" diff --git a/server/opensandbox_server/services/k8s/batchsandbox_provider.py b/server/opensandbox_server/services/k8s/batchsandbox_provider.py index 8a5dd7d2f..2248fdbf9 100644 --- a/server/opensandbox_server/services/k8s/batchsandbox_provider.py +++ b/server/opensandbox_server/services/k8s/batchsandbox_provider.py @@ -61,6 +61,26 @@ logger = logging.getLogger(__name__) +def _merge_security_context( + template_sc: Dict[str, Any], runtime_sc: Dict[str, Any] +) -> Dict[str, Any]: + """Merge the template's container securityContext into the runtime one. + + Nested dicts (capabilities, seccompProfile, ...) merge recursively so a + template member on one key (e.g. capabilities.add) survives even when the + runtime populates another key of the same field (e.g. capabilities.drop from + network-policy wiring). On actual conflicting leaves, the runtime value wins. + """ + merged = dict(template_sc) + for key, runtime_value in runtime_sc.items(): + template_value = merged.get(key) + if isinstance(runtime_value, dict) and isinstance(template_value, dict): + merged[key] = _merge_security_context(template_value, runtime_value) + else: + merged[key] = runtime_value + return merged + + class BatchSandboxProvider(WorkloadProvider): """Workload provider for BatchSandbox CRDs.""" @@ -78,6 +98,7 @@ def __init__( logger.info(f"Using BatchSandbox template file: {template_file_path}") self.execd_init_resources = k8s_config.execd_init_resources if k8s_config else None self.image_pull_policy = k8s_config.image_pull_policy if k8s_config else "IfNotPresent" + self.execd_run_as_init = bool(app_config and app_config.runtime.execd_run_as_init) self.resolver = SecureRuntimeResolver(app_config) if app_config else None self.runtime_class = ( @@ -162,7 +183,7 @@ def create_workload( annotations=annotations, ) - extra_volumes, extra_mounts = self._extract_template_pod_extras() + extra_volumes, extra_mounts, extra_security_context = self._extract_template_pod_extras() if windows_profile: validate_windows_profile_resource_limits(resource_limits) @@ -179,6 +200,9 @@ def create_workload( ) main_env = dict(env) + main_env["OPENSANDBOX_ID"] = sandbox_id + if self.execd_run_as_init: + main_env["EXECD_INIT"] = "1" if credential_proxy_enabled: main_env[OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT] = "true" @@ -285,7 +309,9 @@ def create_workload( batchsandbox["spec"].pop("expireTime", None) else: batchsandbox["spec"]["expireTime"] = expires_at.isoformat() - self._merge_pod_spec_extras(batchsandbox, extra_volumes, extra_mounts) + self._merge_pod_spec_extras( + batchsandbox, extra_volumes, extra_mounts, extra_security_context + ) merged_pod_spec = batchsandbox.get("spec", {}).get("template", {}).get("spec", {}) ensure_egress_runtime_compatible( network_policy, @@ -373,9 +399,23 @@ def _create_workload_from_pool( "replicas": 1, "poolRef": pool_ref, } - needs_task_template = env or entrypoint != DEFAULT_ENTRYPOINT + needs_task_template = ( + env + or entrypoint != DEFAULT_ENTRYPOINT + or self.execd_run_as_init + ) if needs_task_template: - spec["taskTemplate"] = self._build_task_template(entrypoint, env) + spec["taskTemplate"] = self._build_task_template(entrypoint, env, batchsandbox_name) + else: + # Fast path: the pre-created pool pod keeps running its own warm + # entrypoint, so no per-allocation env can reach execd. The + # authoritative BatchSandbox id cannot be injected here; eBPF + # audit attribution reports unsupported for this allocation. + logger.info( + "pool sandbox %s: default allocation without a task template cannot inject " + "OPENSANDBOX_ID; eBPF audit sandbox_id attribution is unsupported on this path", + batchsandbox_name, + ) if expires_at is not None: spec["expireTime"] = expires_at.isoformat() runtime_manifest = { @@ -406,14 +446,17 @@ def _create_workload_from_pool( "kind": "BatchSandbox", } - def _extract_template_pod_extras(self) -> tuple[list[Dict[str, Any]], list[Dict[str, Any]]]: - """Extract extra template volumes and mounts for runtime merge.""" + def _extract_template_pod_extras( + self, + ) -> tuple[list[Dict[str, Any]], list[Dict[str, Any]], Optional[Dict[str, Any]]]: + """Extract extra template volumes, mounts, and container securityContext for runtime merge.""" template = self.template_manager.get_base_template() spec = template.get("spec", {}) if isinstance(template, dict) else {} template_spec = spec.get("template", {}).get("spec", {}) extra_volumes = template_spec.get("volumes", []) or [] extra_mounts: list[Dict[str, Any]] = [] + extra_security_context: Optional[Dict[str, Any]] = None containers = template_spec.get("containers", []) or [] if containers: target = None @@ -424,20 +467,24 @@ def _extract_template_pod_extras(self) -> tuple[list[Dict[str, Any]], list[Dict[ if target is None: target = containers[0] extra_mounts = target.get("volumeMounts", []) or [] + security_context = target.get("securityContext") + if isinstance(security_context, dict): + extra_security_context = security_context if not isinstance(extra_volumes, list): extra_volumes = [] if not isinstance(extra_mounts, list): extra_mounts = [] - return extra_volumes, extra_mounts + return extra_volumes, extra_mounts, extra_security_context def _merge_pod_spec_extras( self, batchsandbox: Dict[str, Any], extra_volumes: list[Dict[str, Any]], extra_mounts: list[Dict[str, Any]], + extra_security_context: Optional[Dict[str, Any]] = None, ) -> None: - """Merge template-provided volumes and mounts into runtime pod spec.""" + """Merge template-provided volumes, mounts, and securityContext into runtime pod spec.""" try: spec = batchsandbox["spec"]["template"]["spec"] except KeyError: @@ -460,6 +507,18 @@ def _merge_pod_spec_extras( if not containers or not isinstance(containers, list): return main_container = containers[0] + if extra_security_context and isinstance(main_container, dict): + # The template's container securityContext is a base default: merge it + # into the runtime container's own securityContext (runtime leaves win, + # nested dicts merge so template members like capabilities.add survive), + # and fill the whole context when the runtime sets none. + runtime_security_context = main_container.get("securityContext") + if isinstance(runtime_security_context, dict): + main_container["securityContext"] = _merge_security_context( + extra_security_context, runtime_security_context + ) + else: + main_container["securityContext"] = extra_security_context mounts = main_container.get("volumeMounts", []) or [] if isinstance(mounts, list) and extra_mounts: existing = {m.get("name") for m in mounts if isinstance(m, dict)} @@ -477,14 +536,31 @@ def _build_task_template( self, entrypoint: List[str], env: Dict[str, str], + sandbox_id: str, ) -> Dict[str, Any]: - """Build pool taskTemplate with shell-escaped bootstrap command.""" + """Build pool taskTemplate with shell-escaped bootstrap command. + + With execd_run_as_init enabled, the task is NOT backgrounded: the + shim's shell execs bootstrap.sh, which execs `execd --init` (the + EXECD_INIT env is injected below), so execd becomes the root of the + task process tree. It reaps orphaned task children (subreaper) and + propagates the entrypoint exit code back to the shim. Without it, + the classic background-and-wait topology is preserved. + """ escaped_entrypoint = ' '.join(shlex.quote(arg) for arg in entrypoint) - user_process_cmd = f"/opt/opensandbox/bootstrap.sh {escaped_entrypoint} &" - + if self.execd_run_as_init: + # exec: the task-executor shim's TERM trap signals its direct + # child, which must be execd (not an intermediate shell). + user_process_cmd = f"exec /opt/opensandbox/bootstrap.sh {escaped_entrypoint}" + else: + user_process_cmd = f"/opt/opensandbox/bootstrap.sh {escaped_entrypoint} &" + wrapped_command = ["/bin/sh", "-c", user_process_cmd] + if self.execd_run_as_init: + env = {**env, "EXECD_INIT": "1"} env_list = [{"name": k, "value": v} for k, v in env.items()] if env else [] + env_list.append({"name": "OPENSANDBOX_ID", "value": sandbox_id}) return { "spec": { diff --git a/server/opensandbox_server/services/k8s/client.py b/server/opensandbox_server/services/k8s/client.py index cc7cae416..32886aba7 100644 --- a/server/opensandbox_server/services/k8s/client.py +++ b/server/opensandbox_server/services/k8s/client.py @@ -365,16 +365,30 @@ def patch_pvc( The pinned kubernetes-client picks ``application/json-patch+json`` by default (first entry in the generated content-type list) which would - reject our merge-shaped body. Force strategic-merge so list fields - like ``ownerReferences`` merge by key. + reject our merge-shaped body. Its generated high-level method does not + accept a content-type override, so use the underlying ``ApiClient`` to + preserve strategic-merge semantics for list fields like + ``ownerReferences``. """ if self._write_limiter: self._write_limiter.acquire() - return self.get_core_v1_api().patch_namespaced_persistent_volume_claim( - name=name, - namespace=namespace, + api_client = self.get_core_v1_api().api_client + return api_client.call_api( + "/api/v1/namespaces/{namespace}/persistentvolumeclaims/{name}", + "PATCH", + path_params={"namespace": namespace, "name": name}, + query_params=[], + header_params={ + "Accept": "application/json", + "Content-Type": "application/strategic-merge-patch+json", + }, body=body, - _content_type="application/strategic-merge-patch+json", + post_params=[], + files={}, + response_type="V1PersistentVolumeClaim", + auth_settings=["BearerToken"], + _return_http_data_only=True, + collection_formats={}, ) # ------------------------------------------------------------------ diff --git a/server/opensandbox_server/services/k8s/informer.py b/server/opensandbox_server/services/k8s/informer.py index 5e31c2cf0..9fd288bdf 100644 --- a/server/opensandbox_server/services/k8s/informer.py +++ b/server/opensandbox_server/services/k8s/informer.py @@ -25,6 +25,11 @@ logger = logging.getLogger(__name__) +# An idle watch sends nothing until the server closes the stream at +# ``timeout_seconds``, so the client read timeout must sit above it. +_WATCH_READ_TIMEOUT_BUFFER_SECONDS = 10 +_WATCH_CONNECT_TIMEOUT_SECONDS = 10 + class WorkloadInformer: """Maintain an in-memory cache of a namespaced custom resource via watch.""" @@ -58,13 +63,33 @@ def __init__( self._lock = threading.RLock() self._resource_version: Optional[str] = None self._has_synced = False + self._last_contact_at: Optional[float] = None self._stop_event = threading.Event() self._thread: Optional[threading.Thread] = None @property def has_synced(self) -> bool: - """Return True once an initial list has completed.""" - return self._has_synced + """Return True while the cache is populated and still being maintained. + + A watch can stop delivering without raising โ€” a dead connection leaves the + reader parked forever โ€” so a "listed once" latch would keep readers on a + frozen cache. Going stale makes them fall back to a live request. + + A stopped informer is likewise considered not synced, however recent its + last contact, since nothing will refresh the cache again. + """ + if self._stop_event.is_set(): + # Stopped: nothing will refresh the cache again, however recent it is. + return False + with self._lock: + if not self._has_synced or self._last_contact_at is None: + return False + return time.monotonic() - self._last_contact_at <= self._staleness_limit_seconds + + @property + def _staleness_limit_seconds(self) -> float: + """One resync period, by which the cache should have been rebuilt, plus a watch cycle.""" + return self.resync_period_seconds + self.watch_timeout_seconds def start(self) -> None: """Start the background watch thread if not already running.""" @@ -201,6 +226,7 @@ def _full_resync(self) -> None: self._cache = new_cache self._advance_resource_version(resource_version) self._has_synced = True + self._last_contact_at = time.monotonic() def _run_watch_loop(self, timeout_seconds: int) -> None: """Stream watch events to keep the cache fresh.""" @@ -210,6 +236,11 @@ def _run_watch_loop(self, timeout_seconds: int) -> None: self.list_fn, resource_version=self._resource_version, timeout_seconds=timeout_seconds, + # Without this a half-open connection parks the thread forever. + _request_timeout=( + _WATCH_CONNECT_TIMEOUT_SECONDS, + timeout_seconds + _WATCH_READ_TIMEOUT_BUFFER_SECONDS, + ), ): if self._stop_event.is_set(): break @@ -217,6 +248,10 @@ def _run_watch_loop(self, timeout_seconds: int) -> None: finally: w.stop() + # The stream ran to completion, so the API server is still reachable. + with self._lock: + self._last_contact_at = time.monotonic() + def _handle_event(self, event: Dict[str, Any]) -> None: obj = event.get("object") if obj is None: diff --git a/server/opensandbox_server/services/k8s/k8s_diagnostics.py b/server/opensandbox_server/services/k8s/k8s_diagnostics.py index 00684a388..9d90631dd 100644 --- a/server/opensandbox_server/services/k8s/k8s_diagnostics.py +++ b/server/opensandbox_server/services/k8s/k8s_diagnostics.py @@ -22,6 +22,7 @@ from __future__ import annotations import re +from typing import Any from fastapi import HTTPException, status from kubernetes.client.exceptions import ApiException @@ -30,6 +31,16 @@ SANDBOX_ID_LABEL, SandboxErrorCodes, ) +from opensandbox_server.services.diagnostics import ( + DiagnosticResult, + limit_diagnostic_lines, + unsupported_scope_error, +) + +_SUPPORTED_LOG_SCOPES = ("container", "all") +_SUPPORTED_EVENT_SCOPES = ("runtime", "all") +_STABLE_LOG_LINE_LIMIT = 100 +_STABLE_EVENT_LINE_LIMIT = 50 #: Default container to pull logs from when the caller does not specify one. #: OSB-managed sandbox pods canonically run the user workload in a container @@ -52,6 +63,72 @@ def _parse_since(since: str) -> int: class K8sDiagnosticsMixin: """Mixin that implements diagnostics methods for the Kubernetes backend.""" + def get_sandbox_log_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable log diagnostics using Kubernetes capabilities.""" + normalized_scope = scope.strip().lower() + if normalized_scope not in _SUPPORTED_LOG_SCOPES: + raise unsupported_scope_error("logs", scope, _SUPPORTED_LOG_SCOPES) + + content = self.get_sandbox_logs( + sandbox_id, + tail=_STABLE_LOG_LINE_LIMIT + 1, + since=None, + container=None, + ) + content, truncated = limit_diagnostic_lines( + content, + _STABLE_LOG_LINE_LIMIT, + keep_tail=True, + ) + warnings: tuple[str, ...] = () + if normalized_scope == "all": + warnings = ( + "The current backend only contributes sandbox container logs to the all scope.", + ) + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="logs", + scope=normalized_scope, + content=content, + truncated=truncated, + warnings=warnings, + ) + + def get_sandbox_event_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable event diagnostics using Kubernetes capabilities.""" + normalized_scope = scope.strip().lower() + if normalized_scope not in _SUPPORTED_EVENT_SCOPES: + raise unsupported_scope_error("events", scope, _SUPPORTED_EVENT_SCOPES) + + content = self.get_sandbox_events( + sandbox_id, + limit=_STABLE_EVENT_LINE_LIMIT + 1, + ) + content, truncated = limit_diagnostic_lines( + content, + _STABLE_EVENT_LINE_LIMIT, + keep_tail=False, + ) + warnings: tuple[str, ...] = () + if normalized_scope == "all": + warnings = ("The current backend only contributes runtime events to the all scope.",) + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="events", + scope=normalized_scope, + content=content, + truncated=truncated, + warnings=warnings, + ) + def _find_pod_for_sandbox(self, sandbox_id: str): """Find the Pod associated with a sandbox ID via label selector.""" label_selector = f"{SANDBOX_ID_LABEL}={sandbox_id}" @@ -240,17 +317,38 @@ def get_sandbox_events(self, sandbox_id: str, limit: int = 50) -> str: pod_name = pod.metadata.name core_v1 = self.k8s_client.get_core_v1_api() - events_resp = core_v1.list_namespaced_event( - namespace=pod.metadata.namespace, - field_selector=f"involvedObject.name={pod_name}", - limit=limit, - ) + events: list[Any] = [] + continuation: str | None = None + try: + while len(events) < limit: + if continuation is None: + events_resp = core_v1.list_namespaced_event( + namespace=pod.metadata.namespace, + field_selector=f"involvedObject.name={pod_name}", + limit=limit, + ) + else: + events_resp = core_v1.list_namespaced_event( + namespace=pod.metadata.namespace, + field_selector=f"involvedObject.name={pod_name}", + limit=limit, + _continue=continuation, + ) + + events.extend(events_resp.items or []) + metadata = getattr(events_resp, "metadata", None) + next_continuation = getattr(metadata, "_continue", None) + if not next_continuation or next_continuation == continuation: + break + continuation = next_continuation + except ApiException as exc: + raise _map_pod_event_error(pod_name, exc) from exc - if not events_resp.items: + if not events: return "(no events)" lines: list[str] = [] - for ev in events_resp.items: + for ev in events[:limit]: ts = ev.last_timestamp or ev.event_time or ev.first_timestamp or "N/A" lines.append( f"[{ts}] {ev.type:8s} {ev.reason or 'N/A':20s} {ev.message or ''}" @@ -303,7 +401,7 @@ def _map_pod_log_error(pod_name: str, container: str, exc: ApiException) -> HTTP }, ) return HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={ "code": SandboxErrorCodes.K8S_API_ERROR, "message": ( @@ -312,3 +410,46 @@ def _map_pod_log_error(pod_name: str, container: str, exc: ApiException) -> HTTP ), }, ) + + +def _map_pod_event_error(pod_name: str, exc: ApiException) -> HTTPException: + """Translate a Kubernetes event ApiException into a contract error.""" + raw_status = getattr(exc, "status", None) or 0 + body = getattr(exc, "body", None) or str(exc) + + if raw_status == 400: + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": SandboxErrorCodes.K8S_API_ERROR, + "message": ( + f"Kubernetes rejected event request for pod '{pod_name}': {body}" + ), + }, + ) + if raw_status in (401, 403): + return HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "code": SandboxErrorCodes.K8S_API_ERROR, + "message": f"Kubernetes denied event access for pod '{pod_name}': {body}", + }, + ) + if raw_status == 404: + return HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND, + "message": f"Pod '{pod_name}' not found when reading events: {body}", + }, + ) + return HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.K8S_API_ERROR, + "message": ( + f"Kubernetes returned {raw_status} when reading events for pod " + f"'{pod_name}': {body}" + ), + }, + ) diff --git a/server/opensandbox_server/services/k8s/provider_common.py b/server/opensandbox_server/services/k8s/provider_common.py index 96589c5ff..c16a542f2 100644 --- a/server/opensandbox_server/services/k8s/provider_common.py +++ b/server/opensandbox_server/services/k8s/provider_common.py @@ -129,7 +129,11 @@ def _build_execd_init_container( "(test ! -e /usr/local/libexec/opensandbox-session-gate || " "(cp /usr/local/libexec/opensandbox-session-gate " "/opt/opensandbox/opensandbox-session-gate && " - "chmod 0555 /opt/opensandbox/opensandbox-session-gate))" + "chmod 0555 /opt/opensandbox/opensandbox-session-gate)) && " + "(test ! -e /usr/local/libexec/opensandbox-launcher || " + "(cp /usr/local/libexec/opensandbox-launcher " + "/opt/opensandbox/opensandbox-launcher && " + "chmod 0555 /opt/opensandbox/opensandbox-launcher))" ) security_context = None if disable_ipv6_for_egress: diff --git a/server/opensandbox_server/services/sandbox_service.py b/server/opensandbox_server/services/sandbox_service.py index d90ed19aa..5c64cd1fb 100644 --- a/server/opensandbox_server/services/sandbox_service.py +++ b/server/opensandbox_server/services/sandbox_service.py @@ -35,6 +35,7 @@ RenewSandboxExpirationResponse, Sandbox, ) +from opensandbox_server.services.diagnostics import DiagnosticResult from opensandbox_server.services.validators import ensure_valid_port @@ -258,6 +259,40 @@ def _apply_metadata_patch(labels: dict, patch: dict) -> dict: # Diagnostics (DevOps) # ------------------------------------------------------------------ + @abstractmethod + def get_sandbox_log_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable log diagnostics using runtime-specific policy. + + Args: + sandbox_id: Unique sandbox identifier. + scope: Diagnostic scope requested by the caller. + + Returns: + Runtime-provided stable diagnostic result. + """ + pass + + @abstractmethod + def get_sandbox_event_diagnostics( + self, + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + """Collect stable event diagnostics using runtime-specific policy. + + Args: + sandbox_id: Unique sandbox identifier. + scope: Diagnostic scope requested by the caller. + + Returns: + Runtime-provided stable diagnostic result. + """ + pass + @abstractmethod def get_sandbox_logs( self, diff --git a/server/opensandbox_server/tenants/__init__.py b/server/opensandbox_server/tenants/__init__.py index ce1466589..0693aa8d1 100644 --- a/server/opensandbox_server/tenants/__init__.py +++ b/server/opensandbox_server/tenants/__init__.py @@ -26,6 +26,11 @@ from opensandbox_server.tenants.models import TenantEntry from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable +import logging +from typing import Iterable + +logger = logging.getLogger(__name__) + def validate_tenant_config(app_config) -> None: """Validate tenant configuration against runtime and auth settings. @@ -49,6 +54,90 @@ def validate_tenant_config(app_config) -> None: ) +def validate_tenant_namespaces( + tenants: Iterable[TenantEntry], core_v1_api +) -> None: + """Validate that every tenant namespace exists and is accessible. + + Enforces the OSEP-0014 startup requirement that all tenant namespaces + exist and are accessible before the server accepts traffic (fail-fast). + + Args: + tenants: Tenant entries to validate. + core_v1_api: A Kubernetes ``CoreV1Api`` used to read namespaces. + + Raises: + ValueError: If any tenant namespace is missing or inaccessible. The + error aggregates all failing namespaces so operators can fix the + configuration in a single pass. + """ + from kubernetes.client import ApiException + + failures: list[str] = [] + checked: set[str] = set() + for tenant in tenants: + namespace = tenant.namespace + if namespace in checked: + continue + checked.add(namespace) + try: + core_v1_api.read_namespace(name=namespace) + except ApiException as exc: + if exc.status == 404: + failures.append( + f"tenant '{tenant.name}': namespace '{namespace}' does not exist" + ) + elif exc.status in (401, 403): + failures.append( + f"tenant '{tenant.name}': namespace '{namespace}' is not accessible " + f"(HTTP {exc.status})" + ) + else: + failures.append( + f"tenant '{tenant.name}': failed to read namespace '{namespace}' " + f"(HTTP {exc.status})" + ) + except Exception as exc: # noqa: BLE001 - surface any client error as fatal + failures.append( + f"tenant '{tenant.name}': failed to read namespace '{namespace}': {exc}" + ) + + if failures: + raise ValueError( + "Tenant namespace validation failed; all tenant namespaces must exist " + "and be accessible at startup:\n - " + "\n - ".join(failures) + ) + + logger.info("Validated %d tenant namespace(s) at startup", len(checked)) + + +def validate_tenant_namespaces_on_startup(provider, core_v1_api) -> None: + """Validate tenant namespaces at startup if the provider can enumerate them. + + Enforces the OSEP-0014 fail-fast requirement that all tenant namespaces + exist and are accessible before the server accepts traffic. Providers that + resolve tenants per API key (e.g. the HTTP provider) cannot enumerate all + tenants at startup; validating their empty set would silently report + success, so validation is skipped with a warning instead. + + Args: + provider: The configured tenant provider. + core_v1_api: A Kubernetes ``CoreV1Api`` used to read namespaces. + + Raises: + ValueError: If any tenant namespace is missing or inaccessible (for + enumerable providers). + """ + if not getattr(provider, "supports_enumeration", True): + logger.warning( + "Skipping tenant namespace startup validation: the tenant provider " + "cannot enumerate all tenants. Ensure tenant namespaces exist and " + "are accessible before issuing tenant API keys." + ) + return + validate_tenant_namespaces(provider.list_tenants(), core_v1_api) + + __all__ = [ "TenantEntry", "TenantProvider", @@ -62,4 +151,6 @@ def validate_tenant_config(app_config) -> None: "set_current_tenant", "resolve_tenants_path", "validate_tenant_config", + "validate_tenant_namespaces", + "validate_tenant_namespaces_on_startup", ] diff --git a/server/opensandbox_server/tenants/file_provider.py b/server/opensandbox_server/tenants/file_provider.py index 607daf203..282236699 100644 --- a/server/opensandbox_server/tenants/file_provider.py +++ b/server/opensandbox_server/tenants/file_provider.py @@ -98,6 +98,11 @@ def __init__(self, path: Optional[str | Path] = None) -> None: self._watcher_stop = threading.Event() self._watcher_thread: Optional[threading.Thread] = None + @property + def supports_enumeration(self) -> bool: + """The file provider lists its full config at startup.""" + return True + @property def path(self) -> Path: return self._path diff --git a/server/opensandbox_server/tenants/http_provider.py b/server/opensandbox_server/tenants/http_provider.py index 425348ae8..8110f8296 100644 --- a/server/opensandbox_server/tenants/http_provider.py +++ b/server/opensandbox_server/tenants/http_provider.py @@ -96,6 +96,11 @@ def __init__(self, config: HTTPTenantProviderConfig) -> None: self._callbacks: List[Callable[[List[TenantEntry]], None]] = [] self._client: Optional[httpx.Client] = None + @property + def supports_enumeration(self) -> bool: + """The HTTP provider only knows tenants seen in prior per-key lookups.""" + return False + def lookup(self, api_key: str) -> Optional[TenantEntry]: now = time.monotonic() diff --git a/server/opensandbox_server/tenants/provider.py b/server/opensandbox_server/tenants/provider.py index f5241bd11..38bac1e2b 100644 --- a/server/opensandbox_server/tenants/provider.py +++ b/server/opensandbox_server/tenants/provider.py @@ -40,6 +40,14 @@ def list_tenants(self) -> List[TenantEntry]: """Return all known tenant entries (used for startup validation).""" ... + supports_enumeration: bool + """True if ``list_tenants()`` returns the complete tenant set at startup. + + File-backed providers enumerate their config at startup. Per-key HTTP + providers only know tenants discovered through prior lookups, so their + startup list is empty and must not be treated as validated. + """ + def ready(self) -> bool: """True once the provider has loaded initial state and can serve lookups.""" ... diff --git a/server/pyproject.toml b/server/pyproject.toml index fa04a388b..a5da1655b 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -58,6 +58,8 @@ dependencies = [ "tomli; python_version < \"3.11\"", "uvicorn[standard]", "websockets>=14.0", + "grpcio>=1.83.0", + "protobuf>=7.35.1", ] [project.urls] @@ -89,6 +91,7 @@ packages = ["opensandbox_server"] [dependency-groups] dev = [ + "grpcio-tools>=1.83.0", "pytest>=7.0.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", @@ -100,6 +103,7 @@ dev = [ target-version = "py310" line-length = 100 src = ["opensandbox_server", "tests"] +exclude = ["opensandbox_server/services/fleets/generated"] [tool.ruff.lint] select = ["E4", "E7", "E9", "F"] diff --git a/server/tests/k8s/test_agent_sandbox_provider.py b/server/tests/k8s/test_agent_sandbox_provider.py index 33fa105e9..357288ef7 100644 --- a/server/tests/k8s/test_agent_sandbox_provider.py +++ b/server/tests/k8s/test_agent_sandbox_provider.py @@ -865,7 +865,7 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", credential_proxy_enabled=True, ) @@ -879,7 +879,7 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) # Find sidecar container sidecar = next((c for c in containers if c["name"] == "egress"), None) assert sidecar is not None - assert sidecar["image"] == "opensandbox/egress:v1.1.5" + assert sidecar["image"] == "opensandbox/egress:v1.1.6" # Verify sidecar has environment variable env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} @@ -939,7 +939,7 @@ def test_create_workload_with_network_policy_persists_annotation_and_sidecar_tok expires_at=None, execd_image="execd:latest", network_policy=NetworkPolicy(default_action="deny", egress=[]), - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", annotations={SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "egress-token"}, egress_auth_token="egress-token", ) @@ -977,7 +977,7 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): expires_at=None, execd_image="execd:latest", network_policy=NetworkPolicy(default_action="deny", egress=[]), - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", egress_mode=EGRESS_MODE_DNS_NFT, ) @@ -1016,7 +1016,7 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls( expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -1060,7 +1060,7 @@ def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured( expires_at=None, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -1095,7 +1095,7 @@ def test_create_workload_with_network_policy_drops_net_admin_from_main_container expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -1172,7 +1172,7 @@ def test_egress_sidecar_contains_network_policy_in_env(self, mock_k8s_client): expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py index 5b4e07345..bfc7a7083 100644 --- a/server/tests/k8s/test_batchsandbox_provider.py +++ b/server/tests/k8s/test_batchsandbox_provider.py @@ -530,8 +530,8 @@ def test_create_workload_converts_env_to_list(self, mock_k8s_client): body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] env_vars = body["spec"]["template"]["spec"]["containers"][0]["env"] - # Should have user env vars plus EXECD - assert len(env_vars) == 3 + # Should have user env vars plus OPENSANDBOX_ID and EXECD + assert len(env_vars) == 4 env_dict = {e["name"]: e["value"] for e in env_vars} assert env_dict["FOO"] == "bar" assert env_dict["BAZ"] == "qux" @@ -644,6 +644,198 @@ def test_create_workload_dedupes_template_volume_and_mount_names( assert mount_names.count("opensandbox-bin") == 1 assert "sandbox-shared-data" in mount_names + def test_create_workload_applies_template_container_security_context(self, mock_k8s_client, tmp_path): + template_file = tmp_path / "template.yaml" + template_file.write_text( + """ +spec: + template: + spec: + containers: + - name: sandbox + image: ubuntu:latest + securityContext: + runAsNonRoot: true + seccompProfile: + type: Unconfined +""" + ) + provider = BatchSandboxProvider( + mock_k8s_client, _app_config_with_template(str(template_file)) + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "sandbox-test", "uid": "uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc), + execd_image="execd:latest", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + container = body["spec"]["template"]["spec"]["containers"][0] + + # Template image must not override the runtime image, but its container + # securityContext should be propagated to the generated Pod. + assert container["name"] == "sandbox" + assert container["image"] == "python:3.11" + assert container["securityContext"] == { + "runAsNonRoot": True, + "seccompProfile": {"type": "Unconfined"}, + } + + def test_create_workload_merges_template_security_context_with_runtime_network_policy( + self, mock_k8s_client, tmp_path + ): + template_file = tmp_path / "template.yaml" + template_file.write_text( + """ +spec: + template: + spec: + containers: + - name: sandbox + image: ubuntu:latest + securityContext: + runAsNonRoot: true +""" + ) + provider = BatchSandboxProvider( + mock_k8s_client, _app_config_with_template(str(template_file)) + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "sandbox-test", "uid": "uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc), + execd_image="execd:latest", + network_policy=NetworkPolicy(default_action="deny", egress=[]), + egress_image="opensandbox/egress:v1.1.6", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + container = body["spec"]["template"]["spec"]["containers"][0] + + # Runtime-provided securityContext keys (network policy capabilities) win; + # template-provided keys supplement rather than replace them. + assert container["securityContext"] == { + "runAsNonRoot": True, + "capabilities": {"drop": ["NET_ADMIN"]}, + } + + def test_create_workload_merges_template_nested_capabilities_with_runtime_network_policy( + self, mock_k8s_client, tmp_path + ): + template_file = tmp_path / "template.yaml" + template_file.write_text( + """ +spec: + template: + spec: + containers: + - name: sandbox + image: ubuntu:latest + securityContext: + capabilities: + add: + - SYS_PTRACE +""" + ) + provider = BatchSandboxProvider( + mock_k8s_client, _app_config_with_template(str(template_file)) + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "sandbox-test", "uid": "uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc), + execd_image="execd:latest", + network_policy=NetworkPolicy(default_action="deny", egress=[]), + egress_image="opensandbox/egress:v1.1.6", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + container = body["spec"]["template"]["spec"]["containers"][0] + + # Nested dicts merge: the template's capabilities.add survives even though + # network-policy wiring populates a different member (capabilities.drop). + assert container["securityContext"] == { + "capabilities": { + "add": ["SYS_PTRACE"], + "drop": ["NET_ADMIN"], + }, + } + + def test_create_workload_runtime_capabilities_win_over_template_conflicts( + self, mock_k8s_client, tmp_path + ): + template_file = tmp_path / "template.yaml" + template_file.write_text( + """ +spec: + template: + spec: + containers: + - name: sandbox + image: ubuntu:latest + securityContext: + capabilities: + drop: + - ALL +""" + ) + provider = BatchSandboxProvider( + mock_k8s_client, _app_config_with_template(str(template_file)) + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "sandbox-test", "uid": "uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc), + execd_image="execd:latest", + network_policy=NetworkPolicy(default_action="deny", egress=[]), + egress_image="opensandbox/egress:v1.1.6", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + container = body["spec"]["template"]["spec"]["containers"][0] + + # Conflicting leaves keep the runtime value (network-policy requirement). + assert container["securityContext"] == { + "capabilities": {"drop": ["NET_ADMIN"]}, + } + def test_create_workload_sets_resource_limits_and_requests(self, mock_k8s_client): provider = BatchSandboxProvider(mock_k8s_client) mock_k8s_client.create_custom_object.return_value = { @@ -1454,7 +1646,47 @@ def test_create_workload_poolref_allows_entrypoint_and_env(self, mock_k8s_client # Example: /opt/opensandbox/bootstrap.sh python app.py & assert "/opt/opensandbox/bootstrap.sh python app.py" in command[2] assert command[2].endswith(" &") - assert task_template["spec"]["process"]["env"] == [{"name": "FOO", "value": "bar"}] + assert task_template["spec"]["process"]["env"] == [ + {"name": "FOO", "value": "bar"}, + {"name": "OPENSANDBOX_ID", "value": "test-id"}, + ] + + def test_create_workload_poolref_default_fast_path_skips_task_template(self, mock_k8s_client, monkeypatch): + """ + The default pool allocation (no env, default entrypoint, no init mode) + must skip the task template and keep the warm fast path, while logging + that OPENSANDBOX_ID cannot be injected (eBPF audit attribution + unsupported on this path). + """ + import opensandbox_server.services.k8s.batchsandbox_provider as provider_module + + mock_logger = MagicMock() + monkeypatch.setattr(provider_module, "logger", mock_logger) + provider = BatchSandboxProvider(mock_k8s_client) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "sandbox-test-id", "uid": "test-uid"} + } + + result = provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri=""), + entrypoint=[], # default entrypoint + env={}, + resource_limits={}, + labels={}, + expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc), + execd_image="execd:latest", + extensions={"poolRef": "my-pool"}, + ) + + assert result == {"name": "sandbox-test-id", "uid": "test-uid", "apiVersion": "sandbox.opensandbox.io/v1alpha1", "kind": "BatchSandbox"} + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + assert body["spec"]["poolRef"] == "my-pool" + assert "taskTemplate" not in body["spec"] + log_messages = " ".join(str(call) for call in mock_logger.info.call_args_list) + assert "OPENSANDBOX_ID" in log_messages def test_build_task_template_with_env(self, mock_k8s_client): """ @@ -1471,7 +1703,7 @@ def test_build_task_template_with_env(self, mock_k8s_client): provider = BatchSandboxProvider(mock_k8s_client) result = provider._build_task_template( - entrypoint=["/usr/bin/python", "app.py"], env={"KEY1": "value1", "KEY2": "value2"} + entrypoint=["/usr/bin/python", "app.py"], env={"KEY1": "value1", "KEY2": "value2"}, sandbox_id="bs-1" ) assert "spec" in result @@ -1493,8 +1725,34 @@ def test_build_task_template_with_env(self, mock_k8s_client): assert process_task["env"] == [ {"name": "KEY1", "value": "value1"}, {"name": "KEY2", "value": "value2"}, + {"name": "OPENSANDBOX_ID", "value": "bs-1"}, ] + def test_build_task_template_injects_execd_run_as_init_when_enabled(self, mock_k8s_client): + config = AppConfig( + runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test", execd_run_as_init=True), + kubernetes=KubernetesRuntimeConfig(namespace="test-ns"), + ) + provider = BatchSandboxProvider(mock_k8s_client, config) + + result = provider._build_task_template( + entrypoint=["/usr/bin/python", "app.py"], env={"KEY1": "value1"}, sandbox_id="bs-1" + ) + + env = result["spec"]["process"]["env"] + assert {"name": "EXECD_INIT", "value": "1"} in env + assert {"name": "KEY1", "value": "value1"} in env + assert {"name": "OPENSANDBOX_ID", "value": "bs-1"} in env + + # With execd_run_as_init the task is NOT backgrounded: the shim's + # shell execs bootstrap.sh, which execs `execd --init` as the root of + # the task process tree (orphan reaping + exit-code propagation). + command = result["spec"]["process"]["command"] + assert command[0] == "/bin/sh" + assert command[1] == "-c" + assert "/opt/opensandbox/bootstrap.sh /usr/bin/python app.py" in command[2] + assert " &" not in command[2] + def test_build_task_template_without_env(self, mock_k8s_client): """ Test _build_task_template without environment variables. @@ -1506,12 +1764,12 @@ def test_build_task_template_without_env(self, mock_k8s_client): """ provider = BatchSandboxProvider(mock_k8s_client) - result = provider._build_task_template(entrypoint=["/usr/bin/python", "app.py"], env={}) + result = provider._build_task_template(entrypoint=["/usr/bin/python", "app.py"], env={}, sandbox_id="bs-1") assert "spec" in result assert "process" in result["spec"] process_task = result["spec"]["process"] - assert process_task["env"] == [] + assert process_task["env"] == [{"name": "OPENSANDBOX_ID", "value": "bs-1"}] # Without env, command directly calls bootstrap.sh in background command = process_task["command"] assert command[0] == "/bin/sh" @@ -1533,7 +1791,7 @@ def test_build_task_template_uses_default_env_path(self, mock_k8s_client): provider = BatchSandboxProvider(mock_k8s_client) result = provider._build_task_template( - entrypoint=["python", "app.py"], env={"TEST_VAR": "test_value"} + entrypoint=["python", "app.py"], env={"TEST_VAR": "test_value"}, sandbox_id="bs-1" ) command = result["spec"]["process"]["command"][2] @@ -1555,6 +1813,7 @@ def test_build_task_template_escapes_special_characters(self, mock_k8s_client): result = provider._build_task_template( entrypoint=["python", "-c", 'print("hello world")'], env={"KEY": "value with spaces", "QUOTE": "it's fine"}, + sandbox_id="bs-1", ) command = result["spec"]["process"]["command"][2] @@ -1668,7 +1927,10 @@ def test_create_workload_poolref_default_entrypoint_with_env_includes_task_templ assert body["spec"]["poolRef"] == "my-pool" assert "taskTemplate" in body["spec"] task_template = body["spec"]["taskTemplate"] - assert task_template["spec"]["process"]["env"] == [{"name": "VERSION", "value": "11"}] + assert task_template["spec"]["process"]["env"] == [ + {"name": "VERSION", "value": "11"}, + {"name": "OPENSANDBOX_ID", "value": "test-id"}, + ] def test_create_workload_poolref_none_entrypoint_no_env_omits_task_template(self, mock_k8s_client): """When entrypoint is None and env is empty, taskTemplate is omitted. @@ -1762,7 +2024,7 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", credential_proxy_enabled=True, ) @@ -1776,7 +2038,7 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) # Find sidecar container sidecar = next((c for c in containers if c["name"] == "egress"), None) assert sidecar is not None - assert sidecar["image"] == "opensandbox/egress:v1.1.5" + assert sidecar["image"] == "opensandbox/egress:v1.1.6" # Verify sidecar has environment variable env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} @@ -1840,7 +2102,7 @@ def test_create_workload_windows_profile_with_network_policy_keeps_ipv6_disable( execd_image="execd:latest", platform=PlatformSpec(os="windows", arch="amd64"), network_policy=NetworkPolicy(default_action="deny", egress=[]), - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -1879,7 +2141,7 @@ def test_create_workload_with_network_policy_persists_annotation_and_sidecar_tok expires_at=None, execd_image="execd:latest", network_policy=NetworkPolicy(default_action="deny", egress=[]), - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", annotations={SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "egress-token"}, egress_auth_token="egress-token", ) @@ -1917,7 +2179,7 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): expires_at=None, execd_image="execd:latest", network_policy=NetworkPolicy(default_action="deny", egress=[]), - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", egress_mode=EGRESS_MODE_DNS_NFT, ) @@ -1957,7 +2219,7 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls( expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -2001,7 +2263,7 @@ def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured( expires_at=None, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -2036,7 +2298,7 @@ def test_create_workload_with_network_policy_drops_net_admin_from_main_container expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -2113,7 +2375,7 @@ def test_egress_sidecar_contains_network_policy_in_env(self, mock_k8s_client): expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] @@ -2203,7 +2465,7 @@ def test_create_workload_with_network_policy_works_with_template( expires_at=expires_at, execd_image="execd:latest", network_policy=network_policy, - egress_image="opensandbox/egress:v1.1.5", + egress_image="opensandbox/egress:v1.1.6", ) body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index 912b17fd1..d19375fc9 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -63,7 +63,7 @@ class TestEgressSidecarViaApply: def test_builds_container_with_basic_config(self): """Test that container is built with correct basic configuration.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[ @@ -80,7 +80,7 @@ def test_builds_container_with_basic_config(self): def test_contains_egress_rules_environment_variable(self): """Test that container includes OPENSANDBOX_EGRESS_RULES environment variable.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], @@ -96,7 +96,7 @@ def test_contains_egress_rules_environment_variable(self): def test_always_mounts_runtime_volume(self): container = _egress_container( - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", NetworkPolicy(default_action="deny", egress=[]), ) assert container["volumeMounts"] == [ @@ -107,7 +107,7 @@ def test_always_mounts_runtime_volume(self): ] def test_contains_transparent_mitm_env_when_credential_proxy_enabled(self): - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], @@ -129,7 +129,7 @@ def test_contains_transparent_mitm_env_when_credential_proxy_enabled(self): ] def test_contains_egress_token_when_provided(self): - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], @@ -149,7 +149,7 @@ def test_contains_egress_token_when_provided(self): ] def test_egress_mode_dns_nft(self): - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], @@ -166,7 +166,7 @@ def test_egress_mode_dns_nft(self): def test_serializes_network_policy_correctly(self): """Test that network policy is correctly serialized to JSON.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[ @@ -191,7 +191,7 @@ def test_serializes_network_policy_correctly(self): def test_handles_empty_egress_rules(self): """Test that empty egress rules are handled correctly.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="allow", egress=[], @@ -207,7 +207,7 @@ def test_handles_empty_egress_rules(self): def test_handles_missing_default_action(self): """Test that missing default_action is handled (exclude_none=True).""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( egress=[NetworkRule(action="allow", target="example.com")], ) @@ -222,7 +222,7 @@ def test_handles_missing_default_action(self): def test_security_context_adds_net_admin_not_privileged(self): """Egress sidecar uses NET_ADMIN only (IPv6 is disabled in execd init when egress is on).""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[], @@ -236,14 +236,14 @@ def test_security_context_adds_net_admin_not_privileged(self): def test_no_command_uses_image_entrypoint(self): container = _egress_container( - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", NetworkPolicy(default_action="deny", egress=[]), ) assert "command" not in container def test_container_spec_is_valid_kubernetes_format(self): """Test that returned container spec is in valid Kubernetes format.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], @@ -266,7 +266,7 @@ def test_container_spec_is_valid_kubernetes_format(self): def test_handles_wildcard_domains(self): """Test that wildcard domains in egress rules are handled correctly.""" - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", egress=[ @@ -308,7 +308,7 @@ def test_adds_egress_sidecar_container(self): default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], ) - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" apply_egress_to_spec( containers, @@ -327,7 +327,7 @@ def test_does_not_touch_unrelated_pod_state(self): default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], ) - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" apply_egress_to_spec( containers, @@ -352,7 +352,7 @@ def test_preserves_existing_pod_sysctls_when_not_passed_in(self): default_action="deny", egress=[NetworkRule(action="allow", target="example.com")], ) - egress_image = "opensandbox/egress:v1.1.5" + egress_image = "opensandbox/egress:v1.1.6" apply_egress_to_spec( containers, @@ -374,7 +374,7 @@ def test_no_op_when_no_network_policy(self): apply_egress_to_spec( containers, None, - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", ) assert len(containers) == 0 @@ -409,7 +409,7 @@ def test_extra_env_injected_into_sidecar(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", extra_env=extra, ) @@ -427,7 +427,7 @@ def test_extra_env_none_value_becomes_empty_string(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", extra_env={"OPENSANDBOX_EGRESS_LOG_LEVEL": None}, ) @@ -444,7 +444,7 @@ def test_extra_env_mitm_ignored_when_credential_proxy_enabled(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", credential_proxy_enabled=True, extra_env={"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "false"}, ) @@ -466,7 +466,7 @@ def test_extra_env_empty_dict_is_noop(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.5", + "opensandbox/egress:v1.1.6", extra_env={}, ) @@ -484,7 +484,7 @@ def test_sandbox_id_injected_as_env(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.4", + "opensandbox/egress:v1.1.6", sandbox_id="sbx-abc123", ) @@ -502,7 +502,7 @@ def test_sandbox_id_omitted_when_not_provided(self): apply_egress_to_spec( containers, network_policy, - "opensandbox/egress:v1.1.4", + "opensandbox/egress:v1.1.6", ) env_names = {e["name"] for e in containers[0]["env"]} diff --git a/server/tests/k8s/test_informer.py b/server/tests/k8s/test_informer.py index de53a1c43..533af7482 100644 --- a/server/tests/k8s/test_informer.py +++ b/server/tests/k8s/test_informer.py @@ -15,6 +15,9 @@ import time from unittest.mock import MagicMock, patch +import pytest +from kubernetes.client import ApiException + from opensandbox_server.services.k8s.informer import WorkloadInformer @@ -213,6 +216,101 @@ def test_handle_event_does_not_downgrade_resource_version(self): assert informer._resource_version == "200" +class TestWorkloadInformerStaleness: + """has_synced reflects whether the cache is still being maintained.""" + + def _synced_informer(self) -> WorkloadInformer: + informer = _make_informer( + list_fn=MagicMock(return_value=_list_response("x")), + resync_period_seconds=300, + watch_timeout_seconds=60, + ) + informer._full_resync() + return informer + + def test_has_synced_false_once_contact_goes_stale(self): + """A watch that stalls without raising must not leave readers on a frozen cache.""" + informer = self._synced_informer() + assert informer.has_synced is True + + informer._last_contact_at -= informer._staleness_limit_seconds + 1 + assert informer.has_synced is False + + def test_has_synced_true_within_staleness_limit(self): + """Recent contact keeps the cache usable.""" + informer = self._synced_informer() + informer._last_contact_at -= informer._staleness_limit_seconds - 1 + assert informer.has_synced is True + + def test_has_synced_false_after_stop(self): + """A stopped informer will never refresh again, however recent its last contact.""" + informer = self._synced_informer() + informer.stop() + assert informer.has_synced is False + + def test_completed_watch_stream_refreshes_contact(self): + """An idle watch that closes cleanly still proves the API server is reachable.""" + informer = self._synced_informer() + informer._last_contact_at -= informer._staleness_limit_seconds + 1 + + fake_watch = MagicMock() + fake_watch.stream.return_value = iter(()) + with patch( + "opensandbox_server.services.k8s.informer.watch.Watch", + return_value=fake_watch, + ): + informer._run_watch_loop(60) + + assert informer.has_synced is True + + +class TestWorkloadInformerWatchResilience: + """The watch stream cannot silently park the informer thread.""" + + def test_watch_stream_sets_client_side_request_timeout(self): + """A client read timeout is passed, above the server-side watch timeout.""" + informer = _make_informer() + fake_watch = MagicMock() + fake_watch.stream.return_value = iter(()) + + with patch( + "opensandbox_server.services.k8s.informer.watch.Watch", + return_value=fake_watch, + ): + informer._run_watch_loop(60) + + kwargs = fake_watch.stream.call_args.kwargs + connect_timeout, read_timeout = kwargs["_request_timeout"] + assert connect_timeout > 0 + assert read_timeout > kwargs["timeout_seconds"] + + def test_raising_watch_stream_does_not_refresh_contact(self): + """A stream that raises proves nothing about reachability. + + This is the live error path: the client raises ApiException(410) rather + than yielding an ERROR event, and _run's handler then forces a relist. + """ + informer = _make_informer( + list_fn=MagicMock(return_value=_list_response("x")), + resync_period_seconds=300, + watch_timeout_seconds=60, + ) + informer._full_resync() + informer._last_contact_at -= informer._staleness_limit_seconds + 1 + + fake_watch = MagicMock() + fake_watch.stream.side_effect = ApiException(status=410) + + with patch( + "opensandbox_server.services.k8s.informer.watch.Watch", + return_value=fake_watch, + ): + with pytest.raises(ApiException): + informer._run_watch_loop(60) + + assert informer.has_synced is False + + class TestWorkloadInformerStartStop: """start/stop thread lifecycle.""" diff --git a/server/tests/k8s/test_k8s_client.py b/server/tests/k8s/test_k8s_client.py index a2c7aff88..eda7cb705 100644 --- a/server/tests/k8s/test_k8s_client.py +++ b/server/tests/k8s/test_k8s_client.py @@ -15,7 +15,7 @@ import pytest from unittest.mock import MagicMock, patch -from kubernetes.client import ApiException +from kubernetes.client import ApiClient, ApiException, CoreV1Api from opensandbox_server.config import KubernetesRuntimeConfig from opensandbox_server.services.k8s.client import K8sClient @@ -390,20 +390,36 @@ def test_patch_custom_object_delegates_to_api(self, k8s_runtime_config): name="foo-1", body=body ) - def test_patch_pvc_uses_strategic_merge_content_type(self, k8s_runtime_config): - """patch_pvc must pin strategic-merge content type so a merge-shaped - body (e.g. ``{"metadata": {"ownerReferences": [...]}}``) is accepted. - The generated kubernetes client defaults to ``application/json-patch+json`` - which would reject the body as malformed JSON Patch ops.""" + def test_patch_pvc_uses_supported_strategic_merge_request( + self, k8s_runtime_config + ): + """patch_pvc sends a strategic-merge request accepted by ApiClient.""" c = self._make_client(k8s_runtime_config) + api_client = ApiClient() + c._core_v1_api = CoreV1Api(api_client) body = {"metadata": {"ownerReferences": [{"name": "x"}]}} - c.patch_pvc("ns", "pvc-a", body) - c._core_v1_api.patch_namespaced_persistent_volume_claim.assert_called_once_with( - name="pvc-a", - namespace="ns", - body=body, - _content_type="application/strategic-merge-patch+json", - ) + try: + with patch.object( + api_client, "call_api", return_value="patched" + ) as mock_call_api: + result = c.patch_pvc("ns", "pvc-a", body) + + assert result == "patched" + mock_call_api.assert_called_once() + args, kwargs = mock_call_api.call_args + assert args == ( + "/api/v1/namespaces/{namespace}/persistentvolumeclaims/{name}", + "PATCH", + ) + assert kwargs["path_params"] == {"namespace": "ns", "name": "pvc-a"} + assert kwargs["header_params"]["Content-Type"] == ( + "application/strategic-merge-patch+json" + ) + assert kwargs["body"] == body + assert kwargs["auth_settings"] == ["BearerToken"] + assert kwargs["_return_http_data_only"] is True + finally: + api_client.close() def test_create_secret_delegates_to_api(self, k8s_runtime_config): """create_secret forwards to CoreV1Api.create_namespaced_secret.""" diff --git a/server/tests/k8s/test_k8s_diagnostics.py b/server/tests/k8s/test_k8s_diagnostics.py index eeca85c5a..9f9f81884 100644 --- a/server/tests/k8s/test_k8s_diagnostics.py +++ b/server/tests/k8s/test_k8s_diagnostics.py @@ -13,13 +13,14 @@ # limitations under the License. from types import SimpleNamespace -from unittest.mock import MagicMock +from typing import cast +from unittest.mock import call, MagicMock import pytest from fastapi import HTTPException from kubernetes.client import V1ResourceRequirements -from opensandbox_server.services.constants import SANDBOX_ID_LABEL +from opensandbox_server.services.constants import SANDBOX_ID_LABEL, SandboxErrorCodes from opensandbox_server.services.k8s.k8s_diagnostics import ( K8sDiagnosticsMixin, _parse_since, @@ -280,6 +281,27 @@ def test_get_sandbox_logs_maps_kubernetes_403_to_forbidden_response() -> None: assert exc.value.status_code == 403 +@pytest.mark.parametrize("api_status", [None, 503]) +def test_get_sandbox_logs_maps_undocumented_kubernetes_failures_to_500( + api_status: int | None, +) -> None: + from kubernetes.client.exceptions import ApiException + + service = _DiagnosticsService([_multi_container_pod()]) + api_exc = ApiException(status=api_status, reason="Kubernetes log API error") + api_exc.body = "log API unavailable" + service.core_v1.read_namespaced_pod_log.side_effect = api_exc + + with pytest.raises(HTTPException) as exc: + service.get_sandbox_logs("sbx-1") + + assert exc.value.status_code == 500 + detail = cast(dict[str, str], exc.value.detail) + assert detail["code"] == SandboxErrorCodes.K8S_API_ERROR + assert "pod-1" in detail["message"] + assert api_exc.body in detail["message"] + + def test_get_sandbox_inspect_formats_runtime_statuses_and_resources() -> None: running_status = _status(running=SimpleNamespace(started_at="2026-01-01T00:00:01Z")) waiting_status = _status(waiting=SimpleNamespace(reason="ImagePullBackOff", message="pull failed")) @@ -365,3 +387,144 @@ def test_get_sandbox_events_uses_found_pod_namespace() -> None: field_selector="involvedObject.name=pod-1", limit=50, ) + + +def test_get_sandbox_events_follows_continuation_until_limit() -> None: + service = _DiagnosticsService([_pod()]) + + def event(index: int) -> SimpleNamespace: + return SimpleNamespace( + last_timestamp=f"2026-01-01T00:00:{index:02d}Z", + event_time=None, + first_timestamp=None, + type="Normal", + reason="Started", + message=f"event-{index}", + ) + + service.core_v1.list_namespaced_event.side_effect = [ + SimpleNamespace( + items=[event(index) for index in range(50)], + metadata=SimpleNamespace(_continue="next-page-token"), + ), + SimpleNamespace( + items=[event(50)], + metadata=SimpleNamespace(_continue=None), + ), + ] + + output = service.get_sandbox_events("sbx-1", limit=51) + + assert len(output.splitlines()) == 51 + assert "event-50" in output + assert service.core_v1.list_namespaced_event.call_args_list == [ + call( + namespace="sandbox-system", + field_selector="involvedObject.name=pod-1", + limit=51, + ), + call( + namespace="sandbox-system", + field_selector="involvedObject.name=pod-1", + limit=51, + _continue="next-page-token", + ), + ] + + +def test_stable_event_diagnostics_policy_is_owned_by_kubernetes_service() -> None: + service = _DiagnosticsService([_pod()]) + events = [f"event {index}" for index in range(51)] + service.get_sandbox_events = MagicMock(return_value="\n".join(events)) + + result = service.get_sandbox_event_diagnostics("sbx-1", scope="ALL") + + assert result.scope == "all" + assert result.content.splitlines() == events[:50] + assert result.truncated is True + assert result.warnings == ( + "The current backend only contributes runtime events to the all scope.", + ) + service.get_sandbox_events.assert_called_once_with("sbx-1", limit=51) + + +def test_stable_log_diagnostics_policy_is_owned_by_kubernetes_service() -> None: + service = _DiagnosticsService([_pod()]) + lines = [f"line {index}" for index in range(101)] + service.get_sandbox_logs = MagicMock(return_value="\n".join(lines)) + + result = service.get_sandbox_log_diagnostics("sbx-1", scope="ALL") + + assert result.scope == "all" + assert result.content.splitlines() == lines[-100:] + assert result.truncated is True + assert result.warnings == ( + "The current backend only contributes sandbox container logs to the all scope.", + ) + service.get_sandbox_logs.assert_called_once_with( + "sbx-1", + tail=101, + since=None, + container=None, + ) + + +@pytest.mark.parametrize( + ("method_name", "scope", "kind", "supported"), + [ + ("get_sandbox_log_diagnostics", "lifecycle", "logs", "container, all"), + ("get_sandbox_event_diagnostics", "network", "events", "runtime, all"), + ], +) +def test_stable_diagnostics_reject_unsupported_kubernetes_scopes( + method_name: str, + scope: str, + kind: str, + supported: str, +) -> None: + service = _DiagnosticsService([_pod()]) + + with pytest.raises(HTTPException) as exc: + getattr(service, method_name)("sbx-1", scope) + + assert exc.value.status_code == 400 + assert exc.value.detail == { + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported {kind} diagnostics scope {scope!r}. Supported scopes: {supported}." + ), + } + service.k8s_client.list_pods.assert_not_called() + + +@pytest.mark.parametrize( + ("api_status", "expected_status", "expected_code"), + [ + (400, 400, SandboxErrorCodes.K8S_API_ERROR), + (403, 403, SandboxErrorCodes.K8S_API_ERROR), + (404, 404, SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND), + (503, 500, SandboxErrorCodes.K8S_API_ERROR), + ], +) +def test_get_sandbox_events_maps_kubernetes_api_errors( + api_status: int, + expected_status: int, + expected_code: str, +) -> None: + from kubernetes.client.exceptions import ApiException + + service = _DiagnosticsService([_pod()]) + api_exc = ApiException(status=api_status, reason="Kubernetes event API error") + api_exc.body = f"event API failed with status {api_status}" + service.core_v1.list_namespaced_event.side_effect = api_exc + + with pytest.raises(HTTPException) as exc: + service.get_sandbox_events("sbx-1") + + assert exc.value.status_code == expected_status + detail = exc.value.detail + assert isinstance(detail, dict) + typed_detail = cast(dict[str, str], detail) + assert typed_detail["code"] == expected_code + assert "pod-1" in typed_detail["message"] + assert api_exc.body in typed_detail["message"] diff --git a/server/tests/k8s/test_kubernetes_service.py b/server/tests/k8s/test_kubernetes_service.py index 1c44dbfbe..a1fa02b58 100644 --- a/server/tests/k8s/test_kubernetes_service.py +++ b/server/tests/k8s/test_kubernetes_service.py @@ -96,7 +96,7 @@ def test_credential_proxy_requires_dns_nft_mode( create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[]) create_sandbox_request.credential_proxy = CredentialProxyConfig(enabled=True) k8s_service.app_config.egress = EgressConfig( - image="opensandbox/egress:v1.1.5", mode=EGRESS_MODE_DNS + image="opensandbox/egress:v1.1.6", mode=EGRESS_MODE_DNS ) with pytest.raises(HTTPException) as exc_info: @@ -323,7 +323,7 @@ async def test_create_sandbox_with_network_policy_passes_egress_token_and_annota self, k8s_service, create_sandbox_request ): create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[]) - k8s_service.app_config.egress = EgressConfig(image="opensandbox/egress:v1.1.5") + k8s_service.app_config.egress = EgressConfig(image="opensandbox/egress:v1.1.6") k8s_service.workload_provider.create_workload.return_value = { "name": "test-id", "uid": "uid-1" } @@ -397,7 +397,7 @@ async def test_create_sandbox_with_network_policy_passes_egress_mode_dns_nft_fro ): create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[]) k8s_service.app_config.egress = EgressConfig( - image="opensandbox/egress:v1.1.5", + image="opensandbox/egress:v1.1.6", mode=EGRESS_MODE_DNS_NFT, ) k8s_service.workload_provider.create_workload.return_value = { diff --git a/server/tests/k8s/test_pool_service.py b/server/tests/k8s/test_pool_service.py index b3f481e4c..951be403a 100644 --- a/server/tests/k8s/test_pool_service.py +++ b/server/tests/k8s/test_pool_service.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest +from copy import deepcopy from unittest.mock import MagicMock from kubernetes.client import ApiException @@ -161,6 +162,50 @@ def test_create_pool_calls_k8s_api_with_correct_manifest(self): assert result.name == "ci-pool" + def test_create_pool_preserves_static_pvc_template(self): + svc, mock_api = _make_pool_service(namespace="opensandbox") + mock_api.create_namespaced_custom_object.return_value = _make_raw_pool( + name="shared-workspace-pool", + namespace="opensandbox", + ) + template = { + "spec": { + "containers": [ + { + "name": "sandbox-container", + "image": "python:3.11", + "volumeMounts": [ + { + "name": "shared-workspace", + "mountPath": "/workspace", + } + ], + } + ], + "volumes": [ + { + "name": "shared-workspace", + "persistentVolumeClaim": { + "claimName": "shared-workspace-pvc", + }, + } + ], + } + } + request = CreatePoolRequest( + name="shared-workspace-pool", + template=template, + capacitySpec=_capacity_spec(), + ) + expected_template = deepcopy(request.template) + + svc.create_pool(request) + + body = mock_api.create_namespaced_custom_object.call_args.kwargs["body"] + assert body["spec"]["template"] == expected_template + assert request.template == expected_template + assert template == expected_template + def test_create_pool_returns_pool_response(self): svc, mock_api = _make_pool_service() raw = _make_raw_pool() diff --git a/server/tests/test_config.py b/server/tests/test_config.py index a450c0a1c..02cbb61fd 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -76,6 +76,7 @@ def test_load_config_from_file(tmp_path, monkeypatch): assert loaded.server.max_sandbox_timeout_seconds == 172800 assert loaded.runtime.type == "kubernetes" assert loaded.runtime.execd_image == "opensandbox/execd:test" + assert loaded.runtime.execd_run_as_init is False assert loaded.ingress is not None assert loaded.ingress.mode == "gateway" assert loaded.ingress.gateway is not None @@ -151,6 +152,33 @@ def test_load_config_without_env_uses_toml_api_key(tmp_path, monkeypatch): assert loaded.server.api_key == "toml-secret-key" +def test_runtime_execd_run_as_init_parses_from_toml(tmp_path): + toml = """ + [runtime] + type = "docker" + execd_image = "opensandbox/execd:test" + execd_run_as_init = true + """ + config_path = tmp_path / "config.toml" + config_path.write_text(toml) + + loaded = config_module.load_config(config_path) + assert loaded.runtime.execd_run_as_init is True + + +def test_runtime_execd_run_as_init_defaults_false(tmp_path): + toml = """ + [runtime] + type = "docker" + execd_image = "opensandbox/execd:test" + """ + config_path = tmp_path / "config.toml" + config_path.write_text(toml) + + loaded = config_module.load_config(config_path) + assert loaded.runtime.execd_run_as_init is False + + def test_docker_runtime_disallows_kubernetes_block(): server_cfg = ServerConfig() runtime_cfg = RuntimeConfig(type="docker", execd_image="busybox:latest") @@ -891,10 +919,41 @@ def test_egress_config_mode_literal(): base = EgressConfig(image="opensandbox/egress:v1") assert base.mode == EGRESS_MODE_DNS assert base.disable_ipv6 is True + assert base.readiness_timeout_seconds == 30.0 cfg = EgressConfig(image="opensandbox/egress:v1", mode=EGRESS_MODE_DNS_NFT) assert cfg.mode == EGRESS_MODE_DNS_NFT +def test_egress_config_readiness_timeout_must_be_positive(): + cfg = EgressConfig(readiness_timeout_seconds=75.5) + assert cfg.readiness_timeout_seconds == 75.5 + + with pytest.raises(ValidationError): + EgressConfig(readiness_timeout_seconds=0) + + +def test_load_config_with_egress_readiness_timeout(tmp_path, monkeypatch): + _reset_config(monkeypatch) + toml = textwrap.dedent( + """ + [runtime] + type = "docker" + execd_image = "opensandbox/execd:test" + + [egress] + image = "opensandbox/egress:test" + readiness_timeout_seconds = 75.5 + """ + ) + config_path = tmp_path / "config.toml" + config_path.write_text(toml) + + loaded = config_module.load_config(config_path) + + assert loaded.egress is not None + assert loaded.egress.readiness_timeout_seconds == 75.5 + + def test_log_config_defaults(): """LogConfig should have sensible defaults.""" cfg = LogConfig() @@ -1555,4 +1614,3 @@ def test_env_secure_access_active_key_must_exist(self, tmp_path, monkeypatch) -> with pytest.raises(ValidationError, match="not found in secure_access.keys"): config_module.load_config(config_path) - diff --git a/server/tests/test_devops.py b/server/tests/test_devops.py index 6272e235e..b607c9bd4 100644 --- a/server/tests/test_devops.py +++ b/server/tests/test_devops.py @@ -12,25 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. +from fastapi import HTTPException from fastapi.testclient import TestClient from opensandbox_server.api import devops +from opensandbox_server.services.diagnostics import DiagnosticResult -def test_diagnostics_logs_with_scope_returns_not_implemented( +def test_diagnostics_logs_with_scope_returns_stable_inline_descriptor( client: TestClient, auth_headers: dict, monkeypatch, ) -> None: + content = "sandbox log: ๆต‹่ฏ•" + class StubService: @staticmethod - def get_sandbox_logs( + def get_sandbox_log_diagnostics( sandbox_id: str, - tail: int, - since: str | None = None, - container: str | None = None, - ) -> str: - raise AssertionError("stable diagnostics requests must not call legacy logs") + scope: str, + ) -> DiagnosticResult: + assert sandbox_id == "sbx-001" + assert scope == "container" + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="logs", + scope=scope, + content=content, + ) monkeypatch.setattr(devops, "sandbox_service", StubService()) @@ -39,9 +48,221 @@ def get_sandbox_logs( headers=auth_headers, ) - assert response.status_code == 501 + assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") - assert response.json()["code"] == "DIAGNOSTICS_NOT_IMPLEMENTED" + assert response.json() == { + "sandboxId": "sbx-001", + "kind": "logs", + "scope": "container", + "delivery": "inline", + "contentType": "text/plain; charset=utf-8", + "content": content, + "contentLength": len(content.encode("utf-8")), + "truncated": False, + } + + +def test_diagnostics_logs_serializes_service_truncation_result( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + lines = [f"line {index}" for index in range(101)] + + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + assert sandbox_id == "sbx-001" + assert scope == "container" + return DiagnosticResult( + sandbox_id=sandbox_id, + kind="logs", + scope=scope, + content="\n".join(lines[-100:]), + truncated=True, + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/logs?scope=container", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["content"].splitlines() == lines[-100:] + assert response.json()["truncated"] is True + + +def test_diagnostics_logs_with_scope_ignores_legacy_container_selector( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + scopes: list[str] = [] + + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + scopes.append(scope) + return DiagnosticResult(sandbox_id, "logs", scope, "sandbox logs") + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + for scope in ("container", "all"): + response = client.get( + f"/v1/sandboxes/sbx-001/diagnostics/logs?scope={scope}&container=egress", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["scope"] == scope + + assert scopes == ["container", "all"] + + +def test_diagnostics_logs_with_scope_ignores_legacy_since_filter( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + scopes: list[str] = [] + + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + scopes.append(scope) + return DiagnosticResult(sandbox_id, "logs", scope, "sandbox logs") + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + for scope in ("container", "all"): + response = client.get( + f"/v1/sandboxes/sbx-001/diagnostics/logs?scope={scope}&since=5m", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["scope"] == scope + + assert scopes == ["container", "all"] + + +def test_diagnostics_logs_with_scope_ignores_legacy_tail_bound( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + scopes: list[str] = [] + + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + scopes.append(scope) + return DiagnosticResult( + sandbox_id, + "logs", + scope, + "first line\nsecond line", + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + for scope in ("container", "all"): + for tail in (1, 10000): + response = client.get( + f"/v1/sandboxes/sbx-001/diagnostics/logs?scope={scope}&tail={tail}", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["content"] == "first line\nsecond line" + + assert scopes == ["container", "container", "all", "all"] + + +def test_diagnostics_logs_rejects_unsupported_scope( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + raise HTTPException( + status_code=400, + detail={ + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported logs diagnostics scope {scope!r}. " + "Supported scopes: container, all." + ), + }, + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/logs?scope=lifecycle", + headers=auth_headers, + ) + + assert response.status_code == 400 + assert response.json() == { + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + "Unsupported logs diagnostics scope 'lifecycle'. Supported scopes: container, all." + ), + } + + +def test_diagnostics_logs_all_scope_discloses_backend_limit( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_sandbox_log_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + return DiagnosticResult( + sandbox_id, + "logs", + scope, + "container logs only", + warnings=( + "The current backend only contributes sandbox container logs to the all scope.", + ), + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/logs?scope=all", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["warnings"] == [ + "The current backend only contributes sandbox container logs to the all scope." + ] def test_diagnostics_logs_without_scope_preserves_deprecated_plain_text( @@ -106,26 +327,249 @@ def get_sandbox_logs( assert captured == {"container": "egress"} -def test_diagnostics_events_with_scope_returns_not_implemented( +def test_diagnostics_events_with_scope_returns_stable_inline_descriptor( client: TestClient, auth_headers: dict, monkeypatch, ) -> None: class StubService: @staticmethod - def get_sandbox_events(sandbox_id: str, limit: int) -> str: - raise AssertionError("stable diagnostics requests must not call legacy events") + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + assert sandbox_id == "sbx-001" + assert scope == "RUNTIME" + return DiagnosticResult( + sandbox_id, + "events", + scope.lower(), + "runtime event", + ) monkeypatch.setattr(devops, "sandbox_service", StubService()) response = client.get( - "/v1/sandboxes/sbx-001/diagnostics/events?scope=runtime", + "/v1/sandboxes/sbx-001/diagnostics/events?scope=RUNTIME", headers=auth_headers, ) - assert response.status_code == 501 + assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") - assert response.json()["code"] == "DIAGNOSTICS_NOT_IMPLEMENTED" + assert response.json() == { + "sandboxId": "sbx-001", + "kind": "events", + "scope": "runtime", + "delivery": "inline", + "contentType": "text/plain; charset=utf-8", + "content": "runtime event", + "contentLength": 13, + "truncated": False, + } + + +def test_diagnostics_events_serializes_service_truncation_result( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + events = [f"event {index}" for index in range(51)] + + class StubService: + @staticmethod + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + assert sandbox_id == "sbx-001" + assert scope == "runtime" + return DiagnosticResult( + sandbox_id, + "events", + scope, + "\n".join(events[:50]), + truncated=True, + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/events?scope=runtime", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["content"].splitlines() == events[:50] + assert response.json()["truncated"] is True + + +def test_diagnostics_events_with_scope_ignores_legacy_limit_bound( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + scopes: list[str] = [] + + class StubService: + @staticmethod + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + scopes.append(scope) + return DiagnosticResult( + sandbox_id, + "events", + scope, + "first event\nsecond event", + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + for scope in ("runtime", "all"): + for limit in (1, 500): + response = client.get( + f"/v1/sandboxes/sbx-001/diagnostics/events?scope={scope}&limit={limit}", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["content"] == "first event\nsecond event" + + assert scopes == ["runtime", "runtime", "all", "all"] + + +def test_diagnostics_events_rejects_unavailable_lifecycle_scope( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + calls: list[tuple[str, str]] = [] + + class StubService: + @staticmethod + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + calls.append((sandbox_id, scope)) + raise HTTPException( + status_code=400, + detail={ + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported events diagnostics scope {scope!r}. " + "Supported scopes: runtime, all." + ), + }, + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/events?scope=lifecycle", + headers=auth_headers, + ) + + assert response.status_code == 400 + assert response.json() == { + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + "Unsupported events diagnostics scope 'lifecycle'. " + "Supported scopes: runtime, all." + ), + } + assert calls == [("sbx-001", "lifecycle")] + + +def test_diagnostics_events_all_scope_discloses_backend_limit( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + return DiagnosticResult( + sandbox_id, + "events", + scope, + "runtime event", + warnings=("The current backend only contributes runtime events to the all scope.",), + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/events?scope=all", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["warnings"] == [ + "The current backend only contributes runtime events to the all scope." + ] + + +def test_diagnostics_events_rejects_unsupported_scope( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_sandbox_event_diagnostics( + sandbox_id: str, + scope: str, + ) -> DiagnosticResult: + raise HTTPException( + status_code=400, + detail={ + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported events diagnostics scope {scope!r}. " + "Supported scopes: runtime, all." + ), + }, + ) + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/events?scope=network", + headers=auth_headers, + ) + + assert response.status_code == 400 + assert response.json()["code"] == "DIAGNOSTICS_SCOPE_UNSUPPORTED" + + +def test_diagnostics_events_without_scope_preserves_deprecated_plain_text( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_sandbox_events(sandbox_id: str, limit: int) -> str: + assert sandbox_id == "sbx-001" + assert limit == 10 + return "legacy events" + + monkeypatch.setattr(devops, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/diagnostics/events?limit=10", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/plain") + assert response.headers["deprecation"] == "true" + assert response.text == "legacy events" def test_diagnostics_summary_redacts_unexpected_exception_details( diff --git a/server/tests/test_docker_diagnostics.py b/server/tests/test_docker_diagnostics.py index 32bc1a9e1..5dec9300f 100644 --- a/server/tests/test_docker_diagnostics.py +++ b/server/tests/test_docker_diagnostics.py @@ -13,8 +13,14 @@ # limitations under the License. from types import SimpleNamespace +from typing import cast from unittest.mock import MagicMock, patch +from docker.errors import DockerException +from fastapi import HTTPException, status +import pytest + +from opensandbox_server.services.constants import SandboxErrorCodes from opensandbox_server.services.docker.docker_diagnostics import ( DockerDiagnosticsMixin, _parse_since_to_timestamp, @@ -68,6 +74,82 @@ def test_get_sandbox_logs_returns_placeholder_for_empty_output() -> None: assert service.get_sandbox_logs("sbx-1") == "(no logs)" +def test_get_sandbox_logs_maps_docker_errors_to_contract_response() -> None: + container = _container({"State": {}}) + container.logs.side_effect = DockerException("daemon disconnected") + service = _DiagnosticsService(container) + + with pytest.raises(HTTPException) as exc: + service.get_sandbox_logs("sbx-1") + + assert exc.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + detail = cast(dict[str, str], exc.value.detail) + assert detail["code"] == SandboxErrorCodes.CONTAINER_QUERY_FAILED + assert detail["message"] == "Failed to read logs for sandbox sbx-1: daemon disconnected" + + +def test_stable_log_diagnostics_policy_is_owned_by_docker_service() -> None: + container = _container({"State": {}}) + lines = [f"line {index}" for index in range(101)] + container.logs.return_value = "\n".join(lines) + service = _DiagnosticsService(container) + + result = service.get_sandbox_log_diagnostics("sbx-1", scope="ALL") + + assert result.scope == "all" + assert result.content.splitlines() == lines[-100:] + assert result.truncated is True + assert result.warnings == ( + "The current backend only contributes sandbox container logs to the all scope.", + ) + container.logs.assert_called_once_with(tail=101, timestamps=True) + + +def test_stable_event_diagnostics_policy_is_owned_by_docker_service() -> None: + service = _DiagnosticsService(_container({"State": {}})) + events = [f"event {index}" for index in range(51)] + service.get_sandbox_events = MagicMock(return_value="\n".join(events)) + + result = service.get_sandbox_event_diagnostics("sbx-1", scope="ALL") + + assert result.scope == "all" + assert result.content.splitlines() == events[:50] + assert result.truncated is True + assert result.warnings == ( + "The current backend only contributes runtime events to the all scope.", + ) + service.get_sandbox_events.assert_called_once_with("sbx-1", limit=51) + + +@pytest.mark.parametrize( + ("method_name", "scope", "kind", "supported"), + [ + ("get_sandbox_log_diagnostics", "lifecycle", "logs", "container, all"), + ("get_sandbox_event_diagnostics", "network", "events", "runtime, all"), + ], +) +def test_stable_diagnostics_reject_unsupported_docker_scopes( + method_name: str, + scope: str, + kind: str, + supported: str, +) -> None: + container = _container({"State": {}}) + service = _DiagnosticsService(container) + + with pytest.raises(HTTPException) as exc: + getattr(service, method_name)("sbx-1", scope) + + assert exc.value.status_code == status.HTTP_400_BAD_REQUEST + assert exc.value.detail == { + "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED", + "message": ( + f"Unsupported {kind} diagnostics scope {scope!r}. Supported scopes: {supported}." + ), + } + container.logs.assert_not_called() + + def test_get_sandbox_inspect_formats_state_resources_ports_and_safe_env() -> None: container = _container( { diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index 3c9f79b96..940a923e6 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -25,6 +25,7 @@ from opensandbox_server.config import ( AppConfig, + DockerConfig, EGRESS_MODE_DNS, EgressConfig, RuntimeConfig, @@ -95,8 +96,18 @@ def test_parse_memory_limit_handles_units(): def test_parse_nano_cpus(): assert parse_nano_cpus("500m") == 500_000_000 assert parse_nano_cpus("2") == 2_000_000_000 + assert parse_nano_cpus("1.5") == 1_500_000_000 + assert parse_nano_cpus("250.5m") == 250_500_000 assert parse_nano_cpus("bad") is None + +@pytest.mark.parametrize( + "value", ["nan", "inf", "-inf", "1e10", "1e308", "1e309", "-1e309"] +) +def test_parse_nano_cpus_rejects_non_finite_and_overflow_values(value: str): + assert parse_nano_cpus(value) is None + + def test_parse_gpu_request(): assert parse_gpu_request("1") == 1 assert parse_gpu_request("4") == 4 @@ -181,6 +192,55 @@ async def test_create_sandbox_applies_security_defaults(mock_docker): assert host_config.get("cap_drop") == service.app_config.docker.drop_capabilities assert host_config.get("pids_limit") == service.app_config.docker.pids_limit +@pytest.mark.asyncio +@patch("opensandbox_server.services.docker.docker_service.docker") +async def test_create_sandbox_applies_config_sandbox_env_and_binds(mock_docker): + """docker.sandbox_env / docker.sandbox_binds apply to every sandbox; request env wins.""" + mock_client = MagicMock() + mock_client.containers.list.return_value = [] + mock_client.api.create_host_config.return_value = {} + mock_client.api.create_container.return_value = {"Id": "cid"} + mock_client.containers.get.return_value = MagicMock() + mock_docker.from_env.return_value = mock_client + + config = _app_config() + config.docker = DockerConfig( + sandbox_env={ + "NODE_EXTRA_CA_CERTS": "/etc/ssl/private-ca/root-ca.crt", + "SHARED": "config", + }, + sandbox_binds=["/opt/certs/root-ca.crt:/etc/ssl/private-ca/root-ca.crt:ro"], + ) + service = DockerSandboxService(config=config) + request = CreateSandboxRequest( + image=ImageSpec(uri="python:3.11"), + timeout=120, + resourceLimits=ResourceLimits(root={}), + env={"SHARED": "request"}, + metadata={}, + entrypoint=["python"], + ) + + with ( + patch.object(service, "_ensure_image_available"), + patch.object(service, "_prepare_sandbox_runtime"), + patch( + "opensandbox_server.services.docker.docker_service.allocate_port_bindings", + return_value={ + "44772": ("0.0.0.0", 40001), + "8080": ("0.0.0.0", 40002), + }, + ), + ): + await service.create_sandbox(request) + + environment = mock_client.api.create_container.call_args.kwargs["environment"] + assert "NODE_EXTRA_CA_CERTS=/etc/ssl/private-ca/root-ca.crt" in environment + assert "SHARED=request" in environment # request overrides the config default + assert "SHARED=config" not in environment + binds = mock_client.api.create_host_config.call_args.kwargs.get("binds") + assert binds == ["/opt/certs/root-ca.crt:/etc/ssl/private-ca/root-ca.crt:ro"] + @pytest.mark.asyncio @patch("opensandbox_server.services.docker.docker_service.docker") async def test_create_sandbox_passes_gpu_device_requests(mock_docker): @@ -809,7 +869,7 @@ def host_cfg_side_effect(**kwargs): cfg = _app_config() cfg.docker.network_mode = "bridge" - cfg.egress = EgressConfig(image="egress:latest") + cfg.egress = EgressConfig(image="egress:latest", readiness_timeout_seconds=75.5) service = DockerSandboxService(config=cfg) req = CreateSandboxRequest( @@ -829,14 +889,19 @@ def host_cfg_side_effect(**kwargs): return_value={ "44772": ("0.0.0.0", 44772), "8080": ("0.0.0.0", 8080), + "18080": ("0.0.0.0", 18080), }, ), patch.object(service, "_ensure_image_available"), patch.object(service, "_prepare_sandbox_runtime"), - patch.object(service, "_wait_for_egress_sidecar_ready"), + patch.object(service, "_wait_for_egress_sidecar_ready") as wait_for_egress_ready, ): await service.create_sandbox(req) + wait_for_egress_ready.assert_called_once() + assert wait_for_egress_ready.call_args.args[1:] == (18080, "egress-token") + assert wait_for_egress_ready.call_args.kwargs == {"timeout_seconds": 75.5} + assert len(mock_client.api.create_container.call_args_list) == 2 sidecar_call = mock_client.api.create_container.call_args_list[0] main_call = mock_client.api.create_container.call_args_list[1] @@ -1517,6 +1582,37 @@ def test_build_labels_marks_manual_cleanup_without_expiration(): assert labels[SANDBOX_MANUAL_CLEANUP_LABEL] == "true" assert "opensandbox.io/expires-at" not in labels + +def test_build_env_omits_execd_run_as_init_by_default(): + service = DockerSandboxService(config=_app_config()) + request = CreateSandboxRequest( + image=ImageSpec(uri="python:3.11"), + resourceLimits=ResourceLimits(root={}), + env={"FOO": "bar"}, + entrypoint=["python"], + ) + + _, environment = service._build_labels_and_env("sandbox-manual", request, None) + + assert "FOO=bar" in environment + assert not any(e.startswith("EXECD_INIT=") for e in environment) + + +def test_build_env_injects_execd_run_as_init_when_enabled(): + config = _app_config() + config.runtime.execd_run_as_init = True + service = DockerSandboxService(config=config) + request = CreateSandboxRequest( + image=ImageSpec(uri="python:3.11"), + resourceLimits=ResourceLimits(root={}), + env={}, + entrypoint=["python"], + ) + + _, environment = service._build_labels_and_env("sandbox-manual", request, None) + + assert "EXECD_INIT=1" in environment + def test_build_labels_stores_extensions_json(): service = DockerSandboxService(config=_app_config()) request = CreateSandboxRequest( @@ -1842,7 +1938,7 @@ async def test_create_sandbox_windows_profile_injects_runtime_defaults(mock_dock mock_docker.from_env.return_value = mock_client cfg = _app_config() - cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.21" + cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.22" cfg.docker.network_mode = "bridge" service = DockerSandboxService(config=cfg) request = CreateSandboxRequest( @@ -1925,7 +2021,7 @@ async def test_create_sandbox_windows_profile_rejects_missing_runtime_devices(mo mock_docker.from_env.return_value = mock_client cfg = _app_config() - cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.21" + cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.22" cfg.docker.network_mode = "bridge" service = DockerSandboxService(config=cfg) request = CreateSandboxRequest( @@ -1964,7 +2060,7 @@ async def test_create_sandbox_windows_profile_rejects_below_minimum_resource_lim mock_docker.from_env.return_value = mock_client cfg = _app_config() - cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.21" + cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.22" cfg.docker.network_mode = "bridge" service = DockerSandboxService(config=cfg) request = CreateSandboxRequest( @@ -2001,7 +2097,7 @@ async def test_create_sandbox_windows_profile_accepts_dockur_demo_like_request(m mock_docker.from_env.return_value = mock_client cfg = _app_config() - cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.21" + cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.22" cfg.docker.network_mode = "bridge" service = DockerSandboxService(config=cfg) request = CreateSandboxRequest( @@ -2055,7 +2151,7 @@ async def test_create_sandbox_windows_profile_with_network_policy_maps_windows_p mock_docker.from_env.return_value = mock_client cfg = _app_config() - cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.21" + cfg.runtime.execd_image = "ghcr.io/opensandbox/execd:v1.0.22" cfg.docker.network_mode = "bridge" cfg.egress = EgressConfig(image="opensandbox/egress:latest") service = DockerSandboxService(config=cfg) diff --git a/server/tests/test_fleets_fastpath_client.py b/server/tests/test_fleets_fastpath_client.py new file mode 100644 index 000000000..b170ec1c6 --- /dev/null +++ b/server/tests/test_fleets_fastpath_client.py @@ -0,0 +1,341 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the FastPath v2 gRPC client.""" + +import grpc +import pytest +import pytest_asyncio +from grpc import aio + +from opensandbox_server.services.fleets import fastpath_client +from opensandbox_server.services.fleets.fastpath_client import ( + FastPathClient, + FastPathConflict, + FastPathError, + FastPathInvalidArgument, + FastPathNotFound, + FastPathUnavailable, + component_target, + namespaced_reference, + port_target, +) +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2 as pb2, +) +from opensandbox_server.services.fleets.generated import ( + fastpath_pb2_grpc as pb2_grpc, +) + + +class _FakeFastPathService(pb2_grpc.FastPathServiceServicer): + """In-process FastPath server with scripted responses.""" + + def __init__(self): + self.created: list[pb2.CreateRequest] = [] + self.last_delete: tuple[str, str] | None = None + self.sandbox = pb2.SandboxInfo( + sandbox_uid="uid-1", + sandbox_name="sbx-1", + namespace="ns-1", + runtime_state="Ready", + data_plane_state="Ready", + image="python:3.11", + pool_ref="default-pool", + ) + self.get_error: grpc.StatusCode | None = None + + async def CreateSandbox(self, request, context): + self.created.append(request) + info = pb2.SandboxInfo() + info.CopyFrom(self.sandbox) + info.sandbox_name = request.request_id + info.namespace = request.namespace + return info + + async def GetSandbox(self, request, context): + if self.get_error is not None: + await context.abort(self.get_error, "scripted failure") + info = pb2.SandboxInfo() + info.CopyFrom(self.sandbox) + info.sandbox_name = request.sandbox_name + info.namespace = request.namespace + return info + + async def DeleteSandbox(self, request, context): + self.last_delete = (request.namespace, request.sandbox_name) + return pb2.DeleteResponse(success=True) + + async def ListSandboxes(self, request, context): + response = pb2.ListResponse() + info = pb2.SandboxInfo() + info.CopyFrom(self.sandbox) + info.namespace = request.namespace + response.items.append(info) + return response + + async def UpdateSandbox(self, request, context): + return pb2.UpdateResponse( + success=True, + sandbox=pb2.SandboxInfo( + sandbox_uid="uid-1", + sandbox_name=request.sandbox_name, + namespace=request.namespace, + runtime_state="Ready", + data_plane_state="Ready", + expires_at_unix_seconds=request.expires_at_unix_seconds, + ), + ) + + async def GetSandboxDiagnostics(self, request, context): + return pb2.SandboxDiagnosticsResponse( + sandbox=self.sandbox, assignment_state="assigned" + ) + + async def WaitSandboxReady(self, request, context): + info = pb2.SandboxInfo() + info.CopyFrom(self.sandbox) + return info + + async def ResolveEndpoint(self, request, context): + return pb2.ResolveEndpointResponse( + sandbox_uid="uid-1", + protocol="HTTP", + resolved_port=44772, + proxy_endpoint="http://sandbox-proxy:8080", + route_generation=1, + expires_at_unix_seconds=1750000000, + required_headers={"x-fast-sandbox-route-credential": "token-1"}, + ) + + async def GetPool(self, request, context): + return pb2.PoolInfo(namespace=request.namespace, name=request.pool_name) + + async def ListPools(self, request, context): + return pb2.ListPoolsResponse( + items=[pb2.PoolInfo(namespace=request.namespace, name="default-pool")] + ) + + +@pytest_asyncio.fixture +async def client_and_server(): + service = _FakeFastPathService() + server = aio.server() + pb2_grpc.add_FastPathServiceServicer_to_server(service, server) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + channel = aio.insecure_channel(f"127.0.0.1:{port}") + stub = pb2_grpc.FastPathServiceStub(channel) + client = FastPathClient(endpoint=f"127.0.0.1:{port}") + client._channel = channel # noqa: SLF001 - share the fixture channel + client._stub = stub + try: + yield client, service + finally: + await channel.close() + await server.stop(None) + + +@pytest.mark.asyncio +async def test_create_sandbox_passes_through_fields(client_and_server): + client, service = client_and_server + request = pb2.CreateRequest( + request_id="sbx-1", + namespace="ns-1", + image="python:3.11", + pool_ref="default-pool", + command=["python", "-m", "http.server"], + args=["8000"], + envs={"PYTHONUNBUFFERED": "1"}, + expires_at_unix_seconds=1750000000, + ) + request.metadata.update({"team": "agents"}) + + info = await client.create_sandbox(request) + + assert info.sandbox_name == "sbx-1" + assert info.namespace == "ns-1" + assert service.created[-1].image == "python:3.11" + assert service.created[-1].expires_at_unix_seconds == 1750000000 + assert service.created[-1].envs["PYTHONUNBUFFERED"] == "1" + + +@pytest.mark.asyncio +async def test_get_sandbox_round_trips_identity(client_and_server): + client, _ = client_and_server + info = await client.get_sandbox("ns-1", "sbx-1") + assert info.sandbox_name == "sbx-1" + assert info.namespace == "ns-1" + + +@pytest.mark.asyncio +async def test_get_sandbox_not_found_maps_to_fastpath_not_found(client_and_server): + client, service = client_and_server + service.get_error = grpc.StatusCode.NOT_FOUND + with pytest.raises(FastPathNotFound) as exc_info: + await client.get_sandbox("ns-1", "missing") + assert exc_info.value.code == "NOT_FOUND" + + +@pytest.mark.asyncio +async def test_delete_sandbox_passes_namespace_and_name(client_and_server): + client, service = client_and_server + await client.delete_sandbox("ns-1", "sbx-1") + assert service.last_delete == ("ns-1", "sbx-1") + + +@pytest.mark.asyncio +async def test_list_sandboxes_applies_filter_and_paging(client_and_server): + client, _ = client_and_server + response = await client.list_sandboxes( + "ns-1", metadata={"team": "agents"}, page_size=10, page_token="tok" + ) + assert response.items[0].namespace == "ns-1" + + +@pytest.mark.asyncio +async def test_update_expiration_returns_sandbox(client_and_server): + client, _ = client_and_server + info = await client.update_expiration("ns-1", "sbx-1", 1750000000) + assert info.expires_at_unix_seconds == 1750000000 + + +@pytest.mark.asyncio +async def test_update_metadata_upsert_and_delete_keys(client_and_server): + client, _ = client_and_server + info = await client.update_metadata( + "ns-1", "sbx-1", upsert={"a": "1"}, delete_keys=["b"] + ) + assert info.sandbox_name == "sbx-1" + + +@pytest.mark.asyncio +async def test_diagnostics_and_ready_and_endpoint_calls(client_and_server): + client, _ = client_and_server + + diag = await client.get_sandbox_diagnostics("ns-1", "sbx-1", limit=10) + assert diag.assignment_state == "assigned" + + ready = await client.wait_sandbox_ready( + namespaced_reference("ns-1", "sbx-1"), data_plane=True + ) + assert ready.runtime_state == "Ready" + + resolved = await client.resolve_endpoint( + namespaced_reference("ns-1", "sbx-1"), component_target("execd") + ) + assert resolved.resolved_port == 44772 + assert resolved.required_headers["x-fast-sandbox-route-credential"] == "token-1" + + +@pytest.mark.asyncio +async def test_pool_calls(client_and_server): + client, _ = client_and_server + pool = await client.get_pool("ns-1", "default-pool") + assert pool.name == "default-pool" + pools = await client.list_pools("ns-1") + assert pools.items[0].name == "default-pool" + + +class _TimeoutRecordingStub: + """Wrapper stub recording the transport deadline of each call.""" + + def __init__(self, inner): + self._inner = inner + self.deadlines: list[float | None] = [] + + def _record(self, method_name): + async def call(request, timeout=None): + self.deadlines.append(timeout) + return await getattr(self._inner, method_name)(request, None) + + return call + + def __getattr__(self, name): + return self._record(name) + + +@pytest.mark.asyncio +async def test_deadline_accounts_for_readiness_wait(client_and_server): + client, service = client_and_server + recording = _TimeoutRecordingStub(service) + client._stub = recording # noqa: SLF001 + ref = namespaced_reference("ns-1", "sbx-1") + + # Non-waiting endpoint lookups stay on the configured deadline even when a + # large server-side wait window is supplied. + await client.resolve_endpoint( + ref, component_target("execd"), wait_until_ready=False, wait_timeout_millis=60000 + ) + assert recording.deadlines[-1] == 30.0 + + # Wait-enabled calls extend the deadline beyond the server-side wait. + await client.resolve_endpoint( + ref, component_target("execd"), wait_until_ready=True, wait_timeout_millis=60000 + ) + assert recording.deadlines[-1] == 65.0 + + await client.wait_sandbox_ready(ref, data_plane=True, wait_timeout_millis=60000) + assert recording.deadlines[-1] == 65.0 + + # Plain lifecycle calls always use the configured deadline. + await client.get_sandbox("ns-1", "sbx-1") + assert recording.deadlines[-1] == 30.0 + + +@pytest.mark.asyncio +async def test_error_mapping_covers_common_codes(): + cases = [ + (grpc.StatusCode.NOT_FOUND, FastPathNotFound), + (grpc.StatusCode.INVALID_ARGUMENT, FastPathInvalidArgument), + (grpc.StatusCode.ALREADY_EXISTS, FastPathConflict), + (grpc.StatusCode.UNAVAILABLE, FastPathUnavailable), + (grpc.StatusCode.DEADLINE_EXCEEDED, FastPathUnavailable), + (grpc.StatusCode.PERMISSION_DENIED, FastPathError), + ] + for code, expected in cases: + error = fastpath_client._to_fastpath_error( # noqa: SLF001 + _abort_error(code) + ) + assert isinstance(error, expected) + assert error.code == code.name + + +@pytest.mark.asyncio +async def test_client_requires_connect_before_calls(): + client = FastPathClient(endpoint="127.0.0.1:1") + with pytest.raises(FastPathUnavailable): + await client.create_sandbox(pb2.CreateRequest(request_id="x")) + + +def test_reference_and_target_helpers(): + ref = namespaced_reference("ns-1", "sbx-1") + assert ref.namespaced_name.namespace == "ns-1" + assert ref.namespaced_name.name == "sbx-1" + + assert component_target("execd").component_name == "execd" + assert port_target(8000).port == 8000 + + +def _abort_error(code: grpc.StatusCode) -> aio.AioRpcError: + """Build a minimal AioRpcError carrying only the status code.""" + return aio.AioRpcError( + code, + aio.Metadata(), # initial_metadata + aio.Metadata(), # trailing_metadata + f"scripted {code.name}", + ) diff --git a/server/tests/test_fleets_mapping.py b/server/tests/test_fleets_mapping.py new file mode 100644 index 000000000..3914d31f6 --- /dev/null +++ b/server/tests/test_fleets_mapping.py @@ -0,0 +1,367 @@ +# pyright: reportAttributeAccessIssue=false +# protobuf-generated modules expose dynamic attributes. + +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for fleets create/status mapping (OSEP-0007 simplified create).""" + +from datetime import datetime, timezone + +import pytest + +from opensandbox_server.api.schema import ( + CredentialProxyConfig, + ImageSpec, + NetworkPolicy, + NetworkRule, + PlatformSpec, + ResourceLimits, + Volume, +) +from opensandbox_server.services.fleets.create_mapping import ( + RENEW_EXTEND_SECONDS_METADATA_KEY, + UnsupportedFieldError, + map_create_request, +) +from opensandbox_server.services.fleets.generated import fastpath_pb2 as pb2 +from opensandbox_server.services.fleets.status_mapping import ( + RESERVED_METADATA_KEYS, + map_sandbox, + map_state, +) + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) +EXPECTED_EXPIRY = int(NOW.timestamp()) + 3600 + + +def _host_volume(): + from opensandbox_server.api.schema import Host + + return Volume(name="data", host=Host(path="/tmp/data"), mountPath="/data") + + +def _auth(): + from opensandbox_server.api.schema import ImageAuth + + return ImageAuth(username="u", password="p") + + +def _base_request(**overrides): + from opensandbox_server.api.schema import CreateSandboxRequest + + payload = { + "image": ImageSpec(uri="python:3.11"), + "entrypoint": ["python", "-m", "http.server"], + "timeout": 3600, + "resource_limits": ResourceLimits(root={"cpu": "500m"}), + } + payload.update(overrides) + return CreateSandboxRequest(**payload) + + +# -- create mapping ----------------------------------------------------------- + + +def test_map_create_request_maps_core_fields(): + request = _base_request( + env={"PYTHONUNBUFFERED": "1"}, + metadata={"team": "agents"}, + extensions={"poolRef": "ml-pool"}, + ) + + create = map_create_request(request, "sbx-1", "ns-1", now=NOW) + + assert create.request_id == "sbx-1" + assert create.namespace == "ns-1" + assert create.image == "python:3.11" + assert create.command == ["python", "-m", "http.server"] + assert create.envs["PYTHONUNBUFFERED"] == "1" + assert create.metadata["team"] == "agents" + assert create.pool_ref == "ml-pool" + assert create.expires_at_unix_seconds == EXPECTED_EXPIRY + + +def test_map_create_request_defaults_pool_ref(): + request = _base_request() + create = map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert create.pool_ref == "default-pool" + + +def test_map_create_request_strips_pool_ref(): + # A whitespace-only poolRef must not reach FastPath; a padded name is + # normalized before forwarding. + blank = map_create_request( + _base_request(extensions={"poolRef": " "}), "sbx-1", "ns-1", now=NOW + ) + assert blank.pool_ref == "default-pool" + + padded = map_create_request( + _base_request(extensions={"poolRef": " ml-pool "}), "sbx-1", "ns-1", now=NOW + ) + assert padded.pool_ref == "ml-pool" + + +def test_map_create_request_renew_extension_goes_to_reserved_metadata(): + request = _base_request(extensions={"access.renew.extend.seconds": "300"}) + create = map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert create.metadata[RENEW_EXTEND_SECONDS_METADATA_KEY] == "300" + + +@pytest.mark.parametrize( + "field_name,payload", + [ + # snapshotId is mutually exclusive with image at the schema layer, so + # build the request image-less to reach the fleets mapping rejection. + ("snapshotId", {"image": None, "snapshot_id": "snap-1"}), + ("platform", {"platform": PlatformSpec(os="linux", arch="amd64")}), + ( + "resourceRequests", + {"resource_requests": ResourceLimits(root={"cpu": "1"})}, + ), + ( + "credentialProxy", + { + # schema requires networkPolicy when credentialProxy is + # enabled; the fleets mapping still rejects credentialProxy + # first. + "credential_proxy": CredentialProxyConfig(enabled=True), + "network_policy": NetworkPolicy( + egress=[NetworkRule(action="allow", target="a.com")] + ), + }, + ), + ( + "networkPolicy", + {"network_policy": NetworkPolicy(egress=[NetworkRule(action="allow", target="a.com")])}, + ), + ("secureAccess", {"secure_access": True}), + ("volumes", {"volumes": [_host_volume()]}), + ], +) +def test_map_create_request_rejects_unsupported_fields(field_name, payload): + request = _base_request(**payload) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == field_name + + +def test_map_create_request_rejects_image_auth(): + request = _base_request(image=ImageSpec(uri="private/reg:1", auth=_auth())) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == "image.auth" + + +def test_map_create_request_rejects_missing_image_even_with_pool_ref(): + # A fast-sandbox SandboxPool does not define the workload image. + request = _base_request(image=None, extensions={"poolRef": "ml-pool"}) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == "image" + + +def test_map_create_request_rejects_null_timeout(): + request = _base_request(timeout=None) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == "timeout" + + +def test_map_create_request_rejects_null_env_value(): + request = _base_request(env={"EMPTY": None}) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == "env" + + +def test_map_create_request_rejects_unknown_extension_key(): + request = _base_request(extensions={"bootstrap.execd.isolation": "per-slot"}) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert "extensions" in exc_info.value.field + + +@pytest.mark.parametrize( + "metadata", + [ + {"team.io/project": "agents"}, # dots and slashes are not DNS labels + {"Team": "agents"}, # uppercase is not a DNS label + {"a" * 64: "v"}, # key too long + {"ok-key": "bad value!"}, # value with spaces + {"ok-key": "v" * 64}, # value too long + ], +) +def test_map_create_request_rejects_non_label_metadata(metadata): + request = _base_request(metadata=metadata) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert exc_info.value.field == "metadata" + + +def test_map_create_request_accepts_label_compliant_metadata(): + request = _base_request( + metadata={"team": "agents", "region-us-east-1": "prod"}, + env={"K": "v"}, + ) + create = map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert create.metadata["team"] == "agents" + assert create.metadata["region-us-east-1"] == "prod" + + +def test_map_create_request_reuses_absolute_expiry_on_remap(): + # A transport retry of the same sandbox_id must reuse the first expiry, + # even when the clock has advanced, or FastPath rejects the changed intent. + first = map_create_request(_base_request(), "sbx-1", "ns-1", now=NOW) + retry = map_create_request( + _base_request(), + "sbx-1", + "ns-1", + now=datetime(2026, 8, 18, 13, 0, 0, tzinfo=timezone.utc), + expires_at_unix_seconds=first.expires_at_unix_seconds, + ) + assert retry.expires_at_unix_seconds == first.expires_at_unix_seconds == EXPECTED_EXPIRY + + +def test_map_create_request_accepts_matching_pool_resources(): + request = _base_request(resource_limits=ResourceLimits(root={"cpu": "500m", "memory": "512Mi"})) + create = map_create_request( + request, + "sbx-1", + "ns-1", + now=NOW, + pool_resources={"cpu": "500m", "memory": "512Mi", "pids": "256"}, + ) + assert create.image == "python:3.11" + + +def test_map_create_request_rejects_mismatched_pool_resources(): + request = _base_request(resource_limits=ResourceLimits(root={"cpu": "1"})) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request( + request, + "sbx-1", + "ns-1", + now=NOW, + pool_resources={"cpu": "500m", "memory": "512Mi"}, + ) + assert exc_info.value.field == "resourceLimits" + + +def test_map_create_request_rejects_undefinted_pool_resource_key(): + request = _base_request(resource_limits=ResourceLimits(root={"gpu": "1"})) + with pytest.raises(UnsupportedFieldError) as exc_info: + map_create_request( + request, "sbx-1", "ns-1", now=NOW, pool_resources={"cpu": "500m"} + ) + assert exc_info.value.field == "resourceLimits" + + +def test_map_create_request_compares_quantities_canonically(): + # Same quantity expressed differently must not be rejected. + request = _base_request( + resource_limits=ResourceLimits(root={"cpu": "0.5", "memory": "1Gi"}) + ) + create = map_create_request( + request, + "sbx-1", + "ns-1", + now=NOW, + pool_resources={"cpu": "500m", "memory": "1024Mi", "pids": "256"}, + ) + assert create.image == "python:3.11" + + +def test_map_create_request_skips_pool_check_when_profile_unknown(): + request = _base_request(resource_limits=ResourceLimits(root={"cpu": "1"})) + create = map_create_request(request, "sbx-1", "ns-1", now=NOW) + assert create.image == "python:3.11" + + +# -- status mapping ----------------------------------------------------------- + + +def _info(**overrides): + info = pb2.SandboxInfo( + sandbox_uid="uid-1", + sandbox_name="sbx-1", + namespace="ns-1", + runtime_state="Ready", + data_plane_state="Ready", + image="python:3.11", + created_at_unix_seconds=int(NOW.timestamp()), + ) + if "expires_at_unix_seconds" in overrides: + info.expires_at_unix_seconds = overrides["expires_at_unix_seconds"] + if "metadata" in overrides: + info.metadata.update(overrides["metadata"]) + if "runtime_state" in overrides: + info.runtime_state = overrides["runtime_state"] + if "data_plane_state" in overrides: + info.data_plane_state = overrides["data_plane_state"] + return info + + +@pytest.mark.parametrize( + "runtime,data_plane,expected", + [ + ("Ready", "Ready", "Running"), + ("Ready", "", "Pending"), + ("Pending", "", "Pending"), + ("Creating", "", "Pending"), + ("Draining", "", "Stopping"), + ("Stopped", "", "Terminated"), + ("Failed", "", "Failed"), + ("Unavailable", "", "Failed"), + ("", "", "Pending"), + ], +) +def test_map_state_matrix(runtime, data_plane, expected): + assert map_state(_info(runtime_state=runtime, data_plane_state=data_plane)) == expected + + +def test_map_sandbox_builds_public_model(): + sandbox = map_sandbox( + _info( + expires_at_unix_seconds=EXPECTED_EXPIRY, + metadata={"team": "agents", RENEW_EXTEND_SECONDS_METADATA_KEY: "300"}, + ) + ) + + assert sandbox.id == "sbx-1" + assert sandbox.image is not None + assert sandbox.image.uri == "python:3.11" + assert sandbox.status.state == "Running" + assert sandbox.metadata == {"team": "agents"} + assert sandbox.expires_at == datetime.fromtimestamp(EXPECTED_EXPIRY, tz=timezone.utc) + assert sandbox.created_at == NOW + + +def test_map_sandbox_terminated_on_retained_stopped_crd(): + # Retained Stopped objects map to Terminated, but the Expired reason + # cannot be confirmed from SandboxInfo (no Conditions field), so it stays + # unset. + sandbox = map_sandbox(_info(runtime_state="Stopped")) + assert sandbox.status.state == "Terminated" + assert sandbox.status.reason is None + + +def test_map_sandbox_omits_empty_metadata_and_expiry(): + sandbox = map_sandbox(_info()) + assert sandbox.metadata is None + assert sandbox.expires_at is None + + +def test_reserved_metadata_keys_are_defined(): + assert RENEW_EXTEND_SECONDS_METADATA_KEY in RESERVED_METADATA_KEYS diff --git a/server/tests/test_http_metrics.py b/server/tests/test_http_metrics.py new file mode 100644 index 000000000..a5529b98f --- /dev/null +++ b/server/tests/test_http_metrics.py @@ -0,0 +1,204 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for low-cardinality Server HTTP request metrics.""" + +from collections.abc import AsyncIterator +from unittest.mock import MagicMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import Histogram, InMemoryMetricReader +from starlette.responses import StreamingResponse + +import opensandbox_server.integrations.otel.metrics as otel_metrics +from opensandbox_server.middleware.http_metrics import HttpMetricsMiddleware + + +def _test_app() -> FastAPI: + app = FastAPI() + + @app.get("/items/{item_id}") + async def get_item(item_id: int) -> dict[str, int]: + return {"item_id": item_id} + + @app.get("/explode") + async def explode() -> None: + raise RuntimeError("boom") + + @app.get("/stream-error") + async def stream_error() -> StreamingResponse: + async def broken_body() -> AsyncIterator[bytes]: + yield b"partial" + raise RuntimeError("stream boom") + + return StreamingResponse(broken_body()) + + app.add_middleware(HttpMetricsMiddleware) + return app + + +@pytest.mark.parametrize( + ("path", "expected_status", "expected_route"), + [ + ("/items/42", 200, "/items/{item_id}"), + ("/items/not-an-int", 422, "/items/{item_id}"), + ("/missing", 404, "unknown"), + ("/explode", 500, "/explode"), + ], +) +def test_http_middleware_records_status_and_route_template( + path: str, + expected_status: int, + expected_route: str, +) -> None: + client = TestClient(_test_app(), raise_server_exceptions=False) + + with patch("opensandbox_server.middleware.http_metrics.record_http_request_duration") as record: + response = client.get(path) + + assert response.status_code == expected_status + record.assert_called_once() + assert record.call_args.kwargs["method"] == "GET" + assert record.call_args.kwargs["route"] == expected_route + assert record.call_args.kwargs["status_code"] == expected_status + assert record.call_args.kwargs["duration_ms"] >= 0 + + +@pytest.mark.parametrize("path", ["/docs", "/redoc", "/openapi.json"]) +def test_http_middleware_records_registered_starlette_routes(path: str) -> None: + client = TestClient(_test_app()) + + with patch("opensandbox_server.middleware.http_metrics.record_http_request_duration") as record: + response = client.get(path) + + assert response.status_code == 200 + record.assert_called_once() + assert record.call_args.kwargs["route"] == path + + +def test_http_middleware_covers_auth_rejection(client: TestClient) -> None: + with patch("opensandbox_server.middleware.http_metrics.record_http_request_duration") as record: + response = client.get("/v1/sandboxes") + + assert response.status_code == 401 + record.assert_called_once() + assert record.call_args.kwargs["route"] == "unknown" + assert record.call_args.kwargs["status_code"] == 401 + + +def test_http_middleware_does_not_fail_request_when_recorder_raises() -> None: + client = TestClient(_test_app()) + + with patch( + "opensandbox_server.middleware.http_metrics.record_http_request_duration", + side_effect=RuntimeError("boom"), + ): + response = client.get("/items/42") + + assert response.status_code == 200 + assert response.json() == {"item_id": 42} + + +def test_http_middleware_records_streaming_failures_as_500() -> None: + client = TestClient(_test_app(), raise_server_exceptions=False) + + with patch( + "opensandbox_server.middleware.http_metrics.record_http_request_duration" + ) as record: + client.get("/stream-error") + + record.assert_called_once() + assert record.call_args.kwargs["status_code"] == 500 + + +def test_record_http_request_duration_uses_low_cardinality_attributes() -> None: + histogram = MagicMock() + + with patch.object(otel_metrics, "_http_request_duration_histogram", histogram): + otel_metrics.record_http_request_duration( + duration_ms=12.5, + method="GET", + route="/sandboxes/{sandbox_id}", + status_code=200, + ) + + histogram.record.assert_called_once_with( + 12.5, + attributes={ + "http_method": "GET", + "http_route": "/sandboxes/{sandbox_id}", + "http_status_code": 200, + }, + ) + + +def test_record_http_request_duration_bounds_unknown_methods() -> None: + histogram = MagicMock() + + with patch.object(otel_metrics, "_http_request_duration_histogram", histogram): + otel_metrics.record_http_request_duration( + duration_ms=12.5, + method="BREW-sandbox-123", + route="/sandboxes/{sandbox_id}", + status_code=200, + ) + + assert histogram.record.call_args.kwargs["attributes"]["http_method"] == "OTHER" + + +def test_http_request_histogram_is_collectable() -> None: + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + histogram = otel_metrics._http_request_histogram_from_provider(provider) + + with patch.object(otel_metrics, "_http_request_duration_histogram", histogram): + otel_metrics.record_http_request_duration( + duration_ms=12.5, + method="GET", + route="/sandboxes/{sandbox_id}", + status_code=200, + ) + + metrics_data = reader.get_metrics_data() + assert metrics_data is not None + metric = metrics_data.resource_metrics[0].scope_metrics[0].metrics[0] + assert isinstance(metric.data, Histogram) + point = metric.data.data_points[0] + assert metric.name == "server.http.request.duration" + assert metric.unit == "ms" + assert point.count == 1 + assert point.attributes == { + "http_method": "GET", + "http_route": "/sandboxes/{sandbox_id}", + "http_status_code": 200, + } + provider.shutdown() + + +def test_record_http_request_duration_swallows_errors() -> None: + histogram = MagicMock() + histogram.record.side_effect = RuntimeError("boom") + + with patch.object(otel_metrics, "_http_request_duration_histogram", histogram): + otel_metrics.record_http_request_duration( + duration_ms=1.0, + method="GET", + route="/health", + status_code=200, + ) + + histogram.record.assert_called_once() diff --git a/server/tests/test_routes.py b/server/tests/test_routes.py index 6793c4637..489b31068 100644 --- a/server/tests/test_routes.py +++ b/server/tests/test_routes.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from email.utils import parsedate_to_datetime + from fastapi.testclient import TestClient +from opensandbox_server.main import app + class TestHealthCheck: @@ -21,6 +25,30 @@ def test_health_check(self, client: TestClient): response = client.get("/health") assert response.status_code == 200 assert response.json() == {"status": "healthy"} + assert parsedate_to_datetime(response.headers["date"]).tzinfo is not None + + def test_unhandled_error_has_date_header(self, auth_headers: dict): + async def raise_unhandled_error(): + raise RuntimeError("test unhandled error") + + app.add_api_route("/_test/unhandled-error", raise_unhandled_error) + route = app.router.routes.pop() + app.router.routes.insert(0, route) + error_client = TestClient(app, raise_server_exceptions=False) + + try: + response = error_client.get( + "/_test/unhandled-error", + headers=auth_headers, + ) + finally: + error_client.close() + app.router.routes.remove(route) + + assert response.status_code == 500 + dates = response.headers.get_list("date") + assert len(dates) == 1 + assert parsedate_to_datetime(dates[0]).tzinfo is not None class TestVersionInfo: diff --git a/server/tests/test_routes_endpoint_behavior.py b/server/tests/test_routes_endpoint_behavior.py index 6d7e6fb24..e552471d1 100644 --- a/server/tests/test_routes_endpoint_behavior.py +++ b/server/tests/test_routes_endpoint_behavior.py @@ -18,6 +18,10 @@ from opensandbox_server.api import lifecycle from opensandbox_server.api.schema import Endpoint +from opensandbox_server.services.constants import ( + OPEN_SANDBOX_INGRESS_HEADER, + OPEN_SANDBOX_SECURE_ACCESS_HEADER, +) def test_get_endpoint_returns_service_result( @@ -45,6 +49,30 @@ def get_endpoint(sandbox_id: str, port: int, **kwargs) -> Endpoint: assert calls == [("sbx-001", 44772)] +def test_get_endpoint_preserves_ingress_header( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_endpoint(sandbox_id: str, port: int, **kwargs) -> Endpoint: + return Endpoint( + endpoint="gateway.example.com", + headers={OPEN_SANDBOX_INGRESS_HEADER: "sbx-001-44772"}, + ) + + monkeypatch.setattr(lifecycle, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/endpoints/44772", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["headers"] == {OPEN_SANDBOX_INGRESS_HEADER: "sbx-001-44772"} + + def test_get_endpoint_use_server_proxy_rewrites_url( client: TestClient, auth_headers: dict, @@ -70,6 +98,34 @@ def get_endpoint(sandbox_id: str, port: int, **kwargs) -> Endpoint: assert response.json()["endpoint"] == "testserver/v1/sandboxes/sbx-001/proxy/44772" +def test_get_endpoint_use_server_proxy_omits_ingress_header( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_endpoint(sandbox_id: str, port: int, **kwargs) -> Endpoint: + return Endpoint( + endpoint="gateway.example.com", + headers={ + OPEN_SANDBOX_INGRESS_HEADER.lower(): "sbx-001-44772", + OPEN_SANDBOX_SECURE_ACCESS_HEADER: "secure-token", + }, + ) + + monkeypatch.setattr(lifecycle, "sandbox_service", StubService()) + + response = client.get( + "/v1/sandboxes/sbx-001/endpoints/44772", + params={"use_server_proxy": "true"}, + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["headers"] == {OPEN_SANDBOX_SECURE_ACCESS_HEADER: "secure-token"} + + def test_get_endpoint_use_server_proxy_without_mount_prefix( client: TestClient, auth_headers: dict, diff --git a/server/tests/test_routes_proxy.py b/server/tests/test_routes_proxy.py index 5ba019fe2..4a9bba275 100644 --- a/server/tests/test_routes_proxy.py +++ b/server/tests/test_routes_proxy.py @@ -14,10 +14,14 @@ import asyncio import gzip +from types import SimpleNamespace from typing import Any, cast import httpx +import pytest from fastapi.testclient import TestClient +from starlette.requests import ClientDisconnect +from starlette.types import Message from websockets.typing import Origin import opensandbox_server.api.proxy as proxy_api @@ -55,9 +59,21 @@ async def aiter_raw(self): yield chunk async def aclose(self): + await asyncio.sleep(0) self.aclose_called = True +class _BlockingStreamingResponse(_FakeStreamingResponse): + def __init__(self) -> None: + super().__init__() + self.body_started = asyncio.Event() + + async def aiter_raw(self): + self.body_started.set() + await asyncio.Future() + yield b"unreachable" + + class _FakeAsyncClient: def __init__(self): self.built = None @@ -208,6 +224,153 @@ def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> assert fake_client.response.aclose_called is True +def test_proxy_preserves_origin_date_and_filters_server_header( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> Endpoint: + assert sandbox_id == "sbx-123" + assert port == 44772 + assert resolve_internal is True + return Endpoint(endpoint="backend.example:40109") + + monkeypatch.setattr(lifecycle, "sandbox_service", StubService()) + + fake_client = _FakeAsyncClient() + origin_date = "Wed, 21 Oct 2015 07:28:00 GMT" + fake_client.response = _FakeStreamingResponse( + headers={ + "Date": origin_date, + "Server": "backend-server", + "X-Backend": "yes", + }, + chunks=[b"proxy-ok"], + ) + _set_http_client(client, fake_client) + + response = client.get( + "/v1/sandboxes/sbx-123/proxy/44772", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.headers.get("x-backend") == "yes" + assert response.headers.get("date") == origin_date + assert "server" not in response.headers + + +@pytest.mark.parametrize( + ("request_path", "location", "expected_location"), + [ + ( + "/v1/sandboxes/sbx-123/proxy/44772/", + "/login?next=%2F", + "/v1/sandboxes/sbx-123/proxy/44772/login?next=%2F", + ), + ( + "/sandboxes/sbx-123/proxy/44772/", + "/login?next=%2F", + "/sandboxes/sbx-123/proxy/44772/login?next=%2F", + ), + ( + "/v1/sandboxes/sbx-123/proxy/44772/nested/page", + "/login?next=%2F", + "/v1/sandboxes/sbx-123/proxy/44772/login?next=%2F", + ), + ("/v1/sandboxes/sbx-123/proxy/44772/", "login?next=%2F", "login?next=%2F"), + ( + "/v1/sandboxes/sbx-123/proxy/44772/", + "https://example.com/login", + "https://example.com/login", + ), + ( + "/v1/sandboxes/sbx-123/proxy/44772/", + "//example.com/login", + "//example.com/login", + ), + ("/v1/sandboxes/sbx-123/proxy/44772/", "?next=%2F", "?next=%2F"), + ], +) +def test_proxy_rewrites_only_root_relative_redirects( + client: TestClient, + auth_headers: dict, + monkeypatch, + request_path: str, + location: str, + expected_location: str, +) -> None: + class StubService: + @staticmethod + def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> Endpoint: + assert sandbox_id == "sbx-123" + assert port == 44772 + assert resolve_internal is True + return Endpoint(endpoint="backend.example:40109") + + monkeypatch.setattr(lifecycle, "sandbox_service", StubService()) + + fake_client = _FakeAsyncClient() + fake_client.response = _FakeStreamingResponse( + status_code=302, + headers={"Location": location}, + ) + _set_http_client(client, fake_client) + + response = client.get( + request_path, + headers=auth_headers, + follow_redirects=False, + ) + + assert response.status_code == 302 + assert response.headers["location"] == expected_location + + +def test_proxy_rewrites_root_relative_redirect_with_server_eip_path( + client: TestClient, + auth_headers: dict, + monkeypatch, +) -> None: + class StubService: + @staticmethod + def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> Endpoint: + assert sandbox_id == "sbx-123" + assert port == 44772 + assert resolve_internal is True + return Endpoint(endpoint="backend.example:40109") + + monkeypatch.setattr(lifecycle, "sandbox_service", StubService()) + monkeypatch.setattr( + lifecycle, + "get_config", + lambda: SimpleNamespace( + server=SimpleNamespace(eip="sandbox.example.com/opensandbox/") + ), + ) + + fake_client = _FakeAsyncClient() + fake_client.response = _FakeStreamingResponse( + status_code=302, + headers={"Location": "/login?next=%2F"}, + ) + _set_http_client(client, fake_client) + + response = client.get( + "/v1/sandboxes/sbx-123/proxy/44772/", + headers=auth_headers, + follow_redirects=False, + ) + + assert response.status_code == 302 + assert ( + response.headers["location"] + == "/opensandbox/sandboxes/sbx-123/proxy/44772/login?next=%2F" + ) + + def test_proxy_root_path_forwards_endpoint_headers_and_query( client: TestClient, auth_headers: dict, @@ -357,6 +520,7 @@ def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> ) assert response.status_code == 200 + assert fake_client.built is not None lowered_headers = { key.lower(): value for key, value in fake_client.built["headers"].items() } @@ -528,6 +692,79 @@ def get_endpoint(sandbox_id: str, port: int, resolve_internal: bool = False) -> assert fake_client.response.aclose_called is True +def test_proxy_closes_backend_response_when_downstream_rejects_headers() -> None: + """A disconnect before body iteration must not retain the backend connection.""" + + async def run() -> None: + backend_response = _FakeStreamingResponse() + response = proxy_api._ProxyStreamingResponse( + cast(httpx.Response, backend_response), + status_code=200, + headers={}, + ) + + async def receive() -> Message: + return {"type": "http.disconnect"} + + async def reject_response_start(message: Message) -> None: + assert message["type"] == "http.response.start" + raise ConnectionError("downstream disconnected") + + try: + await response( + {"type": "http", "asgi": {"spec_version": "2.4"}}, + receive, + reject_response_start, + ) + except ClientDisconnect: + pass + else: + raise AssertionError("expected the downstream send to fail") + + assert backend_response.aclose_called is True + + asyncio.run(run()) + + +def test_proxy_closes_backend_response_when_stream_is_cancelled() -> None: + """Task cancellation must not interrupt returning the backend connection.""" + + async def run() -> None: + backend_response = _BlockingStreamingResponse() + response = proxy_api._ProxyStreamingResponse( + cast(httpx.Response, backend_response), + status_code=200, + headers={}, + ) + + async def send(message: Message) -> None: + return None + + async def receive() -> Message: + return {"type": "http.disconnect"} + + task = asyncio.create_task( + response( + {"type": "http", "asgi": {"spec_version": "2.4"}}, + receive, + send, + ) + ) + await backend_response.body_started.wait() + task.cancel() + + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("expected stream task cancellation") + + assert backend_response.aclose_called is True + + asyncio.run(run()) + + def test_proxy_rejects_websocket_upgrade( client: TestClient, auth_headers: dict, diff --git a/server/tests/test_tenants.py b/server/tests/test_tenants.py index 58b417732..78f8a8741 100644 --- a/server/tests/test_tenants.py +++ b/server/tests/test_tenants.py @@ -22,7 +22,11 @@ import pytest -from opensandbox_server.tenants import validate_tenant_config +from opensandbox_server.tenants import ( + validate_tenant_config, + validate_tenant_namespaces, + validate_tenant_namespaces_on_startup, +) from opensandbox_server.tenants.context import get_current_tenant, set_current_tenant from opensandbox_server.tenants.file_provider import ( FileTenantProvider, @@ -494,3 +498,125 @@ def test_validate_tenant_config_ok(): cfg.runtime.type = "kubernetes" cfg.server.api_key = None validate_tenant_config(cfg) + + +# --- validate_tenant_namespaces --- + + +def _tenant(name: str, namespace: str) -> TenantEntry: + return TenantEntry(name=name, namespace=namespace, api_keys=("k",)) + + +def test_validate_tenant_namespaces_ok(): + core_v1 = MagicMock() + tenants = [_tenant("alpha", "ns-alpha"), _tenant("beta", "ns-beta")] + validate_tenant_namespaces(tenants, core_v1) + assert core_v1.read_namespace.call_count == 2 + + +def test_validate_tenant_namespaces_dedupes_shared_namespace(): + core_v1 = MagicMock() + tenants = [_tenant("alpha", "shared"), _tenant("beta", "shared")] + validate_tenant_namespaces(tenants, core_v1) + assert core_v1.read_namespace.call_count == 1 + + +def test_validate_tenant_namespaces_missing_raises(): + from kubernetes.client import ApiException + + core_v1 = MagicMock() + core_v1.read_namespace.side_effect = ApiException(status=404) + with pytest.raises(ValueError, match="does not exist"): + validate_tenant_namespaces([_tenant("alpha", "ns-alpha")], core_v1) + + +def test_validate_tenant_namespaces_forbidden_raises(): + from kubernetes.client import ApiException + + core_v1 = MagicMock() + core_v1.read_namespace.side_effect = ApiException(status=403) + with pytest.raises(ValueError, match="not accessible"): + validate_tenant_namespaces([_tenant("alpha", "ns-alpha")], core_v1) + + +def test_validate_tenant_namespaces_aggregates_failures(): + from kubernetes.client import ApiException + + core_v1 = MagicMock() + + def _read(name: str): + if name == "ok": + return MagicMock() + raise ApiException(status=404) + + core_v1.read_namespace.side_effect = lambda name: _read(name) + tenants = [ + _tenant("alpha", "ok"), + _tenant("beta", "missing-1"), + _tenant("gamma", "missing-2"), + ] + with pytest.raises(ValueError) as exc_info: + validate_tenant_namespaces(tenants, core_v1) + message = str(exc_info.value) + assert "missing-1" in message + assert "missing-2" in message + + +# --- validate_tenant_namespaces_on_startup --- + + +def test_startup_validation_file_provider_validates(tmp_path): + f = tmp_path / "tenants.toml" + f.write_text(SAMPLE_TOML) + provider = FileTenantProvider(f) + provider.start() + try: + assert provider.supports_enumeration + core_v1 = MagicMock() + validate_tenant_namespaces_on_startup(provider, core_v1) + assert core_v1.read_namespace.call_count == 2 + finally: + provider.close() + + +def test_startup_validation_file_provider_missing_raises(tmp_path): + from kubernetes.client import ApiException + + f = tmp_path / "tenants.toml" + f.write_text(SAMPLE_TOML) + provider = FileTenantProvider(f) + provider.start() + try: + core_v1 = MagicMock() + core_v1.read_namespace.side_effect = ApiException(status=404) + with pytest.raises(ValueError, match="does not exist"): + validate_tenant_namespaces_on_startup(provider, core_v1) + finally: + provider.close() + + +def test_startup_validation_http_provider_skipped_with_warning(monkeypatch): + cfg = HTTPTenantProviderConfig(endpoint="http://localhost:9999/tenants") + provider = HTTPTenantProvider(cfg) + provider.start() + try: + assert not provider.supports_enumeration + core_v1 = MagicMock() + + warnings = [] + + def _capture_warning(message: str, *args) -> None: + warnings.append(message % args) + + monkeypatch.setattr( + "opensandbox_server.tenants.logger.warning", + _capture_warning, + ) + validate_tenant_namespaces_on_startup(provider, core_v1) + core_v1.read_namespace.assert_not_called() + assert any( + "Skipping tenant namespace startup validation" in message + for message in warnings + ) + finally: + provider.close() diff --git a/server/uv.lock b/server/uv.lock index a6e2f84af..a4216687d 100644 --- a/server/uv.lock +++ b/server/uv.lock @@ -338,6 +338,130 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/fd/655c8a773d728bc3c93fb4713ae4bf79ffc75996f86fb78b2974c8e1dfbd/grpcio-1.83.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fba099b716e73512d61b97f71ea3c31a72abb36904036e316bf4dd148ca8dcc8", size = 6334247, upload-time = "2026-07-23T15:18:53.099Z" }, + { url = "https://files.pythonhosted.org/packages/d4/9a/1ce5760d35a04a992006dd2f79afff2db548f93ee7426fa95c9f1fc90c61/grpcio-1.83.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6755ed67cc3e454d51ae9f6e1915b80d3942fa4de956ef48dacd45ab7f40b727", size = 12168650, upload-time = "2026-07-23T15:18:56.348Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ab/bbcb5be0a1a6cb21f036e2afdd4f7a70147cfb7a7b42648a310d7c43acfc/grpcio-1.83.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5882c1a721b50ce0123ee5e839e1ab059ad72a7ade76cdf2d5bd833b56791acf", size = 6916899, upload-time = "2026-07-23T15:18:58.339Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/4c27977ecb3b3f9f363b93f570e001cb24ef264a9a907d7fd0f949ed59f0/grpcio-1.83.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e3eedfc92b6b9f2960115e7e620cf0cbf80bb7849a51ce3820dc54dfd88b6b9", size = 7648761, upload-time = "2026-07-23T15:19:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/23/49/0c823a7627ff2e69a61e4a53c4edf215272892fc2c47c6431f033d46f4cc/grpcio-1.83.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4fcaa7c45c45b4a89e2867d1f1785d9481a788399d915e341ed2eb49aeef9dd4", size = 7074920, upload-time = "2026-07-23T15:19:02.293Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ce/963f01ff7c789a76909c9691b704112e02ca1e11c10405cd99c2bd7c40f1/grpcio-1.83.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6b6c666a1d5613ff360c9e90f44665e3a88b25a815209ddbc0917eec281931cb", size = 7598046, upload-time = "2026-07-23T15:19:03.921Z" }, + { url = "https://files.pythonhosted.org/packages/eb/de/1ce6bdefc847a7973040d10cebc8996c653a2a687c0a4da8d05dcab4e397/grpcio-1.83.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6be5c807b717be3dd649446f021301fd7907e376318675d2147823071034112a", size = 8634792, upload-time = "2026-07-23T15:19:05.633Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8b/7fe6a73895e3bdd788101d1276e48e0d262ebb165afacec1ec4efebcd785/grpcio-1.83.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c834e86d8fd2f03d7e4db49a027f7c5b89c5b88eed305543a5295bd6fee61e40", size = 8000286, upload-time = "2026-07-23T15:19:07.739Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b0/9a779de2bcda8722501a056fad1bec3d1117977af0c080ab1fc0655fdf35/grpcio-1.83.0-cp310-cp310-win32.whl", hash = "sha256:35a5b1c192496b6c25956eebfa963468935612206fd2543ac3ce981e6a5e0f03", size = 4404616, upload-time = "2026-07-23T15:19:09.988Z" }, + { url = "https://files.pythonhosted.org/packages/f0/8e/ce9a23590cac33a6c24e6386cc0ffc55821cc13212acc822e98f00a67161/grpcio-1.83.0-cp310-cp310-win_amd64.whl", hash = "sha256:8f6c395e493d20c39b29392ca200e9aaeb78d0bc2f04db0c0a7da7ddc939aa57", size = 5162304, upload-time = "2026-07-23T15:19:11.467Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/3b781cd07a715ea5f5125ae264226e7fc4d87603d6d3955022cabfdc5da2/grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f", size = 6338720, upload-time = "2026-07-23T15:19:13.177Z" }, + { url = "https://files.pythonhosted.org/packages/21/cc/d14833d15d5984e366f1b027fa78bd038c9b028c66880bffb0f5a4d25ee2/grpcio-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:4772402f43517b4824980be4b3b2274a81eec0004a70009473c31b340d43e223", size = 12178773, upload-time = "2026-07-23T15:19:15.401Z" }, + { url = "https://files.pythonhosted.org/packages/6b/98/8acbb416544e7871132d8e42a07ed70c802d70e6a16c6009e505a34d32a4/grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0", size = 6921203, upload-time = "2026-07-23T15:19:17.824Z" }, + { url = "https://files.pythonhosted.org/packages/45/9c/0fdbfaf4fc54e5c88f6bce4008a065092fe7fbc4460eb5617ae8b20fd505/grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d", size = 7648508, upload-time = "2026-07-23T15:19:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ea/107b9dbb2ed3ad14dd774fd3dde7d29ff9938a6c198654becb2c3a0e9a6a/grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9", size = 7079466, upload-time = "2026-07-23T15:19:21.478Z" }, + { url = "https://files.pythonhosted.org/packages/3b/06/9fa9941089e6fae83b060b6ce61c1e81053e52decae43197245f45e07d36/grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745", size = 7605583, upload-time = "2026-07-23T15:19:23.74Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/f10fb56062dc2771c630827a82d9ad0ecd05cad572ea3b08d49f6631680a/grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617", size = 8637810, upload-time = "2026-07-23T15:19:25.536Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/f84927258f6a1b6ea6dea661fdc6de859b35e560c96f3012d15ccd39f85e/grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969", size = 8008021, upload-time = "2026-07-23T15:19:27.863Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6b/cdf72161397ccd29d4ca2192f641524536c9cf54ad948c9dd0e0e01138fa/grpcio-1.83.0-cp311-cp311-win32.whl", hash = "sha256:cb056f6e171c42639a50460b2929c82241fda51f71cf3dcdd68090fe45095a45", size = 4404376, upload-time = "2026-07-23T15:19:30.137Z" }, + { url = "https://files.pythonhosted.org/packages/df/ed/e0ffeb4c848699c194dc9fb6a29ab29bcb2b6aac8c416bf18c51bfe8242c/grpcio-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:7416952ca770477990257206276999056f8316d79196f2f25942393e58a20b49", size = 5164469, upload-time = "2026-07-23T15:19:31.941Z" }, + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + +[[package]] +name = "grpcio-tools" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/b1/50b17b3a2ba8970dbb00b2035a4df218bc6ec88d2d77fc7da4e42e1c7b19/grpcio_tools-1.83.0.tar.gz", hash = "sha256:515907265d14fa9975d0c7723f95a9da01463d7ac607546a03f8741f86a1bb07", size = 6400437, upload-time = "2026-07-23T15:22:18.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/00/4a0b426262c8488a02d7fcabe72a070c4691ea79ebf050d543d5cf054234/grpcio_tools-1.83.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:f281cb706999676bb841bcd57129a69091b9286236c89d6114c752ebf6cd5a1b", size = 2652630, upload-time = "2026-07-23T15:20:49.826Z" }, + { url = "https://files.pythonhosted.org/packages/10/22/c5ebf22b6975b9846d54b0e5328eea1780ba5906419e43d305415e110611/grpcio_tools-1.83.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:3647c6adae9528dd56183061371151e3a96f71c299dc69c540318c3af2233a88", size = 5967265, upload-time = "2026-07-23T15:20:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/3d/06/c0a9bad5cc5b1eef47647c480fc9aac82ae67e723db8f5e90e56bdb5adf9/grpcio_tools-1.83.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6346c688d25bcf264e55f0c5f48ae825f7a2906ab969ed3b4a93df53e48bf07", size = 2704488, upload-time = "2026-07-23T15:20:53.779Z" }, + { url = "https://files.pythonhosted.org/packages/3f/8e/e54c7475e0fb528861f66a2e2a0b892a86b5a636905336337d29e091c2c1/grpcio_tools-1.83.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:281d5d056d7ba839f4fff9b63f9ad239fe3353fe10e28e782251bdae6ba68306", size = 3032303, upload-time = "2026-07-23T15:20:55.589Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4f/2221bc79d7c2315d5e3604f353c470b380386fe5d7f8d4740a7d00630394/grpcio_tools-1.83.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35af1be2fe409abf9817ec43c34aab8a99189b15530eb78f3a94a6b1266d8b12", size = 2773919, upload-time = "2026-07-23T15:20:57.187Z" }, + { url = "https://files.pythonhosted.org/packages/93/94/50b1e7b1526e11e8e7cc6796cf71707ce6bc8714d9bacdd7290a2039b140/grpcio_tools-1.83.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7f5f5b6e1a91422069601fbf94f7fc970a7647fa69d2bc9f59e38913523117af", size = 3226535, upload-time = "2026-07-23T15:20:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/fa/df/3de83d76f30a76ba85464d72a579199a1191ad027f5c89786db767b22f83/grpcio_tools-1.83.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:cad9333c0d5afcc2ffb26bebc5e8f097e3218b964758c3c65609dbcb77ec2aa7", size = 3798914, upload-time = "2026-07-23T15:21:00.05Z" }, + { url = "https://files.pythonhosted.org/packages/f4/51/957d040037a142b7232b5f1bfc0abfdac0acbdaec74f2ba3f39cc570ccb2/grpcio_tools-1.83.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2106b29b9dae5068acab7ee2f6b104d3054b4f31289f105b4afe1fcfbf1966c4", size = 3457747, upload-time = "2026-07-23T15:21:01.511Z" }, + { url = "https://files.pythonhosted.org/packages/49/27/7199a053df071d5c9924aee050d6aea391892f05cb6eb266365a72c8b05c/grpcio_tools-1.83.0-cp310-cp310-win32.whl", hash = "sha256:a47e674e6afac5d73ee3a87d57ed53f7b79cc01f44d602fe6ee90919aa171583", size = 1022559, upload-time = "2026-07-23T15:21:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/32/d4/6201e4948618a60b5fdfecd3b193da9b86c76e1aa8555573da78649900a5/grpcio_tools-1.83.0-cp310-cp310-win_amd64.whl", hash = "sha256:88fc53ee3ce28d3ea2fe8fe1d3ed57854d0d25d6dac18e74c1f24e0a377bb509", size = 1192122, upload-time = "2026-07-23T15:21:04.398Z" }, + { url = "https://files.pythonhosted.org/packages/b9/f0/d55c59b3af39d2ca975a2bafc5be1a60e0460a1b507bdbcf7bb8a17567db/grpcio_tools-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:72471a4a46909f1d798836c0a0aa2f568e10f6404585d7b22ac7330dd6a7bc74", size = 2652835, upload-time = "2026-07-23T15:21:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/82/97/59bdf27c5f99848087b3b268c90b3fcc557d400c8d0224cf49c7ac573e60/grpcio_tools-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:7da547dd1e0b1fe3d6d5677e9c1848969ecdbd51a92c342cf82d885c5935de7d", size = 5967887, upload-time = "2026-07-23T15:21:08.19Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c4/117d6688c4cb40240ce48da1ffac005e8da8bf63785634bdbb9143ca367e/grpcio_tools-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33350bd94c4913f8eaf6ca79ce69cc673bb46ca5f800e6474b1d6899ed321dad", size = 2704869, upload-time = "2026-07-23T15:21:09.857Z" }, + { url = "https://files.pythonhosted.org/packages/1a/51/e4d1a89f69072bb93b975548c40fe9524219ab5777cce113c1e2183b1f2d/grpcio_tools-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0f27da6dc58c910f31dcee4fcbfd05659c10c14b9e132f4f17da48d867e75eb4", size = 3032318, upload-time = "2026-07-23T15:21:11.55Z" }, + { url = "https://files.pythonhosted.org/packages/ae/26/f9a082b79ae7f7dac7050047614000721dcf3d1c6d14bffa881b1f7a9774/grpcio_tools-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b3a5000e8540efb80f74d9c760f36ed9408294d76edb0fa4b87fd287b0a8a258", size = 2774113, upload-time = "2026-07-23T15:21:13.037Z" }, + { url = "https://files.pythonhosted.org/packages/de/59/b715d431218f0e8382490db7b1d01b3d32392c359c1886eee7e2551edaa8/grpcio_tools-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f710032264ed114c9d3b52f4c5cf71d68ccf70f25c4cb5776fe60388374bc01b", size = 3226715, upload-time = "2026-07-23T15:21:14.88Z" }, + { url = "https://files.pythonhosted.org/packages/5f/99/c4e91a2116062f7b42ab99459eeda15ec59a07daad01f4f38753fb1584ac/grpcio_tools-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ad00de87334154901e9af50a7f3f95261a0133502c6c0cea1f4e6107245154c3", size = 3799000, upload-time = "2026-07-23T15:21:16.352Z" }, + { url = "https://files.pythonhosted.org/packages/b8/f7/0bb5e45a1f51822c0bcf5a355a06132c43b9dbe5c283edc8d0b63bec7626/grpcio_tools-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de2b0c645363f7c4b005f145e06e790870dfb21bb7e162dec6fee2f054de9291", size = 3457774, upload-time = "2026-07-23T15:21:17.953Z" }, + { url = "https://files.pythonhosted.org/packages/e7/51/52aec42a3059bad4263a7a82c715a0fc08220f7533be41f0b9177b05468a/grpcio_tools-1.83.0-cp311-cp311-win32.whl", hash = "sha256:6d1a1c9e62689d04b63b227558b759a55dce8fb81a3934d3b5f95d1ee26a2b45", size = 1022798, upload-time = "2026-07-23T15:21:19.854Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/034d6daa5b2fae4e4c1718111c1b791589c3743d7d8fb07984e0b59c7f2c/grpcio_tools-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:a6ac3cc2c2d77f869a96dfaf2b1315852878babddfe2dc49b9fd47afbf502865", size = 1192474, upload-time = "2026-07-23T15:21:21.467Z" }, + { url = "https://files.pythonhosted.org/packages/27/4d/02e4d809d880785a613697e4c0f8134a436ec0142dc3c11989ad1a1c787a/grpcio_tools-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:bd93bbe18c4424805fd2e39854f75d76f80655175621254dc43cb45ec8e91e85", size = 2653283, upload-time = "2026-07-23T15:21:22.973Z" }, + { url = "https://files.pythonhosted.org/packages/01/c9/23e8423ac54c3858a5cfbca8a954fa292cab7b8a9a1ee9dc3259b906b763/grpcio_tools-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:dc2d370563ee1ee6c1769e49df35ef3f6e75cea8f25acf4ac6e54b335e6f788f", size = 5965938, upload-time = "2026-07-23T15:21:24.686Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a1/6eb17cf322bfbb76a9b9a8a5ca4a6b27f0af84821145969fc464feedaa0e/grpcio_tools-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8350e236470700b02bc4ba7f27a8796559630170e6527dda41c0344fbe988e56", size = 2705429, upload-time = "2026-07-23T15:21:26.519Z" }, + { url = "https://files.pythonhosted.org/packages/d4/84/f3c7e5e91e5d40ee792f112260bd329db5381de6f83b21387dd6163ebe51/grpcio_tools-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b1649f47c4675c1540ad2a77005f4d08392c06202686a0b1bb6b894f96cb75fe", size = 3033412, upload-time = "2026-07-23T15:21:28.183Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ec/acd6c1800925b0d60f8a670b68cf5bc3566aee60e7cb90178b34253bf53a/grpcio_tools-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4edc6ba9fdca70bbf585ff6ab5971b8cd6140b4b316df66fd91d168bc1b617", size = 2774500, upload-time = "2026-07-23T15:21:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/36/e6/c7c6f4e25b7344f1fbc5df69606bfd4b9692d4e32d17781c665d3ed45e70/grpcio_tools-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:762a9f8a4a4a39bda02feebed94efb8d778e0e5a82d0c8f786dce5ddcb950c7f", size = 3229875, upload-time = "2026-07-23T15:21:31.617Z" }, + { url = "https://files.pythonhosted.org/packages/77/fc/9cbdc4606f378a9c2b569c0b6b57f181f97787006be4131a5820d469b70a/grpcio_tools-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c6c469c928a183f1a99ab26e263fe307347ee7023fa623b55bc778846b2f51b9", size = 3803163, upload-time = "2026-07-23T15:21:33.35Z" }, + { url = "https://files.pythonhosted.org/packages/7b/0b/3b754886a02ead1487a967fd11ff13e920218fcf6c8e174f6de0c26dd819/grpcio_tools-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7980b3ca9dd31c42468c5af8cec97037f83715ea6efbe1b936ecb9c6832ac0f5", size = 3461815, upload-time = "2026-07-23T15:21:34.982Z" }, + { url = "https://files.pythonhosted.org/packages/eb/29/4629c154853f6f677299cde014e615084c15ad1d4fedd6845d2e7d354c7e/grpcio_tools-1.83.0-cp312-cp312-win32.whl", hash = "sha256:fd2ff46917f566b3b63dae191d1b05ef2188fe51e756ef321cbdd707ab29dbfb", size = 1022490, upload-time = "2026-07-23T15:21:36.696Z" }, + { url = "https://files.pythonhosted.org/packages/bb/59/696b9671e32a693c67299724a1f70f81dfb78aca6a3283ea6a65d54e92b8/grpcio_tools-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:92d2343806b5c21162a57fbaad24fcd8d935ef530f95a0f706a4e2546fdc0662", size = 1192286, upload-time = "2026-07-23T15:21:38.446Z" }, + { url = "https://files.pythonhosted.org/packages/63/2f/a7a4465b2a5b74b479373bf44d86da5840d7d20871764a39fb300e55e093/grpcio_tools-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:3277cfbb7cbbd2d72921fbcd7aba6c8ab1c91a9ab27e8045ac0a0f2e0517cec9", size = 2652845, upload-time = "2026-07-23T15:21:40.173Z" }, + { url = "https://files.pythonhosted.org/packages/ab/69/ed8ebaae3bd0ecd2387b0fdb3a696bd4b4d4565d18589ee3ba7c6affcbed/grpcio_tools-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1aa9617ce9c2bcbfb8f2fa08e6259e1b3cadaab0316e41f71496847af2f0a664", size = 5963575, upload-time = "2026-07-23T15:21:41.943Z" }, + { url = "https://files.pythonhosted.org/packages/13/23/75bddae583077f1374c64e2a87b5924bdbffe162b3855af30c30a0fc8c5c/grpcio_tools-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ad37c786ea92825534466052f5f22f1f29983b1d00ca71ad43e256715a86bba3", size = 2705094, upload-time = "2026-07-23T15:21:43.675Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ac/7e1b12b1c5afff4f7d578f3c4eafbb1849f8004ddcc236cd3ff95c8be607/grpcio_tools-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:c82216864d435ecf6f535798d03e9f6b9025a672a2e815715d629aba4ba70349", size = 3033061, upload-time = "2026-07-23T15:21:45.657Z" }, + { url = "https://files.pythonhosted.org/packages/b8/6d/113291a7aad0c47a1e2ba2375595dfc3c2ab648e0ea05ff3d6f0f89055b3/grpcio_tools-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:04284627655629387b63278591e5efd0aded28d3a08432fe8a8765e4daf2d5b2", size = 2773649, upload-time = "2026-07-23T15:21:47.404Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c2/ce1c6c2475ed5cb0f7c6689bde57e3c105714676227da11b423ff37620a9/grpcio_tools-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fe627a5248d8e712f5ec3e019420050e61b10600ed241264aefb379a0f1b338", size = 3229788, upload-time = "2026-07-23T15:21:49.441Z" }, + { url = "https://files.pythonhosted.org/packages/c5/53/823fd52c29630398706de400ce7003b03e17ce91df024fc53cde810d2758/grpcio_tools-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1d5a9a9664d2f4bdda000652e7febf9ab4545a6391c8f05babd52ecb27d7e03e", size = 3802531, upload-time = "2026-07-23T15:21:51.37Z" }, + { url = "https://files.pythonhosted.org/packages/30/85/942ee07caf97b75ead416c6ad5f2fb12b16b8bc92fa3d60bbbea4c06e076/grpcio_tools-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4bf0e421e1ab5f2cd638de44fc903aed3ba4a2fcb19b93c6f528ff0ec63e3a6a", size = 3461032, upload-time = "2026-07-23T15:21:53.343Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0b/54cb428ea18912fa9541509fbc2e10f9807d96e88cf435f86e63c540ac25/grpcio_tools-1.83.0-cp313-cp313-win32.whl", hash = "sha256:7b1bd6db403b38addded54866187eba6f9ab9afadf72bb8d0515ed13f0b16c5c", size = 1022116, upload-time = "2026-07-23T15:21:55.036Z" }, + { url = "https://files.pythonhosted.org/packages/77/80/2369320766091f6daedb924d133037a9f8b84bfb3e4d02d6ccffcd57b0cd/grpcio_tools-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:d654c645af7cf608a30644bffb8d1ef6b14e8846482c3d0131d0dda91f6fb590", size = 1191934, upload-time = "2026-07-23T15:21:56.862Z" }, + { url = "https://files.pythonhosted.org/packages/b2/bd/3abb9c200f90110805553ca0e8f7908a0b89485ea99bb271daa37eaafb72/grpcio_tools-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:1ea047ff4bd2bb32fe5268042cb9c9e7bb054e932b52ad756642622f032ef656", size = 2652841, upload-time = "2026-07-23T15:21:58.726Z" }, + { url = "https://files.pythonhosted.org/packages/a2/46/d5beb04f0ffe552e55eaddd3413786e17a1fa68edb2fd8398969c38bc7e8/grpcio_tools-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7a8ac9cb3fbf7a5e4fe59f211e77b4fa4d51279c9f480e6ff98037cf56da1ad8", size = 5963493, upload-time = "2026-07-23T15:22:00.547Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/d8e01ca3281cb59722372c415024a7e70e8a653e70e2075e875394e4f761/grpcio_tools-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a5fa95fb33a600d2491867a1048f47baa27a830eac01f475043b8ccf63a471eb", size = 2705303, upload-time = "2026-07-23T15:22:02.448Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/75b382d8f116e274ce639ec55a6908dc792627b01d3c47b4f8991701203b/grpcio_tools-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0ca67941524662e01adea91571bb79df1ac9b4b2641812ef8636e21945119bee", size = 3033047, upload-time = "2026-07-23T15:22:04.445Z" }, + { url = "https://files.pythonhosted.org/packages/0f/9f/200fb55729b735192fded061f53aa37c88cf9f58933cb108e16f5c1fd967/grpcio_tools-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e2a5816a5b6b06b42a6989f02944841c2a8b3daaa7f033dac9267f70078028ef", size = 2773830, upload-time = "2026-07-23T15:22:06.519Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bc/e585c4e255599ec45f2e07e0f570352158354285afd65ef30ceda97b445e/grpcio_tools-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f4a83f002895b11c4a862c366d77165825e0064aff14bf2c7453f59f66599b0e", size = 3229907, upload-time = "2026-07-23T15:22:08.358Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/0c098b4ff64d948666b79d0036ce104b8c2209e6f4d1594046044dcfa25d/grpcio_tools-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1cfee967ae073bc064862971871229248965422f38556096aec76db19d8a8c79", size = 3802600, upload-time = "2026-07-23T15:22:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e1/249047aab0da8c6b3b2e0156e6868aa0b598973ddf53f59186c43664ff96/grpcio_tools-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f7e82ee718ae09f879cb832e4517a56691de24383a2da673be184ad8b18e452f", size = 3461307, upload-time = "2026-07-23T15:22:12.33Z" }, + { url = "https://files.pythonhosted.org/packages/39/aa/11a228c47acecde2e05715bda4480e5a695690e00776c1db391e5a02c1f7/grpcio_tools-1.83.0-cp314-cp314-win32.whl", hash = "sha256:846fd211ebb72f50d39d3874cc0d616c2b9bcb71db51121ca86af29eec013c74", size = 1045047, upload-time = "2026-07-23T15:22:14.23Z" }, + { url = "https://files.pythonhosted.org/packages/30/b1/d1f150b2ab3b4ae9932c05104fe1edbcb7fbf505587ea8db99e49341a05f/grpcio_tools-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8c9686b0c19f70b63d8d6cfeff5ad3480bdedecd60f14711fe43950f5397253", size = 1224199, upload-time = "2026-07-23T15:22:16.064Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -485,11 +609,13 @@ source = { editable = "." } dependencies = [ { name = "docker" }, { name = "fastapi" }, + { name = "grpcio" }, { name = "httpx", extra = ["socks"] }, { name = "kubernetes" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, + { name = "protobuf" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-multipart" }, @@ -503,6 +629,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "grpcio-tools" }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -514,11 +641,13 @@ dev = [ requires-dist = [ { name = "docker" }, { name = "fastapi", specifier = ">=0.137.0" }, + { name = "grpcio", specifier = ">=1.83.0" }, { name = "httpx", extras = ["socks"] }, { name = "kubernetes" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, + { name = "protobuf", specifier = ">=7.35.1" }, { name = "pydantic" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, { name = "python-multipart", specifier = ">=0.0.31" }, @@ -532,6 +661,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "grpcio-tools", specifier = ">=1.83.0" }, { name = "pyright", specifier = ">=1.1.0" }, { name = "pytest", specifier = ">=7.0.0" }, { name = "pytest-asyncio", specifier = ">=0.21.0" }, @@ -1050,6 +1180,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, ] +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + [[package]] name = "six" version = "1.17.0" diff --git a/specs/execd-api.yaml b/specs/execd-api.yaml index b62ea653f..ccfbce2d4 100644 --- a/specs/execd-api.yaml +++ b/specs/execd-api.yaml @@ -2526,6 +2526,53 @@ components: type: boolean diff_supported: type: boolean + hardening: + type: object + description: >- + execd init-mode and workload-hardening state (OSEP-0018): whether + execd is the sandbox init and which of its controls are in effect. + Not an isolation capability; reported here so operators see + enforcement state in one place. + properties: + init_mode: + type: string + enum: [pid1, subreaper, none] + description: >- + How execd supervises the sandbox process tree. pid1: execd is + the kernel init of the container. subreaper: execd reaps + orphans but lacks the PID 1 kernel signal shield. none: init + mode is off (default). + signal_shield: + type: boolean + description: >- + Whether the kernel PID 1 signal shield protects execd from + in-namespace signals (true only in init_mode pid1). + cap_drop: + $ref: "#/components/schemas/HardeningLayerState" + description: Capability/bounding-set reduction on user code. + seccomp: + $ref: "#/components/schemas/HardeningLayerState" + description: Seccomp floor installed on user code. + landlock: + $ref: "#/components/schemas/HardeningLayerState" + description: Landlock filesystem confinement on user code. + ebpf: + $ref: "#/components/schemas/HardeningLayerState" + description: eBPF exec/connect/privilege observation. + + HardeningLayerState: + type: object + description: >- + Whether one hardening layer is actually enforced. state is "active" | + "disabled" (not configured) | "degraded" (configured but a + prerequisite is missing) | "unsupported" (kernel/build cannot provide + it). message gives the concrete reason whenever state is not active. + properties: + state: + type: string + enum: [active, disabled, degraded, unsupported] + message: + type: string responses: ServiceUnavailable: diff --git a/tests/javascript/package.json b/tests/javascript/package.json index 8b4f6f021..dea4e5403 100644 --- a/tests/javascript/package.json +++ b/tests/javascript/package.json @@ -13,9 +13,9 @@ "brace-expansion@^5.0.0": "5.0.7", "flatted@^3.0.0": "3.4.2", "esbuild": "0.25.2", - "postcss": "8.5.12", + "postcss": "8.5.23", "vite": "6.4.3", - "js-yaml": "5.2.1" + "js-yaml": "5.2.2" } }, "scripts": { diff --git a/tests/javascript/pnpm-lock.yaml b/tests/javascript/pnpm-lock.yaml index 3554c578c..1a2c06f56 100644 --- a/tests/javascript/pnpm-lock.yaml +++ b/tests/javascript/pnpm-lock.yaml @@ -12,9 +12,9 @@ overrides: brace-expansion@^5.0.0: 5.0.7 flatted@^3.0.0: 3.4.2 esbuild: 0.25.2 - postcss: 8.5.12 + postcss: 8.5.23 vite: 6.4.3 - js-yaml: 5.2.1 + js-yaml: 5.2.2 importers: @@ -292,66 +292,79 @@ packages: resolution: {integrity: sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.60.2': resolution: {integrity: sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.60.2': resolution: {integrity: sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.60.2': resolution: {integrity: sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.60.2': resolution: {integrity: sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.60.2': resolution: {integrity: sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.60.2': resolution: {integrity: sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.60.2': resolution: {integrity: sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.60.2': resolution: {integrity: sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.60.2': resolution: {integrity: sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.60.2': resolution: {integrity: sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.60.2': resolution: {integrity: sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.60.2': resolution: {integrity: sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.60.2': resolution: {integrity: sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==} @@ -713,8 +726,8 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - js-yaml@5.2.1: - resolution: {integrity: sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==} + js-yaml@5.2.2: + resolution: {integrity: sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==} hasBin: true json-buffer@3.0.1: @@ -753,8 +766,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -799,8 +812,8 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} - postcss@8.5.12: - resolution: {integrity: sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==} + postcss@8.5.23: + resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} engines: {node: ^10 || ^12 || >=14} prelude-ls@1.2.1: @@ -1098,7 +1111,7 @@ snapshots: globals: 14.0.0 ignore: 5.3.2 import-fresh: 3.3.1 - js-yaml: 5.2.1 + js-yaml: 5.2.2 minimatch: 3.1.5 strip-json-comments: 3.1.1 transitivePeerDependencies: @@ -1579,7 +1592,7 @@ snapshots: isexe@2.0.0: {} - js-yaml@5.2.1: + js-yaml@5.2.2: dependencies: argparse: 2.0.1 @@ -1618,7 +1631,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.12: {} + nanoid@3.3.18: {} natural-compare@1.4.0: {} @@ -1655,9 +1668,9 @@ snapshots: picomatch@4.0.4: {} - postcss@8.5.12: + postcss@8.5.23: dependencies: - nanoid: 3.3.12 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -1763,7 +1776,7 @@ snapshots: esbuild: 0.25.2 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.12 + postcss: 8.5.23 rollup: 4.60.2 tinyglobby: 0.2.17 optionalDependencies: diff --git a/tests/javascript/tests/test_wait_until_ready_diagnostics.test.ts b/tests/javascript/tests/test_wait_until_ready_diagnostics.test.ts index 61bb8659f..f0573f7d1 100644 --- a/tests/javascript/tests/test_wait_until_ready_diagnostics.test.ts +++ b/tests/javascript/tests/test_wait_until_ready_diagnostics.test.ts @@ -45,7 +45,6 @@ test("waitUntilReady timeout includes last health-check error and connection con expect(message).toContain("Last health check error"); expect(message).toContain("domain=localhost:8080"); expect(message).toContain("useServerProxy=false"); - expect(message).toContain("useServerProxy=true"); }); test("waitUntilReady timeout includes false-continuously hint when ping returns false", async () => { diff --git a/tests/python/Makefile b/tests/python/Makefile index d84a6e182..9207e7672 100644 --- a/tests/python/Makefile +++ b/tests/python/Makefile @@ -7,7 +7,7 @@ sync-dev: uv sync --group dev test: - uv run pytest + uv run pytest --ignore=tests/test_execd_init_e2e.py --ignore=tests/test_execd_hardening_e2e.py test-kubernetes-mini: uv run pytest \ diff --git a/tests/python/tests/test_execd_hardening_e2e.py b/tests/python/tests/test_execd_hardening_e2e.py new file mode 100644 index 000000000..045d9fd07 --- /dev/null +++ b/tests/python/tests/test_execd_hardening_e2e.py @@ -0,0 +1,510 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +E2E tests for the server-path hardening floor (OSEP-0018 R-i). + +Requires a server running with ``runtime.execd_run_as_init = true``. Docker +bridge (scripts/python-execd-hardening-e2e.sh) injects the hardened TOML via +server config:: + + [docker] + sandbox_env = { EXECD_ISOLATION_CONFIG = "/etc/opensandbox/isolation.toml" } + sandbox_binds = [ + "/tmp/opensandbox-e2e/workspace:/workspace", + "/tmp/opensandbox-e2e/isolation.hardened.toml:/etc/opensandbox/isolation.toml", + ] + +Kubernetes (scripts/python-k8s-execd-init-e2e.sh) delivers the TOML in a +ConfigMap mounted by the e2e batchsandbox template and points execd at it per +request (``EXECD_ISOLATION_CONFIG`` env + the workspace PVC mounted at +/mnt/workspace-exec; /workspace itself is a runtime-provided noexec tmpfs on +k8s). + +Verifies through the SDK, with the whole server -> sandbox -> execd path +running with ``[hardening]``/``[landlock]`` enabled: + +- every execd-spawned path (entrypoint + /command) runs reduced: + no effective caps, bounding set trimmed, seccomp filter mode, no_new_privs +- execd's config env is stripped from the workload (EXECD_ISOLATION_CONFIG + and EXECD_ACCESS_TOKEN absent from /command; EXECD_ACCESS_TOKEN absent + from the entrypoint) +- Landlock confinement: /tmp writable, /etc/passwd not writable, the + bind-mounted workspace writable AND executable (exercises the launcher's + mount expansion), /proc/1/environ denied +- GET /v1/isolated/capabilities reports init_mode=pid1 with + cap_drop/seccomp active and landlock active|unsupported + +A second class covers the fail-open degradation with CAP_SETPCAP dropped +from the container ceiling (run by the script's phase 2, gated on +OPENSANDBOX_HARDENING_DEGRADATION=true): + +- cap_drop reports degraded with a concrete reason; seccomp/landlock stay + active +- the floor still applies (CapEff=0, seccomp, NNP) but the bounding set is + NOT trimmed (fail-open: workloads keep the container ceiling's bounding + set) + +A third class covers bwrap isolated sessions under init mode + the floor +(OSEP-0018 R-o): sessions run with the bwrap namespace + seccomp/NNP floor +and the credential env strip, and the hardening report stays intact around +session create/run/delete. +""" + +import logging +import os +import time + +import pytest +from opensandbox import SandboxSync +from opensandbox.models.isolated import ( + CreateIsolatedSessionRequest, + HardeningStatus, + IsolatedWorkspaceSpec, +) +from opensandbox.models.sandboxes import PVC, Volume + +from tests.base_e2e_test import ( + get_test_pvc_name, + is_kubernetes_runtime, +) +from tests.test_execd_init_e2e import ( + _create_sandbox, + _destroy, + _run_command, +) + +logger = logging.getLogger(__name__) + +WORKSPACE_HOST = os.environ.get( + "OPENSANDBOX_HARDENING_WORKSPACE_HOST", "/tmp/opensandbox-e2e/workspace" +) + +# The entrypoint is launched with bootstrapEnv: EXECD_ACCESS_TOKEN is +# stripped, everything else (incl. EXECD_ISOLATION_CONFIG) is kept for +# image entrypoint scripts. /command uses the full config blacklist. +ENTRYPOINT_BOOTSTRAP_ENV_STRIPPED = ["EXECD_ACCESS_TOKEN"] +COMMAND_ENV_STRIPPED = [ + "EXECD_ACCESS_TOKEN", + "EXECD_ISOLATION_CONFIG", + "JUPYTER_HOST", + "JUPYTER_TOKEN", + "EXECD_ENVS", +] + +# Kubernetes: the hardened TOML travels in the opensandbox-e2e-execd-isolation +# ConfigMap, mounted by the e2e batchsandbox template at this path. +K8S_EXECD_ISOLATION_CONFIG = "/etc/opensandbox/execd-isolation/isolation.hardened.toml" + +# Kubernetes: the e2e PVC is backed by a hostPath PV on the kind node. The PV +# used to live under the node's /tmp, which is a noexec tmpfs, so every mount +# of the PVC was not executable (writes/reads fine, exec EACCES regardless of +# Landlock). The e2e harness now places the PV on the node rootfs +# (/var/opensandbox-e2e, scripts/common/kubernetes-e2e.sh); the workspace is +# mounted at /mnt/workspace-exec, which sits in the allowed_writable Landlock +# set so the mount-expansion rule grants write+exec. +K8S_WORKSPACE_EXEC = "/mnt/workspace-exec" + + +def _hardened_sandbox_options() -> dict: + """Sandbox create kwargs that point execd at the hardened TOML. + + Docker: the server injects EXECD_ISOLATION_CONFIG via [docker] sandbox_env + (config-level bind mount of isolation.hardened.toml at + /etc/opensandbox/isolation.toml) โ€” no request args needed. Kubernetes: the + TOML arrives via the template-mounted ConfigMap, so the request env points + execd at it; the workspace PVC is mounted at /mnt/workspace-exec so the + Landlock bind-mount expansion is exercised like the docker bind mount + (/workspace itself is a runtime-provided noexec tmpfs on k8s). + """ + if not is_kubernetes_runtime(): + return {} + return { + "env": {"EXECD_ISOLATION_CONFIG": K8S_EXECD_ISOLATION_CONFIG}, + "volumes": [ + Volume( + name="hardening-workspace", + pvc=PVC(claimName=get_test_pvc_name()), + mountPath=K8S_WORKSPACE_EXEC, + ), + ], + } + +_HARDENING_REPORT: HardeningStatus | None = None + + +def _hardening_report(sandbox: SandboxSync, refresh: bool = False) -> HardeningStatus: + """Probe execd's capabilities endpoint via the SDK model and cache. + ``refresh=True`` forces a live probe (used where the assertion must + observe the endpoint AFTER a state change, e.g. session teardown). + Consuming ``IsolatedCapabilities.hardening`` also pins the spec -> SDK + -> implementation alignment of the hardening object (OSEP-0018 R-r).""" + global _HARDENING_REPORT + if _HARDENING_REPORT is None or refresh: + caps = sandbox.isolation.capabilities() + if caps.hardening is None: + pytest.fail("capabilities endpoint returned no hardening object") + _HARDENING_REPORT = caps.hardening + return _HARDENING_REPORT + + +def _landlock_state(sandbox: SandboxSync) -> str: + report = _hardening_report(sandbox) + assert report.landlock is not None + return report.landlock.state + + +def _status_fields(sandbox: SandboxSync, fields: list[str]) -> dict: + """Parse selected /proc/self/status fields of the /command shell. + + Two constraints: (1) the read happens with the shell's own read loop, + NOT with forked helpers โ€” under Landlock the ruleset only grants the + launcher's own /proc/ (documented OSEP-0018 limitation), so a + forked grep/cat would get EACCES on its own /proc/self; (2) execd's + /command SSE output strips newlines, so the values must come out as a + single line (space-separated key=value pairs). + """ + arms = "\n".join( + f' {name}:*) {name}="${{line#*:\t}}" ;;' for name in fields + ) + echo = " ".join(f'"{n}=${n}"' for n in fields) + script = ( + 'while IFS= read -r line; do\n' + ' case "$line" in\n' + f"{arms}\n" + ' esac\n' + 'done < /proc/self/status\n' + f'echo {echo}' + ) + out = _run_command(sandbox, script) + parsed: dict[str, str] = {} + for token in out.split(): + key, _, value = token.partition("=") + parsed[key.strip()] = value.strip() + return parsed + + +def _read_entrypoint_dump(sbx: SandboxSync) -> str: + """Fetch the entrypoint's status + env dump written to /workspace. + + Docker: read the host side of the bind mount. Kubernetes: the dump lands + in the workspace PVC (mounted at /workspace), which is not host-readable + from the runner โ€” read it back through the SDK files API instead. + """ + path = f"/workspace/state-{sbx.id}.txt" + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if not is_kubernetes_runtime(): + host_path = os.path.join(WORKSPACE_HOST, f"state-{sbx.id}.txt") + if os.path.exists(host_path): + with open(host_path, encoding="utf-8") as f: + return f.read() + else: + try: + return sbx.files.read_file(path) + except Exception: # noqa: BLE001 # file may not be flushed yet + pass + time.sleep(1) + pytest.fail(f"entrypoint dump {path} never appeared (runtime kubernetes={is_kubernetes_runtime()})") + + +def _parse_entrypoint_dump(content: str) -> tuple[dict, dict]: + """Split the entrypoint dump into status fields and an env dict.""" + status_part, _, env_part = content.partition("=== env ===") + status: dict[str, str] = {} + for line in status_part.splitlines(): + key, _, value = line.partition(":") + status[key.strip()] = value.strip() + env = dict(line.split("=", 1) for line in env_part.splitlines() if "=" in line) + return status, env + + +class TestHardeningE2E: + @pytest.fixture(scope="module", autouse=True) + def sandbox(self): + # Entrypoint: dump its own /proc/self/status + env to the + # bind-mounted workspace (host-readable), then stay alive. The dump + # is written once at startup, before any /command churn, so it + # reflects exactly the launcher-applied floor. The status is read + # with the shell's own loop, NOT `cat`: under Landlock a forked + # descendant resolves its own /proc/, which the inherited + # ruleset does not grant (documented OSEP-0018 limitation), so a + # forked helper would get EACCES. + sbx = _create_sandbox( + entrypoint=[ + "sh", + "-c", + "out=/workspace/state-$OPENSANDBOX_ID.txt; " + "{ echo '=== status ==='; " + "while IFS= read -r line; do echo \"$line\"; done < /proc/self/status; " + "echo '=== env ==='; env | sort; } > \"$out\" 2>&1; " + "while :; do sleep 1; done", + ], + tag="execd-hardening-e2e", + **_hardened_sandbox_options(), + ) + logger.info("โœ“ hardening sandbox created: %s", sbx.id) + yield sbx + _destroy(sbx) + + def test_capabilities_endpoint_reports_hardening(self, sandbox) -> None: + report = _hardening_report(sandbox) + assert report.init_mode == "pid1", f"init_mode = {report.init_mode}" + assert report.signal_shield is True + assert report.cap_drop is not None and report.cap_drop.state == "active", ( + report.cap_drop + ) + assert report.seccomp is not None and report.seccomp.state == "active", ( + report.seccomp + ) + assert report.landlock is not None + assert report.landlock.state in ("active", "unsupported"), report.landlock + assert report.ebpf is not None and report.ebpf.state == "disabled", report.ebpf + logger.info( + "hardening report: init_mode=%s cap_drop=%s seccomp=%s landlock=%s", + report.init_mode, + report.cap_drop, + report.seccomp, + report.landlock, + ) + + def test_command_path_is_reduced(self, sandbox) -> None: + # /command children go through the same launcher prelude as the + # entrypoint: zero effective caps, bounding set trimmed, seccomp + # filter mode, no_new_privs. + status = _status_fields(sandbox, ["CapEff", "CapBnd", "Seccomp", "NoNewPrivs"]) + assert status["CapEff"] == "0000000000000000", status + assert status["CapBnd"] == "0000000000000000", status + assert status["Seccomp"] == "2", status + assert status["NoNewPrivs"] == "1", status + + def test_command_path_strips_execd_config_env(self, sandbox) -> None: + env = _run_command(sandbox, "env") + for name in COMMAND_ENV_STRIPPED: + assert f"{name}=" not in env, f"/command env leaked {name}" + + def test_entrypoint_is_reduced_and_env_stripped(self, sandbox) -> None: + status, env = _parse_entrypoint_dump(_read_entrypoint_dump(sandbox)) + assert status["CapEff"] == "0000000000000000", status + assert status["CapBnd"] == "0000000000000000", status + assert status["Seccomp"] == "2", status + assert status["NoNewPrivs"] == "1", status + for name in ENTRYPOINT_BOOTSTRAP_ENV_STRIPPED: + assert name not in env, f"entrypoint env leaked {name}" + + def test_workload_cannot_read_execd_environ(self, sandbox) -> None: + # PR_SET_DUMPABLE shield: same-uid workload without CAP_SYS_PTRACE + # cannot read execd's environment even before Landlock. + result = sandbox.commands.run("cat /proc/1/environ") + assert result.error is not None, "reading execd's /proc/1/environ must be denied" + stderr = "".join(msg.text for msg in result.logs.stderr) + assert "Permission denied" in stderr or "Operation not permitted" in stderr + + def test_tmp_is_writable(self, sandbox) -> None: + if _landlock_state(sandbox) != "active": + pytest.skip("landlock not active on this kernel") + _run_command(sandbox, "echo ok > /tmp/hardening-e2e-write && rm /tmp/hardening-e2e-write") + + def test_etc_passwd_is_not_writable(self, sandbox) -> None: + if _landlock_state(sandbox) != "active": + pytest.skip("landlock not active on this kernel") + result = sandbox.commands.run("echo x >> /etc/passwd") + assert result.error is not None, "writing /etc/passwd must be denied by landlock" + + def test_workspace_bind_mount_writable_and_executable(self, sandbox) -> None: + # The workspace is a separate mount from /: executing a script from it + # exercises the launcher's mount expansion (the grants beneath the + # mount point must be merged onto it), and writing to it exercises + # the workspace read/write rule. On k8s the exec workspace is + # /mnt/workspace-exec (the PVC); the e2e PV lives on the node rootfs + # since the previous hostPath location (/tmp) was a noexec tmpfs. + if _landlock_state(sandbox) != "active": + pytest.skip("landlock not active on this kernel") + workspace = K8S_WORKSPACE_EXEC if is_kubernetes_runtime() else "/workspace" + script = f"{workspace}/hardening-e2e.sh" + if is_kubernetes_runtime(): + # k8s regression probe: confirm the PVC mount is really present + # and executable. /proc must be read with the shell's own loop: + # the Landlock /proc/self rule pins the launcher's pid, so forked + # helpers get EACCES on their own procfs (documented OSEP-0018 + # limitation). + diag = _run_command( + sandbox, + f"id; " + f"printf '#!/bin/sh\\necho workspace-exec-ok\\n' > {script}" + f" && chmod +x {script}; " + "while IFS= read -r line; do case \"$line\" in " + f"*workspace-exec*) echo \"$line\" ;; esac; " + "done < /proc/self/mounts; " + f"stat -c '%A %a %U:%G %n' {script}", + ) + logger.info("workspace exec diagnostics:\n%s", diag) + _run_command( + sandbox, + f"printf '#!/bin/sh\\necho workspace-exec-ok\\n' > {script}" + f" && chmod +x {script} && {script}", + ) + + +@pytest.mark.skipif( + os.environ.get("OPENSANDBOX_HARDENING_DEGRADATION") != "true", + reason="requires the degradation server (CAP_SETPCAP dropped); run via " + "scripts/python-execd-hardening-e2e.sh phase 2", +) +@pytest.mark.skipif( + is_kubernetes_runtime(), + reason="CAP_SETPCAP ceiling degradation is docker-only (kubernetes " + "securityContext caps are not tuned in the k8s e2e)", +) +class TestHardeningDegradationE2E: + @pytest.fixture(scope="module", autouse=True) + def sandbox(self): + sbx = _create_sandbox(tag="execd-hardening-degradation-e2e") + logger.info("โœ“ degradation sandbox created: %s", sbx.id) + yield sbx + _destroy(sbx) + + def test_cap_drop_reports_degraded_with_reason(self, sandbox) -> None: + report = _hardening_report(sandbox) + assert report.init_mode == "pid1", f"init_mode = {report.init_mode}" + assert report.cap_drop is not None + cap_drop = report.cap_drop + assert cap_drop.state == "degraded", cap_drop + assert cap_drop.message is not None and "SETPCAP" in cap_drop.message, cap_drop + # The remaining layers must not cascade: fail-open is per layer. + assert report.seccomp is not None and report.seccomp.state == "active", ( + report.seccomp + ) + assert report.landlock is not None + assert report.landlock.state in ("active", "unsupported"), report.landlock + + def test_floor_still_applies_without_setpcap(self, sandbox) -> None: + # Bounding-set trim is skipped without CAP_SETPCAP, but capset (drop + # own caps), seccomp and NNP still apply โ€” the workload is reduced + # even in the degraded state. + status = _status_fields(sandbox, ["CapEff", "CapBnd", "Seccomp", "NoNewPrivs"]) + assert status["CapEff"] == "0000000000000000", status + assert status["Seccomp"] == "2", status + assert status["NoNewPrivs"] == "1", status + # Fail-open: the bounding set keeps the container ceiling caps. + assert status["CapBnd"] != "0000000000000000", status + + +class TestIsolatedSessionHardeningE2E: + """bwrap isolated sessions under init mode + the hardening floor + (OSEP-0018 R-o). + + The sandbox runs with ``[hardening]``/``[landlock]`` enabled and execd as + PID 1, so the whole server -> sandbox -> execd -> launcher -> bwrap chain + is active. bwrap itself is launcher-exempt (``withoutHardening``: its + workload is already reduced by bwrap's own seccomp + namespaces), so this + pins that the isolated-session path composes with the floor: sessions + run, their workload carries bwrap's seccomp/NNP floor and the credential + env strip, and the sandbox-level hardening report stays intact around + session create/run/delete. + """ + + @pytest.fixture(scope="module", autouse=True) + def sandbox(self): + # The isolation extension grants the container ceiling CAP_SYS_ADMIN + # (bwrap needs it to build namespaces); the floor still applies to + # every user-code child, and bwrap remains launcher-exempt. + sbx = _create_sandbox( + extensions={"bootstrap.execd.isolation": "enable"}, + tag="execd-hardening-isolated-e2e", + ) + logger.info("โœ“ hardening+isolated sandbox created: %s", sbx.id) + yield sbx + _destroy(sbx) + + def _create_session(self, sandbox): + return sandbox.isolation.create( + CreateIsolatedSessionRequest( + workspace=IsolatedWorkspaceSpec(path="/tmp", mode="rw"), + ) + ) + + def test_capabilities_available_with_hardening(self, sandbox) -> None: + caps = sandbox.isolation.capabilities() + assert caps.available, caps.message + assert caps.isolator == "bwrap" + # The floor must not be disturbed by the isolation extension or by + # probing bwrap inside the hardened sandbox. + report = _hardening_report(sandbox) + assert report.init_mode == "pid1", f"init_mode = {report.init_mode}" + assert report.cap_drop is not None and report.cap_drop.state == "active", ( + report.cap_drop + ) + assert report.seccomp is not None and report.seccomp.state == "active", ( + report.seccomp + ) + + def test_isolated_session_workload_has_floor(self, sandbox) -> None: + # Inside the bwrap namespace the workload carries bwrap's own floor: + # the shared seccomp denylist in filter mode, no_new_privs, and the + # credential env strip (bwrap --unsetenv blacklist). Read the fields + # with the session shell's own loop so no forked helper is involved. + session = self._create_session(sandbox) + try: + code = ( + "while IFS= read -r line; do " + "case \"$line\" in " + "Seccomp:*) echo \"sec=${line#*:\t}\" ;; " + "NoNewPrivs:*) echo \"nnp=${line#*:\t}\" ;; " + "esac; done < /proc/self/status\n" + "if env | grep -q '^EXECD_ACCESS_TOKEN='; then " + "echo token_leaked; else echo token_stripped; fi" + ) + result = session.run(code) + assert "sec=2" in result.text, result.text + assert "nnp=1" in result.text, result.text + assert "token_stripped" in result.text, result.text + finally: + session.delete() + + def test_isolated_session_pid_isolation(self, sandbox) -> None: + session = self._create_session(sandbox) + try: + result = session.run("echo $$") + pid = int(result.text.strip()) + assert pid <= 2, f"session pid = {pid}, want PID 1 or 2 in the namespace" + finally: + session.delete() + + def test_isolated_session_state_persists(self, sandbox) -> None: + session = self._create_session(sandbox) + try: + session.run("export PERSIST_HARDENED=abc123") + result = session.run("echo $PERSIST_HARDENED") + assert "abc123" in result.text, result.text + finally: + session.delete() + + def test_session_delete_while_workload_busy(self, sandbox) -> None: + # A backgrounded workload inside the session must be torn down with + # the session under reaper dispatch; the sandbox stays healthy and + # the floor stays active afterwards. + session = self._create_session(sandbox) + session.run("sleep 30 &") + session.delete() + + # Fresh probe: the process-global cache was populated by earlier + # tests, so a cached report would make this assertion vacuous. + report = _hardening_report(sandbox, refresh=True) + assert report.init_mode == "pid1", f"init_mode = {report.init_mode}" + status = _status_fields(sandbox, ["CapEff", "Seccomp", "NoNewPrivs"]) + assert status["CapEff"] == "0000000000000000", status + assert status["Seccomp"] == "2", status + assert status["NoNewPrivs"] == "1", status diff --git a/tests/python/tests/test_execd_init_e2e.py b/tests/python/tests/test_execd_init_e2e.py new file mode 100644 index 000000000..fab19f2cb --- /dev/null +++ b/tests/python/tests/test_execd_init_e2e.py @@ -0,0 +1,300 @@ +# +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +E2E tests for execd-as-init mode (OSEP-0018). + +Requires a server running with ``runtime.execd_run_as_init = true`` (the +dedicated real-e2e and kubernetes-nightly jobs set it). Verifies the +container-level init contract through the SDK: + +- execd is PID 1 and the workload's parent +- orphaned children are reaped (no zombie accumulation under PID 1) +- a fork-heavy workload keeps the process table bounded +- in-namespace ``kill -9 1`` is inert (kernel signal shield) +- application signals (HUP/USR1/USR2/WINCH) are forwarded to the entrypoint +- the entrypoint's exit code propagates to the container/runtime (docker + bridge; on Kubernetes the test skips โ€” BatchSandbox does not surface the + container exit code or a terminal lifecycle state, OSEP-0018 R-l) +- an in-namespace ``kill 1`` (SIGTERM) still stops the sandbox โ€” interim + behavior pin for OSEP-0018 ยง3 (R-a: trusted out-of-band stop channel); + on Kubernetes the pin asserts execd becomes unreachable instead of a + lifecycle state transition +- the workload cannot read execd's environment (``/proc/1/environ`` denied by + non-dumpable, independent of Landlock) +- ``GET /v1/isolated/capabilities`` reports ``hardening.init_mode = pid1`` + and ``hardening.signal_shield = true`` +""" + +import json +import logging +import time +from datetime import timedelta + +import pytest +from opensandbox import SandboxSync +from opensandbox.models.execd import RunCommandOpts +from opensandbox.models.sandboxes import SandboxImageSpec + +from tests.base_e2e_test import ( + create_connection_config_sync, + get_e2e_sandbox_resource, + get_sandbox_image, + is_kubernetes_runtime, +) + +logger = logging.getLogger(__name__) + +EXECD_CAPABILITIES_URL = "http://127.0.0.1:44772/v1/isolated/capabilities" + + +def _run_command(sandbox, command: str) -> str: + """Run a command and return its combined stdout.""" + result = sandbox.commands.run(command, opts=RunCommandOpts()) + assert result.error is None, f"command failed: {result.error}" + return "".join(msg.text for msg in result.logs.stdout) + + +def _zombie_count(sandbox) -> int: + """Count processes in state Z whose parent is PID 1.""" + out = _run_command( + sandbox, + "z=0; for p in /proc/[0-9]*; do " + "stat=$(cat \"$p/stat\" 2>/dev/null) || continue; " + "stat=${stat#*)}; set -- $stat; " + "[ \"$1\" = Z ] && [ \"$2\" = 1 ] && z=$((z+1)); done; echo $z", + ) + return int(out.strip()) + + +def _process_count(sandbox) -> int: + return int(_run_command(sandbox, "ls -d /proc/[0-9]* | wc -l").strip()) + + +def _create_sandbox( + entrypoint: list[str] | None = None, + tag: str = "execd-init-e2e", + extensions: dict[str, str] | None = None, + env: dict[str, str] | None = None, + volumes: list | None = None, +) -> SandboxSync: + connection_config = create_connection_config_sync() + return SandboxSync.create( + image=SandboxImageSpec(get_sandbox_image()), + resource=get_e2e_sandbox_resource(), + connection_config=connection_config, + timeout=timedelta(minutes=5), + ready_timeout=timedelta(seconds=60), + entrypoint=entrypoint, + extensions=extensions, + env=env, + volumes=volumes, + metadata={"tag": tag}, + ) + + +def _destroy(sandbox) -> None: + try: + sandbox.kill() + except Exception as exc: # noqa: BLE001 + logger.warning("Teardown: sandbox.kill() failed: %s", exc, exc_info=True) + try: + sandbox.close() + except Exception as exc: # noqa: BLE001 + logger.warning("Teardown: sandbox.close() failed: %s", exc, exc_info=True) + + +class TestExecdInitE2E: + @pytest.fixture(scope="module", autouse=True) + def sandbox(self): + sbx = _create_sandbox(tag="execd-init-e2e") + logger.info("โœ“ execd-init sandbox created: %s", sbx.id) + yield sbx + _destroy(sbx) + + def test_pid1_is_execd(self, sandbox) -> None: + assert _run_command(sandbox, "cat /proc/1/comm").strip() == "execd" + + def test_workload_is_direct_child_of_execd(self, sandbox) -> None: + # /proc/$$/stat field 4 is the parent pid; the run-command shell is + # a direct child of execd (PID 1). + ppid = _run_command(sandbox, "awk '{print $4}' /proc/$$/stat").strip() + assert ppid == "1", f"workload ppid = {ppid}, want 1" + + def test_orphans_are_reaped(self, sandbox) -> None: + # Background children reparent to PID 1 and must be reaped by execd. + _run_command(sandbox, "for i in $(seq 1 5); do ( sleep 0.1 ) & done") + time.sleep(2) + zombies = _zombie_count(sandbox) + assert zombies == 0, f"zombies under pid 1: {zombies}" + + def test_kill9_pid1_is_inert(self, sandbox) -> None: + assert "alive" in _run_command(sandbox, "kill -9 1; echo alive") + + def test_workload_cannot_read_execd_environ(self, sandbox) -> None: + result = sandbox.commands.run("cat /proc/1/environ", opts=RunCommandOpts()) + assert result.error is not None, "reading execd's /proc/1/environ must be denied" + stderr = "".join(msg.text for msg in result.logs.stderr) + assert "Permission denied" in stderr or "Operation not permitted" in stderr + + @pytest.fixture(scope="module") + def signal_sandbox(self): + """Sandbox whose entrypoint traps HUP/USR1/USR2/WINCH โ€” used to + observe forwarding of the whole application-signal set.""" + sbx = _create_sandbox( + entrypoint=[ + "sh", + "-c", + "trap 'echo got-hup >> /tmp/execd-hup.log' HUP; " + "trap 'echo got-usr1 >> /tmp/execd-usr1.log' USR1; " + "trap 'echo got-usr2 >> /tmp/execd-usr2.log' USR2; " + "trap 'echo got-winch >> /tmp/execd-winch.log' WINCH; " + "while :; do sleep 1; done", + ], + tag="execd-init-e2e-signal", + ) + yield sbx + _destroy(sbx) + + @pytest.fixture(scope="module") + def kill1_sandbox(self): + """Sandbox destroyed by its own in-namespace ``kill 1``.""" + sbx = _create_sandbox(tag="execd-init-e2e-kill1") + yield sbx + _destroy(sbx) + + def test_application_signals_forwarded_to_entrypoint(self, signal_sandbox) -> None: + # In-namespace HUP/USR1/USR2/WINCH to PID 1 are delivered (execd + # installs handlers, so the kernel signal shield does not apply) and + # forwarded to the entrypoint process group. The /command shell runs + # in its own process group and must not receive them. + out = _run_command( + signal_sandbox, + "sleep 1; kill -HUP 1; kill -USR1 1; kill -USR2 1; kill -WINCH 1; " + "sleep 2; cat /tmp/execd-hup.log /tmp/execd-usr1.log " + "/tmp/execd-usr2.log /tmp/execd-winch.log", + ) + for marker in ("got-hup", "got-usr1", "got-usr2", "got-winch"): + assert marker in out, f"forwarded signal marker {marker} missing: {out}" + + def test_entrypoint_exit_code_propagates(self) -> None: + # When the user entrypoint exits, execd exits with the same status so + # Docker/kubelet observe it (OSEP-0018 ยง2 "entrypoint owns the + # container lifecycle"). The sleep gives the sandbox time to become + # ready before the entrypoint exits. + if is_kubernetes_runtime(): + # BatchSandbox stays Pending after the pod completes and does not + # surface the container exit code (OSEP-0018 R-l): the lifecycle + # state-transition assertion below is docker-runtime-specific. + pytest.skip( + "BatchSandbox does not surface the container exit code " + "(OSEP-0018 R-l)" + ) + sbx = _create_sandbox( + entrypoint=["sh", "-c", "sleep 20; exit 42"], + tag="execd-init-e2e-exit42", + ) + try: + deadline = time.monotonic() + 75 + state = None + while time.monotonic() < deadline: + state = sbx.get_info().status.state + if state in {"Failed", "Terminated"}: + break + time.sleep(2) + if state not in {"Failed", "Terminated"}: + pytest.fail(f"entrypoint-exit sandbox stuck in state {state}") + info = sbx.get_info() + assert info.status.state == "Failed", info.status + assert info.status.message and "exited with code 42" in info.status.message, ( + info.status.message + ) + finally: + _destroy(sbx) + + def test_in_namespace_sigterm_kill1_stops_sandbox(self, kill1_sandbox) -> None: + # Interim-behavior pin (OSEP-0018 ยง3, R-a): today an in-namespace + # `kill 1` SIGTERM reaches execd's forwarding loop and stops the + # sandbox, matching the pre-OSEP bootstrap behavior. Once the trusted + # out-of-band stop channel lands, execd must ignore in-namespace + # SIGTERM and this test flips to asserting the sandbox stays Running. + try: + kill1_sandbox.commands.run( + "kill 1; sleep 5; echo alive", opts=RunCommandOpts() + ) + except Exception: # noqa: BLE001 # the execd connection dies mid-stream + pass + if is_kubernetes_runtime(): + self._assert_execd_unreachable(kill1_sandbox) + return + deadline = time.monotonic() + 45 + state = None + while time.monotonic() < deadline: + state = kill1_sandbox.get_info().status.state + if state in {"Failed", "Terminated"}: + return + time.sleep(2) + pytest.fail(f"sandbox did not stop after in-namespace kill 1 (state={state})") + + def _assert_execd_unreachable(self, kill1_sandbox) -> None: + """k8s leg of the kill-1 pin: the pod exits after ``kill 1`` but + BatchSandbox stays Pending (it never transitions to a terminal + lifecycle state โ€” OSEP-0018 R-l), so assert the observable effect + instead: execd is gone and the sandbox is unusable. + """ + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + try: + kill1_sandbox.commands.run("echo alive", opts=RunCommandOpts()) + except Exception: # noqa: BLE001 # execd unreachable -> sandbox stopped + # Confirm on a fresh attempt so a single stale-pooled- + # connection error (the kill-1 proxy stream dies mid-flight) + # cannot false-pass while execd is still alive. + time.sleep(1) + try: + kill1_sandbox.commands.run("echo alive", opts=RunCommandOpts()) + except Exception: # noqa: BLE001 + return + continue + time.sleep(2) + pytest.fail("execd still reachable after in-namespace kill 1") + + def test_fork_heavy_keeps_process_table_bounded(self, sandbox) -> None: + # Sustained fork churn: short-lived background children (reparented + # orphans) plus a few long sleepers. The reaper must keep the process + # table bounded and zombie-free over many cycles. + baseline = _process_count(sandbox) + for _ in range(20): + _run_command(sandbox, "for i in $(seq 1 5); do ( sleep 0.1 ) & done; sleep 0.3") + _run_command(sandbox, "sleep 15 & sleep 15 & sleep 15 &") + time.sleep(2) + zombies = _zombie_count(sandbox) + assert zombies == 0, f"zombies under pid 1: {zombies}" + total = _process_count(sandbox) + assert total <= baseline + 10, f"process table grew: {baseline} -> {total}" + + def test_hardening_reports_pid1(self, sandbox) -> None: + # execd's /command SSE output strips newlines, so the probe emits a + # single JSON object instead of multi-line prints. + probe = ( + "python3 -c \"import json,urllib.request;" + f"h=json.load(urllib.request.urlopen('{EXECD_CAPABILITIES_URL}'))['hardening'];" + "print(json.dumps({'init_mode': h['init_mode'], 'signal_shield': h['signal_shield']}))\"" + ) + report = json.loads(_run_command(sandbox, probe)) + assert report["init_mode"] == "pid1", f"hardening.init_mode = {report['init_mode']}" + assert report["signal_shield"] is True, ( + f"hardening.signal_shield = {report['signal_shield']}" + ) diff --git a/tests/python/uv.lock b/tests/python/uv.lock index abc3446a1..11748ef33 100644 --- a/tests/python/uv.lock +++ b/tests/python/uv.lock @@ -122,6 +122,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + [[package]] name = "idna" version = "3.15" @@ -155,6 +164,7 @@ source = { editable = "../../sdks/sandbox/python" } dependencies = [ { name = "attrs" }, { name = "httpx" }, + { name = "httpx-sse" }, { name = "pydantic" }, { name = "python-dateutil" }, ] @@ -163,6 +173,7 @@ dependencies = [ requires-dist = [ { name = "attrs", specifier = ">=21.3.0" }, { name = "httpx", specifier = ">=0.27.0,<1.0" }, + { name = "httpx-sse", specifier = ">=0.4.3,<0.5" }, { name = "pydantic", specifier = ">=2.4.2,<3.0" }, { name = "pyjwt", marker = "extra == 'pool-redis'", specifier = ">=2.13.0" }, { name = "python-dateutil", specifier = ">=2.8.2,<3.0" },