diff --git a/.github/workflows/real-e2e.yml b/.github/workflows/real-e2e.yml index 12cf3a64..8ab7fe74 100644 --- a/.github/workflows/real-e2e.yml +++ b/.github/workflows/real-e2e.yml @@ -62,6 +62,7 @@ jobs: execd_image = "opensandbox/execd:local" [egress] image = "opensandbox/egress:local" + mode = "dns" [docker] network_mode = "bridge" [storage] @@ -144,6 +145,7 @@ jobs: execd_image = "opensandbox/execd:local" [egress] image = "opensandbox/egress:local" + mode = "dns+nft" [docker] network_mode = "bridge" [storage] diff --git a/kubernetes/charts/opensandbox-server/values.yaml b/kubernetes/charts/opensandbox-server/values.yaml index 5025a1a4..2ca4b350 100644 --- a/kubernetes/charts/opensandbox-server/values.yaml +++ b/kubernetes/charts/opensandbox-server/values.yaml @@ -72,4 +72,8 @@ configToml: | informer_watch_timeout_seconds = 60 workload_provider = "batchsandbox" batchsandbox_template_file = "/etc/opensandbox/example.batchsandbox-template.yaml" + + [egress] + image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.3" + mode = "dns+nft" diff --git a/server/README.md b/server/README.md index 99abea7c..95ee18b2 100644 --- a/server/README.md +++ b/server/README.md @@ -255,38 +255,74 @@ EOF - Informer settings are **beta** and enabled by default to reduce API calls; set `informer_enabled = false` to turn off. - Resync and watch timeouts control how often the cache refreshes; tune for your cluster API limits. -### Egress sidecar for `networkPolicy` +### Egress configuration -- **Required when using `networkPolicy`**: Configure the sidecar image. The `egress.image` setting is mandatory when requests include `networkPolicy`: - ```toml - [runtime] - type = "docker" - execd_image = "opensandbox/execd:v1.0.7" - - [egress] - image = "opensandbox/egress:v1.0.3" - ``` -- Supported only in Docker bridge mode; requests with `networkPolicy` are rejected when `network_mode=host` or when `egress.image` is not configured. -- Main container shares the sidecar netns and explicitly drops `NET_ADMIN`; the sidecar keeps `NET_ADMIN` to manage iptables. -- IPv6 is disabled in the shared namespace when the egress sidecar is injected to keep policy enforcement consistent. -- Sidecar image is pulled before start; delete/expire/failure paths attempt to clean up the sidecar as well. -- Request example (`CreateSandboxRequest` with `networkPolicy`): - ```json - { - "image": {"uri": "python:3.11-slim"}, - "entrypoint": ["python", "-m", "http.server", "8000"], - "timeout": 3600, - "resourceLimits": {"cpu": "500m", "memory": "512Mi"}, - "networkPolicy": { - "defaultAction": "deny", - "egress": [ - {"action": "allow", "target": "pypi.org"}, - {"action": "allow", "target": "*.python.org"} - ] - } - } - ``` -- When `networkPolicy` is empty or omitted, no sidecar is injected (allow-all at start). +The **`[egress]`** block configures the **egress sidecar** image and enforcement mode. The server only starts this sidecar when a sandbox is created **with** a `networkPolicy` (outbound allow/deny rules). If the create request omits `networkPolicy`, no egress sidecar is added and outbound traffic is not restricted by this mechanism. + +#### Keys + +| Key | Type | Default | Required | Description | +|-----|------|---------|----------|-------------| +| `image` | string | — | **Yes** whenever `networkPolicy` is used in a create request | OCI image containing the egress binary. Pulled before the sidecar starts. | +| `mode` | `dns` or `dns+nft` | `dns` | No | How the sidecar enforces policy. Written to the sidecar as `OPENSANDBOX_EGRESS_MODE` (see below). | + +#### `mode` values + +- **`dns`**: DNS-based enforcement via the in-sidecar DNS proxy. No nftables layer-2 rules from this path. **CIDR and static IP targets in the policy are not enforced** (use domain-style rules only if you rely on `dns` mode). +- **`dns+nft`**: Same DNS path, plus nftables where available (see the [egress component README](../components/egress/README.md) for capabilities and fallbacks). **CIDR and static IP allow/deny rules are supported** via nftables when the table is applied successfully. + +#### Per-request `networkPolicy` + +- Rules are defined on **`CreateSandboxRequest.networkPolicy`** (default action and ordered egress rules: hostnames / patterns, and IP or CIDR entries when using **`dns+nft`**). +- The serialized policy is passed into the sidecar as **`OPENSANDBOX_EGRESS_RULES`** (JSON). +- An auth token may be attached for the egress HTTP API; see runtime behavior below. + +#### Docker runtime + +- **`egress.image` must be set** in config when clients send `networkPolicy`; otherwise the request is rejected. +- Outbound policy requires **`docker.network_mode = "bridge"`**. Requests with `networkPolicy` are rejected for `network_mode=host` or for user-defined Docker networks that are incompatible with the sidecar attachment model. +- The main sandbox container shares the sidecar’s network namespace, **drops `NET_ADMIN`**, and relies on the sidecar for policy; the sidecar **keeps `NET_ADMIN`**. +- **IPv6** is disabled in the shared namespace so allow/deny behavior stays consistent. + +#### Kubernetes runtime + +- When `networkPolicy` is present, the workload pod includes an **egress** sidecar built from `egress.image`, in addition to the main sandbox container. +- **`egress.image`** is required in the same way as for Docker. + +#### Operational notes + +- The sidecar image is pulled (or validated) before start; delete, expiry, and failure paths attempt to remove the sidecar. +- For deeper behavior (DNS proxy, nftables, limits), refer to the **egress** component documentation under `components/egress/`. + +#### Example (`~/.sandbox.toml`) + +```toml +[runtime] +type = "docker" +execd_image = "opensandbox/execd:v1.0.7" + +[egress] +image = "opensandbox/egress:v1.0.3" +mode = "dns" +``` + +#### Example create request with `networkPolicy` + +```json +{ + "image": {"uri": "python:3.11-slim"}, + "entrypoint": ["python", "-m", "http.server", "8000"], + "timeout": 3600, + "resourceLimits": {"cpu": "500m", "memory": "512Mi"}, + "networkPolicy": { + "defaultAction": "deny", + "egress": [ + {"action": "allow", "target": "pypi.org"}, + {"action": "allow", "target": "*.python.org"} + ] + } +} +``` ### Run the server @@ -501,9 +537,10 @@ curl -X DELETE \ ### Egress configuration -| Key | Type | Required | Description | -|---------------|--------|----------|--------------------------------| -| `egress.image` | string | **Required when using `networkPolicy`** | Container image with egress binary. Must be configured when `networkPolicy` is provided in sandbox creation requests. | +| Key | Type | Default | Required if using `networkPolicy` | Description | +|-----|------|---------|-----------------------------------|-------------| +| `egress.image` | string | — | Yes | Egress sidecar image (OCI reference). | +| `egress.mode` | `dns` \| `dns+nft` | `dns` | No | `OPENSANDBOX_EGRESS_MODE`. CIDR/IP rules need `dns+nft`; `dns` is domain-oriented only. | ### Docker configuration diff --git a/server/README_zh.md b/server/README_zh.md index 142097e1..102e1c67 100644 --- a/server/README_zh.md +++ b/server/README_zh.md @@ -231,23 +231,59 @@ mode = "direct" # Docker 运行时仅支持 direct(直连,无 L7 网关) - Informer 配置为 **Beta**,默认开启以减少 API 压力;若需关闭设置 `informer_enabled = false`。 - resync / watch 超时用于控制缓存刷新频率,可根据集群 API 限流调优。 -### Egress sidecar 配置与使用 +### Egress 配置(`[egress]` 配置块) + +**`[egress]`** 用于配置 **egress 侧车** 的镜像与执行模式。仅当创建沙箱的请求中带有 **`networkPolicy`**(出站允许/拒绝规则)时,服务器才会注入该侧车;若请求未带 `networkPolicy`,不会添加 egress 侧车,也不会通过该机制限制出站流量。 + +#### 配置项 + +| 键 | 类型 | 默认值 | 何时必填 | 说明 | +|----|------|--------|----------|------| +| `image` | string | — | 任意一次创建请求携带 `networkPolicy` 时 **必填** | 包含 egress 可执行文件的容器镜像;侧车启动前会拉取或校验镜像。 | +| `mode` | `dns` 或 `dns+nft` | `dns` | 否 | 侧车如何执行策略,写入环境变量 `OPENSANDBOX_EGRESS_MODE`(见下)。 | + +#### `mode` 取值 + +- **`dns`**:通过侧车内 DNS 代理做基于域名的策略;不依赖本路径下的 nftables 二层规则。**策略中的 CIDR、静态 IP 类目标不会被强制执行**(若只用 `dns` 模式,请使用域名类规则)。 +- **`dns+nft`**:在 `dns` 的基础上启用 nftables(能力与回退行为见 [egress 组件说明](../components/egress/README.md))。**支持 CIDR 与静态 IP 的放行/拒绝规则**(nftables 表成功下发时生效)。 + +#### 请求体中的 `networkPolicy` + +- 规则在 **`CreateSandboxRequest.networkPolicy`** 中声明(默认动作与有序的 egress 规则:域名/通配符;在使用 **`dns+nft`** 时还可包含 IP 或 CIDR 条目)。 +- 序列化后的策略以 JSON 形式注入侧车环境变量 **`OPENSANDBOX_EGRESS_RULES`**。 +- 可能同时下发用于 egress HTTP API 的鉴权信息(与运行时行为一致)。 + +#### Docker 运行时 + +- 客户端传入 `networkPolicy` 时,配置中必须设置 **`egress.image`**,否则请求会被拒绝。 +- 出站策略要求 **`docker.network_mode = "bridge"`**;`network_mode=host` 或与侧车挂载模型不兼容的用户自定义网络下,携带 `networkPolicy` 的请求会被拒绝。 +- 主沙箱容器与侧车 **共享网络命名空间**,主容器 **drop `NET_ADMIN`**,由侧车保留 **`NET_ADMIN`** 完成策略相关操作。 +- 共享 netns 内会 **禁用 IPv6**,以保证放行/拒绝行为一致。 + +#### Kubernetes 运行时 + +- 当请求带有 `networkPolicy` 时,工作负载 Pod 中除主容器外,还会增加基于 **`egress.image`** 的 **egress** 侧车。 +- **`egress.image`** 的必填规则与 Docker 相同。 + +#### 运维说明 + +- 侧车镜像在启动前拉取或校验;删除、过期、失败等路径会尽量清理侧车。 +- DNS 代理、nftables、能力边界等详见仓库内 **`components/egress/`** 文档。 + +#### 配置示例(`~/.sandbox.toml`) -- **使用 `networkPolicy` 时必需**:配置 sidecar 镜像。当请求携带 `networkPolicy` 时,`egress.image` 配置项是必需的: ```toml [runtime] type = "docker" -execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.7" +execd_image = "opensandbox/execd:v1.0.7" [egress] -image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.3" +image = "opensandbox/egress:v1.0.3" +mode = "dns" ``` -- 仅支持 Docker bridge 模式(`network_mode=host` 时会拒绝携带 `networkPolicy` 的请求,或当 `egress.image` 未配置时也会拒绝)。 -- 主容器共享 sidecar 网络命名空间,主容器会显式 drop `NET_ADMIN`,sidecar 保留 `NET_ADMIN` 完成 iptables。 -- 注入 sidecar 时会在共享 netns 内默认禁用 IPv6,以保持策略生效一致性。 -- 侧车镜像会在启动前自动拉取;删除/过期/失败时会尝试同步清理 sidecar。 -- 请求体示例(`CreateSandboxRequest` 中携带 `networkPolicy`): +#### 带 `networkPolicy` 的创建请求示例 + ```json { "image": {"uri": "python:3.11-slim"}, @@ -263,7 +299,6 @@ image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0 } } ``` -- `networkPolicy` 为空/缺省时不注入 sidecar,默认 allow-all。 ### 启动服务 @@ -477,9 +512,10 @@ curl -X DELETE \ ### Egress 配置 -| 键 | 类型 | 必需 | 描述 | -|---------------|--------|----|--------------------------------| -| `egress.image` | string | **使用 `networkPolicy` 时必需** | 包含 egress 二进制文件的容器镜像。当创建沙箱的请求中包含 `networkPolicy` 时,必须配置此项。 | +| 键 | 类型 | 默认值 | 使用 `networkPolicy` 时是否必填 | 说明 | +|----|------|--------|--------------------------------|------| +| `egress.image` | string | — | 是 | Egress 侧车镜像(OCI 引用)。 | +| `egress.mode` | `dns` \| `dns+nft` | `dns` | 否 | `OPENSANDBOX_EGRESS_MODE`。CIDR/IP 类规则需 `dns+nft`;`dns` 仅面向域名类策略。 | ### Docker 配置 diff --git a/server/example.config.k8s.toml b/server/example.config.k8s.toml index b7e6fda5..f0a04908 100644 --- a/server/example.config.k8s.toml +++ b/server/example.config.k8s.toml @@ -65,3 +65,10 @@ batchsandbox_template_file = "~/batchsandbox-template.yaml" [ingress] # Ingress exposure mode: direct (default) or gateway mode = "direct" + +[egress] +# Egress configuration +# ----------------------------------------------------------------- +image = "opensandbox/egress:v1.0.3" +# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). +mode = "dns" diff --git a/server/example.config.k8s.zh.toml b/server/example.config.k8s.zh.toml index 7c82b780..a61d752c 100644 --- a/server/example.config.k8s.zh.toml +++ b/server/example.config.k8s.zh.toml @@ -66,3 +66,10 @@ batchsandbox_template_file = "~/batchsandbox-template.yaml" [ingress] # Ingress exposure mode: direct (default) or gateway mode = "direct" + +[egress] +# Egress configuration +# ----------------------------------------------------------------- +image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.3" +# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). +mode = "dns" diff --git a/server/example.config.toml b/server/example.config.toml index efefb990..0599a4d5 100644 --- a/server/example.config.toml +++ b/server/example.config.toml @@ -37,6 +37,8 @@ execd_image = "opensandbox/execd:v1.0.7" # Egress configuration # ----------------------------------------------------------------- image = "opensandbox/egress:v1.0.3" +# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). +mode = "dns" [storage] # Volume and storage configuration diff --git a/server/example.config.zh.toml b/server/example.config.zh.toml index cf3c52e2..8ba43df8 100644 --- a/server/example.config.zh.toml +++ b/server/example.config.zh.toml @@ -34,6 +34,8 @@ execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd # Egress configuration # ----------------------------------------------------------------- image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.3" +# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). +mode = "dns" [storage] # 卷存储配置 diff --git a/server/src/config.py b/server/src/config.py index cf2843ac..201e4ee9 100644 --- a/server/src/config.py +++ b/server/src/config.py @@ -50,6 +50,9 @@ GATEWAY_ROUTE_MODE_HEADER = "header" GATEWAY_ROUTE_MODE_URI = "uri" +EGRESS_MODE_DNS = "dns" +EGRESS_MODE_DNS_NFT = "dns+nft" + def _is_valid_ip(host: str) -> bool: try: @@ -350,6 +353,13 @@ class EgressConfig(BaseModel): description="Container image for the egress sidecar (used when network policy is requested).", min_length=1, ) + mode: Literal[ + EGRESS_MODE_DNS, + EGRESS_MODE_DNS_NFT, + ] = Field( + default=EGRESS_MODE_DNS, + description="Egress enforcement passed to the sidecar as OPENSANDBOX_EGRESS_MODE (dns or dns+nft).", + ) class RuntimeConfig(BaseModel): @@ -617,6 +627,8 @@ def get_config_path() -> Path: "StorageConfig", "KubernetesRuntimeConfig", "EgressConfig", + "EGRESS_MODE_DNS", + "EGRESS_MODE_DNS_NFT", "SecureRuntimeConfig", "DEFAULT_CONFIG_PATH", "CONFIG_ENV_VAR", diff --git a/server/src/services/constants.py b/server/src/services/constants.py index 08459eaa..27ed4bdb 100644 --- a/server/src/services/constants.py +++ b/server/src/services/constants.py @@ -27,6 +27,14 @@ OPEN_SANDBOX_EGRESS_AUTH_HEADER = "OPENSANDBOX-EGRESS-AUTH" SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY = "opensandbox.io/egress-auth-token" +# Environment variable name for passing network policy to egress sidecar +EGRESS_RULES_ENV = "OPENSANDBOX_EGRESS_RULES" +# Must match components/egress/pkg/constants/configuration.go EnvEgressMode +EGRESS_MODE_ENV = "OPENSANDBOX_EGRESS_MODE" +# Must match components/egress/pkg/constants/configuration.go EnvEgressToken +OPENSANDBOX_EGRESS_TOKEN = "OPENSANDBOX_EGRESS_TOKEN" + + class SandboxErrorCodes: """Canonical error codes for sandbox service operations.""" @@ -101,5 +109,8 @@ class SandboxErrorCodes: "OPEN_SANDBOX_INGRESS_HEADER", "OPEN_SANDBOX_EGRESS_AUTH_HEADER", "SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY", + "EGRESS_RULES_ENV", + "EGRESS_MODE_ENV", + "OPENSANDBOX_EGRESS_TOKEN", "SandboxErrorCodes", ] diff --git a/server/src/services/docker.py b/server/src/services/docker.py index c7cf785e..59ef583a 100644 --- a/server/src/services/docker.py +++ b/server/src/services/docker.py @@ -59,6 +59,9 @@ ) from src.config import AppConfig, get_config from src.services.constants import ( + EGRESS_MODE_ENV, + EGRESS_RULES_ENV, + OPENSANDBOX_EGRESS_TOKEN, SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY, SANDBOX_EMBEDDING_PROXY_PORT_LABEL, SANDBOX_EXPIRES_AT_LABEL, @@ -92,7 +95,6 @@ ensure_valid_host_path, ensure_volumes_valid, ) - logger = logging.getLogger(__name__) @@ -110,7 +112,6 @@ def _running_inside_docker_container() -> bool: HOST_NETWORK_MODE = "host" BRIDGE_NETWORK_MODE = "bridge" PENDING_FAILURE_TTL_SECONDS = int(os.environ.get("PENDING_FAILURE_TTL", "3600")) -EGRESS_RULES_ENV = "OPENSANDBOX_EGRESS_RULES" EGRESS_SIDECAR_LABEL = "opensandbox.io/egress-sidecar-for" @@ -1989,9 +1990,12 @@ def _start_egress_sidecar( self._ensure_image_available(egress_image, None, sandbox_id) policy_payload = json.dumps(network_policy.model_dump(by_alias=True, exclude_none=True)) + assert self.app_config.egress is not None # validated by ensure_egress_configured with networkPolicy + egress_mode = self.app_config.egress.mode sidecar_env = [ f"{EGRESS_RULES_ENV}={policy_payload}", - f"OPENSANDBOX_EGRESS_TOKEN={egress_token}", + f"{EGRESS_MODE_ENV}={egress_mode}", + f"{OPENSANDBOX_EGRESS_TOKEN}={egress_token}", ] sidecar_host_config_kwargs: dict[str, Any] = { diff --git a/server/src/services/k8s/agent_sandbox_provider.py b/server/src/services/k8s/agent_sandbox_provider.py index b1e25a31..e7636be9 100644 --- a/server/src/services/k8s/agent_sandbox_provider.py +++ b/server/src/services/k8s/agent_sandbox_provider.py @@ -29,7 +29,7 @@ V1VolumeMount, ) -from src.config import AppConfig +from src.config import AppConfig, EGRESS_MODE_DNS from src.services.helpers import format_ingress_endpoint from src.api.schema import Endpoint, ImageSpec, NetworkPolicy, Volume from src.services.k8s.agent_sandbox_template import AgentSandboxTemplateManager @@ -136,6 +136,7 @@ def create_workload( volumes: Optional[List[Volume]] = None, annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """Create an agent-sandbox Sandbox CRD workload.""" if self.runtime_class: @@ -154,6 +155,7 @@ def create_workload( network_policy=network_policy, egress_image=egress_image, egress_auth_token=egress_auth_token, + egress_mode=egress_mode, ) # Add user-specified volumes if provided @@ -217,6 +219,7 @@ def _build_pod_spec( network_policy: Optional[NetworkPolicy] = None, egress_image: Optional[str] = None, egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """Build pod spec dict for the Sandbox CRD.""" init_container = self._build_execd_init_container(execd_image) @@ -254,8 +257,9 @@ def _build_pod_spec( network_policy=network_policy, egress_image=egress_image, egress_auth_token=egress_auth_token, + egress_mode=egress_mode, ) - + return pod_spec def _build_execd_init_container(self, execd_image: str) -> V1Container: diff --git a/server/src/services/k8s/batchsandbox_provider.py b/server/src/services/k8s/batchsandbox_provider.py index a4a2eab4..942e7b8a 100644 --- a/server/src/services/k8s/batchsandbox_provider.py +++ b/server/src/services/k8s/batchsandbox_provider.py @@ -29,7 +29,7 @@ V1VolumeMount, ) -from src.config import AppConfig, INGRESS_MODE_GATEWAY +from src.config import AppConfig, EGRESS_MODE_DNS, INGRESS_MODE_GATEWAY from src.services.helpers import format_ingress_endpoint from src.api.schema import Endpoint, ImageSpec, NetworkPolicy, Volume from src.services.k8s.image_pull_secret_helper import ( @@ -115,6 +115,7 @@ def create_workload( volumes: Optional[List[Volume]] = None, annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """ Create a BatchSandbox workload. @@ -223,6 +224,7 @@ def create_workload( network_policy=network_policy, egress_image=egress_image, egress_auth_token=egress_auth_token, + egress_mode=egress_mode, ) # Add user-specified volumes if provided diff --git a/server/src/services/k8s/egress_helper.py b/server/src/services/k8s/egress_helper.py index 0e3f5efe..f79bf18f 100644 --- a/server/src/services/k8s/egress_helper.py +++ b/server/src/services/k8s/egress_helper.py @@ -23,36 +23,48 @@ from typing import Dict, Any, List, Optional from src.api.schema import NetworkPolicy +from src.config import EGRESS_MODE_DNS +from src.services.constants import ( + EGRESS_MODE_ENV, + EGRESS_RULES_ENV, + OPENSANDBOX_EGRESS_TOKEN, +) -# Environment variable name for passing network policy to egress sidecar -EGRESS_RULES_ENV = "OPENSANDBOX_EGRESS_RULES" +# Privileged sidecar: IPv6 disable is applied at container start (see EGRESS_K8S_START_COMMAND), not via Pod sysctls. +EGRESS_K8S_START_COMMAND = [ + "/bin/sh", + "-c", + "sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 && /egress", +] def build_egress_sidecar_container( egress_image: str, network_policy: NetworkPolicy, egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """ Build egress sidecar container specification for Kubernetes Pod. This function creates a container spec that can be added to a Pod's containers list. The sidecar container will: - - Run the egress image + - Run **privileged** with a startup command that runs ``sysctl`` then ``/egress`` - Receive network policy via OPENSANDBOX_EGRESS_RULES environment variable - - Have NET_ADMIN capability to manage iptables - + Note: In Kubernetes, containers in the same Pod share the network namespace, so the main container can access the sidecar's ports (44772 for execd, 8080 for HTTP) via localhost without explicit port declarations. - - Important: IPv6 should be disabled at the Pod level (not container level) using - build_ipv6_disable_sysctls() and adding the result to Pod's securityContext.sysctls. + + IPv6 for ``all`` interfaces is disabled inside the sidecar start script; Pod-level + ``net.ipv6.conf.all.disable_ipv6`` sysctl injection is not used. Args: egress_image: Container image for the egress sidecar network_policy: Network policy configuration to enforce - + egress_auth_token: Optional bearer token for egress HTTP API + egress_mode: ``dns`` or ``dns+nft`` (``OPENSANDBOX_EGRESS_MODE``) + Returns: Dict containing container specification compatible with Kubernetes Pod spec. This dict can be directly added to the Pod's containers list. @@ -67,15 +79,6 @@ def build_egress_sidecar_container( ) ) pod_spec["containers"].append(sidecar) - - # Disable IPv6 at Pod level (extends existing sysctls) - if "securityContext" not in pod_spec: - pod_spec["securityContext"] = {} - existing_sysctls = pod_spec["securityContext"].get("sysctls") - new_sysctls = build_ipv6_disable_sysctls() - pod_spec["securityContext"]["sysctls"] = _merge_sysctls( - existing_sysctls, new_sysctls - ) ``` """ # Serialize network policy to JSON for environment variable @@ -87,12 +90,16 @@ def build_egress_sidecar_container( { "name": EGRESS_RULES_ENV, "value": policy_payload, - } + }, + { + "name": EGRESS_MODE_ENV, + "value": egress_mode, + }, ] if egress_auth_token: env.append( { - "name": "OPENSANDBOX_EGRESS_TOKEN", + "name": OPENSANDBOX_EGRESS_TOKEN, "value": egress_auth_token, } ) @@ -101,29 +108,26 @@ def build_egress_sidecar_container( container_spec: Dict[str, Any] = { "name": "egress", "image": egress_image, + "command": EGRESS_K8S_START_COMMAND, "env": env, "securityContext": _build_security_context_for_egress(), } - + return container_spec def _build_security_context_for_egress() -> Dict[str, Any]: """ Build security context for egress sidecar container. - - The egress sidecar needs NET_ADMIN capability to manage iptables rules - for network policy enforcement. - - This is an internal helper function used by build_egress_sidecar_container(). - + + The sidecar runs **privileged** so it can manage iptables/nft and run sysctl + inside the container network namespace at startup. + Returns: - Dict containing security context configuration with NET_ADMIN capability. + Dict containing security context with ``privileged: true``. """ return { - "capabilities": { - "add": ["NET_ADMIN"], - }, + "privileged": True, } @@ -154,63 +158,28 @@ def build_security_context_for_sandbox_container( } -def _merge_sysctls( - existing_sysctls: Optional[List[Dict[str, str]]], - new_sysctls: List[Dict[str, str]], -) -> List[Dict[str, str]]: - """ - Merge new sysctls into existing sysctls, avoiding duplicates. - - If a sysctl with the same name already exists, the new value will - override the existing one (last write wins). - - Args: - existing_sysctls: Existing sysctls list or None - new_sysctls: New sysctls to merge in - - Returns: - Merged list of sysctls with no duplicate names - """ - if not existing_sysctls: - return new_sysctls.copy() - - # Create a dict to track sysctls by name (for deduplication) - sysctls_dict: Dict[str, str] = {} - - # First, add existing sysctls - for sysctl in existing_sysctls: - if isinstance(sysctl, dict) and "name" in sysctl: - sysctls_dict[sysctl["name"]] = sysctl.get("value", "") - - # Then, add/override with new sysctls - for sysctl in new_sysctls: - if isinstance(sysctl, dict) and "name" in sysctl: - sysctls_dict[sysctl["name"]] = sysctl.get("value", "") - - # Convert back to list format - return [{"name": name, "value": value} for name, value in sysctls_dict.items()] - - def apply_egress_to_spec( pod_spec: Dict[str, Any], containers: List[Dict[str, Any]], network_policy: Optional[NetworkPolicy], egress_image: Optional[str], egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> None: """ Apply egress sidecar configuration to Pod spec. - This function adds the egress sidecar container to the containers list - and configures IPv6 disable sysctls at the Pod level when network policy - is provided. Existing sysctls are preserved and merged with the new ones. + This function adds the egress sidecar container to the containers list. + It does **not** mutate Pod ``securityContext.sysctls`` for IPv6; the sidecar + disables ``net.ipv6.conf.all.disable_ipv6`` via its startup command. Args: pod_spec: Pod specification dict (will be modified in place) containers: List of container dicts (will be modified in place) network_policy: Optional network policy configuration egress_image: Optional egress sidecar image - + egress_mode: ``dns`` or ``dns+nft`` (default ``EGRESS_MODE_DNS``). + Example: ```python containers = [main_container_dict] @@ -224,31 +193,18 @@ def apply_egress_to_spec( ) ``` - Note: - This function extends existing sysctls rather than overwriting them. - If a sysctl with the same name already exists, the egress-related - sysctls will override it (last write wins). """ if not network_policy or not egress_image: return - + # Build and add egress sidecar container sidecar_container = build_egress_sidecar_container( egress_image=egress_image, network_policy=network_policy, egress_auth_token=egress_auth_token, + egress_mode=egress_mode, ) containers.append(sidecar_container) - - # Disable IPv6 at Pod level, merging with existing sysctls - if "securityContext" not in pod_spec: - pod_spec["securityContext"] = {} - - existing_sysctls = pod_spec["securityContext"].get("sysctls") - new_sysctls = build_ipv6_disable_sysctls() - pod_spec["securityContext"]["sysctls"] = _merge_sysctls( - existing_sysctls, new_sysctls - ) def build_security_context_from_dict( @@ -282,9 +238,9 @@ def build_security_context_from_dict( """ if not security_context_dict: return None - + from kubernetes.client import V1SecurityContext, V1Capabilities - + capabilities = None if "capabilities" in security_context_dict: caps_dict = security_context_dict["capabilities"] @@ -294,8 +250,16 @@ def build_security_context_from_dict( add=add_caps if add_caps else None, drop=drop_caps if drop_caps else None, ) - - return V1SecurityContext(capabilities=capabilities) + + privileged = security_context_dict.get("privileged") + + if capabilities is None and privileged is None: + return None + + return V1SecurityContext( + capabilities=capabilities, + privileged=privileged, + ) def serialize_security_context_to_dict( @@ -330,7 +294,7 @@ def serialize_security_context_to_dict( return None result: Dict[str, Any] = {} - + if security_context.capabilities: caps: Dict[str, Any] = {} if security_context.capabilities.add: @@ -339,28 +303,8 @@ def serialize_security_context_to_dict( caps["drop"] = security_context.capabilities.drop if caps: result["capabilities"] = caps - - return result if result else None + if security_context.privileged is not None: + result["privileged"] = security_context.privileged -def build_ipv6_disable_sysctls() -> list[Dict[str, str]]: - """ - Build sysctls configuration to disable IPv6 in the Pod. - - When egress sidecar is used, IPv6 should be disabled in the shared network - namespace to keep policy enforcement consistent. This matches the Docker - implementation behavior. - - Returns: - List of sysctl configurations to disable IPv6 at Pod level. - - Note: - These sysctls need to be set at the Pod's securityContext level, not - at the container level. The calling code should merge this into the - Pod spec's securityContext.sysctls field. - """ - return [ - {"name": "net.ipv6.conf.all.disable_ipv6", "value": "1"}, - {"name": "net.ipv6.conf.default.disable_ipv6", "value": "1"}, - {"name": "net.ipv6.conf.lo.disable_ipv6", "value": "1"}, - ] + return result if result else None diff --git a/server/src/services/k8s/kubernetes_service.py b/server/src/services/k8s/kubernetes_service.py index 288340fc..b4b6d30e 100644 --- a/server/src/services/k8s/kubernetes_service.py +++ b/server/src/services/k8s/kubernetes_service.py @@ -39,7 +39,7 @@ Sandbox, SandboxStatus, ) -from src.config import AppConfig, get_config +from src.config import AppConfig, EGRESS_MODE_DNS, get_config from src.services.constants import ( SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY, SANDBOX_ID_LABEL, @@ -302,6 +302,11 @@ def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse resource_limits = request.resource_limits.root try: + egress_mode = ( + self.app_config.egress.mode + if self.app_config.egress + else EGRESS_MODE_DNS + ) # Get egress image if network policy is provided egress_image = None egress_auth_token = None @@ -332,6 +337,7 @@ def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse network_policy=request.network_policy, egress_image=egress_image, egress_auth_token=egress_auth_token, + egress_mode=egress_mode, volumes=request.volumes, ) diff --git a/server/src/services/k8s/workload_provider.py b/server/src/services/k8s/workload_provider.py index a73b326a..3884db4f 100644 --- a/server/src/services/k8s/workload_provider.py +++ b/server/src/services/k8s/workload_provider.py @@ -21,6 +21,7 @@ from typing import Dict, List, Any, Optional from src.api.schema import Endpoint, ImageSpec, NetworkPolicy, Volume +from src.config import EGRESS_MODE_DNS class WorkloadProvider(ABC): @@ -49,6 +50,7 @@ def create_workload( volumes: Optional[List[Volume]] = None, annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, + egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """ Create a new workload resource. @@ -68,6 +70,7 @@ def create_workload( network_policy: Optional network policy for egress traffic control. When provided, an egress sidecar container will be added to the Pod. egress_image: Optional egress sidecar image. Required when network_policy is provided. + egress_mode: Sidecar ``OPENSANDBOX_EGRESS_MODE`` (from app ``[egress].mode`` when using network policy). volumes: Optional list of volume mounts for the sandbox. Returns: diff --git a/server/tests/k8s/test_agent_sandbox_provider.py b/server/tests/k8s/test_agent_sandbox_provider.py index 7f57247b..425d1385 100644 --- a/server/tests/k8s/test_agent_sandbox_provider.py +++ b/server/tests/k8s/test_agent_sandbox_provider.py @@ -24,9 +24,18 @@ from kubernetes.client import ApiException from src.api.schema import ImageSpec, NetworkPolicy, NetworkRule -from src.config import AppConfig, ExecdInitResources, KubernetesRuntimeConfig, AgentSandboxRuntimeConfig, RuntimeConfig +from src.config import ( + AppConfig, + AgentSandboxRuntimeConfig, + EGRESS_MODE_DNS, + EGRESS_MODE_DNS_NFT, + ExecdInitResources, + KubernetesRuntimeConfig, + RuntimeConfig, +) from src.services.constants import SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY from src.services.k8s.agent_sandbox_provider import AgentSandboxProvider +from src.services.constants import OPENSANDBOX_EGRESS_TOKEN def _app_config(shutdown_policy: str = "Delete", service_account: str | None = None, execd_init_resources: ExecdInitResources | None = None) -> AppConfig: @@ -529,12 +538,10 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) # Verify sidecar has environment variable env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} assert "OPENSANDBOX_EGRESS_RULES" in env_vars - - # Verify sidecar has NET_ADMIN capability - assert "securityContext" in sidecar - assert "capabilities" in sidecar["securityContext"] - assert "add" in sidecar["securityContext"]["capabilities"] - assert "NET_ADMIN" in sidecar["securityContext"]["capabilities"]["add"] + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS + + assert sidecar.get("securityContext", {}).get("privileged") is True + assert "net.ipv6.conf.all.disable_ipv6=1" in sidecar["command"][2] def test_create_workload_with_network_policy_persists_annotation_and_sidecar_token(self, mock_k8s_client): provider = AgentSandboxProvider(mock_k8s_client) @@ -565,12 +572,38 @@ def test_create_workload_with_network_policy_persists_annotation_and_sidecar_tok sidecar = next((c for c in containers if c["name"] == "egress"), None) assert sidecar is not None env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} - assert env_vars["OPENSANDBOX_EGRESS_TOKEN"] == "egress-token" + assert env_vars[OPENSANDBOX_EGRESS_TOKEN] == "egress-token" + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS - def test_create_workload_with_network_policy_adds_ipv6_disable_sysctls(self, mock_k8s_client): - """ - Test case: Verify IPv6 disable sysctls are added to Pod spec - """ + def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): + provider = AgentSandboxProvider(mock_k8s_client) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "test-id", "uid": "test-uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=None, + execd_image="execd:latest", + network_policy=NetworkPolicy(default_action="deny", egress=[]), + egress_image="opensandbox/egress:v1.0.3", + egress_mode=EGRESS_MODE_DNS_NFT, + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + containers = body["spec"]["podTemplate"]["spec"]["containers"] + sidecar = next((c for c in containers if c["name"] == "egress"), None) + assert sidecar is not None + env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS_NFT + + def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client): provider = AgentSandboxProvider(mock_k8s_client) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} @@ -598,22 +631,11 @@ def test_create_workload_with_network_policy_adds_ipv6_disable_sysctls(self, moc body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] pod_spec = body["spec"]["podTemplate"]["spec"] - - # Verify securityContext with sysctls exists - assert "securityContext" in pod_spec - assert "sysctls" in pod_spec["securityContext"] - - sysctls = pod_spec["securityContext"]["sysctls"] - sysctl_names = {s["name"] for s in sysctls} - - # Verify all IPv6 disable sysctls are present - assert "net.ipv6.conf.all.disable_ipv6" in sysctl_names - assert "net.ipv6.conf.default.disable_ipv6" in sysctl_names - assert "net.ipv6.conf.lo.disable_ipv6" in sysctl_names - - # Verify all values are "1" - for sysctl in sysctls: - assert sysctl["value"] == "1" + + assert "securityContext" not in pod_spec or "sysctls" not in pod_spec.get("securityContext", {}) + + sidecar = next(c for c in pod_spec["containers"] if c["name"] == "egress") + assert sidecar["command"][2].startswith("sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1") def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client): """ diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py index 654556ed..ddc66b4a 100644 --- a/server/tests/k8s/test_batchsandbox_provider.py +++ b/server/tests/k8s/test_batchsandbox_provider.py @@ -22,9 +22,17 @@ from kubernetes.client import ApiException from src.api.schema import ImageSpec, ImageAuth, NetworkPolicy, NetworkRule -from src.config import AppConfig, ExecdInitResources, KubernetesRuntimeConfig, RuntimeConfig +from src.config import ( + AppConfig, + EGRESS_MODE_DNS, + EGRESS_MODE_DNS_NFT, + ExecdInitResources, + KubernetesRuntimeConfig, + RuntimeConfig, +) from src.services.constants import SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY from src.services.k8s.batchsandbox_provider import BatchSandboxProvider +from src.services.constants import OPENSANDBOX_EGRESS_TOKEN from src.services.k8s.image_pull_secret_helper import IMAGE_AUTH_SECRET_PREFIX from src.services.k8s.volume_helper import apply_volumes_to_pod_spec @@ -1235,12 +1243,12 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) # Verify sidecar has environment variable env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} assert "OPENSANDBOX_EGRESS_RULES" in env_vars - - # Verify sidecar has NET_ADMIN capability - assert "securityContext" in sidecar - assert "capabilities" in sidecar["securityContext"] - assert "add" in sidecar["securityContext"]["capabilities"] - assert "NET_ADMIN" in sidecar["securityContext"]["capabilities"]["add"] + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS + + # Egress sidecar: privileged + sysctl + /egress startup (Kubernetes) + assert sidecar.get("securityContext", {}).get("privileged") is True + assert sidecar.get("command") is not None + assert "net.ipv6.conf.all.disable_ipv6=1" in sidecar["command"][2] def test_create_workload_with_network_policy_persists_annotation_and_sidecar_token(self, mock_k8s_client): provider = BatchSandboxProvider(mock_k8s_client) @@ -1271,12 +1279,39 @@ def test_create_workload_with_network_policy_persists_annotation_and_sidecar_tok sidecar = next((c for c in containers if c["name"] == "egress"), None) assert sidecar is not None env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} - assert env_vars["OPENSANDBOX_EGRESS_TOKEN"] == "egress-token" + assert env_vars[OPENSANDBOX_EGRESS_TOKEN] == "egress-token" + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS - def test_create_workload_with_network_policy_adds_ipv6_disable_sysctls(self, mock_k8s_client): - """ - Test case: Verify IPv6 disable sysctls are added to Pod spec - """ + def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): + provider = BatchSandboxProvider(mock_k8s_client) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "test-id", "uid": "test-uid"} + } + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=None, + execd_image="execd:latest", + network_policy=NetworkPolicy(default_action="deny", egress=[]), + egress_image="opensandbox/egress:v1.0.3", + egress_mode=EGRESS_MODE_DNS_NFT, + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + containers = body["spec"]["template"]["spec"]["containers"] + sidecar = next((c for c in containers if c["name"] == "egress"), None) + assert sidecar is not None + env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])} + assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS_NFT + + def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client): + """IPv6 all.disable is applied in egress sidecar start command, not Pod sysctls.""" provider = BatchSandboxProvider(mock_k8s_client) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} @@ -1304,22 +1339,11 @@ def test_create_workload_with_network_policy_adds_ipv6_disable_sysctls(self, moc body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] pod_spec = body["spec"]["template"]["spec"] - - # Verify securityContext with sysctls exists - assert "securityContext" in pod_spec - assert "sysctls" in pod_spec["securityContext"] - - sysctls = pod_spec["securityContext"]["sysctls"] - sysctl_names = {s["name"] for s in sysctls} - - # Verify all IPv6 disable sysctls are present - assert "net.ipv6.conf.all.disable_ipv6" in sysctl_names - assert "net.ipv6.conf.default.disable_ipv6" in sysctl_names - assert "net.ipv6.conf.lo.disable_ipv6" in sysctl_names - - # Verify all values are "1" - for sysctl in sysctls: - assert sysctl["value"] == "1" + + assert "securityContext" not in pod_spec or "sysctls" not in pod_spec.get("securityContext", {}) + + sidecar = next(c for c in pod_spec["containers"] if c["name"] == "egress") + assert sidecar["command"][2].startswith("sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1") def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client): """ @@ -1535,10 +1559,9 @@ def test_create_workload_with_network_policy_works_with_template(self, mock_k8s_ sidecar = next((c for c in containers if c["name"] == "egress"), None) assert sidecar is not None - # Verify IPv6 sysctls are present - assert "securityContext" in pod_spec - assert "sysctls" in pod_spec["securityContext"] - + # Pod-level IPv6 sysctls are not injected for egress (sidecar startup handles all.disable) + assert "securityContext" not in pod_spec or "sysctls" not in pod_spec.get("securityContext", {}) + # Verify template volumes are still merged volume_names = [v["name"] for v in pod_spec["volumes"]] assert "sandbox-shared-data" in volume_names diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index 72c49dc4..00d29be4 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -19,12 +19,13 @@ import json from src.api.schema import NetworkPolicy, NetworkRule +from src.config import EGRESS_MODE_DNS, EGRESS_MODE_DNS_NFT +from src.services.constants import EGRESS_MODE_ENV, EGRESS_RULES_ENV, OPENSANDBOX_EGRESS_TOKEN from src.services.k8s.egress_helper import ( - EGRESS_RULES_ENV, + EGRESS_K8S_START_COMMAND, apply_egress_to_spec, build_egress_sidecar_container, build_security_context_for_sandbox_container, - build_ipv6_disable_sysctls, ) @@ -59,9 +60,11 @@ def test_contains_egress_rules_environment_variable(self): container = build_egress_sidecar_container(egress_image, network_policy) env_vars = container["env"] - assert len(env_vars) == 1 + assert len(env_vars) == 2 assert env_vars[0]["name"] == EGRESS_RULES_ENV assert env_vars[0]["value"] is not None + assert env_vars[1]["name"] == EGRESS_MODE_ENV + assert env_vars[1]["value"] == EGRESS_MODE_DNS def test_contains_egress_token_when_provided(self): egress_image = "opensandbox/egress:v1.0.3" @@ -77,7 +80,24 @@ def test_contains_egress_token_when_provided(self): ) env_vars = {env["name"]: env["value"] for env in container["env"]} - assert env_vars["OPENSANDBOX_EGRESS_TOKEN"] == "egress-token" + assert env_vars[OPENSANDBOX_EGRESS_TOKEN] == "egress-token" + assert env_vars[EGRESS_MODE_ENV] == EGRESS_MODE_DNS + + def test_egress_mode_dns_nft(self): + egress_image = "opensandbox/egress:v1.0.3" + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + container = build_egress_sidecar_container( + egress_image, + network_policy, + egress_mode=EGRESS_MODE_DNS_NFT, + ) + + env_vars = {env["name"]: env["value"] for env in container["env"]} + assert env_vars[EGRESS_MODE_ENV] == EGRESS_MODE_DNS_NFT def test_serializes_network_policy_correctly(self): """Test that network policy is correctly serialized to JSON.""" @@ -138,8 +158,8 @@ def test_handles_missing_default_action(self): assert "defaultAction" not in policy_dict or policy_dict.get("defaultAction") is None assert "egress" in policy_dict - def test_security_context_has_net_admin_capability(self): - """Test that security context includes NET_ADMIN capability.""" + def test_security_context_is_privileged(self): + """Egress sidecar runs privileged (Kubernetes).""" egress_image = "opensandbox/egress:v1.0.3" network_policy = NetworkPolicy( default_action="deny", @@ -149,9 +169,16 @@ def test_security_context_has_net_admin_capability(self): container = build_egress_sidecar_container(egress_image, network_policy) security_context = container["securityContext"] - assert "capabilities" in security_context - assert "add" in security_context["capabilities"] - assert "NET_ADMIN" in security_context["capabilities"]["add"] + assert security_context.get("privileged") is True + + def test_start_command_runs_sysctl_then_egress(self): + container = build_egress_sidecar_container( + "opensandbox/egress:v1.0.3", + NetworkPolicy(default_action="deny", egress=[]), + ) + assert container["command"] == EGRESS_K8S_START_COMMAND + assert "net.ipv6.conf.all.disable_ipv6=1" in container["command"][2] + assert container["command"][2].endswith("&& /egress") def test_container_spec_is_valid_kubernetes_format(self): """Test that returned container spec is in valid Kubernetes format.""" @@ -174,6 +201,7 @@ def test_container_spec_is_valid_kubernetes_format(self): assert len(container["env"]) > 0 assert "name" in container["env"][0] assert "value" in container["env"][0] + assert "command" in container def test_handles_wildcard_domains(self): """Test that wildcard domains in egress rules are handled correctly.""" @@ -213,49 +241,6 @@ def test_drops_net_admin_when_network_policy_enabled(self): assert "NET_ADMIN" in result["capabilities"]["drop"] -class TestBuildIpv6DisableSysctls: - """Tests for build_ipv6_disable_sysctls function.""" - - def test_returns_list_of_sysctls(self): - """Test that function returns a list of sysctl configurations.""" - sysctls = build_ipv6_disable_sysctls() - - assert isinstance(sysctls, list) - assert len(sysctls) == 3 - - def test_contains_all_required_ipv6_disable_sysctls(self): - """Test that all required IPv6 disable sysctls are present.""" - sysctls = build_ipv6_disable_sysctls() - - sysctl_names = {s["name"] for s in sysctls} - expected_names = { - "net.ipv6.conf.all.disable_ipv6", - "net.ipv6.conf.default.disable_ipv6", - "net.ipv6.conf.lo.disable_ipv6", - } - - assert sysctl_names == expected_names - - def test_all_sysctls_have_value_one(self): - """Test that all sysctls have value "1".""" - sysctls = build_ipv6_disable_sysctls() - - for sysctl in sysctls: - assert sysctl["value"] == "1" - assert "name" in sysctl - - def test_sysctls_are_in_valid_kubernetes_format(self): - """Test that sysctls are in valid Kubernetes format.""" - sysctls = build_ipv6_disable_sysctls() - - for sysctl in sysctls: - assert isinstance(sysctl, dict) - assert "name" in sysctl - assert "value" in sysctl - assert isinstance(sysctl["name"], str) - assert isinstance(sysctl["value"], str) - - class TestApplyEgressToSpec: """Tests for apply_egress_to_spec function.""" @@ -280,8 +265,8 @@ def test_adds_egress_sidecar_container(self): assert containers[0]["name"] == "egress" assert containers[0]["image"] == egress_image - def test_adds_ipv6_disable_sysctls(self): - """Test that IPv6 disable sysctls are added to Pod spec.""" + def test_does_not_add_pod_sysctls_for_ipv6(self): + """IPv6 disable is not merged into Pod securityContext.sysctls (sidecar start script).""" pod_spec: dict = {} containers: list = [] network_policy = NetworkPolicy( @@ -297,20 +282,15 @@ def test_adds_ipv6_disable_sysctls(self): egress_image=egress_image, ) - assert "securityContext" in pod_spec - assert "sysctls" in pod_spec["securityContext"] - sysctls = pod_spec["securityContext"]["sysctls"] - assert len(sysctls) == 3 - sysctl_names = {s["name"] for s in sysctls} - assert "net.ipv6.conf.all.disable_ipv6" in sysctl_names + assert "securityContext" not in pod_spec - def test_extends_existing_sysctls(self): - """Test that existing sysctls are preserved and merged.""" + def test_preserves_existing_pod_sysctls_without_merging_ipv6(self): + """Existing Pod sysctls are left unchanged when egress is applied.""" pod_spec: dict = { "securityContext": { "sysctls": [ {"name": "net.core.somaxconn", "value": "1024"}, - {"name": "net.ipv6.conf.all.disable_ipv6", "value": "0"}, # Will be overridden + {"name": "net.ipv6.conf.all.disable_ipv6", "value": "0"}, ] } } @@ -330,19 +310,10 @@ def test_extends_existing_sysctls(self): sysctls = pod_spec["securityContext"]["sysctls"] sysctl_dict = {s["name"]: s["value"] for s in sysctls} - - # Verify existing sysctl is preserved - assert "net.core.somaxconn" in sysctl_dict + assert sysctl_dict["net.core.somaxconn"] == "1024" - - # Verify IPv6 sysctls are added/updated - assert "net.ipv6.conf.all.disable_ipv6" in sysctl_dict - assert sysctl_dict["net.ipv6.conf.all.disable_ipv6"] == "1" # Overridden - assert "net.ipv6.conf.default.disable_ipv6" in sysctl_dict - assert "net.ipv6.conf.lo.disable_ipv6" in sysctl_dict - - # Verify total count (1 existing + 3 IPv6, but one was overridden, so 4 total) - assert len(sysctls) == 4 + assert sysctl_dict["net.ipv6.conf.all.disable_ipv6"] == "0" + assert len(sysctls) == 2 def test_no_op_when_no_network_policy(self): """Test that function does nothing when network_policy is None.""" diff --git a/server/tests/k8s/test_kubernetes_service.py b/server/tests/k8s/test_kubernetes_service.py index 7c642a0f..101aa03d 100644 --- a/server/tests/k8s/test_kubernetes_service.py +++ b/server/tests/k8s/test_kubernetes_service.py @@ -29,7 +29,7 @@ SandboxErrorCodes, ) from src.api.schema import ImageAuth, ListSandboxesRequest, NetworkPolicy -from src.config import EgressConfig +from src.config import EGRESS_MODE_DNS, EGRESS_MODE_DNS_NFT, EgressConfig from src.api.schema import Endpoint @@ -244,8 +244,35 @@ def test_create_sandbox_with_network_policy_passes_egress_token_and_annotations( _, kwargs = k8s_service.workload_provider.create_workload.call_args assert kwargs["egress_auth_token"] == "egress-token" + assert kwargs["egress_mode"] == EGRESS_MODE_DNS assert kwargs["annotations"][SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] == "egress-token" + def test_create_sandbox_with_network_policy_passes_egress_mode_dns_nft_from_config( + self, k8s_service, create_sandbox_request + ): + create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[]) + k8s_service.app_config.egress = EgressConfig( + image="opensandbox/egress:v1.0.3", + mode=EGRESS_MODE_DNS_NFT, + ) + k8s_service.workload_provider.create_workload.return_value = { + "name": "test-id", "uid": "uid-1" + } + k8s_service.workload_provider.get_workload.return_value = MagicMock() + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", "reason": "", "message": "", + "last_transition_at": datetime.now(timezone.utc), + } + + with patch( + "src.services.k8s.kubernetes_service.generate_egress_token", + return_value="egress-token", + ): + k8s_service.create_sandbox(create_sandbox_request) + + _, kwargs = k8s_service.workload_provider.create_workload.call_args + assert kwargs["egress_mode"] == EGRESS_MODE_DNS_NFT + def test_get_endpoint_merges_egress_auth_header_from_instance_metadata( self, k8s_service ): diff --git a/server/tests/test_config.py b/server/tests/test_config.py index f069362d..33b69ad7 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -19,12 +19,15 @@ from src import config as config_module from src.config import ( AppConfig, + EGRESS_MODE_DNS, + EGRESS_MODE_DNS_NFT, + EgressConfig, GatewayConfig, GatewayRouteModeConfig, IngressConfig, RuntimeConfig, ServerConfig, - StorageConfig + StorageConfig, ) @@ -601,3 +604,9 @@ def test_kubernetes_runtime_with_firecracker_is_valid(): assert cfg.secure_runtime is not None assert cfg.secure_runtime.type == "firecracker" assert cfg.secure_runtime.k8s_runtime_class == "kata-fc" + + +def test_egress_config_mode_literal(): + assert EgressConfig(image="opensandbox/egress:v1").mode == EGRESS_MODE_DNS + cfg = EgressConfig(image="opensandbox/egress:v1", mode=EGRESS_MODE_DNS_NFT) + assert cfg.mode == EGRESS_MODE_DNS_NFT diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index c414e7ca..dbc61a19 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -24,12 +24,14 @@ from src.config import ( AppConfig, + EGRESS_MODE_DNS, EgressConfig, RuntimeConfig, ServerConfig, StorageConfig, IngressConfig, ) +from src.services.constants import EGRESS_MODE_ENV, OPENSANDBOX_EGRESS_TOKEN from src.services.constants import ( SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY, SANDBOX_EXPIRES_AT_LABEL, @@ -398,7 +400,8 @@ def host_cfg_side_effect(**kwargs): assert labels[SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] == "egress-token" sidecar_env = sidecar_kwargs["environment"] - assert "OPENSANDBOX_EGRESS_TOKEN=egress-token" in sidecar_env + assert f"{OPENSANDBOX_EGRESS_TOKEN}=egress-token" in sidecar_env + assert f"{EGRESS_MODE_ENV}={EGRESS_MODE_DNS}" in sidecar_env # ---------------------------------------------------------------------------