feat(egress): add nameserver exempt for direct DNS forwarding (#356)

* feat(egress): add nameserver exempt for direct DNS forwarding

* fix(egress): nameserver exempt to IP-only and ensure nft allow set

* chore(egress): add unittest for ParseNameserverExemptList

* fix(egress): cache nameserver exempt addrs and split exempt tests

Author: Pangjiping

components/egress/README.md

@@ -42,6 +42,8 @@ The egress control is implemented as a **Sidecar** that shares the network names
 - Mode (`OPENSANDBOX_EGRESS_MODE`, default `dns`):
   - `dns`: DNS proxy only, no nftables (IP/CIDR rules have no effect at L2).
   - `dns+nft`: enable nftables; if nft apply fails, fallback to `dns`. IP/CIDR enforcement and DoH/DoT blocking require this mode.
+- **Nameserver exempt**  
+  Set `OPENSANDBOX_EGRESS_NAMESERVER_EXEMPT` to a comma-separated list of **nameserver IPs** (e.g. `26.26.26.26` or `26.26.26.26,100.100.2.116`). Only single IPs are supported; CIDR entries are ignored. Traffic to these IPs on port 53 is not redirected to the proxy (iptables RETURN). In `dns+nft` mode, these IPs are also merged into the nft allow set so proxy upstream traffic to them (sent without SO_MARK) is accepted. Use when the upstream is reachable only via a specific route (e.g. tunnel) and SO_MARK would send proxy traffic elsewhere.
 - **DNS and nft mode (nameserver whitelist)**  
   In `dns+nft` mode, the sidecar automatically allows:
   - **127.0.0.1** — so packets redirected by iptables to the proxy (127.0.0.1:15353) are accepted by nft.
@@ -178,6 +180,5 @@ More details in [docs/benchmark.md](docs/benchmark.md).
 
 - **"iptables setup failed"**: Ensure the sidecar container has `--cap-add=NET_ADMIN`.
 - **DNS resolution fails for all domains**:  
-  - Check if the upstream DNS (from `/etc/resolv.conf`) is reachable.  
-  - In `dns+nft` mode, the sidecar whitelists nameserver IPs from resolv.conf at startup; check logs for `[dns] whitelisting proxy listen + N nameserver(s)` and ensure `/etc/resolv.conf` is readable and contains valid, reachable nameservers. The proxy prefers the first non-loopback nameserver from resolv.conf; if only loopback exists (e.g. Docker 127.0.0.11), it is used (proxy upstream traffic bypasses the redirect). Fallback to 8.8.8.8 only when resolv.conf is empty or unreadable.
+  Check upstream reachability from the sidecar (`ip route`, `dig @<upstream> . NS +timeout=3`). In `dns+nft` mode, check logs for `[dns] whitelisting proxy listen + N nameserver(s)`.
 - **Traffic not blocked**: If nftables apply fails, the sidecar falls back to dns; check logs, `nft list table inet opensandbox`, and `CAP_NET_ADMIN`.

components/egress/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"context"
+	"net/netip"
 	"os"
 	"os/signal"
 	"strings"
@@ -44,6 +45,12 @@ func main() {
 	}
 
 	allowIPs := AllowIPsForNft("/etc/resolv.conf")
+	// Merge nameserver exempt IPs into nft allow set so proxy traffic to them (no SO_MARK) is allowed in dns+nft mode.
+	for _, addr := range dnsproxy.ParseNameserverExemptList() {
+		if !containsAddr(allowIPs, addr) {
+			allowIPs = append(allowIPs, addr)
+		}
+	}
 
 	mode := parseMode()
 	log.Infof("enforcement mode: %s", mode)
@@ -57,7 +64,11 @@ func main() {
 	}
 	log.Infof("dns proxy started on 127.0.0.1:15353")
 
-	if err := iptables.SetupRedirect(15353); err != nil {
+	exemptDst := dnsproxy.ParseNameserverExemptList()
+	if len(exemptDst) > 0 {
+		log.Infof("nameserver exempt list: %v (proxy upstream in this list will not set SO_MARK)", exemptDst)
+	}
+	if err := iptables.SetupRedirect(15353, exemptDst); err != nil {
 		log.Fatalf("failed to install iptables redirect: %v", err)
 	}
 	log.Infof("iptables redirect configured (OUTPUT 53 -> 15353) with SO_MARK bypass for proxy upstream traffic")
@@ -98,6 +109,15 @@ func isTruthy(v string) bool {
 	}
 }
 
+func containsAddr(addrs []netip.Addr, a netip.Addr) bool {
+	for _, x := range addrs {
+		if x == a {
+			return true
+		}
+	}
+	return false
+}
+
 func parseMode() string {
 	mode := strings.ToLower(strings.TrimSpace(os.Getenv(constants.EnvEgressMode)))
 	switch mode {

components/egress/pkg/constants/configuration.go

@@ -23,6 +23,9 @@ const (
 	EnvEgressRules    = "OPENSANDBOX_EGRESS_RULES"
 	EnvEgressLogLevel = "OPENSANDBOX_EGRESS_LOG_LEVEL"
 	EnvMaxNameservers = "OPENSANDBOX_EGRESS_MAX_NS"
+
+	// EnvNameserverExempt comma-separated IPs; proxy upstream to these is not marked and is allowed in nft allow set
+	EnvNameserverExempt = "OPENSANDBOX_EGRESS_NAMESERVER_EXEMPT"
 )
 
 const (

components/egress/pkg/dnsproxy/exempt.go

@@ -0,0 +1,75 @@
+// Copyright 2026 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dnsproxy
+
+import (
+	"net/netip"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/alibaba/opensandbox/egress/pkg/constants"
+)
+
+var (
+	exemptListOnce sync.Once
+	exemptAddrs    []netip.Addr
+	exemptSet      map[netip.Addr]struct{}
+)
+
+// ParseNameserverExemptList returns IPs from OPENSANDBOX_EGRESS_NAMESERVER_EXEMPT (comma-separated).
+// Only single IPs are accepted; invalid or CIDR entries are skipped. Result is cached. Used for nft allow set, iptables, and UpstreamInExemptList.
+func ParseNameserverExemptList() []netip.Addr {
+	exemptListOnce.Do(func() { parseNameserverExemptListUncached() })
+	return exemptAddrs
+}
+
+func parseNameserverExemptListUncached() {
+	raw := strings.TrimSpace(os.Getenv(constants.EnvNameserverExempt))
+	if raw == "" {
+		exemptAddrs = nil
+		exemptSet = nil
+		return
+	}
+	set := make(map[netip.Addr]struct{})
+	var out []netip.Addr
+	for _, s := range strings.Split(raw, ",") {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		if addr, err := netip.ParseAddr(s); err == nil {
+			if _, exists := set[addr]; exists {
+				continue
+			}
+			set[addr] = struct{}{}
+			out = append(out, addr)
+		}
+	}
+	exemptAddrs = out
+	exemptSet = set
+}
+
+// UpstreamInExemptList returns true when upstreamHost is in the nameserver exempt list (exact IP match).
+// When true, the proxy should not set SO_MARK so upstream traffic follows normal routing (e.g. via tun).
+func UpstreamInExemptList(upstreamHost string) bool {
+	addr, err := netip.ParseAddr(upstreamHost)
+	if err != nil {
+		return false
+	}
+	ParseNameserverExemptList() // ensure cache is initialized
+	_, ok := exemptSet[addr]
+	return ok
+}

components/egress/pkg/dnsproxy/exempt_test.go

@@ -0,0 +1,77 @@
+// Copyright 2026 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dnsproxy
+
+import (
+	"net/netip"
+	"reflect"
+	"sync"
+	"testing"
+
+	"github.com/alibaba/opensandbox/egress/pkg/constants"
+)
+
+func resetNameserverExemptCache(t *testing.T) {
+	t.Helper()
+	exemptAddrs = nil
+	exemptSet = nil
+	exemptListOnce = sync.Once{}
+}
+
+func TestParseNameserverExemptList_IPOnly(t *testing.T) {
+	t.Setenv(constants.EnvNameserverExempt, "1.1.1.1, 2001:db8::1 ,invalid, 10.0.0.0/8, ,")
+	resetNameserverExemptCache(t)
+
+	got := ParseNameserverExemptList()
+	want := []netip.Addr{netip.MustParseAddr("1.1.1.1"), netip.MustParseAddr("2001:db8::1")}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("ParseNameserverExemptList() = %v, want %v", got, want)
+	}
+
+	// Cached result should stay the same on subsequent calls.
+	if got2 := ParseNameserverExemptList(); !reflect.DeepEqual(got2, want) {
+		t.Fatalf("cached ParseNameserverExemptList() = %v, want %v", got2, want)
+	}
+}
+
+func TestUpstreamInExemptList_IPOnly(t *testing.T) {
+	t.Setenv(constants.EnvNameserverExempt, "1.1.1.1,2001:db8::1")
+	resetNameserverExemptCache(t)
+
+	if !UpstreamInExemptList("1.1.1.1") {
+		t.Fatalf("expected IPv4 upstream to be exempt")
+	}
+	if !UpstreamInExemptList("2001:db8::1") {
+		t.Fatalf("expected IPv6 upstream to be exempt")
+	}
+	if UpstreamInExemptList("10.0.0.2") {
+		t.Fatalf("unexpected exempt match for non-listed IP")
+	}
+	if UpstreamInExemptList("not-an-ip") {
+		t.Fatalf("invalid IP string should not match")
+	}
+}
+
+func TestUpstreamInExemptList_CIDRIgnored(t *testing.T) {
+	t.Setenv(constants.EnvNameserverExempt, "10.0.0.0/24")
+	resetNameserverExemptCache(t)
+
+	if got := ParseNameserverExemptList(); len(got) != 0 {
+		t.Fatalf("CIDR should be ignored in exempt list, got %v", got)
+	}
+	if UpstreamInExemptList("10.0.0.5") {
+		t.Fatalf("CIDR should not make upstream exempt")
+	}
+}

components/egress/pkg/dnsproxy/proxy_linux.go

@@ -18,17 +18,30 @@ package dnsproxy
 
 import (
 	"net"
+	"sync"
 	"syscall"
 	"time"
 
 	"golang.org/x/sys/unix"
 
 	"github.com/alibaba/opensandbox/egress/pkg/constants"
+	"github.com/alibaba/opensandbox/egress/pkg/log"
 )
 
+var exemptDialerLogOnce sync.Once
+
 // dialerWithMark sets SO_MARK so iptables can RETURN marked packets (bypass
-// redirect for proxy's own upstream DNS queries).
+// redirect for proxy's own upstream DNS queries). When upstream is in the nameserver
+// exempt list, returns a plain dialer (no mark) so upstream traffic follows normal
+// routing (e.g. via tun); iptables still does not redirect by destination exempt.
 func (p *Proxy) dialerWithMark() *net.Dialer {
+	if UpstreamInExemptList(p.UpstreamHost()) {
+		exemptDialerLogOnce.Do(func() {
+			log.Infof("[dns] upstream %s in nameserver exempt list, not setting SO_MARK", p.UpstreamHost())
+		})
+		return &net.Dialer{Timeout: 5 * time.Second}
+	}
+
 	return &net.Dialer{
 		Timeout: 5 * time.Second,
 		Control: func(network, address string, c syscall.RawConn) error {

components/egress/pkg/iptables/redirect.go

@@ -16,6 +16,7 @@ package iptables
 
 import (
 	"fmt"
+	"net/netip"
 	"os/exec"
 	"strconv"
 
@@ -24,14 +25,30 @@ import (
 )
 
 // SetupRedirect installs OUTPUT nat redirect for DNS (udp/tcp 53 -> port).
-// Packets carrying mark bypassMark will RETURN (used by the proxy's own upstream
-// queries to avoid redirect loops). Requires CAP_NET_ADMIN inside the namespace.
-func SetupRedirect(port int) error {
+//
+// exemptDst: optional list of destination IPs; traffic to these is not redirected. Packets carrying mark are also RETURNed (proxy's own upstream). Requires CAP_NET_ADMIN.
+func SetupRedirect(port int, exemptDst []netip.Addr) error {
 	log.Infof("installing iptables DNS redirect: OUTPUT port 53 -> %d (mark %s bypass)", port, constants.MarkHex)
 	targetPort := strconv.Itoa(port)
 
-	rules := [][]string{
-		// Bypass packets marked by the proxy itself (see dnsproxy dialer).
+	var rules [][]string
+	for _, d := range exemptDst {
+		addr := d
+		dStr := d.String()
+		if addr.Is4() {
+			rules = append(rules,
+				[]string{"iptables", "-t", "nat", "-A", "OUTPUT", "-p", "udp", "--dport", "53", "-d", dStr, "-j", "RETURN"},
+				[]string{"iptables", "-t", "nat", "-A", "OUTPUT", "-p", "tcp", "--dport", "53", "-d", dStr, "-j", "RETURN"},
+			)
+		} else {
+			rules = append(rules,
+				[]string{"ip6tables", "-t", "nat", "-A", "OUTPUT", "-p", "udp", "--dport", "53", "-d", dStr, "-j", "RETURN"},
+				[]string{"ip6tables", "-t", "nat", "-A", "OUTPUT", "-p", "tcp", "--dport", "53", "-d", dStr, "-j", "RETURN"},
+			)
+		}
+	}
+	// Bypass packets marked by the proxy itself (see dnsproxy dialer).
+	markAndRedirect := [][]string{
 		{"iptables", "-t", "nat", "-A", "OUTPUT", "-p", "udp", "--dport", "53", "-m", "mark", "--mark", constants.MarkHex, "-j", "RETURN"},
 		{"iptables", "-t", "nat", "-A", "OUTPUT", "-p", "tcp", "--dport", "53", "-m", "mark", "--mark", constants.MarkHex, "-j", "RETURN"},
 		// Redirect all other DNS traffic to local proxy port.
@@ -43,6 +60,7 @@ func SetupRedirect(port int) error {
 		{"ip6tables", "-t", "nat", "-A", "OUTPUT", "-p", "udp", "--dport", "53", "-j", "REDIRECT", "--to-port", targetPort},
 		{"ip6tables", "-t", "nat", "-A", "OUTPUT", "-p", "tcp", "--dport", "53", "-j", "REDIRECT", "--to-port", targetPort},
 	}
+	rules = append(rules, markAndRedirect...)
 
 	for _, args := range rules {
 		if output, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil {

components/egress/pkg/nftables/manager.go

@@ -77,6 +77,7 @@ func NewManagerWithOptions(opts Options) *Manager {
 }
 
 // ApplyStatic reconciles static allow/deny IP and CIDR entries into nftables.
+//
 // It creates a dedicated table/chain and overwrites previous state.
 // Uses the same mutex as AddResolvedIPs so a /policy update never overlaps a DNS
 // callback: without this, add-element could run while the table is being deleted/recreated