|
| 1 | +package extractor |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "errors" |
| 6 | + "fmt" |
| 7 | + "net" |
| 8 | + "net/http" |
| 9 | + "net/url" |
| 10 | + "syscall" |
| 11 | + "time" |
| 12 | +) |
| 13 | + |
| 14 | +// errBlockedAddress is returned when a fetch targets a non-public address and private-network |
| 15 | +// blocking is enabled. the check runs at connect time against the actual resolved IP, so it also |
| 16 | +// defends against DNS rebinding and redirects that point back at internal hosts. |
| 17 | +var errBlockedAddress = errors.New("connection to non-public address blocked") |
| 18 | + |
| 19 | +// extraBlockedCIDRs are IANA special-purpose ranges that the net.IP helpers below do not classify |
| 20 | +// as private/loopback/link-local but which are still not public, routable destinations. Blocking |
| 21 | +// them closes SSRF paths into CGNAT, benchmarking, documentation, reserved and broadcast space. |
| 22 | +var extraBlockedCIDRs = parseCIDRs( |
| 23 | + "0.0.0.0/8", // RFC1122 "this network" (IsUnspecified only covers 0.0.0.0 itself) |
| 24 | + "100.64.0.0/10", // RFC6598 carrier-grade NAT |
| 25 | + "192.0.0.0/24", // RFC6890 IETF protocol assignments |
| 26 | + "192.0.2.0/24", // TEST-NET-1 documentation |
| 27 | + "198.18.0.0/15", // RFC2544 benchmarking |
| 28 | + "198.51.100.0/24", // TEST-NET-2 documentation |
| 29 | + "203.0.113.0/24", // TEST-NET-3 documentation |
| 30 | + "240.0.0.0/4", // RFC1112 reserved (former class E) |
| 31 | + "255.255.255.255/32", // limited broadcast |
| 32 | + "64:ff9b::/96", // RFC6052 NAT64 |
| 33 | + "100::/64", // RFC6666 discard-only |
| 34 | + "2001:db8::/32", // IPv6 documentation |
| 35 | +) |
| 36 | + |
| 37 | +func parseCIDRs(cidrs ...string) []*net.IPNet { |
| 38 | + nets := make([]*net.IPNet, 0, len(cidrs)) |
| 39 | + for _, c := range cidrs { |
| 40 | + if _, n, err := net.ParseCIDR(c); err == nil { |
| 41 | + nets = append(nets, n) |
| 42 | + } |
| 43 | + } |
| 44 | + return nets |
| 45 | +} |
| 46 | + |
| 47 | +// isBlockedIP reports whether ip is outside the public routable range and must not be dialed. |
| 48 | +func isBlockedIP(ip net.IP) bool { |
| 49 | + if ip == nil { |
| 50 | + return true |
| 51 | + } |
| 52 | + if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || |
| 53 | + ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() { |
| 54 | + return true |
| 55 | + } |
| 56 | + for _, n := range extraBlockedCIDRs { |
| 57 | + if n.Contains(ip) { |
| 58 | + return true |
| 59 | + } |
| 60 | + } |
| 61 | + return false |
| 62 | +} |
| 63 | + |
| 64 | +// safeControl is a net.Dialer Control hook that rejects connections to non-public addresses. |
| 65 | +func safeControl(_, address string, _ syscall.RawConn) error { |
| 66 | + host, _, err := net.SplitHostPort(address) |
| 67 | + if err != nil { |
| 68 | + return err |
| 69 | + } |
| 70 | + ip := net.ParseIP(host) |
| 71 | + if ip == nil { |
| 72 | + return fmt.Errorf("%w: cannot parse address %q", errBlockedAddress, host) |
| 73 | + } |
| 74 | + if isBlockedIP(ip) { |
| 75 | + return fmt.Errorf("%w: %s", errBlockedAddress, ip) |
| 76 | + } |
| 77 | + return nil |
| 78 | +} |
| 79 | + |
| 80 | +// validatePublicHost rejects reqURL when its host is, or resolves to, a non-public address. It |
| 81 | +// enforces the SSRF policy uniformly across all retrievers, including Cloudflare Browser Rendering |
| 82 | +// which fetches remotely and so is not covered by the connect-time dialer guard. |
| 83 | +func validatePublicHost(ctx context.Context, reqURL string) error { |
| 84 | + u, err := url.Parse(reqURL) |
| 85 | + if err != nil { |
| 86 | + return err |
| 87 | + } |
| 88 | + host := u.Hostname() |
| 89 | + if host == "" { |
| 90 | + return fmt.Errorf("%w: no host in %q", errBlockedAddress, reqURL) |
| 91 | + } |
| 92 | + if ip := net.ParseIP(host); ip != nil { |
| 93 | + if isBlockedIP(ip) { |
| 94 | + return fmt.Errorf("%w: %s", errBlockedAddress, ip) |
| 95 | + } |
| 96 | + return nil |
| 97 | + } |
| 98 | + ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) |
| 99 | + if err != nil { |
| 100 | + return fmt.Errorf("resolve %q: %w", host, err) |
| 101 | + } |
| 102 | + for _, ipa := range ips { |
| 103 | + if isBlockedIP(ipa.IP) { |
| 104 | + return fmt.Errorf("%w: %s resolves to %s", errBlockedAddress, host, ipa.IP) |
| 105 | + } |
| 106 | + } |
| 107 | + return nil |
| 108 | +} |
| 109 | + |
| 110 | +// safeTransport returns an http.Transport whose dialer blocks non-public addresses. Proxy support |
| 111 | +// is disabled: an env proxy would make the dialer validate the proxy's IP instead of the real |
| 112 | +// destination, defeating the guard. As a consequence, deployments that require HTTP_PROXY/HTTPS_PROXY |
| 113 | +// for outbound access must run with --allow-private-networks (which keeps the default proxy-aware |
| 114 | +// transport) and rely on network-level egress controls instead. |
| 115 | +func safeTransport(timeout time.Duration) *http.Transport { |
| 116 | + dialer := &net.Dialer{Timeout: timeout, Control: safeControl} |
| 117 | + tr, ok := http.DefaultTransport.(*http.Transport) |
| 118 | + if !ok { |
| 119 | + return &http.Transport{DialContext: dialer.DialContext, Proxy: nil} |
| 120 | + } |
| 121 | + cloned := tr.Clone() |
| 122 | + cloned.Proxy = nil |
| 123 | + cloned.DialContext = dialer.DialContext |
| 124 | + return cloned |
| 125 | +} |
0 commit comments