|
| 1 | +""" |
| 2 | +PodWorker | serverless | utils | rp_ssrf.py |
| 3 | +
|
| 4 | +SSRF-safe fetching of user-supplied (job-input) URLs. |
| 5 | +
|
| 6 | +Job input carries arbitrary URLs that the worker downloads. Without restriction |
| 7 | +a job can point the worker at cloud instance-metadata (169.254.169.254) or other |
| 8 | +non-public addresses (CWE-918). This module blocks any destination that is not a |
| 9 | +globally-routable IP, pins connections to a pre-validated address to defeat DNS |
| 10 | +rebinding, re-validates redirect hops, and caps download size. |
| 11 | +""" |
| 12 | + |
| 13 | +import ipaddress |
| 14 | +import os |
| 15 | +import socket |
| 16 | +from typing import Iterator, List, Optional |
| 17 | +from urllib.parse import urljoin, urlparse |
| 18 | + |
| 19 | +from requests.adapters import HTTPAdapter |
| 20 | + |
| 21 | +from runpod.http_client import SyncClientSession |
| 22 | + |
| 23 | +_ALLOWED_SCHEMES = ("http", "https") |
| 24 | +_REDIRECT_STATUS = frozenset((301, 302, 303, 307, 308)) |
| 25 | +_DEFAULT_MAX_REDIRECTS = 5 |
| 26 | + |
| 27 | +# Size cap. Generous default that does not break legitimate image/zip/model |
| 28 | +# downloads but bounds pathological abuse and the in-memory read in file(). |
| 29 | +_MAX_BYTES_ENV = "RUNPOD_MAX_DOWNLOAD_BYTES" |
| 30 | +_DEFAULT_MAX_BYTES = 5 * 1024 ** 3 # 5 GiB |
| 31 | + |
| 32 | +# CGNAT / shared address space (RFC 6598). Not reported by IPv4Address.is_private |
| 33 | +# before CPython 3.12.4, so it is blocked explicitly rather than relied upon. |
| 34 | +_EXTRA_BLOCKED_NETWORKS = [ipaddress.ip_network("100.64.0.0/10")] |
| 35 | + |
| 36 | +# Escape hatch for workers that legitimately fetch from a same-VPC / self-hosted |
| 37 | +# endpoint. Off by default; scheme allowlist and size cap still apply when set. |
| 38 | +_ALLOW_PRIVATE_ENV = "RUNPOD_ALLOW_PRIVATE_DOWNLOAD_URLS" |
| 39 | + |
| 40 | + |
| 41 | +class SSRFError(ValueError): |
| 42 | + """ |
| 43 | + Raised when a URL is rejected as unsafe (non-public destination, blocked |
| 44 | + scheme, or oversized body). |
| 45 | +
|
| 46 | + Subclasses ValueError and deliberately NOT requests.RequestException so it |
| 47 | + bypasses the download path's backoff retry and `except RequestException` |
| 48 | + handler: a blocked internal URL must fail loudly, not be retried or silently |
| 49 | + reported as an ordinary failed download. |
| 50 | + """ |
| 51 | + |
| 52 | + |
| 53 | +def _ssrf_protection_enabled() -> bool: |
| 54 | + return os.environ.get(_ALLOW_PRIVATE_ENV, "").strip().lower() not in ( |
| 55 | + "1", |
| 56 | + "true", |
| 57 | + "yes", |
| 58 | + ) |
| 59 | + |
| 60 | + |
| 61 | +def is_safe_address(ip: str) -> bool: |
| 62 | + """ |
| 63 | + Return True only for a globally-routable unicast IP address. |
| 64 | +
|
| 65 | + Blocks loopback, link-local (incl. 169.254.169.254 metadata), RFC1918, |
| 66 | + CGNAT, reserved, multicast, unspecified, IPv6 unique-local/link-local, and |
| 67 | + IPv4-mapped IPv6 forms of any of the above. Unparseable input is unsafe. |
| 68 | + """ |
| 69 | + try: |
| 70 | + address = ipaddress.ip_address(ip) |
| 71 | + except ValueError: |
| 72 | + return False |
| 73 | + |
| 74 | + # Unwrap IPv4-mapped IPv6 (e.g. ::ffff:10.0.0.1) and classify the real IPv4. |
| 75 | + mapped = getattr(address, "ipv4_mapped", None) |
| 76 | + if mapped is not None: |
| 77 | + address = mapped |
| 78 | + |
| 79 | + if any(address in network for network in _EXTRA_BLOCKED_NETWORKS): |
| 80 | + return False |
| 81 | + |
| 82 | + if ( |
| 83 | + address.is_private |
| 84 | + or address.is_loopback |
| 85 | + or address.is_link_local |
| 86 | + or address.is_reserved |
| 87 | + or address.is_multicast |
| 88 | + or address.is_unspecified |
| 89 | + ): |
| 90 | + return False |
| 91 | + |
| 92 | + return address.is_global |
| 93 | + |
| 94 | + |
| 95 | +def resolve_and_validate(host: str, port: int) -> List[str]: |
| 96 | + """ |
| 97 | + Resolve `host` and return its IPs, failing closed on any unsafe address. |
| 98 | +
|
| 99 | + Raises SSRFError if the host cannot be resolved, resolves to nothing, or |
| 100 | + resolves to any non-global address (a host resolving to both a public and a |
| 101 | + private IP is treated as hostile). Validation is skipped when the escape |
| 102 | + hatch env var is set, but resolution still occurs so callers can pin. |
| 103 | + """ |
| 104 | + try: |
| 105 | + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) |
| 106 | + except socket.gaierror as err: |
| 107 | + raise SSRFError(f"could not resolve host {host!r}: {err}") from err |
| 108 | + |
| 109 | + ips: List[str] = [] |
| 110 | + for info in infos: |
| 111 | + ip = info[4][0] |
| 112 | + if ip not in ips: |
| 113 | + ips.append(ip) |
| 114 | + |
| 115 | + if not ips: |
| 116 | + raise SSRFError(f"no addresses resolved for host {host!r}") |
| 117 | + |
| 118 | + if _ssrf_protection_enabled(): |
| 119 | + for ip in ips: |
| 120 | + if not is_safe_address(ip): |
| 121 | + raise SSRFError( |
| 122 | + f"host {host!r} resolves to non-public address {ip}" |
| 123 | + ) |
| 124 | + |
| 125 | + return ips |
| 126 | + |
| 127 | + |
| 128 | +def max_download_bytes() -> int: |
| 129 | + """Configured per-download byte cap; defaults to 5 GiB.""" |
| 130 | + raw = os.environ.get(_MAX_BYTES_ENV) |
| 131 | + if raw is None: |
| 132 | + return _DEFAULT_MAX_BYTES |
| 133 | + try: |
| 134 | + value = int(raw) |
| 135 | + except ValueError: |
| 136 | + return _DEFAULT_MAX_BYTES |
| 137 | + return value if value > 0 else _DEFAULT_MAX_BYTES |
| 138 | + |
| 139 | + |
| 140 | +class PinnedIPAdapter(HTTPAdapter): |
| 141 | + """ |
| 142 | + Requests adapter that dials a pre-validated IP instead of re-resolving the |
| 143 | + URL host, defeating DNS rebinding. |
| 144 | +
|
| 145 | + The superclass builds the connection pool with its normal TLS/verification |
| 146 | + configuration; this adapter only redirects the socket to the pinned IP and |
| 147 | + keeps the TLS SNI, certificate hostname check, and Host header bound to the |
| 148 | + original URL host. It never alters certificate verification. |
| 149 | + """ |
| 150 | + |
| 151 | + def __init__(self, pinned_ip: str, **kwargs): |
| 152 | + self._pinned_ip = pinned_ip |
| 153 | + super().__init__(**kwargs) |
| 154 | + |
| 155 | + def send(self, request, **kwargs): |
| 156 | + parsed = urlparse(request.url) |
| 157 | + authority = parsed.hostname or "" |
| 158 | + if parsed.port is not None: |
| 159 | + authority = f"{authority}:{parsed.port}" |
| 160 | + request.headers["Host"] = authority |
| 161 | + return super().send(request, **kwargs) |
| 162 | + |
| 163 | + def _pin(self, pool, url: str): |
| 164 | + parsed = urlparse(url) |
| 165 | + if parsed.scheme == "https": |
| 166 | + pool.assert_hostname = parsed.hostname |
| 167 | + pool.conn_kw = dict(pool.conn_kw or {}) |
| 168 | + pool.conn_kw["server_hostname"] = parsed.hostname |
| 169 | + pool.host = self._pinned_ip |
| 170 | + return pool |
| 171 | + |
| 172 | + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): |
| 173 | + pool = super().get_connection_with_tls_context( |
| 174 | + request, verify, proxies=proxies, cert=cert |
| 175 | + ) |
| 176 | + return self._pin(pool, request.url) |
| 177 | + |
| 178 | + def get_connection(self, url, proxies=None): # requests < 2.32 fallback |
| 179 | + pool = super().get_connection(url, proxies=proxies) |
| 180 | + return self._pin(pool, url) |
| 181 | + |
| 182 | + |
| 183 | +def _build_pinned_session(pinned_ip: str) -> SyncClientSession: |
| 184 | + """A session whose http(s) connections dial only `pinned_ip`.""" |
| 185 | + session = SyncClientSession() |
| 186 | + adapter = PinnedIPAdapter(pinned_ip) |
| 187 | + session.mount("http://", adapter) |
| 188 | + session.mount("https://", adapter) |
| 189 | + return session |
| 190 | + |
| 191 | + |
| 192 | +def _enforce_content_length(response, max_bytes: Optional[int]) -> None: |
| 193 | + if max_bytes is None: |
| 194 | + return |
| 195 | + declared = response.headers.get("Content-Length") |
| 196 | + if declared is None: |
| 197 | + return |
| 198 | + try: |
| 199 | + size = int(declared) |
| 200 | + except ValueError: |
| 201 | + return |
| 202 | + if size > max_bytes: |
| 203 | + response.close() |
| 204 | + raise SSRFError(f"download exceeds size cap: {size} > {max_bytes} bytes") |
| 205 | + |
| 206 | + |
| 207 | +def iter_content_capped(response, chunk_size: int, max_bytes: Optional[int]) -> Iterator[bytes]: |
| 208 | + """ |
| 209 | + Yield the response body in chunks, aborting with SSRFError if the running |
| 210 | + total exceeds `max_bytes`. Guards against a lying/absent Content-Length. |
| 211 | + """ |
| 212 | + total = 0 |
| 213 | + for chunk in response.iter_content(chunk_size=chunk_size): |
| 214 | + if not chunk: |
| 215 | + continue |
| 216 | + total += len(chunk) |
| 217 | + if max_bytes is not None and total > max_bytes: |
| 218 | + response.close() |
| 219 | + raise SSRFError(f"download exceeds size cap: >{max_bytes} bytes") |
| 220 | + yield chunk |
| 221 | + |
| 222 | + |
| 223 | +def safe_get( |
| 224 | + url: str, |
| 225 | + *, |
| 226 | + stream: bool = True, |
| 227 | + timeout: int = 30, |
| 228 | + headers: Optional[dict] = None, |
| 229 | + max_bytes: Optional[int] = None, |
| 230 | + max_redirects: int = _DEFAULT_MAX_REDIRECTS, |
| 231 | +): |
| 232 | + """ |
| 233 | + Fetch a user-supplied URL with SSRF protections. |
| 234 | +
|
| 235 | + Enforces an http(s) scheme allowlist, resolves and validates the host to a |
| 236 | + public IP, pins the connection to that IP, and re-validates every redirect |
| 237 | + hop. Rejects a body whose declared size exceeds the cap. Returns the |
| 238 | + (streamed) requests.Response for the caller to consume; use |
| 239 | + iter_content_capped to enforce the cap while reading. Raises SSRFError on |
| 240 | + any block. |
| 241 | + """ |
| 242 | + if max_bytes is None: |
| 243 | + max_bytes = max_download_bytes() |
| 244 | + |
| 245 | + current = url |
| 246 | + for _ in range(max_redirects + 1): |
| 247 | + parsed = urlparse(current) |
| 248 | + if parsed.scheme not in _ALLOWED_SCHEMES: |
| 249 | + raise SSRFError(f"blocked URL scheme {parsed.scheme!r}: {current}") |
| 250 | + if not parsed.hostname: |
| 251 | + raise SSRFError(f"URL has no host: {current}") |
| 252 | + |
| 253 | + port = parsed.port or (443 if parsed.scheme == "https" else 80) |
| 254 | + ips = resolve_and_validate(parsed.hostname, port) |
| 255 | + |
| 256 | + session = _build_pinned_session(ips[0]) |
| 257 | + response = session.get( |
| 258 | + current, |
| 259 | + headers=headers, |
| 260 | + allow_redirects=False, |
| 261 | + stream=stream, |
| 262 | + timeout=timeout, |
| 263 | + ) |
| 264 | + |
| 265 | + if response.status_code in _REDIRECT_STATUS: |
| 266 | + location = response.headers.get("Location") |
| 267 | + response.close() |
| 268 | + if not location: |
| 269 | + raise SSRFError(f"redirect without Location header: {current}") |
| 270 | + current = urljoin(current, location) |
| 271 | + continue |
| 272 | + |
| 273 | + _enforce_content_length(response, max_bytes) |
| 274 | + return response |
| 275 | + |
| 276 | + raise SSRFError(f"exceeded maximum redirects ({max_redirects}): {url}") |
0 commit comments