Skip to content

Commit 853e786

Browse files
committed
fix(serverless): block SSRF in job-input downloads
Job input carries arbitrary URLs that the worker downloads. Without restriction a job can point the worker at cloud instance-metadata (169.254.169.254) or other non-public addresses (CWE-918). Add runpod/serverless/utils/rp_ssrf.py: an http(s) scheme allowlist, DNS resolution that fails closed on any non-global IP (link-local, loopback, RFC1918, CGNAT, reserved, multicast, IPv6 ULA), a connection adapter that pins the socket to the validated IP to defeat DNS rebinding, per-hop redirect re-validation, and a streamed size cap. Route both rp_download fetch sites through it, and stream file() to disk instead of buffering the whole untrusted body in memory. Blocks raise SSRFError (a ValueError, not a RequestException) so they fail loudly rather than being retried or silently dropped. Configurable via RUNPOD_ALLOW_PRIVATE_DOWNLOAD_URLS (escape hatch, off by default) and RUNPOD_MAX_DOWNLOAD_BYTES (default 5 GiB).
1 parent 4128025 commit 853e786

4 files changed

Lines changed: 610 additions & 98 deletions

File tree

‎runpod/serverless/utils/rp_download.py‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,11 @@
1616
import backoff
1717
from requests import RequestException
1818

19-
from runpod.http_client import SyncClientSession
19+
from runpod.serverless.utils.rp_ssrf import (
20+
iter_content_capped,
21+
max_download_bytes,
22+
safe_get,
23+
)
2024

2125
HEADERS = {"User-Agent": "runpod-python/0.0.0 (https://runpod.io; support@runpod.io)"}
2226

@@ -57,7 +61,7 @@ def download_files_from_urls(job_id: str, urls: Union[str, List[str]]) -> List[s
5761

5862
@backoff.on_exception(backoff.expo, RequestException, max_tries=3)
5963
def download_file(url: str, path_to_save: str) -> str:
60-
with SyncClientSession().get(url, headers=HEADERS, stream=True, timeout=5) as response:
64+
with safe_get(url, stream=True, timeout=5, headers=HEADERS) as response:
6165
response.raise_for_status()
6266
content_disposition = response.headers.get("Content-Disposition")
6367
file_extension = ""
@@ -72,11 +76,10 @@ def download_file(url: str, path_to_save: str) -> str:
7276
file_size = int(response.headers.get("Content-Length", 0))
7377
chunk_size = calculate_chunk_size(file_size)
7478

75-
# write the content in chunks to the file
79+
# write the content in chunks to the file, aborting past the size cap
7680
with open(path_to_save + file_extension, "wb") as file_path:
77-
for chunk in response.iter_content(chunk_size=chunk_size):
78-
if chunk: # filter out keep-alive chunks
79-
file_path.write(chunk)
81+
for chunk in iter_content_capped(response, chunk_size, max_download_bytes()):
82+
file_path.write(chunk)
8083

8184
return file_extension
8285

@@ -117,27 +120,32 @@ def file(file_url: str) -> dict:
117120
"""
118121
os.makedirs("job_files", exist_ok=True)
119122

120-
download_response = SyncClientSession().get(file_url, headers=HEADERS, timeout=30)
123+
with safe_get(file_url, stream=True, timeout=30, headers=HEADERS) as download_response:
124+
content_disposition = download_response.headers.get("Content-Disposition")
121125

122-
content_disposition = download_response.headers.get("Content-Disposition")
126+
original_file_name = ""
127+
if content_disposition:
128+
params = extract_disposition_params(content_disposition)
123129

124-
original_file_name = ""
125-
if content_disposition:
126-
params = extract_disposition_params(content_disposition)
130+
original_file_name = params.get("filename", "")
127131

128-
original_file_name = params.get("filename", "")
132+
if not original_file_name:
133+
download_path = urlparse(file_url).path
134+
original_file_name = os.path.basename(download_path)
129135

130-
if not original_file_name:
131-
download_path = urlparse(file_url).path
132-
original_file_name = os.path.basename(download_path)
136+
file_type = os.path.splitext(original_file_name)[1].replace(".", "")
133137

134-
file_type = os.path.splitext(original_file_name)[1].replace(".", "")
138+
file_name = f"{uuid.uuid4()}"
135139

136-
file_name = f"{uuid.uuid4()}"
140+
output_file_path = os.path.join("job_files", f"{file_name}.{file_type}")
137141

138-
output_file_path = os.path.join("job_files", f"{file_name}.{file_type}")
139-
with open(output_file_path, "wb") as output_file:
140-
output_file.write(download_response.content)
142+
# Stream to disk in chunks (aborting past the size cap) instead of
143+
# buffering the entire untrusted body in memory.
144+
file_size = int(download_response.headers.get("Content-Length", 0))
145+
chunk_size = calculate_chunk_size(file_size)
146+
with open(output_file_path, "wb") as output_file:
147+
for chunk in iter_content_capped(download_response, chunk_size, max_download_bytes()):
148+
output_file.write(chunk)
141149

142150
if file_type == "zip":
143151
unzipped_directory = os.path.join("job_files", file_name)

‎runpod/serverless/utils/rp_ssrf.py‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
"""
2+
PodWorker | serverless | utils | rp_ssrf.py
3+
4+
SSRF-safe fetching of user-supplied (job-input) URLs.
5+
6+
Job input carries arbitrary URLs that the worker downloads. Without restriction
7+
a job can point the worker at cloud instance-metadata (169.254.169.254) or other
8+
non-public addresses (CWE-918). This module blocks any destination that is not a
9+
globally-routable IP, pins connections to a pre-validated address to defeat DNS
10+
rebinding, re-validates redirect hops, and caps download size.
11+
"""
12+
13+
import ipaddress
14+
import os
15+
import socket
16+
from typing import Iterator, List, Optional
17+
from urllib.parse import urljoin, urlparse
18+
19+
from requests.adapters import HTTPAdapter
20+
21+
from runpod.http_client import SyncClientSession
22+
23+
_ALLOWED_SCHEMES = ("http", "https")
24+
_REDIRECT_STATUS = frozenset((301, 302, 303, 307, 308))
25+
_DEFAULT_MAX_REDIRECTS = 5
26+
27+
# Size cap. Generous default that does not break legitimate image/zip/model
28+
# downloads but bounds pathological abuse and the in-memory read in file().
29+
_MAX_BYTES_ENV = "RUNPOD_MAX_DOWNLOAD_BYTES"
30+
_DEFAULT_MAX_BYTES = 5 * 1024 ** 3 # 5 GiB
31+
32+
# CGNAT / shared address space (RFC 6598). Not reported by IPv4Address.is_private
33+
# before CPython 3.12.4, so it is blocked explicitly rather than relied upon.
34+
_EXTRA_BLOCKED_NETWORKS = [ipaddress.ip_network("100.64.0.0/10")]
35+
36+
# Escape hatch for workers that legitimately fetch from a same-VPC / self-hosted
37+
# endpoint. Off by default; scheme allowlist and size cap still apply when set.
38+
_ALLOW_PRIVATE_ENV = "RUNPOD_ALLOW_PRIVATE_DOWNLOAD_URLS"
39+
40+
41+
class SSRFError(ValueError):
42+
"""
43+
Raised when a URL is rejected as unsafe (non-public destination, blocked
44+
scheme, or oversized body).
45+
46+
Subclasses ValueError and deliberately NOT requests.RequestException so it
47+
bypasses the download path's backoff retry and `except RequestException`
48+
handler: a blocked internal URL must fail loudly, not be retried or silently
49+
reported as an ordinary failed download.
50+
"""
51+
52+
53+
def _ssrf_protection_enabled() -> bool:
54+
return os.environ.get(_ALLOW_PRIVATE_ENV, "").strip().lower() not in (
55+
"1",
56+
"true",
57+
"yes",
58+
)
59+
60+
61+
def is_safe_address(ip: str) -> bool:
62+
"""
63+
Return True only for a globally-routable unicast IP address.
64+
65+
Blocks loopback, link-local (incl. 169.254.169.254 metadata), RFC1918,
66+
CGNAT, reserved, multicast, unspecified, IPv6 unique-local/link-local, and
67+
IPv4-mapped IPv6 forms of any of the above. Unparseable input is unsafe.
68+
"""
69+
try:
70+
address = ipaddress.ip_address(ip)
71+
except ValueError:
72+
return False
73+
74+
# Unwrap IPv4-mapped IPv6 (e.g. ::ffff:10.0.0.1) and classify the real IPv4.
75+
mapped = getattr(address, "ipv4_mapped", None)
76+
if mapped is not None:
77+
address = mapped
78+
79+
if any(address in network for network in _EXTRA_BLOCKED_NETWORKS):
80+
return False
81+
82+
if (
83+
address.is_private
84+
or address.is_loopback
85+
or address.is_link_local
86+
or address.is_reserved
87+
or address.is_multicast
88+
or address.is_unspecified
89+
):
90+
return False
91+
92+
return address.is_global
93+
94+
95+
def resolve_and_validate(host: str, port: int) -> List[str]:
96+
"""
97+
Resolve `host` and return its IPs, failing closed on any unsafe address.
98+
99+
Raises SSRFError if the host cannot be resolved, resolves to nothing, or
100+
resolves to any non-global address (a host resolving to both a public and a
101+
private IP is treated as hostile). Validation is skipped when the escape
102+
hatch env var is set, but resolution still occurs so callers can pin.
103+
"""
104+
try:
105+
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
106+
except socket.gaierror as err:
107+
raise SSRFError(f"could not resolve host {host!r}: {err}") from err
108+
109+
ips: List[str] = []
110+
for info in infos:
111+
ip = info[4][0]
112+
if ip not in ips:
113+
ips.append(ip)
114+
115+
if not ips:
116+
raise SSRFError(f"no addresses resolved for host {host!r}")
117+
118+
if _ssrf_protection_enabled():
119+
for ip in ips:
120+
if not is_safe_address(ip):
121+
raise SSRFError(
122+
f"host {host!r} resolves to non-public address {ip}"
123+
)
124+
125+
return ips
126+
127+
128+
def max_download_bytes() -> int:
129+
"""Configured per-download byte cap; defaults to 5 GiB."""
130+
raw = os.environ.get(_MAX_BYTES_ENV)
131+
if raw is None:
132+
return _DEFAULT_MAX_BYTES
133+
try:
134+
value = int(raw)
135+
except ValueError:
136+
return _DEFAULT_MAX_BYTES
137+
return value if value > 0 else _DEFAULT_MAX_BYTES
138+
139+
140+
class PinnedIPAdapter(HTTPAdapter):
141+
"""
142+
Requests adapter that dials a pre-validated IP instead of re-resolving the
143+
URL host, defeating DNS rebinding.
144+
145+
The superclass builds the connection pool with its normal TLS/verification
146+
configuration; this adapter only redirects the socket to the pinned IP and
147+
keeps the TLS SNI, certificate hostname check, and Host header bound to the
148+
original URL host. It never alters certificate verification.
149+
"""
150+
151+
def __init__(self, pinned_ip: str, **kwargs):
152+
self._pinned_ip = pinned_ip
153+
super().__init__(**kwargs)
154+
155+
def send(self, request, **kwargs):
156+
parsed = urlparse(request.url)
157+
authority = parsed.hostname or ""
158+
if parsed.port is not None:
159+
authority = f"{authority}:{parsed.port}"
160+
request.headers["Host"] = authority
161+
return super().send(request, **kwargs)
162+
163+
def _pin(self, pool, url: str):
164+
parsed = urlparse(url)
165+
if parsed.scheme == "https":
166+
pool.assert_hostname = parsed.hostname
167+
pool.conn_kw = dict(pool.conn_kw or {})
168+
pool.conn_kw["server_hostname"] = parsed.hostname
169+
pool.host = self._pinned_ip
170+
return pool
171+
172+
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
173+
pool = super().get_connection_with_tls_context(
174+
request, verify, proxies=proxies, cert=cert
175+
)
176+
return self._pin(pool, request.url)
177+
178+
def get_connection(self, url, proxies=None): # requests < 2.32 fallback
179+
pool = super().get_connection(url, proxies=proxies)
180+
return self._pin(pool, url)
181+
182+
183+
def _build_pinned_session(pinned_ip: str) -> SyncClientSession:
184+
"""A session whose http(s) connections dial only `pinned_ip`."""
185+
session = SyncClientSession()
186+
adapter = PinnedIPAdapter(pinned_ip)
187+
session.mount("http://", adapter)
188+
session.mount("https://", adapter)
189+
return session
190+
191+
192+
def _enforce_content_length(response, max_bytes: Optional[int]) -> None:
193+
if max_bytes is None:
194+
return
195+
declared = response.headers.get("Content-Length")
196+
if declared is None:
197+
return
198+
try:
199+
size = int(declared)
200+
except ValueError:
201+
return
202+
if size > max_bytes:
203+
response.close()
204+
raise SSRFError(f"download exceeds size cap: {size} > {max_bytes} bytes")
205+
206+
207+
def iter_content_capped(response, chunk_size: int, max_bytes: Optional[int]) -> Iterator[bytes]:
208+
"""
209+
Yield the response body in chunks, aborting with SSRFError if the running
210+
total exceeds `max_bytes`. Guards against a lying/absent Content-Length.
211+
"""
212+
total = 0
213+
for chunk in response.iter_content(chunk_size=chunk_size):
214+
if not chunk:
215+
continue
216+
total += len(chunk)
217+
if max_bytes is not None and total > max_bytes:
218+
response.close()
219+
raise SSRFError(f"download exceeds size cap: >{max_bytes} bytes")
220+
yield chunk
221+
222+
223+
def safe_get(
224+
url: str,
225+
*,
226+
stream: bool = True,
227+
timeout: int = 30,
228+
headers: Optional[dict] = None,
229+
max_bytes: Optional[int] = None,
230+
max_redirects: int = _DEFAULT_MAX_REDIRECTS,
231+
):
232+
"""
233+
Fetch a user-supplied URL with SSRF protections.
234+
235+
Enforces an http(s) scheme allowlist, resolves and validates the host to a
236+
public IP, pins the connection to that IP, and re-validates every redirect
237+
hop. Rejects a body whose declared size exceeds the cap. Returns the
238+
(streamed) requests.Response for the caller to consume; use
239+
iter_content_capped to enforce the cap while reading. Raises SSRFError on
240+
any block.
241+
"""
242+
if max_bytes is None:
243+
max_bytes = max_download_bytes()
244+
245+
current = url
246+
for _ in range(max_redirects + 1):
247+
parsed = urlparse(current)
248+
if parsed.scheme not in _ALLOWED_SCHEMES:
249+
raise SSRFError(f"blocked URL scheme {parsed.scheme!r}: {current}")
250+
if not parsed.hostname:
251+
raise SSRFError(f"URL has no host: {current}")
252+
253+
port = parsed.port or (443 if parsed.scheme == "https" else 80)
254+
ips = resolve_and_validate(parsed.hostname, port)
255+
256+
session = _build_pinned_session(ips[0])
257+
response = session.get(
258+
current,
259+
headers=headers,
260+
allow_redirects=False,
261+
stream=stream,
262+
timeout=timeout,
263+
)
264+
265+
if response.status_code in _REDIRECT_STATUS:
266+
location = response.headers.get("Location")
267+
response.close()
268+
if not location:
269+
raise SSRFError(f"redirect without Location header: {current}")
270+
current = urljoin(current, location)
271+
continue
272+
273+
_enforce_content_length(response, max_bytes)
274+
return response
275+
276+
raise SSRFError(f"exceeded maximum redirects ({max_redirects}): {url}")

0 commit comments

Comments
 (0)