Repository navigation
Expand file tree
/
Copy pathsecurity.py
More file actions
374 lines (327 loc) · 14.6 KB
/
Copy pathsecurity.py
File metadata and controls
374 lines (327 loc) · 14.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
"""URL validation with SSRF protection — private IP blocking, DNS rebinding detection, scheme validation."""
import asyncio
import httpcore
import ipaddress
import re
import time
from typing import Any
from urllib.parse import urljoin, urlparse
import httpx
_MAX_URL_LENGTH = 8192
_BLOCKED_SCHEMES = frozenset({
"file", "ftp", "gopher", "data", "javascript", "vbscript",
"about", "chrome", "chrome-extension", "jar",
})
_BLOCKED_HOSTNAMES = frozenset({
"localhost", "metadata.google.internal", "169.254.169.254",
"0.0.0.0", "0",
})
_DNS_REBINDING_SUFFIXES = (
".nip.io", ".sslip.io", ".xip.io", ".nip.name", ".1u.ms",
)
_PRIVATE_NETWORKS: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("0.0.0.0/8"),
ipaddress.ip_network("224.0.0.0/4"),
ipaddress.ip_network("240.0.0.0/4"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("::ffff:0:0/96"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
ipaddress.ip_network("::/128"),
]
class SecurityError(ValueError):
"""Raised when input fails security validation."""
pass
# ── DNS-rebinding TOCTOU pinning ─────────────────────────────────
# validate_url stores its validated IPs; PinningNetworkBackend is wired
# into the shared httpx clients so connect_tcp uses exactly those IPs,
# keeping the hostname for TLS SNI/cert validation. Closes the re-resolve
# window a rebinding record would otherwise exploit. Unvalidated hosts
# pass through untouched; pins expire (60s) to plain resolution.
_PIN_TTL = 60.0
_pinned: dict[str, tuple[float, list[str]]] = {}
# Per-attempt connect budget for a pinned IP. This hotspot's IPv6 resolves but
# never answers, so a pinned IPv6 used to burn the whole client timeout
# (25-30 s) with no error and no fallback to a working IPv4.
_PIN_ATTEMPT_TIMEOUT = 5.0
def _pin_host(hostname: str, ips: list[str]) -> None:
_pinned[hostname] = (time.monotonic(), list(ips))
def _ordered_pins(ips: list[str]) -> list[str]:
"""IPv4 first. The pin set contains only VALIDATED public IPs, so
reordering does not weaken the DNS-rebinding protection — we still never
re-resolve the hostname."""
v4 = [ip for ip in ips if ":" not in ip]
v6 = [ip for ip in ips if ":" in ip]
return v4 + v6
class PinningNetworkBackend(httpcore.AsyncNetworkBackend):
"""httpcore network backend that connects to validated IPs."""
def __init__(self) -> None:
self._inner = httpcore.AnyIOBackend()
async def connect_tcp(self, host, port, timeout=None,
local_address=None, socket_options=None):
entry = _pinned.get(host)
if entry:
stamp, ips = entry
if time.monotonic() - stamp <= _PIN_TTL and ips:
# Try the pinned IPs in order (IPv4 first) with a short
# per-attempt timeout, falling through to the next pinned IP
# on failure. ONLY pinned/validated IPs are used — the
# hostname is never re-resolved, so the DNS-rebinding
# TOCTOU hole stays closed.
total = 30.0
if timeout:
try:
total = float(timeout)
except (TypeError, ValueError):
total = 30.0
deadline = time.monotonic() + total
last_exc: Exception | None = None
for ip in _ordered_pins(ips):
remaining = deadline - time.monotonic()
if remaining <= 0:
break
attempt = min(_PIN_ATTEMPT_TIMEOUT, remaining)
try:
return await self._inner.connect_tcp(
ip, port, timeout=attempt,
local_address=local_address,
socket_options=socket_options)
except Exception as e:
last_exc = e
continue
if last_exc is not None:
raise last_exc
return await self._inner.connect_tcp(
host, port, timeout=timeout,
local_address=local_address,
socket_options=socket_options)
def _normalize_ip_notation(host: str) -> str | None:
"""Resolve alternate IP notations that libcurl/httpx would resolve.
Octal, hex, decimal integer, short-form dotted — normalizes to dotted-decimal."""
cleaned = host.strip("[]")
if cleaned.isdigit():
try:
if len(cleaned) > 1 and cleaned[0] == '0' and all(c in '01234567' for c in cleaned):
val = int(cleaned, 8)
else:
val = int(cleaned)
if val <= 0xFFFFFFFF:
return f"{(val >> 24) & 0xFF}.{(val >> 16) & 0xFF}.{(val >> 8) & 0xFF}.{val & 0xFF}"
except (ValueError, OverflowError):
pass
return None
parts = cleaned.split('.')
if not (1 <= len(parts) <= 4):
return None
try:
resolved = []
for p in parts:
if not p:
return None
if len(p) > 1 and p[0] == '0' and all(c in '01234567' for c in p):
resolved.append(int(p, 8))
elif p.lower().startswith('0x'):
resolved.append(int(p, 0))
else:
resolved.append(int(p))
if len(resolved) == 2:
resolved = [resolved[0], 0, 0, resolved[1]]
elif len(resolved) == 3:
resolved = [resolved[0], resolved[1], 0, resolved[2]]
if any(o < 0 or o > 255 for o in resolved):
return None
return '.'.join(str(o) for o in resolved)
except (ValueError, OverflowError):
return None
async def resolve_hostname(hostname: str, timeout: float = 5.0) -> list[str]:
"""Resolve hostname to IP addresses via DNS."""
try:
infos = await asyncio.wait_for(
asyncio.get_running_loop().getaddrinfo(hostname, None),
timeout=timeout
)
return sorted({info[4][0] for info in infos})
except Exception:
return []
def _is_private_address(ip_str: str) -> bool:
"""Check if an IP string is in private/reserved ranges, including
IPv4-mapped IPv6."""
try:
addr = ipaddress.ip_address(ip_str)
except ValueError:
normalized = _normalize_ip_notation(ip_str)
if normalized is None:
return False
try:
addr = ipaddress.ip_address(normalized)
except ValueError:
return False
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
for net in _PRIVATE_NETWORKS:
if isinstance(net, ipaddress.IPv4Network) and addr.ipv4_mapped in net:
return True
for net in _PRIVATE_NETWORKS:
try:
if addr in net:
return True
except TypeError:
pass
return False
def _validate_syntax(url: str, allow_internal: bool = False) -> str:
"""Syntactic URL validation — scheme, length, backslash, brackets, hostname.
Does NOT do DNS resolution (that's async). Returns cleaned URL."""
url = url.strip()
if not url:
raise SecurityError("URL must be a non-empty string")
if len(url) > _MAX_URL_LENGTH:
raise SecurityError(f"URL exceeds maximum length of {_MAX_URL_LENGTH} characters")
if "\\" in url:
raise SecurityError("URL contains backslash character (potential SSRF bypass)")
try:
parsed = urlparse(url)
except Exception as e:
raise SecurityError(f"Malformed URL: {e}")
scheme = (parsed.scheme or "").lower()
if scheme in _BLOCKED_SCHEMES:
raise SecurityError(f"URL scheme '{scheme}' is not allowed")
if scheme not in ("http", "https"):
raise SecurityError(f"Only http and https URLs are supported, got: {scheme}")
netloc_lower = (parsed.netloc or "").lower()
if "[" in netloc_lower or "]" in netloc_lower:
if not (netloc_lower.startswith("[") and "]" in netloc_lower):
raise SecurityError("URL contains malformed bracketed host (potential SSRF bypass)")
bracket_content = netloc_lower[1:netloc_lower.index("]")]
try:
ipaddress.IPv6Address(bracket_content)
except ipaddress.AddressValueError:
if not re.match(r'^v[0-9]+\..+$', bracket_content, re.IGNORECASE):
raise SecurityError("URL contains invalid bracketed host (potential SSRF bypass)")
hostname = parsed.hostname
if not hostname:
raise SecurityError("URL has no valid hostname")
return url
async def validate_url(url: str, allow_internal: bool = False) -> str:
"""Validate a URL: scheme, syntax, private IP (sync+async DNS), DNS rebinding.
Returns the validated URL string or raises SecurityError."""
url = _validate_syntax(url, allow_internal)
parsed = urlparse(url)
hostname = parsed.hostname.lower().rstrip(".") # strip FQDN trailing dot
if allow_internal:
return url
# Check if hostname is a bare IP
is_ip = False
try:
ipaddress.ip_address(hostname)
is_ip = True
except ValueError:
normalized = _normalize_ip_notation(hostname)
if normalized is not None:
try:
ipaddress.ip_address(normalized)
is_ip = True
except ValueError:
pass
if is_ip:
if _is_private_address(hostname):
raise SecurityError(f"URL targets internal/private IP: {hostname}")
return url
# Hostname (not bare IP) — check blocked hostnames and DNS rebinding
if hostname in _BLOCKED_HOSTNAMES:
raise SecurityError(f"URL targets internal service: {hostname}")
# Check after normalization too (e.g., "LOCALHOST" with different case)
lowered_try = hostname.replace("_", "-") # some DNS rebinding obfuscation
if lowered_try in _BLOCKED_HOSTNAMES:
raise SecurityError(f"URL targets internal service: {hostname}")
if hostname.endswith(_DNS_REBINDING_SUFFIXES):
raise SecurityError(f"URL uses DNS rebinding service: {hostname}")
# Resolve DNS — check resolved IPs against private ranges
ips = await resolve_hostname(hostname)
if ips:
for ip_str in ips:
if _is_private_address(ip_str):
raise SecurityError(
f"URL resolves to internal IP ({hostname} -> {ip_str})"
)
# All public — pin so connect reuses these exact IPs (TOCTOU)
_pin_host(hostname, ips)
return url
async def safe_fetch_capped(client: Any, url: str, *, max_bytes: int = 2_000_000,
max_hops: int = 5, allow_internal: bool = False,
meta_out: dict[str, Any] | None = None,
**kwargs) -> tuple[bytes, bool]:
"""safe_fetch variant: streams the body with a hard byte cap, keeping
per-hop SSRF validation. Returns (body_bytes, truncated).
The body is never buffered past max_bytes: the stream is aborted as soon
as the cap is reached and the connection is closed on exit.
Unlike safe_stream (which raises on non-200) the body is returned
whatever the status was. Callers that branch on the status code (or need
the final URL) pass ``meta_out`` — a dict that receives ``status``,
``url`` and ``headers`` of the final hop.
"""
current = url
for _ in range(max_hops + 1):
current = await validate_url(current, allow_internal=allow_internal)
async with client.stream("GET", current, follow_redirects=False, **kwargs) as resp:
if resp.is_redirect and resp.headers.get("location"):
current = urljoin(str(resp.url), resp.headers["location"])
continue
if meta_out is not None:
meta_out["status"] = resp.status_code
meta_out["url"] = str(resp.url)
meta_out["headers"] = dict(resp.headers)
body = b""
truncated = False
async for chunk in resp.aiter_bytes():
room = max_bytes - len(body)
if room <= 0:
truncated = True
break
if len(chunk) > room:
body += chunk[:room]
truncated = True
break
body += chunk
return body, truncated
raise SecurityError(f"Too many redirects (> {max_hops}): {url}")
async def safe_fetch(client: Any, url: str, *, max_hops: int = 5,
allow_internal: bool = False, **kwargs) -> Any:
"""GET with per-hop SSRF validation — follows redirects manually, validating
each hop's URL before requesting it. Returns the final response."""
current = url
for _ in range(max_hops + 1):
current = await validate_url(current, allow_internal=allow_internal)
resp = await client.get(current, follow_redirects=False, **kwargs)
if resp.is_redirect and resp.headers.get("location"):
current = urljoin(str(resp.url), resp.headers["location"])
continue
return resp
raise SecurityError(f"Too many redirects (> {max_hops}): {url}")
async def safe_stream(client: Any, url: str, *, max_bytes: int,
max_hops: int = 5, allow_internal: bool = False,
**kwargs) -> bytes:
"""safe_fetch variant that streams and caps the body at max_bytes, keeping
per-hop SSRF validation. Returns the body bytes (truncated at max_bytes).
Raises OSError-adjacent httpx.HTTPStatusError on non-200 so callers can
distinguish an HTTP error from an empty body."""
current = url
for _ in range(max_hops + 1):
current = await validate_url(current, allow_internal=allow_internal)
async with client.stream("GET", current, follow_redirects=False, **kwargs) as resp:
if resp.is_redirect and resp.headers.get("location"):
current = urljoin(str(resp.url), resp.headers["location"])
continue
if resp.status_code != 200:
raise httpx.HTTPStatusError(
f"HTTP {resp.status_code}", request=resp.request, response=resp)
body = b""
async for chunk in resp.aiter_bytes():
body += chunk
if len(body) > max_bytes:
break
return body
raise SecurityError(f"Too many redirects (> {max_hops}): {url}")