peter
1bf3d6a592
Seen live: yt-dlp closes a connection once it has read what it needed, and the gate printed a traceback for each. A reset ends the tunnel quietly; anything else the gate cannot finish is a log line with its code, not bare stderr.
279 lines
12 KiB
Python
279 lines
12 KiB
Python
"""The egress gate: every fetch of a user-supplied download goes through here, and this is where the
|
|
address is judged — at the moment of the connection, not when the URL was typed (SEC-17).
|
|
|
|
`urlguard` judges the URL the user typed, at enqueue and again in the worker. What that cannot see is
|
|
every request yt-dlp makes after it: a public host answering 3xx with an internal `Location`, a page
|
|
whose formats point inside, a name that resolves public for the check and private for the fetch (DNS
|
|
rebinding). So a guarded download runs with yt-dlp's `proxy` set to this gate, a small HTTP proxy on
|
|
the worker's loopback. For each request it resolves the host ONCE, refuses the connection if any
|
|
address the name has is one the server must not fetch from (`urlguard.is_forbidden`), and otherwise
|
|
connects to the very address it vetted — there is no second lookup for a rebinding answer to slip into.
|
|
|
|
Measured (yt-dlp 2026.8.19, 2026-11-06) on the worker image, with a logging proxy: the `proxy` option
|
|
reaches every path a download takes — the Requests handler, the CurlCFFI one an impersonated request
|
|
uses, the native HLS downloader and an ffmpeg HLS download (segments included). ffmpeg itself also
|
|
speaks rtmp, rtsp, ftp, tcp and more, which no HTTP proxy covers, so a guarded download keeps only the
|
|
formats it fetches over http(s) (`HttpOnlyFormats`, added by `open_ydl`) and fetches HLS natively.
|
|
|
|
A refused connection is answered `403` before anything is dialled, so a live internal port and a dead
|
|
one read the same; `errors.user_error_text` then stores the one redacted sentence for both.
|
|
|
|
Not used for YouTube: those downloads talk to Google's fixed hosts and to the PO-token sidecar, which
|
|
lives on the internal network on purpose (`adapters.YouTubeAdapter.egress_guarded`).
|
|
"""
|
|
|
|
import ipaddress
|
|
import select
|
|
import socket
|
|
import socketserver
|
|
import threading
|
|
from collections.abc import Callable
|
|
from typing import Any, cast
|
|
from urllib.parse import urlsplit
|
|
|
|
import yt_dlp
|
|
from yt_dlp.postprocessor.common import PostProcessor
|
|
|
|
from app import logs
|
|
from app.downloads import urlguard
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
Resolver = Callable[..., list]
|
|
Forbidden = Callable[[ipaddress.IPv4Address | ipaddress.IPv6Address], bool]
|
|
|
|
_BLOCKED = b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_BAD_GATEWAY = b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_BAD_REQUEST = b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_MAX_HEAD = 64 * 1024
|
|
# A tunnel with no traffic either way for this long is closed. Above yt-dlp's own socket_timeout
|
|
# (30 s, `formats.build_ydl_opts`), so the gate never ends a transfer yt-dlp would still wait for.
|
|
_IDLE_S = 120
|
|
# Hop-by-hop headers a proxy must not pass on; `Connection: close` replaces them, so one connection
|
|
# carries one request and a reused client connection can never be steered to a second host unvetted.
|
|
_HOP = (b"proxy-", b"connection:", b"keep-alive:")
|
|
|
|
|
|
def vetted_address(
|
|
host: str, port: int, *, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden
|
|
) -> tuple[str, int] | None:
|
|
"""The address to connect to for `host:port`, or None when it must not be fetched: the name has
|
|
no answer, or ANY of its addresses is forbidden (a round-robin answer is not a coin flip). IPv4
|
|
only, as every download is (`source_address` 0.0.0.0 in `formats.build_ydl_opts`)."""
|
|
try:
|
|
infos = resolve(host, port, socket.AF_INET, socket.SOCK_STREAM)
|
|
except (socket.gaierror, UnicodeError, OSError):
|
|
# silent-ok: "no answer" is a refusal, and the caller's refusal is the line that is logged.
|
|
return None
|
|
addrs = [info[4] for info in infos]
|
|
if not addrs:
|
|
return None
|
|
for addr in addrs:
|
|
try:
|
|
ip = ipaddress.ip_address(addr[0])
|
|
except ValueError:
|
|
# silent-ok: an answer that is not an address is refused like a forbidden one, and the
|
|
# caller logs the refusal.
|
|
return None
|
|
if forbidden(ip):
|
|
return None
|
|
return addrs[0][0], addrs[0][1]
|
|
|
|
|
|
def _read_head(sock: socket.socket) -> bytes | None:
|
|
buf = b""
|
|
while b"\r\n\r\n" not in buf:
|
|
if len(buf) > _MAX_HEAD:
|
|
return None
|
|
chunk = sock.recv(8192)
|
|
if not chunk:
|
|
return None
|
|
buf += chunk
|
|
return buf
|
|
|
|
|
|
def _pipe(a: socket.socket, b: socket.socket) -> None:
|
|
try:
|
|
_relay(a, b)
|
|
except OSError:
|
|
# silent-ok: one side hung up mid-transfer (yt-dlp closes a connection once it has read a
|
|
# file's size, or gives up on a slow host); whichever side cared reports it itself.
|
|
return
|
|
|
|
|
|
def _relay(a: socket.socket, b: socket.socket) -> None:
|
|
socks = [a, b]
|
|
while True:
|
|
ready, _, _ = select.select(socks, [], [], _IDLE_S)
|
|
if not ready:
|
|
return
|
|
for s in ready:
|
|
data = s.recv(65536)
|
|
if not data:
|
|
return
|
|
(b if s is a else a).sendall(data)
|
|
|
|
|
|
class _Handler(socketserver.BaseRequestHandler):
|
|
def handle(self) -> None:
|
|
client: socket.socket = self.request
|
|
head = _read_head(client)
|
|
if head is None:
|
|
return
|
|
head_part, _, body = head.partition(b"\r\n\r\n")
|
|
lines = head_part.split(b"\r\n")
|
|
try:
|
|
method, target, version = lines[0].decode("latin1").split(" ", 2)
|
|
except ValueError:
|
|
# silent-ok: the only client is yt-dlp on this loopback; a malformed request is answered
|
|
# 400 and reaches its own error, which the job records.
|
|
client.sendall(_BAD_REQUEST)
|
|
return
|
|
if method == "CONNECT":
|
|
host, _, port_s = target.rpartition(":")
|
|
host = host.strip("[]")
|
|
upstream = self._dial(host, port_s)
|
|
if upstream is None:
|
|
return
|
|
with upstream:
|
|
client.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
|
if body:
|
|
upstream.sendall(body)
|
|
_pipe(client, upstream)
|
|
return
|
|
u = urlsplit(target)
|
|
if u.scheme != "http" or not u.hostname:
|
|
client.sendall(_BLOCKED)
|
|
return
|
|
try:
|
|
port = u.port or 80
|
|
except ValueError:
|
|
# silent-ok: as for a malformed request line above — answered 400, recorded by the job.
|
|
client.sendall(_BAD_REQUEST)
|
|
return
|
|
upstream = self._dial(u.hostname, str(port))
|
|
if upstream is None:
|
|
return
|
|
path = (u.path or "/") + (f"?{u.query}" if u.query else "")
|
|
headers = [h for h in lines[1:] if not h.lower().startswith(_HOP)]
|
|
request = b"\r\n".join(
|
|
[f"{method} {path} {version}".encode("latin1"), *headers, b"Connection: close"]
|
|
)
|
|
with upstream:
|
|
upstream.sendall(request + b"\r\n\r\n" + body)
|
|
_pipe(client, upstream)
|
|
|
|
def _dial(self, host: str, port_s: str) -> socket.socket | None:
|
|
"""Connect to the address `host` was vetted at, or answer the client and return None."""
|
|
client: socket.socket = self.request
|
|
try:
|
|
port = int(port_s)
|
|
except ValueError:
|
|
# silent-ok: as for a malformed request line above — answered 400, recorded by the job.
|
|
client.sendall(_BAD_REQUEST)
|
|
return None
|
|
gate = cast("_Gate", self.server)
|
|
addr = vetted_address(host, port, resolve=gate.resolve, forbidden=gate.forbidden)
|
|
if addr is None:
|
|
log.warning(
|
|
"egress refused a connection to %s:%s",
|
|
host, # log-pii-ok: the host a download tried to reach, never its path or query
|
|
port,
|
|
extra=logs.event("download.egress.refused", port=port),
|
|
)
|
|
client.sendall(_BLOCKED)
|
|
return None
|
|
try:
|
|
return socket.create_connection(addr, timeout=_IDLE_S)
|
|
except OSError:
|
|
# silent-ok: a public host that does not answer; yt-dlp reports it as the job's own
|
|
# network failure, which the job records.
|
|
client.sendall(_BAD_GATEWAY)
|
|
return None
|
|
|
|
|
|
class _Gate(socketserver.ThreadingTCPServer):
|
|
daemon_threads = True
|
|
allow_reuse_address = True
|
|
|
|
def __init__(self, resolve: Resolver, forbidden: Forbidden):
|
|
self.resolve = resolve
|
|
self.forbidden = forbidden
|
|
super().__init__(("127.0.0.1", 0), _Handler)
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
return f"http://127.0.0.1:{self.server_address[1]}"
|
|
|
|
def handle_error(self, request, client_address) -> None:
|
|
# socketserver's default prints a bare traceback to stderr, outside the log; a connection the
|
|
# gate could not finish is a line with its code instead.
|
|
log.exception("egress gate dropped a connection", extra=logs.event("download.egress.error"))
|
|
|
|
|
|
def start_gate(
|
|
*, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden = urlguard.is_forbidden
|
|
) -> _Gate:
|
|
"""A gate serving on its own thread. The worker uses the one `proxy_url` starts; a test starts
|
|
its own, with a resolver and a judgement it controls."""
|
|
gate = _Gate(resolve, forbidden)
|
|
threading.Thread(target=gate.serve_forever, name="egress-gate", daemon=True).start()
|
|
return gate
|
|
|
|
|
|
# The options key `formats.build_ydl_opts` marks a guarded download with; `open_ydl` reads it. yt-dlp
|
|
# keeps unknown keys in its params and never acts on them.
|
|
GUARDED = "siftlode_egress_guarded"
|
|
|
|
|
|
def open_ydl(opts: dict) -> yt_dlp.YoutubeDL:
|
|
"""The one way this package opens yt-dlp (a contract test holds it): a guarded download also
|
|
gets `HttpOnlyFormats`, which yt-dlp takes only as an instance, not through the options dict."""
|
|
# yt-dlp's stub types its params as a TypedDict; these are the plain dict `formats.build_ydl_opts`
|
|
# assembles, the same every caller handed yt-dlp before.
|
|
ydl = yt_dlp.YoutubeDL(opts) # type: ignore[arg-type]
|
|
if opts.get(GUARDED):
|
|
ydl.add_post_processor(HttpOnlyFormats(ydl), when="pre_process")
|
|
return ydl
|
|
|
|
|
|
_gate: _Gate | None = None
|
|
_gate_lock = threading.Lock()
|
|
|
|
|
|
def proxy_url() -> str:
|
|
"""This process's gate, started on first use (the worker's jobs share it)."""
|
|
global _gate
|
|
with _gate_lock:
|
|
if _gate is None:
|
|
_gate = start_gate()
|
|
return _gate.url
|
|
|
|
|
|
# The protocols yt-dlp fetches over http(s) — the only ones the gate sees. Everything else (rtmp,
|
|
# rtsp, mms, f4m's own fetcher, ws) would be dialled by ffmpeg or another tool directly.
|
|
_HTTP_PROTOCOLS = frozenset({"http", "https", "m3u8_native", "http_dash_segments"})
|
|
|
|
|
|
class HttpOnlyFormats(PostProcessor):
|
|
"""Drops every format a guarded download would fetch past the gate. Runs as `pre_process`, which
|
|
yt-dlp re-reads its formats after ("The pre-processors may have modified the formats"), so the
|
|
selector never sees them. With nothing left, yt-dlp reports no formats — a refusal, not a leak."""
|
|
|
|
def run(self, information):
|
|
# yt-dlp's stub types the info dict as a TypedDict with optional keys; it is a dict here.
|
|
info = cast("dict[str, Any]", information)
|
|
if info.get("formats"):
|
|
info["formats"] = [f for f in info["formats"] if _over_http(f)]
|
|
elif info.get("url") and not _over_http(info):
|
|
info["url"] = None
|
|
return [], information
|
|
|
|
|
|
def _over_http(fmt: dict) -> bool:
|
|
url = fmt.get("manifest_url") or fmt.get("url") or ""
|
|
if urlsplit(url).scheme not in ("http", "https"):
|
|
return False
|
|
protocol = fmt.get("protocol") or urlsplit(fmt.get("url") or "").scheme
|
|
return all(p in _HTTP_PROTOCOLS for p in protocol.split("+"))
|