peter
e7e8d8236e
f4m, ism, a bare m3u8 (native under hls_prefer_native) and the DASH generator fetch through the proxy, but were dropped as if ffmpeg dialled them. The filter also took the job's postprocessor hook, twice, and painted every guarded job "processing" before a byte arrived. Tests now hold an allowed tunnel, the worker's own gate, the single-URL branch, and that the pipeline and the backfill turn the gate on while YouTube does not.
298 lines
12 KiB
Python
298 lines
12 KiB
Python
"""The egress gate: every fetch of a user-supplied download goes through here, and this is where the
|
|
address is judged — at the moment of the connection, not when the URL was typed (SEC-17).
|
|
|
|
`urlguard` judges the URL the user typed, at enqueue and again in the worker. What that cannot see is
|
|
every request yt-dlp makes after it: a public host answering 3xx with an internal `Location`, a page
|
|
whose formats point inside, a name that resolves public for the check and private for the fetch (DNS
|
|
rebinding). So a guarded download runs with yt-dlp's `proxy` set to this gate, a small HTTP proxy on
|
|
the worker's loopback. For each request it resolves the host ONCE, refuses the connection if any
|
|
address the name has is one the server must not fetch from (`urlguard.is_forbidden`), and otherwise
|
|
connects to the very address it vetted — there is no second lookup for a rebinding answer to slip into.
|
|
|
|
Measured (yt-dlp 2026.8.19, 2026-11-06) on the worker image, with a logging proxy: the `proxy` option
|
|
reaches every path a download takes — the Requests handler, the CurlCFFI one an impersonated request
|
|
uses, the native HLS downloader and an ffmpeg HLS download (segments included). ffmpeg itself also
|
|
speaks rtmp, rtsp, ftp, tcp and more, which no HTTP proxy covers, so a guarded download keeps only the
|
|
formats it fetches over http(s) (`HttpOnlyFormats`, added by `open_ydl`) and fetches HLS natively.
|
|
|
|
A refused connection is answered `403` before anything is dialled, so a live internal port and a dead
|
|
one read the same; `errors.user_error_text` then stores the one redacted sentence for both.
|
|
|
|
Not used for YouTube: those downloads talk to Google's fixed hosts and to the PO-token sidecar, which
|
|
lives on the internal network on purpose (`adapters.YouTubeAdapter.egress_guarded`).
|
|
"""
|
|
|
|
import ipaddress
|
|
import select
|
|
import socket
|
|
import socketserver
|
|
import threading
|
|
from collections.abc import Callable
|
|
from typing import Any, cast
|
|
from urllib.parse import urlsplit
|
|
|
|
import yt_dlp
|
|
from yt_dlp.postprocessor.common import PostProcessor
|
|
|
|
from app import logs
|
|
from app.downloads import urlguard
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
Resolver = Callable[..., list]
|
|
Forbidden = Callable[[ipaddress.IPv4Address | ipaddress.IPv6Address], bool]
|
|
|
|
_BLOCKED = b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_BAD_GATEWAY = b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_BAD_REQUEST = b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
_MAX_HEAD = 64 * 1024
|
|
# A tunnel with no traffic either way for this long is closed. Above yt-dlp's own socket_timeout
|
|
# (30 s, `formats.build_ydl_opts`), so the gate never ends a transfer yt-dlp would still wait for.
|
|
_IDLE_S = 120
|
|
# Hop-by-hop headers a proxy must not pass on; `Connection: close` replaces them, so one connection
|
|
# carries one request and a reused client connection can never be steered to a second host unvetted.
|
|
_HOP = (b"proxy-", b"connection:", b"keep-alive:")
|
|
|
|
|
|
def vetted_address(
|
|
host: str, port: int, *, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden
|
|
) -> tuple[str, int] | None:
|
|
"""The address to connect to for `host:port`, or None when it must not be fetched: the name has
|
|
no answer, or ANY of its addresses is forbidden (a round-robin answer is not a coin flip). IPv4
|
|
only, as every download is (`source_address` 0.0.0.0 in `formats.build_ydl_opts`)."""
|
|
try:
|
|
infos = resolve(host, port, socket.AF_INET, socket.SOCK_STREAM)
|
|
except (socket.gaierror, UnicodeError, OSError):
|
|
# silent-ok: "no answer" is a refusal, and the caller's refusal is the line that is logged.
|
|
return None
|
|
addrs = [info[4] for info in infos]
|
|
if not addrs:
|
|
return None
|
|
for addr in addrs:
|
|
try:
|
|
ip = ipaddress.ip_address(addr[0])
|
|
except ValueError:
|
|
# silent-ok: an answer that is not an address is refused like a forbidden one, and the
|
|
# caller logs the refusal.
|
|
return None
|
|
if forbidden(ip):
|
|
return None
|
|
return addrs[0][0], addrs[0][1]
|
|
|
|
|
|
def _read_head(sock: socket.socket) -> bytes | None:
|
|
buf = b""
|
|
while b"\r\n\r\n" not in buf:
|
|
if len(buf) > _MAX_HEAD:
|
|
return None
|
|
chunk = sock.recv(8192)
|
|
if not chunk:
|
|
return None
|
|
buf += chunk
|
|
return buf
|
|
|
|
|
|
def _pipe(a: socket.socket, b: socket.socket) -> None:
|
|
try:
|
|
_relay(a, b)
|
|
except OSError:
|
|
# silent-ok: one side hung up mid-transfer (yt-dlp closes a connection once it has read a
|
|
# file's size, or gives up on a slow host); whichever side cared reports it itself.
|
|
return
|
|
|
|
|
|
def _relay(a: socket.socket, b: socket.socket) -> None:
|
|
socks = [a, b]
|
|
while True:
|
|
ready, _, _ = select.select(socks, [], [], _IDLE_S)
|
|
if not ready:
|
|
return
|
|
for s in ready:
|
|
data = s.recv(65536)
|
|
if not data:
|
|
return
|
|
(b if s is a else a).sendall(data)
|
|
|
|
|
|
class _Handler(socketserver.BaseRequestHandler):
|
|
def handle(self) -> None:
|
|
client: socket.socket = self.request
|
|
head = _read_head(client)
|
|
if head is None:
|
|
return
|
|
head_part, _, body = head.partition(b"\r\n\r\n")
|
|
lines = head_part.split(b"\r\n")
|
|
try:
|
|
method, target, version = lines[0].decode("latin1").split(" ", 2)
|
|
except ValueError:
|
|
# silent-ok: the only client is yt-dlp on this loopback; a malformed request is answered
|
|
# 400 and reaches its own error, which the job records.
|
|
client.sendall(_BAD_REQUEST)
|
|
return
|
|
if method == "CONNECT":
|
|
host, _, port_s = target.rpartition(":")
|
|
host = host.strip("[]")
|
|
upstream = self._dial(host, port_s)
|
|
if upstream is None:
|
|
return
|
|
with upstream:
|
|
client.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
|
if body:
|
|
upstream.sendall(body)
|
|
_pipe(client, upstream)
|
|
return
|
|
u = urlsplit(target)
|
|
if u.scheme != "http" or not u.hostname:
|
|
client.sendall(_BLOCKED)
|
|
return
|
|
try:
|
|
port = u.port or 80
|
|
except ValueError:
|
|
# silent-ok: as for a malformed request line above — answered 400, recorded by the job.
|
|
client.sendall(_BAD_REQUEST)
|
|
return
|
|
upstream = self._dial(u.hostname, str(port))
|
|
if upstream is None:
|
|
return
|
|
path = (u.path or "/") + (f"?{u.query}" if u.query else "")
|
|
headers = [h for h in lines[1:] if not h.lower().startswith(_HOP)]
|
|
request = b"\r\n".join(
|
|
[f"{method} {path} {version}".encode("latin1"), *headers, b"Connection: close"]
|
|
)
|
|
with upstream:
|
|
upstream.sendall(request + b"\r\n\r\n" + body)
|
|
_pipe(client, upstream)
|
|
|
|
def _dial(self, host: str, port_s: str) -> socket.socket | None:
|
|
"""Connect to the address `host` was vetted at, or answer the client and return None."""
|
|
client: socket.socket = self.request
|
|
try:
|
|
port = int(port_s)
|
|
except ValueError:
|
|
# silent-ok: as for a malformed request line above — answered 400, recorded by the job.
|
|
client.sendall(_BAD_REQUEST)
|
|
return None
|
|
gate = cast("_Gate", self.server)
|
|
addr = vetted_address(host, port, resolve=gate.resolve, forbidden=gate.forbidden)
|
|
if addr is None:
|
|
log.warning(
|
|
"egress refused a connection to %s:%s",
|
|
host, # log-pii-ok: the host a download tried to reach, never its path or query
|
|
port,
|
|
extra=logs.event("download.egress.refused", port=port),
|
|
)
|
|
client.sendall(_BLOCKED)
|
|
return None
|
|
try:
|
|
return socket.create_connection(addr, timeout=_IDLE_S)
|
|
except OSError:
|
|
# silent-ok: a public host that does not answer; yt-dlp reports it as the job's own
|
|
# network failure, which the job records.
|
|
client.sendall(_BAD_GATEWAY)
|
|
return None
|
|
|
|
|
|
class _Gate(socketserver.ThreadingTCPServer):
|
|
daemon_threads = True
|
|
allow_reuse_address = True
|
|
|
|
def __init__(self, resolve: Resolver, forbidden: Forbidden):
|
|
self.resolve = resolve
|
|
self.forbidden = forbidden
|
|
super().__init__(("127.0.0.1", 0), _Handler)
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
return f"http://127.0.0.1:{self.server_address[1]}"
|
|
|
|
def handle_error(self, request, client_address) -> None:
|
|
# socketserver's default prints a bare traceback to stderr, outside the log; a connection the
|
|
# gate could not finish is a line with its code instead.
|
|
log.exception("egress gate dropped a connection", extra=logs.event("download.egress.error"))
|
|
|
|
|
|
def start_gate(
|
|
*, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden = urlguard.is_forbidden
|
|
) -> _Gate:
|
|
"""A gate serving on its own thread. The worker uses the one `proxy_url` starts; a test starts
|
|
its own, with a resolver and a judgement it controls."""
|
|
gate = _Gate(resolve, forbidden)
|
|
threading.Thread(target=gate.serve_forever, name="egress-gate", daemon=True).start()
|
|
return gate
|
|
|
|
|
|
# The options key `formats.build_ydl_opts` marks a guarded download with; `open_ydl` reads it. yt-dlp
|
|
# keeps unknown keys in its params and never acts on them.
|
|
GUARDED = "siftlode_egress_guarded"
|
|
|
|
|
|
def open_ydl(opts: dict) -> yt_dlp.YoutubeDL:
|
|
"""The one way this package opens yt-dlp (a contract test holds it): a guarded download also
|
|
gets `HttpOnlyFormats`, which yt-dlp takes only as an instance, not through the options dict."""
|
|
# yt-dlp's stub types its params as a TypedDict; these are the plain dict `formats.build_ydl_opts`
|
|
# assembles, the same every caller handed yt-dlp before.
|
|
ydl = yt_dlp.YoutubeDL(opts) # type: ignore[arg-type]
|
|
if opts.get(GUARDED):
|
|
ydl.add_post_processor(HttpOnlyFormats(), when="pre_process")
|
|
return ydl
|
|
|
|
|
|
_gate: _Gate | None = None
|
|
_gate_lock = threading.Lock()
|
|
|
|
|
|
def proxy_url() -> str:
|
|
"""This process's gate, started on first use (the worker's jobs share it)."""
|
|
global _gate
|
|
with _gate_lock:
|
|
if _gate is None:
|
|
_gate = start_gate()
|
|
return _gate.url
|
|
|
|
|
|
# The protocols fetched over http(s) through the proxy option — the only ones the gate sees: yt-dlp's
|
|
# own fetchers (http, f4m, ism, the DASH segments and their generator, native HLS — which a bare m3u8
|
|
# also gets, as a guarded download sets hls_prefer_native) and ffmpeg's http input (a live stream or
|
|
# a clip), which takes the proxy too. Everything else (rtmp, rtsp, mms, ws) ffmpeg or another tool
|
|
# would dial directly.
|
|
_HTTP_PROTOCOLS = frozenset(
|
|
{
|
|
"http",
|
|
"https",
|
|
"f4m",
|
|
"ism",
|
|
"m3u8",
|
|
"m3u8_native",
|
|
"http_dash_segments",
|
|
"http_dash_segments_generator",
|
|
}
|
|
)
|
|
|
|
|
|
class HttpOnlyFormats(PostProcessor):
|
|
"""Drops every format a guarded download would fetch past the gate. Runs as `pre_process`, which
|
|
yt-dlp re-reads its formats after ("The pre-processors may have modified the formats"), so the
|
|
selector never sees them. With nothing left, yt-dlp reports no formats — a refusal, not a leak."""
|
|
|
|
def set_downloader(self, downloader):
|
|
# Not the job's postprocessor hooks: they would report this filter as a "processing" step
|
|
# before a byte is fetched (yt-dlp attaches them here, once per call).
|
|
self._downloader = downloader
|
|
|
|
def run(self, information):
|
|
# yt-dlp's stub types the info dict as a TypedDict with optional keys; it is a dict here.
|
|
info = cast("dict[str, Any]", information)
|
|
if info.get("formats"):
|
|
info["formats"] = [f for f in info["formats"] if _over_http(f)]
|
|
elif info.get("url") and not _over_http(info):
|
|
info["url"] = None
|
|
return [], information
|
|
|
|
|
|
def _over_http(fmt: dict) -> bool:
|
|
url = fmt.get("manifest_url") or fmt.get("url") or ""
|
|
if urlsplit(url).scheme not in ("http", "https"):
|
|
return False
|
|
protocol = fmt.get("protocol") or urlsplit(fmt.get("url") or "").scheme
|
|
return all(p in _HTTP_PROTOCOLS for p in protocol.split("+"))
|