fix(downloads): a user's download is judged at every connection it makes, not only at the URL typed

The URL guard judged the address the user typed; a public host's 3xx into our
network, a page's formats pointing inside, and a name rebound between the check
and the fetch all got past it, and the stored error told a live internal port
from a dead one. Every non-YouTube download now runs through an egress gate on
the worker's loopback: it resolves each host once, refuses any forbidden answer
before dialling, and connects to the address it vetted. Measured on the worker
image: yt-dlp's proxy reaches the Requests and CurlCFFI handlers, native HLS and
ffmpeg alike. Formats fetched over anything but http(s), which ffmpeg would dial
directly, are dropped, and one factory opens every yt-dlp so none skips it.
This commit is contained in:
2026-11-06 04:54:56 +01:00
parent 995a493bd3
commit 4e4fff87d4
14 changed files with 668 additions and 65 deletions
+8 -2
View File
@@ -14,10 +14,10 @@ Resolution works both before extraction (from source_kind / URL host) and after
import re
from urllib.parse import urlparse
import yt_dlp
from yt_dlp.networking.impersonate import ImpersonateTarget
from app.config import settings
from app.downloads import egress
from app.downloads.stated import StatedFailure
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
@@ -33,6 +33,9 @@ class SourceAdapter:
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
key = "generic"
# SEC-17: every fetch of this source's download goes through the egress gate (`egress`). A
# source's own host list is no guarantee — its pages, redirects and formats can point anywhere.
egress_guarded = True
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
def resolve_url(self, url: str, ydl_opts: dict) -> str:
@@ -92,6 +95,9 @@ class SourceAdapter:
class YouTubeAdapter(SourceAdapter):
key = "youtube"
# Not through the gate: YouTube's fetches go to Google's fixed hosts, and the PO-token provider
# it needs is the sidecar on our internal network, which the gate exists to refuse.
egress_guarded = False
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE = (
"not a bot",
@@ -179,7 +185,7 @@ class FacebookAdapter(SourceAdapter):
post_id = facebook_group_post_id(url)
if post_id is None or not ydl_opts.get("cookiefile"):
return url
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
with egress.open_ydl(ydl_opts) as ydl:
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
page = raw.decode("utf-8", "replace")
# Measured: a refused or expired cookie gets the login page (~450 KB, a login