fix(downloads): a user's download is judged at every connection it makes, not only at the URL typed

The URL guard judged the address the user typed; a public host's 3xx into our
network, a page's formats pointing inside, and a name rebound between the check
and the fetch all got past it, and the stored error told a live internal port
from a dead one. Every non-YouTube download now runs through an egress gate on
the worker's loopback: it resolves each host once, refuses any forbidden answer
before dialling, and connects to the address it vetted. Measured on the worker
image: yt-dlp's proxy reaches the Requests and CurlCFFI handlers, native HLS and
ffmpeg alike. Formats fetched over anything but http(s), which ffmpeg would dial
directly, are dropped, and one factory opens every yt-dlp so none skips it.
This commit is contained in:
2026-11-06 04:54:56 +01:00
parent 995a493bd3
commit 4e4fff87d4
14 changed files with 668 additions and 65 deletions
+12 -2
View File
@@ -359,6 +359,9 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
def __enter__(self):
return self
def add_post_processor(self, pp, when="post_process"):
pass # the egress format filter; nothing here to filter
def __exit__(self, *exc):
return False
@@ -367,7 +370,7 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
f"ERROR: [youtube] dQw4w9WgXcQ: Video unavailable ({target})"
)
monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing)
assert backfill.extract_meta(url, adapters.YouTubeAdapter(), 11) is None
lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
@@ -386,13 +389,16 @@ def test_the_metadata_backfill_takes_out_a_generic_name_whatever_it_holds(sink,
def __enter__(self):
return self
def add_post_processor(self, pp, when="post_process"):
pass # the egress format filter; nothing here to filter
def __exit__(self, *exc):
return False
def extract_info(self, target, download):
raise yt_dlp.utils.DownloadError("ERROR: [generic] Our Trip: Day 1: Unable to download")
monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing)
assert backfill.extract_meta(url, adapters.SourceAdapter(), 11) is None
lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
@@ -449,6 +455,9 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
def __enter__(self):
return self
def add_post_processor(self, pp, when="post_process"):
pass # the egress format filter; nothing here to filter
def __exit__(self, *exc):
return False
@@ -463,6 +472,7 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
resolve_url=lambda target, opts: target,
ydl_opts_extra=lambda clients: {},
retriable=lambda text: True,
egress_guarded=False,
)
monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", _YDL)
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None)