fix(downloads): a user's download is judged at every connection it makes, not only at the URL typed
The URL guard judged the address the user typed; a public host's 3xx into our network, a page's formats pointing inside, and a name rebound between the check and the fetch all got past it, and the stored error told a live internal port from a dead one. Every non-YouTube download now runs through an egress gate on the worker's loopback: it resolves each host once, refuses any forbidden answer before dialling, and connects to the address it vetted. Measured on the worker image: yt-dlp's proxy reaches the Requests and CurlCFFI handlers, native HLS and ffmpeg alike. Formats fetched over anything but http(s), which ffmpeg would dial directly, are dropped, and one factory opens every yt-dlp so none skips it.
This commit is contained in:
@@ -359,6 +359,9 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def add_post_processor(self, pp, when="post_process"):
|
||||
pass # the egress format filter; nothing here to filter
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
@@ -367,7 +370,7 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
|
||||
f"ERROR: [youtube] dQw4w9WgXcQ: Video unavailable ({target})"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
|
||||
monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing)
|
||||
assert backfill.extract_meta(url, adapters.YouTubeAdapter(), 11) is None
|
||||
|
||||
lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
|
||||
@@ -386,13 +389,16 @@ def test_the_metadata_backfill_takes_out_a_generic_name_whatever_it_holds(sink,
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def add_post_processor(self, pp, when="post_process"):
|
||||
pass # the egress format filter; nothing here to filter
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def extract_info(self, target, download):
|
||||
raise yt_dlp.utils.DownloadError("ERROR: [generic] Our Trip: Day 1: Unable to download")
|
||||
|
||||
monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
|
||||
monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing)
|
||||
assert backfill.extract_meta(url, adapters.SourceAdapter(), 11) is None
|
||||
|
||||
lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
|
||||
@@ -449,6 +455,9 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def add_post_processor(self, pp, when="post_process"):
|
||||
pass # the egress format filter; nothing here to filter
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
@@ -463,6 +472,7 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
|
||||
resolve_url=lambda target, opts: target,
|
||||
ydl_opts_extra=lambda clients: {},
|
||||
retriable=lambda text: True,
|
||||
egress_guarded=False,
|
||||
)
|
||||
monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", _YDL)
|
||||
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None)
|
||||
|
||||
Reference in New Issue
Block a user