fix(downloads): the egress filter keeps what yt-dlp fetches itself, and shows no phase

f4m, ism, a bare m3u8 (native under hls_prefer_native) and the DASH generator fetch through the
proxy, but were dropped as if ffmpeg dialled them. The filter also took the job's postprocessor
hook, twice, and painted every guarded job "processing" before a byte arrived. Tests now hold an
allowed tunnel, the worker's own gate, the single-URL branch, and that the pipeline and the
backfill turn the gate on while YouTube does not.
This commit is contained in:
2026-11-07 21:48:38 +01:00
parent fc65b8ab83
commit e7e8d8236e
4 changed files with 106 additions and 6 deletions
+70
View File
@@ -273,6 +273,76 @@ def test_a_format_fetched_past_the_gate_is_never_chosen(tmp_path):
ydl.process_ie_result(only_rtmp, download=False)
def test_a_format_yt_dlp_fetches_itself_is_kept():
# f4m, ism and the DASH generator fetch through yt-dlp's own urlopen; a bare m3u8 goes to the
# native HLS fetcher under hls_prefer_native (set for every guarded download). All ask the gate.
kept = [
{"format_id": p, "url": f"https://{PUBLIC}/{p}", "protocol": p}
for p in ("f4m", "ism", "m3u8", "m3u8_native", "http_dash_segments_generator", "https")
]
raw = {"format_id": "rtmp", "url": f"rtmp://{PUBLIC}/live", "protocol": "rtmp"}
_, info = egress.HttpOnlyFormats().run({"formats": [*kept, raw]})
assert [f["format_id"] for f in info["formats"]] == [f["format_id"] for f in kept]
def test_a_single_url_result_past_the_gate_loses_its_url():
_, info = egress.HttpOnlyFormats().run({"url": f"rtmp://{INTERNAL}/live"})
assert info["url"] is None
_, info = egress.HttpOnlyFormats().run({"url": f"https://{PUBLIC}/a.mp4"})
assert info["url"] == f"https://{PUBLIC}/a.mp4"
def test_the_format_filter_says_no_phase_of_its_own(tmp_path):
# It is a post-processor only to run before the format choice; the job's postprocessor hook
# must not see it (it painted every guarded job "processing" before a byte was fetched).
calls: list[dict] = []
opts = _guarded_opts(tmp_path, "http://127.0.0.1:9", postprocessor_hooks=[calls.append])
with egress.open_ydl(opts) as ydl:
ydl.run_all_pps("pre_process", {"id": "x", "formats": []})
assert calls == []
def test_an_allowed_tunnel_carries_bytes_both_ways(gate):
# The CONNECT path a site served over https takes: a 200, then the bytes the client sent after
# the head (in the same packet here) reach the far end, and its answer comes back.
far = socket.socket()
far.bind((PUBLIC, 0))
far.listen()
def echo():
conn, _ = far.accept()
with conn:
conn.sendall(b"PONG:" + conn.recv(64))
threading.Thread(target=echo, daemon=True).start()
try:
with socket.create_connection(("127.0.0.1", gate.server_address[1]), timeout=5) as s:
s.sendall(f"CONNECT {PUBLIC}:{far.getsockname()[1]} HTTP/1.1\r\n\r\nPING".encode())
got = b""
while b"PONG:PING" not in got:
chunk = s.recv(4096)
if not chunk:
break
got += chunk
assert got.startswith(b"HTTP/1.1 200")
assert got.endswith(b"PONG:PING")
finally:
far.close()
def test_the_workers_own_gate_refuses_our_network():
# Every other gate here is built with a test rule; this is the one `proxy_url()` starts.
internal = _Listener()
try:
port = int(egress.proxy_url().rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port), timeout=5) as s:
s.sendall(f"CONNECT {INTERNAL}:{internal.port} HTTP/1.1\r\n\r\n".encode())
assert s.recv(4096).startswith(b"HTTP/1.1 403")
assert internal.hits == 0
finally:
internal.close()
def _clip_hits(tmp_path: Path, segment: str, proxy: str) -> int:
"""How many connections the internal listener takes while a clip (`download_ranges`, which
yt-dlp hands to ffmpeg even for an HLS format) is cut from a playlist whose one segment is