fix(downloads): the egress filter keeps what yt-dlp fetches itself, and shows no phase
f4m, ism, a bare m3u8 (native under hls_prefer_native) and the DASH generator fetch through the proxy, but were dropped as if ffmpeg dialled them. The filter also took the job's postprocessor hook, twice, and painted every guarded job "processing" before a byte arrived. Tests now hold an allowed tunnel, the worker's own gate, the single-URL branch, and that the pipeline and the backfill turn the gate on while YouTube does not.
This commit is contained in:
@@ -273,6 +273,76 @@ def test_a_format_fetched_past_the_gate_is_never_chosen(tmp_path):
|
||||
ydl.process_ie_result(only_rtmp, download=False)
|
||||
|
||||
|
||||
def test_a_format_yt_dlp_fetches_itself_is_kept():
|
||||
# f4m, ism and the DASH generator fetch through yt-dlp's own urlopen; a bare m3u8 goes to the
|
||||
# native HLS fetcher under hls_prefer_native (set for every guarded download). All ask the gate.
|
||||
kept = [
|
||||
{"format_id": p, "url": f"https://{PUBLIC}/{p}", "protocol": p}
|
||||
for p in ("f4m", "ism", "m3u8", "m3u8_native", "http_dash_segments_generator", "https")
|
||||
]
|
||||
raw = {"format_id": "rtmp", "url": f"rtmp://{PUBLIC}/live", "protocol": "rtmp"}
|
||||
_, info = egress.HttpOnlyFormats().run({"formats": [*kept, raw]})
|
||||
assert [f["format_id"] for f in info["formats"]] == [f["format_id"] for f in kept]
|
||||
|
||||
|
||||
def test_a_single_url_result_past_the_gate_loses_its_url():
|
||||
_, info = egress.HttpOnlyFormats().run({"url": f"rtmp://{INTERNAL}/live"})
|
||||
assert info["url"] is None
|
||||
_, info = egress.HttpOnlyFormats().run({"url": f"https://{PUBLIC}/a.mp4"})
|
||||
assert info["url"] == f"https://{PUBLIC}/a.mp4"
|
||||
|
||||
|
||||
def test_the_format_filter_says_no_phase_of_its_own(tmp_path):
|
||||
# It is a post-processor only to run before the format choice; the job's postprocessor hook
|
||||
# must not see it (it painted every guarded job "processing" before a byte was fetched).
|
||||
calls: list[dict] = []
|
||||
opts = _guarded_opts(tmp_path, "http://127.0.0.1:9", postprocessor_hooks=[calls.append])
|
||||
with egress.open_ydl(opts) as ydl:
|
||||
ydl.run_all_pps("pre_process", {"id": "x", "formats": []})
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_an_allowed_tunnel_carries_bytes_both_ways(gate):
|
||||
# The CONNECT path a site served over https takes: a 200, then the bytes the client sent after
|
||||
# the head (in the same packet here) reach the far end, and its answer comes back.
|
||||
far = socket.socket()
|
||||
far.bind((PUBLIC, 0))
|
||||
far.listen()
|
||||
|
||||
def echo():
|
||||
conn, _ = far.accept()
|
||||
with conn:
|
||||
conn.sendall(b"PONG:" + conn.recv(64))
|
||||
|
||||
threading.Thread(target=echo, daemon=True).start()
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", gate.server_address[1]), timeout=5) as s:
|
||||
s.sendall(f"CONNECT {PUBLIC}:{far.getsockname()[1]} HTTP/1.1\r\n\r\nPING".encode())
|
||||
got = b""
|
||||
while b"PONG:PING" not in got:
|
||||
chunk = s.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
got += chunk
|
||||
assert got.startswith(b"HTTP/1.1 200")
|
||||
assert got.endswith(b"PONG:PING")
|
||||
finally:
|
||||
far.close()
|
||||
|
||||
|
||||
def test_the_workers_own_gate_refuses_our_network():
|
||||
# Every other gate here is built with a test rule; this is the one `proxy_url()` starts.
|
||||
internal = _Listener()
|
||||
try:
|
||||
port = int(egress.proxy_url().rsplit(":", 1)[1])
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=5) as s:
|
||||
s.sendall(f"CONNECT {INTERNAL}:{internal.port} HTTP/1.1\r\n\r\n".encode())
|
||||
assert s.recv(4096).startswith(b"HTTP/1.1 403")
|
||||
assert internal.hits == 0
|
||||
finally:
|
||||
internal.close()
|
||||
|
||||
|
||||
def _clip_hits(tmp_path: Path, segment: str, proxy: str) -> int:
|
||||
"""How many connections the internal listener takes while a clip (`download_ranges`, which
|
||||
yt-dlp hands to ffmpeg even for an HLS format) is cut from a playlist whose one segment is
|
||||
|
||||
Reference in New Issue
Block a user