fix(downloads,sync): bound what one user can make the server fetch, download and spend
SEC-12: the download box accepts an arbitrary http(s) URL and yt-dlp fetches it from the WORKER, so any approved user could aim the server at the compose network, the LAN or a cloud metadata endpoint — and read the result off job.error, where "connection refused" vs "HTTP Error 401" vs "timed out" is a port scanner. downloads/urlguard.py refuses a URL whose host resolves to a loopback/private/link-local/reserved address, at enqueue AND again in the worker (DNS can move in between, and the worker is where the socket opens). A network-class failure on a `url` job now reports one fixed sentence instead of the raw transport text; the error CODE is still classified from the original, so the localized explanation is unchanged. DNS rebinding is explicitly NOT closed — that needs the connection gated inside yt-dlp — and the module says so rather than implying more than it does. SEC-13: `noplaylist` only demotes a video-inside-a-playlist URL to that video, so a URL that is purely a playlist or channel still expanded into every entry — one enqueue, unbounded downloads. `playlist_items` pins it to one. Size was unbounded too: `check_enqueue` compares the CURRENT footprint against the cap, and a download's size is unknown until it lands, so one job could overshoot by any amount; the owner's remaining budget is now handed to yt-dlp as `max_filesize`, applied AFTER the adapter merge so no source adapter can raise a limit the quota layer set. A join's cut-list is capped at 100: an accurate join emits two filters per segment into one ffmpeg filter_complex. SEC-14: the YouTube budget is instance-wide, so one user holding down a manual sync spent the day for everyone. Per-user (not per-IP — the cost belongs to the account) 3-per-10-minutes limiters on the two bulk pulls, subscriptions and playlists, with separate budgets so one does not close the other. The review also listed /rss, /backfill and /enrich; E3 deleted them, and search stays unlimited on purpose — it is interactive, not a bulk pull. Gate: pytest 896 -> 924 (none skipped), vitest 671.
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
"""E12 · S3 — the route-level halves of SEC-12 and SEC-14.
|
||||
|
||||
`test_urlguard.py` pins the address judgement itself; this pins that the ENQUEUE route actually
|
||||
consults it and answers in the structured `{code, reason}` shape the frontend localizes, and that a
|
||||
manual sync trigger stops spending the instance-wide YouTube budget when held down.
|
||||
|
||||
DB-backed (the `as_user` fixture needs real rows) — skipped when no database is reachable.
|
||||
"""
|
||||
|
||||
from app.downloads import urlguard
|
||||
|
||||
|
||||
def test_enqueueing_a_loopback_url_is_refused_with_a_structured_detail(
|
||||
db, as_user, make_user, monkeypatch
|
||||
):
|
||||
user = make_user()
|
||||
db.commit()
|
||||
monkeypatch.setattr(
|
||||
urlguard.socket, "getaddrinfo", lambda *a, **kw: [(None, None, None, "", ("127.0.0.1", 0))]
|
||||
)
|
||||
r = as_user(user).post("/api/downloads", json={"source": "http://redis.internal:6379/x"})
|
||||
assert r.status_code == 422
|
||||
# The shape matters as much as the refusal: a bare English string under a bilingual UI is the
|
||||
# thing the canonical-patterns table forbids.
|
||||
assert r.json()["detail"] == {"code": "url_blocked", "reason": urlguard.REASON_PRIVATE}
|
||||
|
||||
|
||||
def test_a_public_url_gets_past_the_guard(db, as_user, make_user, monkeypatch):
|
||||
# Mutation check for the test above: with a public answer the request must fail LATER (or
|
||||
# succeed), never with url_blocked — otherwise the guard would be refusing everything and the
|
||||
# first test would still pass.
|
||||
user = make_user()
|
||||
db.commit()
|
||||
monkeypatch.setattr(
|
||||
urlguard.socket,
|
||||
"getaddrinfo",
|
||||
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
||||
)
|
||||
r = as_user(user).post("/api/downloads", json={"source": "https://example.com/v"})
|
||||
detail = r.json().get("detail") if r.status_code >= 400 else None
|
||||
assert not (isinstance(detail, dict) and detail.get("code") == "url_blocked")
|
||||
|
||||
|
||||
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
||||
# A YouTube job resolves to a fixed public host from an 11-char id, so there is no
|
||||
# user-controlled destination — resolving one would be pure latency on every enqueue.
|
||||
def boom(*a, **kw):
|
||||
raise AssertionError("the URL guard resolved DNS for a YouTube source")
|
||||
|
||||
user = make_user()
|
||||
db.commit()
|
||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
||||
as_user(user).post("/api/downloads", json={"source": "dQw4w9WgXcQ"})
|
||||
|
||||
|
||||
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
||||
from app.routes import sync as syncroutes
|
||||
|
||||
user = make_user()
|
||||
db.commit()
|
||||
# Neither the YouTube grant nor the import itself is under test — the limiter is.
|
||||
monkeypatch.setattr(syncroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
||||
|
||||
c = as_user(user)
|
||||
codes = [c.post("/api/sync/subscriptions").status_code for _ in range(4)]
|
||||
# 3 per 10 minutes: the first three pass, the fourth is refused.
|
||||
assert codes[:3] == [200, 200, 200]
|
||||
assert codes[3] == 429
|
||||
assert c.post("/api/sync/subscriptions").json()["detail"] == {"code": "sync_cooldown"}
|
||||
|
||||
|
||||
def test_the_cooldown_is_per_user_not_global(db, as_user, make_user, monkeypatch):
|
||||
# Keyed by user id on purpose: the shared YouTube budget is spent per account, and one user
|
||||
# exhausting their allowance must not lock everyone else out of their own sync button.
|
||||
from app.routes import sync as syncroutes
|
||||
|
||||
a, b = make_user(), make_user()
|
||||
db.commit()
|
||||
monkeypatch.setattr(syncroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
||||
|
||||
ca = as_user(a)
|
||||
for _ in range(4):
|
||||
ca.post("/api/sync/subscriptions")
|
||||
assert ca.post("/api/sync/subscriptions").status_code == 429
|
||||
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|
||||
@@ -109,3 +109,52 @@ def test_browser_transcode_flags_universal_playability():
|
||||
assert F("h264", "aac", False) == (True, False, False)
|
||||
# A failed probe / missing stream is never re-encoded.
|
||||
assert F(None, None, True) == (False, False, False)
|
||||
|
||||
|
||||
# --- SEC-13 · one enqueue is one bounded download -------------------------------------------
|
||||
|
||||
|
||||
def test_a_playlist_url_can_only_yield_one_item():
|
||||
# `noplaylist` alone only demotes a video-inside-a-playlist URL to that video; a URL that is
|
||||
# purely a playlist or channel has no such video, so without this it expands into every entry.
|
||||
opts = _opts({})
|
||||
assert opts["noplaylist"] is True
|
||||
assert opts["playlist_items"] == "1"
|
||||
|
||||
|
||||
def test_the_remaining_quota_becomes_a_hard_size_ceiling():
|
||||
opts = formats.build_ydl_opts({}, "/tmp/x", lambda d: None, max_bytes=5_000_000)
|
||||
assert opts["max_filesize"] == 5_000_000
|
||||
|
||||
|
||||
def test_no_budget_means_no_size_filter():
|
||||
# Unlimited users pass None; a non-positive remainder is the quota layer's refusal to make at
|
||||
# enqueue, not an impossible yt-dlp filter to express here.
|
||||
for budget in (None, 0, -1):
|
||||
assert "max_filesize" not in formats.build_ydl_opts(
|
||||
{}, "/tmp/x", lambda d: None, max_bytes=budget
|
||||
)
|
||||
|
||||
|
||||
def test_a_source_adapter_cannot_raise_the_size_ceiling():
|
||||
# The adapter merge happens after max_filesize is set, so this pins that no adapter option can
|
||||
# widen a storage cap — the mirror of the cookiefile rule, in the opposite direction.
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "/tmp/x", lambda d: None, extra={"max_filesize": 999_999_999}, max_bytes=5_000_000
|
||||
)
|
||||
assert opts["max_filesize"] == 5_000_000
|
||||
|
||||
|
||||
def test_a_join_cut_list_is_capped():
|
||||
# An accurate join emits two filters per segment into one ffmpeg filter_complex, so an unbounded
|
||||
# list is an unbounded command line. The surplus is dropped; the legitimate head survives.
|
||||
segs = [{"start_s": i, "end_s": i + 1} for i in range(500)]
|
||||
out = editmod.normalize_edit_spec({"segments": segs})
|
||||
assert len(out["segments"]) == editmod._MAX_SEGMENTS
|
||||
assert out["segments"][0] == {"start_s": 0.0, "end_s": 1.0}
|
||||
|
||||
|
||||
def test_a_normal_cut_list_is_untouched():
|
||||
segs = [{"start_s": 0, "end_s": 5}, {"start_s": 10, "end_s": 12}]
|
||||
out = editmod.normalize_edit_spec({"segments": segs})
|
||||
assert len(out["segments"]) == 2
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""SEC-12 · the download URL guard.
|
||||
|
||||
The Download Center hands a user-supplied URL to yt-dlp on the worker, so the server fetches
|
||||
whatever the user typed. These pin that a URL resolving into our own network is refused, that the
|
||||
refusal reasons stay distinct (an unresolvable host is not the same message as a blocked one), and
|
||||
that a normal public URL still goes through.
|
||||
|
||||
DNS is stubbed rather than live: the point under test is the address JUDGEMENT, and a test that
|
||||
resolves real names would be slow, offline-fragile, and would silently start passing if the guard
|
||||
stopped resolving at all.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.downloads import urlguard
|
||||
|
||||
|
||||
def _stub_dns(monkeypatch, addrs):
|
||||
"""Make every lookup answer with `addrs` (a list of textual IPs)."""
|
||||
|
||||
def fake_getaddrinfo(host, port, *a, **kw):
|
||||
return [(None, None, None, "", (addr, port or 0)) for addr in addrs]
|
||||
|
||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", fake_getaddrinfo)
|
||||
|
||||
|
||||
def _blocked_reason(url):
|
||||
with pytest.raises(urlguard.BlockedUrl) as e:
|
||||
urlguard.check_url(url)
|
||||
return e.value.reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"addr",
|
||||
[
|
||||
"127.0.0.1", # loopback
|
||||
"10.1.2.3", # RFC1918
|
||||
"192.168.0.5", # RFC1918
|
||||
"172.16.9.9", # RFC1918
|
||||
"169.254.169.254", # link-local — the cloud metadata endpoint
|
||||
"0.0.0.0", # unspecified
|
||||
"::1", # IPv6 loopback
|
||||
"fe80::1", # IPv6 link-local
|
||||
"fc00::1", # IPv6 unique-local
|
||||
"::ffff:127.0.0.1", # loopback wearing an IPv6 hat
|
||||
],
|
||||
)
|
||||
def test_addresses_on_our_own_network_are_refused(monkeypatch, addr):
|
||||
_stub_dns(monkeypatch, [addr])
|
||||
assert _blocked_reason("http://whatever.example/video") == urlguard.REASON_PRIVATE
|
||||
|
||||
|
||||
def test_one_bad_answer_in_a_round_robin_blocks_the_whole_host(monkeypatch):
|
||||
# A host answering with both a public and an internal address must not become a coin flip:
|
||||
# yt-dlp would pick for itself, so the guard refuses if ANY answer is forbidden.
|
||||
_stub_dns(monkeypatch, ["93.184.216.34", "127.0.0.1"])
|
||||
assert _blocked_reason("https://mixed.example/v") == urlguard.REASON_PRIVATE
|
||||
|
||||
|
||||
def test_a_public_address_passes(monkeypatch):
|
||||
_stub_dns(monkeypatch, ["93.184.216.34"])
|
||||
urlguard.check_url("https://example.com/watch?v=abc") # does not raise
|
||||
|
||||
|
||||
def test_unresolvable_is_its_own_reason(monkeypatch):
|
||||
import socket as _socket
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise _socket.gaierror("nope")
|
||||
|
||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
||||
# Distinct from PRIVATE on purpose: "we couldn't look that up" and "we refuse to fetch that"
|
||||
# are different things to tell a user, and the frontend renders them differently.
|
||||
assert _blocked_reason("https://nx.example/v") == urlguard.REASON_UNRESOLVABLE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url", ["file:///etc/passwd", "ftp://host/x", "gopher://host/1"])
|
||||
def test_non_http_schemes_are_refused_before_any_lookup(monkeypatch, url):
|
||||
def boom(*a, **kw): # a scheme rejection must not have resolved anything first
|
||||
raise AssertionError("DNS was consulted for a non-http scheme")
|
||||
|
||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
||||
assert _blocked_reason(url) == urlguard.REASON_SCHEME
|
||||
|
||||
|
||||
def test_a_url_without_a_host_is_refused():
|
||||
assert _blocked_reason("http:///just/a/path") == urlguard.REASON_NO_HOST
|
||||
Reference in New Issue
Block a user