peter
0cc450d1c9
SEC-12: the download box accepts an arbitrary http(s) URL and yt-dlp fetches it from the WORKER, so any approved user could aim the server at the compose network, the LAN or a cloud metadata endpoint — and read the result off job.error, where "connection refused" vs "HTTP Error 401" vs "timed out" is a port scanner. downloads/urlguard.py refuses a URL whose host resolves to a loopback/private/link-local/reserved address, at enqueue AND again in the worker (DNS can move in between, and the worker is where the socket opens). A network-class failure on a `url` job now reports one fixed sentence instead of the raw transport text; the error CODE is still classified from the original, so the localized explanation is unchanged. DNS rebinding is explicitly NOT closed — that needs the connection gated inside yt-dlp — and the module says so rather than implying more than it does. SEC-13: `noplaylist` only demotes a video-inside-a-playlist URL to that video, so a URL that is purely a playlist or channel still expanded into every entry — one enqueue, unbounded downloads. `playlist_items` pins it to one. Size was unbounded too: `check_enqueue` compares the CURRENT footprint against the cap, and a download's size is unknown until it lands, so one job could overshoot by any amount; the owner's remaining budget is now handed to yt-dlp as `max_filesize`, applied AFTER the adapter merge so no source adapter can raise a limit the quota layer set. A join's cut-list is capped at 100: an accurate join emits two filters per segment into one ffmpeg filter_complex. SEC-14: the YouTube budget is instance-wide, so one user holding down a manual sync spent the day for everyone. Per-user (not per-IP — the cost belongs to the account) 3-per-10-minutes limiters on the two bulk pulls, subscriptions and playlists, with separate budgets so one does not close the other. The review also listed /rss, /backfill and /enrich; E3 deleted them, and search stays unlimited on purpose — it is interactive, not a bulk pull. Gate: pytest 896 -> 924 (none skipped), vitest 671.
88 lines
3.8 KiB
Python
88 lines
3.8 KiB
Python
"""E12 · S3 — the route-level halves of SEC-12 and SEC-14.
|
|
|
|
`test_urlguard.py` pins the address judgement itself; this pins that the ENQUEUE route actually
|
|
consults it and answers in the structured `{code, reason}` shape the frontend localizes, and that a
|
|
manual sync trigger stops spending the instance-wide YouTube budget when held down.
|
|
|
|
DB-backed (the `as_user` fixture needs real rows) — skipped when no database is reachable.
|
|
"""
|
|
|
|
from app.downloads import urlguard
|
|
|
|
|
|
def test_enqueueing_a_loopback_url_is_refused_with_a_structured_detail(
|
|
db, as_user, make_user, monkeypatch
|
|
):
|
|
user = make_user()
|
|
db.commit()
|
|
monkeypatch.setattr(
|
|
urlguard.socket, "getaddrinfo", lambda *a, **kw: [(None, None, None, "", ("127.0.0.1", 0))]
|
|
)
|
|
r = as_user(user).post("/api/downloads", json={"source": "http://redis.internal:6379/x"})
|
|
assert r.status_code == 422
|
|
# The shape matters as much as the refusal: a bare English string under a bilingual UI is the
|
|
# thing the canonical-patterns table forbids.
|
|
assert r.json()["detail"] == {"code": "url_blocked", "reason": urlguard.REASON_PRIVATE}
|
|
|
|
|
|
def test_a_public_url_gets_past_the_guard(db, as_user, make_user, monkeypatch):
|
|
# Mutation check for the test above: with a public answer the request must fail LATER (or
|
|
# succeed), never with url_blocked — otherwise the guard would be refusing everything and the
|
|
# first test would still pass.
|
|
user = make_user()
|
|
db.commit()
|
|
monkeypatch.setattr(
|
|
urlguard.socket,
|
|
"getaddrinfo",
|
|
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
|
)
|
|
r = as_user(user).post("/api/downloads", json={"source": "https://example.com/v"})
|
|
detail = r.json().get("detail") if r.status_code >= 400 else None
|
|
assert not (isinstance(detail, dict) and detail.get("code") == "url_blocked")
|
|
|
|
|
|
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
|
# A YouTube job resolves to a fixed public host from an 11-char id, so there is no
|
|
# user-controlled destination — resolving one would be pure latency on every enqueue.
|
|
def boom(*a, **kw):
|
|
raise AssertionError("the URL guard resolved DNS for a YouTube source")
|
|
|
|
user = make_user()
|
|
db.commit()
|
|
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
|
as_user(user).post("/api/downloads", json={"source": "dQw4w9WgXcQ"})
|
|
|
|
|
|
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
|
from app.routes import sync as syncroutes
|
|
|
|
user = make_user()
|
|
db.commit()
|
|
# Neither the YouTube grant nor the import itself is under test — the limiter is.
|
|
monkeypatch.setattr(syncroutes, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
|
|
|
c = as_user(user)
|
|
codes = [c.post("/api/sync/subscriptions").status_code for _ in range(4)]
|
|
# 3 per 10 minutes: the first three pass, the fourth is refused.
|
|
assert codes[:3] == [200, 200, 200]
|
|
assert codes[3] == 429
|
|
assert c.post("/api/sync/subscriptions").json()["detail"] == {"code": "sync_cooldown"}
|
|
|
|
|
|
def test_the_cooldown_is_per_user_not_global(db, as_user, make_user, monkeypatch):
|
|
# Keyed by user id on purpose: the shared YouTube budget is spent per account, and one user
|
|
# exhausting their allowance must not lock everyone else out of their own sync button.
|
|
from app.routes import sync as syncroutes
|
|
|
|
a, b = make_user(), make_user()
|
|
db.commit()
|
|
monkeypatch.setattr(syncroutes, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
|
|
|
ca = as_user(a)
|
|
for _ in range(4):
|
|
ca.post("/api/sync/subscriptions")
|
|
assert ca.post("/api/sync/subscriptions").status_code == 429
|
|
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|