fix(clip): a clip is held to its owner's storage cap, and its limits and failures say the server's truth
- The cut file is measured against what its owner has left, for a YouTube range (whose fetch never reads yt-dlp's size ceiling) and for a Plex clip (which fetches nothing), before the fetch and again before it is placed; a refusal leaves the asset pending for a Retry. - A silent Plex file is cut silent; a file the probe cannot read says so. - A second job on a failed or renamed clip gets the same YouTube diagnosis and the fetched title. - The "too long" refusal names the server's cap; a contract test holds the client's copies of the clip limits to clip.py; the shipped player clients are pinned against the installed yt-dlp. - One HDR transfer set and one rating_key lookup; job status words from JobStatus; the display-name guard sees subscript writes and the clip route; stale comments and types follow the clip job.
This commit is contained in:
@@ -175,6 +175,21 @@ def test_the_cookie_capable_clients_match_the_installed_ytdlp():
|
||||
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
|
||||
|
||||
|
||||
def test_the_shipped_player_clients_exist_in_the_installed_ytdlp():
|
||||
# yt-dlp skips a client it does not know with a warning and falls back to its own defaults, so
|
||||
# a bump that renames or drops one (tv_simply, the one measured to deliver HD) would change
|
||||
# what every download is fetched with, silently. Read from the Settings defaults, not from
|
||||
# `settings`, which an environment may override.
|
||||
from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
|
||||
|
||||
from app.config import Settings
|
||||
|
||||
fields = Settings.model_fields
|
||||
for knob in ("download_player_clients", "download_player_clients_fallback"):
|
||||
names = Settings._clients(fields[knob].default)
|
||||
assert names and set(names) <= set(INNERTUBE_CLIENTS), (knob, names)
|
||||
|
||||
|
||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||
|
||||
+139
-6
@@ -11,8 +11,10 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import worker as workermod
|
||||
from app.config import settings
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import pipeline, quota, runner, service
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads.edit import INPUT_GUARD
|
||||
@@ -315,19 +317,20 @@ def test_a_ready_clip_asked_for_again_is_done_at_once(db, make_user, as_user, yt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "status", "reason"),
|
||||
("body", "status", "reason", "params"),
|
||||
[
|
||||
({"end_s": 700}, 422, "too_long"),
|
||||
({"start_s": 299.5, "end_s": 330}, 422, "past_end"),
|
||||
({"ref": "nope"}, 422, "bad_source"),
|
||||
# The cap travels with its refusal, so the client never states a stale one.
|
||||
({"end_s": 700}, 422, "too_long", {"limit_s": clipmod.MAX_CLIP_S}),
|
||||
({"start_s": 299.5, "end_s": 330}, 422, "past_end", {}),
|
||||
({"ref": "nope"}, 422, "bad_source", {}),
|
||||
],
|
||||
)
|
||||
def test_a_youtube_clip_it_cannot_cut_says_why(
|
||||
db, make_user, as_user, yt_video, body, status, reason
|
||||
db, make_user, as_user, yt_video, body, status, reason, params
|
||||
):
|
||||
r = _clip(as_user(make_user()), **{"source": "youtube", "ref": yt_video, **body})
|
||||
assert r.status_code == status
|
||||
assert r.json()["detail"] == {"code": "clip", "reason": reason}
|
||||
assert r.json()["detail"] == {"code": "clip", "reason": reason, **params}
|
||||
assert db.query(DownloadJob).count() == 0
|
||||
|
||||
|
||||
@@ -650,3 +653,133 @@ def test_a_youtube_clip_on_the_bot_wall_is_diagnosed_like_a_download(
|
||||
job, asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
|
||||
assert (job.status, asset.status) == ("error", "error")
|
||||
assert (job.error_code, job.error_detail) == ("login_required", detail)
|
||||
|
||||
|
||||
# --- held to the owner's storage cap, and a clip that cannot or need not go on ---------------------
|
||||
|
||||
|
||||
def _fetch_range(monkeypatch):
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||
f = Path(staging) / "abcdefghijk.mp4"
|
||||
f.write_bytes(b"range")
|
||||
return {"requested_downloads": [{"filepath": str(f)}]}
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["youtube", "plex"])
|
||||
def test_a_clip_for_an_owner_whose_storage_is_full_is_refused_before_it_is_cut(
|
||||
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
|
||||
):
|
||||
_fetch_range(monkeypatch)
|
||||
ref = yt_video if source == "youtube" else "777"
|
||||
job = _clip(as_user(make_user()), source=source, ref=ref).json()
|
||||
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 0)
|
||||
job, asset = worker["run"](job)
|
||||
# Refused like a download at its cap: the job says why, the asset waits for a Retry.
|
||||
assert (job.status, job.error, asset.status) == (
|
||||
"error",
|
||||
dlerrors.QUOTA_FULL_MESSAGE,
|
||||
"pending",
|
||||
)
|
||||
assert "cmd" not in worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["youtube", "plex"])
|
||||
def test_a_clip_larger_than_what_its_owner_has_left_is_refused_and_not_placed(
|
||||
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
|
||||
):
|
||||
# The range fetch's own ceiling never reaches yt-dlp's ffmpeg downloader, and a Plex clip
|
||||
# fetches nothing: the cut file itself is held to the cap.
|
||||
_fetch_range(monkeypatch)
|
||||
ref = yt_video if source == "youtube" else "777"
|
||||
job = _clip(as_user(make_user()), source=source, ref=ref).json()
|
||||
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 3) # the cut is 4 bytes
|
||||
job, asset = worker["run"](job)
|
||||
assert (job.status, job.error, asset.status, asset.rel_path) == (
|
||||
"error",
|
||||
dlerrors.QUOTA_FULL_MESSAGE,
|
||||
"pending",
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def test_a_plex_clip_of_a_silent_file_is_cut_silent(db, make_user, as_user, plex_episode, worker):
|
||||
# The player names the track it plays 0 even when the file has none.
|
||||
worker["probe"] = {"streams": [{"codec_type": "video"}]}
|
||||
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
|
||||
job, asset = worker["run"](job)
|
||||
assert (job.status, asset.status) == ("done", "ready")
|
||||
assert "0:a:0?" in worker["cmd"]
|
||||
|
||||
|
||||
def test_a_plex_clip_of_a_file_the_probe_cannot_read_says_so(
|
||||
db, make_user, as_user, plex_episode, worker
|
||||
):
|
||||
worker["probe"] = None
|
||||
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
|
||||
job, asset = worker["run"](job)
|
||||
assert (job.status, asset.status) == ("error", "error")
|
||||
assert job.error == "The source file could not be read."
|
||||
assert "cmd" not in worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize("where", ["fetch", "encode"])
|
||||
def test_a_paused_clip_frees_its_asset_for_a_later_retry(
|
||||
db, make_user, as_user, yt_video, worker, monkeypatch, where
|
||||
):
|
||||
def stopped(*_a, **_k):
|
||||
raise pipeline._Aborted() if where == "fetch" else pipeline.editmod.EditAborted()
|
||||
|
||||
if where == "fetch":
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", stopped)
|
||||
else:
|
||||
_fetch_range(monkeypatch)
|
||||
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", stopped)
|
||||
job = _clip(as_user(make_user()), source="youtube", ref=yt_video).json()
|
||||
job, asset = worker["run"](job)
|
||||
# The route already wrote the job's own state (paused / canceled); the asset is free again.
|
||||
assert (job.status, job.error, asset.status) == ("running", None, "pending")
|
||||
|
||||
|
||||
# --- a second job on the same clip -----------------------------------------------------------------
|
||||
|
||||
|
||||
def _second_job(db, client, **body):
|
||||
"""Two jobs for one range: the second shares the first's asset."""
|
||||
first = _clip(client, **body).json()
|
||||
second = _clip(client, **body).json()
|
||||
assert first["asset"]["id"] == second["asset"]["id"]
|
||||
return db.get(DownloadJob, first["id"]), db.get(DownloadJob, second["id"])
|
||||
|
||||
|
||||
def test_a_second_clip_job_on_a_failed_clip_gets_the_same_youtube_diagnosis(
|
||||
db, make_user, as_user, yt_video
|
||||
):
|
||||
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref=yt_video)
|
||||
asset = db.get(MediaAsset, second.asset_id)
|
||||
asset.status = "error"
|
||||
asset.error = f"ERROR: [youtube] {yt_video}: Sign in to confirm you’re not a bot."
|
||||
second.status = "running"
|
||||
db.commit()
|
||||
workermod._process_job(second.id)
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, second.id)
|
||||
assert (job.status, job.error_code, job.error_detail) == (
|
||||
"error",
|
||||
"login_required",
|
||||
"youtube.bot_wall",
|
||||
)
|
||||
|
||||
|
||||
def test_a_second_clip_job_takes_the_title_its_sibling_found(db, make_user, as_user):
|
||||
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref="zyxwvutsrqp")
|
||||
assert second.display_name == "Clip (clip)"
|
||||
asset = db.get(MediaAsset, second.asset_id)
|
||||
asset.status, asset.title = "ready", "Found On YouTube (clip)"
|
||||
second.status = "running"
|
||||
db.commit()
|
||||
workermod._finish_from_cache(second.id, asset.id)
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, second.id)
|
||||
assert (job.status, job.display_name) == ("done", "Found On YouTube (clip)")
|
||||
|
||||
@@ -451,7 +451,7 @@ def test_a_non_ascii_filename_survives_the_header_round_trip(
|
||||
def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
|
||||
db, as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
"""The sweep, held end to end rather than at one site.
|
||||
"""The sweep, held end to end rather than at one site (the clip route joined it later).
|
||||
|
||||
`display_text` was introduced for the upload path and applied to three of the five writers of
|
||||
this varchar(255) column; `POST /api/downloads` and `POST /api/downloads/edit` kept the old
|
||||
@@ -503,15 +503,32 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
|
||||
assert patched.json()["display_name"] == bound
|
||||
assert patched.json()["display_uploader"] == bound
|
||||
|
||||
# 6. the clip route — the clip panel's name field.
|
||||
clip = c.post(
|
||||
"/api/downloads/clip",
|
||||
json={
|
||||
"source": "youtube",
|
||||
"ref": "dQw4w9WgXcQ",
|
||||
"start_s": 0,
|
||||
"end_s": 5,
|
||||
"display_name": long_name,
|
||||
},
|
||||
)
|
||||
assert clip.status_code == 200, clip.text
|
||||
assert clip.json()["display_name"] == bound
|
||||
|
||||
|
||||
def _display_write_sites() -> dict[tuple[str, str], set[str]]:
|
||||
"""Every place under `backend/app/` that puts a value into a display field, as
|
||||
(module relative to `app/`, enclosing function) -> the set of value expressions written there.
|
||||
|
||||
TWO SHAPES, because the first version of this guard walked only `ast.Assign` and was therefore
|
||||
THREE SHAPES, because the first version of this guard walked only `ast.Assign` and was therefore
|
||||
blind to the shape the upload route itself uses — `SpooledUpload(..., display_name=...)` is an
|
||||
`ast.keyword`, not an assignment, so the one site with an unbounded REQUEST HEADER behind it was
|
||||
the site the guard could not see.
|
||||
the site the guard could not see. The third, `fields["display_name"] = …`, is a dict a finalize
|
||||
call is spread from (the worker's), which the first two missed as well. A value passed
|
||||
POSITIONALLY is still unseen: a writer binds it to a name or a keyword first, as every route here
|
||||
does.
|
||||
|
||||
ANNOTATED assignments are excluded on purpose: `display_name: Mapped[str | None] =
|
||||
mapped_column(...)` in `models.py` and `display_name: str | None = None` in a Pydantic model are
|
||||
@@ -537,6 +554,13 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
|
||||
t.id if isinstance(t, ast.Name) else t.attr
|
||||
for t in node.targets
|
||||
if isinstance(t, (ast.Name, ast.Attribute))
|
||||
} | {
|
||||
# `fields["display_name"] = …` — a dict the write is then spread from.
|
||||
t.slice.value
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Subscript)
|
||||
and isinstance(t.slice, ast.Constant)
|
||||
and isinstance(t.slice.value, str)
|
||||
}
|
||||
value = node.value
|
||||
elif isinstance(node, ast.keyword):
|
||||
@@ -561,6 +585,9 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
|
||||
# service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback
|
||||
# enqueue_clip slices the source's title to the same width.
|
||||
# downloads.upload_download passes through `spool_upload`'s value, bounded there.
|
||||
# routes enqueue_clip the `display_name=` keyword passes on the name bound above it.
|
||||
# pipeline._download / _process_clip / worker._finish_from_cache
|
||||
# slice a fetched or stored title to the column's width.
|
||||
#
|
||||
# Held as EXACT EQUALITY rather than as a list of offenders. The offender form could only fail
|
||||
# upward: refactoring the writers into a shape the walker did not recognise left it asserting
|
||||
@@ -574,11 +601,15 @@ DISPLAY_WRITE_SITES = {
|
||||
},
|
||||
("downloads/service.py", "enqueue_edit"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"},
|
||||
("downloads/service.py", "enqueue_clip"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"},
|
||||
("downloads/pipeline.py", "_download"): {"meta.title[:DISPLAY_TEXT_MAX]"},
|
||||
("downloads/pipeline.py", "_process_clip"): {"named[:DISPLAY_TEXT_MAX]"},
|
||||
("worker.py", "_finish_from_cache"): {"asset.title[:DISPLAY_TEXT_MAX]"},
|
||||
("downloads/upload.py", "ingest_upload"): {"display_text(display_name)"},
|
||||
("routes/auth.py", "callback"): {"display_text(userinfo.get('name'))"},
|
||||
("routes/auth.py", "_complete_link"): {"display_text(userinfo.get('name'))"},
|
||||
("routes/downloads.py", "enqueue_download"): {"display_text(body.display_name)"},
|
||||
("routes/downloads.py", "enqueue_edit"): {"display_text(body.display_name)"},
|
||||
("routes/downloads.py", "enqueue_clip"): {"display_text(body.display_name)", "display_name"},
|
||||
("routes/downloads.py", "update_download_meta"): {
|
||||
"display_text(body.display_name)",
|
||||
"display_text(body.display_uploader)",
|
||||
|
||||
Reference in New Issue
Block a user