fix(clip): a clip is held to its owner's storage cap, and its limits and failures say the server's truth

- The cut file is measured against what its owner has left, for a YouTube range (whose fetch never
  reads yt-dlp's size ceiling) and for a Plex clip (which fetches nothing), before the fetch and
  again before it is placed; a refusal leaves the asset pending for a Retry.
- A silent Plex file is cut silent; a file the probe cannot read says so.
- A second job on a failed or renamed clip gets the same YouTube diagnosis and the fetched title.
- The "too long" refusal names the server's cap; a contract test holds the client's copies of the
  clip limits to clip.py; the shipped player clients are pinned against the installed yt-dlp.
- One HDR transfer set and one rating_key lookup; job status words from JobStatus; the display-name
  guard sees subscript writes and the clip route; stale comments and types follow the clip job.
This commit is contained in:
2026-10-10 15:35:12 +02:00
parent 72df11f2c0
commit b4bf71ea64
25 changed files with 385 additions and 89 deletions
-8
View File
@@ -1439,14 +1439,6 @@
"lineCount": 1 "lineCount": 1
} }
}, },
{
"code": "reportArgumentType",
"range": {
"startColumn": 38,
"endColumn": 43,
"lineCount": 1
}
},
{ {
"code": "reportArgumentType", "code": "reportArgumentType",
"range": { "range": {
+8 -6
View File
@@ -24,6 +24,7 @@ import re
from pathlib import Path from pathlib import Path
from app.downloads.edit import AAC_ARGS, EVEN_FRAME, INPUT_GUARD, X264_ARGS from app.downloads.edit import AAC_ARGS, EVEN_FRAME, INPUT_GUARD, X264_ARGS
from app.plex.sync import HDR_TRANSFERS
SOURCES = ("youtube", "plex") SOURCES = ("youtube", "plex")
# Owner's cap: a clip is a moment, not a way to export a whole film to a public link. # Owner's cap: a clip is a moment, not a way to export a whole film to a public link.
@@ -39,8 +40,6 @@ _SIG_VERSION = 1
_YOUTUBE_ID = re.compile(r"[A-Za-z0-9_-]{11}") _YOUTUBE_ID = re.compile(r"[A-Za-z0-9_-]{11}")
_RATING_KEY = re.compile(r"[0-9]{1,32}") _RATING_KEY = re.compile(r"[0-9]{1,32}")
_MAX_AUDIO_TRACK = 63 _MAX_AUDIO_TRACK = 63
# The transfer characteristics of HDR video (PQ, HLG): an 8-bit SDR encode must tonemap them.
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
# HDR → SDR as measured on the owner's 4K HDR files (2026-10-10): linearise, tonemap (hable), # HDR → SDR as measured on the owner's 4K HDR files (2026-10-10): linearise, tonemap (hable),
# convert the primaries and transfer to BT.709 and back to 8-bit 4:2:0. # convert the primaries and transfer to BT.709 and back to 8-bit 4:2:0.
_TONEMAP = ( _TONEMAP = (
@@ -59,10 +58,12 @@ _EDGE_SLACK = 1e-3
class ClipError(Exception): class ClipError(Exception):
"""A clip request that cannot be cut; `reason` is an i18n-mappable key.""" """A clip request that cannot be cut; `reason` is an i18n-mappable key, `params` the numbers its
message names (the cap itself, so the client never says a stale one)."""
def __init__(self, reason: str): def __init__(self, reason: str, **params: float):
self.reason = reason self.reason = reason
self.params = params
super().__init__(reason) super().__init__(reason)
@@ -87,7 +88,7 @@ def normalize(
if end - start < MIN_CLIP_S: if end - start < MIN_CLIP_S:
raise ClipError("too_short") raise ClipError("too_short")
if end - start > MAX_CLIP_S: if end - start > MAX_CLIP_S:
raise ClipError("too_long") raise ClipError("too_long", limit_s=MAX_CLIP_S)
spec: dict = {"source": source, "ref": ref, "start_s": start, "end_s": end} spec: dict = {"source": source, "ref": ref, "start_s": start, "end_s": end}
if source == "plex" and audio_track is not None: if source == "plex" and audio_track is not None:
if not 0 <= audio_track <= _MAX_AUDIO_TRACK: if not 0 <= audio_track <= _MAX_AUDIO_TRACK:
@@ -151,7 +152,8 @@ def is_hdr(probe: dict) -> bool:
"""Whether the first video stream of an ffprobe `-show_streams` JSON carries an HDR transfer.""" """Whether the first video stream of an ffprobe `-show_streams` JSON carries an HDR transfer."""
for s in probe.get("streams") or []: for s in probe.get("streams") or []:
if s.get("codec_type") == "video": if s.get("codec_type") == "video":
return s.get("color_transfer") in _HDR_TRANSFERS # PQ / HLG: an 8-bit SDR encode must tonemap them (the one set the Plex stream uses).
return s.get("color_transfer") in HDR_TRANSFERS
return False return False
+3 -2
View File
@@ -281,8 +281,9 @@ def build_ydl_opts(
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra` after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
merge so a source adapter can never clobber it. merge so a source adapter can never clobber it.
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp refuses a format larger `max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp's HTTP downloader refuses
than it rather than filling the disk and only then being caught by the quota accounting.""" a file announced larger than it rather than filling the disk and only then being caught by the
quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it."""
s = normalize(spec) s = normalize(spec)
h = s["max_height"] h = s["max_height"]
opts: dict = { opts: dict = {
+35 -14
View File
@@ -7,7 +7,6 @@ from datetime import date, datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
import yt_dlp import yt_dlp
from sqlalchemy import select
from yt_dlp.utils import download_range_func from yt_dlp.utils import download_range_func
from app import logs, sysconfig from app import logs, sysconfig
@@ -30,6 +29,7 @@ from app.downloads.jobdb import (
from app.downloads.stated import StatedFailure from app.downloads.stated import StatedFailure
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths from app.plex import paths as plex_paths
from app.notifications import create_notification from app.notifications import create_notification
from app.titles import normalize_title from app.titles import normalize_title
@@ -363,7 +363,9 @@ def _extract_with_fallback(
`credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so `credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so
the fallback stays authenticated. the fallback stays authenticated.
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. `max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. A
`section` fetch goes through yt-dlp's ffmpeg downloader, which does not read it: a clip holds
its own output to the cap (`_process_clip`).
`section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the `section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the
range is stream-copied, and its first frame is the marked one (measured), so the clip's own range is stream-copied, and its first frame is the marked one (measured), so the clip's own
@@ -523,8 +525,8 @@ def _download(
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it # SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a # stood when the job was QUEUED — a download's size is unknown until it finishes, so a
# single job could previously blow through the cap by any amount. Handing the remainder # single job could previously blow through the cap by any amount. Handing the remainder
# to yt-dlp as `max_filesize` makes the format selector refuse an oversized file up # to yt-dlp as `max_filesize` makes it refuse an oversized file (its HTTP downloader
# front instead. # checks the announced size) instead.
remaining_bytes = quota.remaining_bytes(cdb, user_id) remaining_bytes = quota.remaining_bytes(cdb, user_id)
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap # …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so # while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
@@ -927,8 +929,18 @@ def _process_edit(job_id: int, asset_id: int) -> None:
shutil.rmtree(staging, ignore_errors=True) shutil.rmtree(staging, ignore_errors=True)
def _clip_room(user_id: int) -> int | None:
"""What the clip's owner may still store (`quota.remaining_bytes`), refusing at zero — see
`_download`: zero is a refusal, not "no limit"."""
with SessionLocal() as db:
remaining = quota.remaining_bytes(db, user_id)
if remaining == 0:
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
return remaining
def _clip_source( def _clip_source(
job_id: int, asset_id: int, spec: dict, staging: Path job_id: int, asset_id: int, spec: dict, staging: Path, remaining: int | None
) -> tuple[Path, float, str | None]: ) -> tuple[Path, float, str | None]:
"""The file a clip is cut from, where in it the clip starts, and the source's title when only """The file a clip is cut from, where in it the clip starts, and the source's title when only
the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at
@@ -939,15 +951,12 @@ def _clip_source(
with SessionLocal() as db: with SessionLocal() as db:
job = db.get(DownloadJob, job_id) job = db.get(DownloadJob, job_id)
asset = db.get(MediaAsset, asset_id) asset = db.get(MediaAsset, asset_id)
remaining = quota.remaining_bytes(db, job.user_id) if job else None
# Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates # Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates
# single videos signed out ("not a bot"), and the clip is theirs alone. # single videos signed out ("not a bot"), and the clip is theirs alone.
if job and asset: if job and asset:
cookiefile = credentials.resolve_clip_cookiefile( cookiefile = credentials.resolve_clip_cookiefile(
db, job.user_id, url, asset, spec, staging db, job.user_id, url, asset, spec, staging
) )
if remaining == 0: # see `_download`: zero is a refusal, not "no limit"
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
_set_job(job_id, phase="downloading", progress=0) _set_job(job_id, phase="downloading", progress=0)
try: try:
info = _extract_with_fallback( info = _extract_with_fallback(
@@ -976,9 +985,7 @@ def _clip_source(
title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info)) title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info))
return _produced_file(info, staging), 0.0, title return _produced_file(info, staging), 0.0, title
with SessionLocal() as db: with SessionLocal() as db:
item = db.execute( item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
select(PlexItem).where(PlexItem.rating_key == spec["ref"])
).scalar_one_or_none()
src = plex_paths.local_media_path(db, item.file_path) if item else None src = plex_paths.local_media_path(db, item.file_path) if item else None
if src is None: if src is None:
raise StatedFailure("The library file is not reachable from here.") raise StatedFailure("The library file is not reachable from here.")
@@ -1013,10 +1020,18 @@ def _process_clip(job_id: int, asset_id: int) -> None:
if staging.exists(): if staging.exists():
shutil.rmtree(staging, ignore_errors=True) shutil.rmtree(staging, ignore_errors=True)
staging.mkdir(parents=True, exist_ok=True) staging.mkdir(parents=True, exist_ok=True)
src, start, fetched_title = _clip_source(job_id, asset_id, spec, staging) # Refused before the fetch and the encode when the owner's storage is already full.
data = probe.probe_json(src) or {} remaining = _clip_room(user_id)
src, start, fetched_title = _clip_source(job_id, asset_id, spec, staging, remaining)
data = probe.probe_json(src)
if not data:
raise StatedFailure("The source file could not be read.")
audio_track = spec.get("audio_track") audio_track = spec.get("audio_track")
if audio_track is not None and audio_track >= clipmod.audio_track_count(data): tracks = clipmod.audio_track_count(data)
if tracks == 0:
# A silent source: the player still names its track 0, and the clip is silent too.
audio_track = None
elif audio_track is not None and audio_track >= tracks:
raise StatedFailure("The chosen audio track is not in the file.") raise StatedFailure("The chosen audio track is not in the file.")
crop = _clip_crop(spec, data) crop = _clip_crop(spec, data)
total_s = clipmod.duration(spec) total_s = clipmod.duration(spec)
@@ -1041,6 +1056,12 @@ def _process_clip(job_id: int, asset_id: int) -> None:
raise StatedFailure("ffmpeg produced no output file") raise StatedFailure("ffmpeg produced no output file")
# The phone-safe shape by construction; still held to the rule every ready file meets. # The phone-safe shape by construction; still held to the rule every ready file meets.
out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s)) out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s))
# Held to the cap by its real size: a range fetch passes yt-dlp's `max_filesize` to its
# ffmpeg downloader, which never reads it, and a Plex clip fetches nothing at all. Read
# again, since other jobs of this owner may have landed during the encode.
remaining = _clip_room(user_id)
if remaining is not None and out.stat().st_size > remaining:
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
ext = out.suffix.lstrip(".").lower() or "mp4" ext = out.suffix.lstrip(".").lower() or "mp4"
rel = storage.edit_rel_path(user_id, asset_id, ext) rel = storage.edit_rel_path(user_id, asset_id, ext)
storage.place_file(out, settings.download_root, rel) storage.place_file(out, settings.download_root, rel)
+2 -1
View File
@@ -15,7 +15,8 @@ the footprint as it stood when the job was created and earlier jobs may have fil
That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same
remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of
max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size
is unknown until it finishes. Edit jobs do not consult it at all (tracked as SEC-19). is unknown until it finishes. CLIP jobs consult it before their fetch or encode and again with
their output's real size before it is placed. Edit jobs do not consult it at all (tracked as SEC-19).
""" """
from dataclasses import dataclass from dataclasses import dataclass
+8 -9
View File
@@ -30,6 +30,7 @@ from app.models import (
PlexShow, PlexShow,
Video, Video,
) )
from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths from app.plex import paths as plex_paths
@@ -193,7 +194,7 @@ def enqueue(
profile_snapshot=spec, profile_snapshot=spec,
display_name=display_name display_name=display_name
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None), or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
status="queued", status=JobStatus.QUEUED,
queue_pos=next_queue_pos(db), queue_pos=next_queue_pos(db),
) )
db.add(job) db.add(job)
@@ -201,7 +202,7 @@ def enqueue(
# Cache hit: the file already exists — no worker round-trip needed. # Cache hit: the file already exists — no worker round-trip needed.
if asset.status == "ready": if asset.status == "ready":
job.status = "done" job.status = JobStatus.DONE
job.progress = 100 job.progress = 100
asset.last_access_at = datetime.now(timezone.utc) asset.last_access_at = datetime.now(timezone.utc)
@@ -272,14 +273,14 @@ def enqueue_edit(
edit_spec=spec, edit_spec=spec,
profile_snapshot={}, profile_snapshot={},
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX], display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
status="queued", status=JobStatus.QUEUED,
queue_pos=next_queue_pos(db), queue_pos=next_queue_pos(db),
) )
db.add(job) db.add(job)
incr_ref_count(db, asset.id) incr_ref_count(db, asset.id)
if asset.status == "ready": # identical clip already produced for this user → instant if asset.status == "ready": # identical clip already produced for this user → instant
job.status = "done" job.status = JobStatus.DONE
job.progress = 100 job.progress = 100
asset.last_access_at = datetime.now(timezone.utc) asset.last_access_at = datetime.now(timezone.utc)
@@ -303,9 +304,7 @@ def _clip_source_facts(db: Session, spec: dict) -> dict:
"thumbnail_url": video.thumbnail_url, "thumbnail_url": video.thumbnail_url,
"length_s": video.duration_seconds, "length_s": video.duration_seconds,
} }
item = db.execute( item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
select(PlexItem).where(PlexItem.rating_key == spec["ref"])
).scalar_one_or_none()
if item is None: if item is None:
raise clipmod.ClipError("not_found") raise clipmod.ClipError("not_found")
if plex_paths.local_media_path(db, item.file_path) is None: if plex_paths.local_media_path(db, item.file_path) is None:
@@ -361,14 +360,14 @@ def enqueue_clip(
edit_spec=spec, edit_spec=spec,
profile_snapshot={}, profile_snapshot={},
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX], display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
status="queued", status=JobStatus.QUEUED,
queue_pos=next_queue_pos(db), queue_pos=next_queue_pos(db),
) )
db.add(job) db.add(job)
incr_ref_count(db, asset.id) incr_ref_count(db, asset.id)
if asset.status == "ready": # the same range was already cut for this user → instant if asset.status == "ready": # the same range was already cut for this user → instant
job.status = "done" job.status = JobStatus.DONE
job.progress = 100 job.progress = 100
asset.last_access_at = datetime.now(timezone.utc) asset.last_access_at = datetime.now(timezone.utc)
+6 -3
View File
@@ -933,8 +933,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
) )
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default), "edit" (a per-user trim/crop derivative of another job) or "upload" (a # "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`). # user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download") job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted). # For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column( source_job_id: Mapped[int | None] = mapped_column(
@@ -943,7 +944,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
source_asset_id: Mapped[int | None] = mapped_column( source_asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL") ForeignKey("media_assets.id", ondelete="SET NULL")
) )
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}}. # For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
edit_spec: Mapped[dict | None] = mapped_column(JSON) edit_spec: Mapped[dict | None] = mapped_column(JSON)
profile_id: Mapped[int | None] = mapped_column( profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL") ForeignKey("download_profiles.id", ondelete="SET NULL")
+3 -2
View File
@@ -35,7 +35,7 @@ from app.config import settings
from app.downloads.edit import encode_thread_cap from app.downloads.edit import encode_thread_cap
from app.models import PlexItem from app.models import PlexItem
from app.plex import paths from app.plex import paths
from app.plex.sync import HW_DECODABLE_CODECS from app.plex.sync import HDR_TRANSFERS, HW_DECODABLE_CODECS
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
_HLS_ROOT = ( _HLS_ROOT = (
@@ -203,7 +203,8 @@ _HW_DECODABLE = HW_DECODABLE_CODECS
# plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph # plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph
# (_venc_opts tonemap branch) when the OpenCL runtime is present, else software libx264. A 10-bit source # (_venc_opts tonemap branch) when the OpenCL runtime is present, else software libx264. A 10-bit source
# WITHOUT one of these transfers is SDR, not HDR, and must not be tone-mapped. See _probe_video_kind. # WITHOUT one of these transfers is SDR, not HDR, and must not be tone-mapped. See _probe_video_kind.
_HDR_TRANSFERS = {"smpte2084", "arib-std-b67"} # THE single source is plex.sync (a clip's tonemap reads it too).
_HDR_TRANSFERS = HDR_TRANSFERS
# The three source classes that drive the transcode-pipeline choice (see _probe_video_kind). # The three source classes that drive the transcode-pipeline choice (see _probe_video_kind).
+6 -4
View File
@@ -64,9 +64,11 @@ def classify_playable(vcodec: str | None, acodec: str | None, container: str | N
# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its # outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its
# own encoder). # own encoder).
HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"}) HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"})
# Color-transfer values marking an HDR source (PQ / HLG). A 10-bit/HDR source breaks the VAAPI # Color-transfer values marking an HDR source (PQ / HLG). THE single source of this set: plex.stream
# scale=nv12 path on Gen 9.5 (see plex.stream) → software transcode, so it counts as NOT HW-capable. # (the tonemap choice) and downloads.clip (a clip's tonemap) import it here, so a transfer added for
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"}) # one is tonemapped by all. A 10-bit/HDR source breaks the VAAPI scale=nv12 path on Gen 9.5 (see
# plex.stream) → software transcode, so it counts as NOT HW-capable.
HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
def transcode_hw_capable( def transcode_hw_capable(
@@ -107,7 +109,7 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
# whole library would drown the sync's own lines. # whole library would drown the sync's own lines.
except (TypeError, ValueError): except (TypeError, ValueError):
depth = None depth = None
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS hdr = str(s.get("colorTrc") or "").lower() in HDR_TRANSFERS
return depth, hdr return depth, hdr
return None, False return None, False
+4 -2
View File
@@ -610,10 +610,12 @@ def enqueue_clip(
spec = clipmod.normalize( spec = clipmod.normalize(
body.source, body.ref, body.start_s, body.end_s, body.audio_track, crop body.source, body.ref, body.start_s, body.end_s, body.audio_track, crop
) )
job = service.enqueue_clip(db, user.id, spec, display_text(body.display_name)) display_name = display_text(body.display_name)
job = service.enqueue_clip(db, user.id, spec, display_name=display_name)
except clipmod.ClipError as e: except clipmod.ClipError as e:
status = 404 if e.reason == "not_found" else 422 status = 404 if e.reason == "not_found" else 422
raise HTTPException(status_code=status, detail={"code": "clip", "reason": e.reason}) from e detail = {"code": "clip", "reason": e.reason, **e.params}
raise HTTPException(status_code=status, detail=detail) from e
except quota.QuotaExceeded as e: except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
return _serialize_job(db, job) return _serialize_job(db, job)
+29 -10
View File
@@ -29,6 +29,7 @@ from datetime import datetime, timezone
from functools import partial from functools import partial
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session
from app import logs from app import logs
from app import logsink from app import logsink
@@ -44,7 +45,7 @@ from app.downloads.jobdb import (
_reset_asset_pending, _reset_asset_pending,
_staging_dir, _staging_dir,
) )
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
@@ -437,7 +438,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
if status is None or status in STOPPED_BY_ROUTE: if status is None or status in STOPPED_BY_ROUTE:
reason = "cancelled" # a route stopped it (or the row vanished) reason = "cancelled" # a route stopped it (or the row vanished)
break break
if status == "running": if status == JobStatus.RUNNING:
if stale is not None and stale > stall_s: if stale is not None and stale > stall_s:
reason = "stalled" reason = "stalled"
break break
@@ -448,7 +449,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
# teardown. A still-running job over the cap is failed ("timeout"); a terminal one (already # teardown. A still-running job over the cap is failed ("timeout"); a terminal one (already
# done/error) just gets its lingering child killed, its DB status left intact ("abandon"). # done/error) just gets its lingering child killed, its DB status left intact ("abandon").
if time.monotonic() - started > wall_s: if time.monotonic() - started > wall_s:
reason = "timeout" if status == "running" else "abandon" reason = "timeout" if status == JobStatus.RUNNING else "abandon"
break break
# else: a terminal status under the cap (done/error, or a re-queued contended asset) means # else: a terminal status under the cap (done/error, or a re-queued contended asset) means
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could # the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
@@ -479,7 +480,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
_reset_asset_pending(asset_id) _reset_asset_pending(asset_id)
if proc.returncode != _CHILD_OK: if proc.returncode != _CHILD_OK:
status, _ = _job_liveness(job_id) status, _ = _job_liveness(job_id)
if status == "running": if status == JobStatus.RUNNING:
_cleanup_staging(asset_id) _cleanup_staging(asset_id)
_fail_running( _fail_running(
job_id, job_id,
@@ -560,15 +561,12 @@ def _process_job(job_id: int) -> None:
asset_id = asset.id if asset else None asset_id = asset.id if asset else None
asset_status = asset.status if asset else None asset_status = asset.status if asset else None
asset_error = asset.error if asset else None asset_error = asset.error if asset else None
if asset_status == "error": if asset is not None and asset_status == "error":
# A job that meets an already-failed asset gets that failure's explanation — not a bare # A job that meets an already-failed asset gets that failure's explanation — not a bare
# "Download failed". Only the text is on the asset, so it is diagnosed again, for this # "Download failed". Only the text is on the asset, so it is diagnosed again, for this
# link, with whether a cookie goes with it asked of the same gate the download child # link, with whether a cookie goes with it asked of the same gate the download child
# used (`credentials.usable_cookie`) — as it stands now, so the advice fits this job's Retry. # used (`credentials.usable_cookie`) — as it stands now, so the advice fits this job's Retry.
url = service.source_url(job.source_kind, job.source_ref) url, cookie = _failed_link_and_cookie(db, job, asset)
cookie = credentials.usable_cookie(
db, job.user_id, url, asset, formats.normalize(job.profile_snapshot)
)
diagnosis = dlerrors.diagnose(asset_error, url=url, cookie_attached=cookie is not None) diagnosis = dlerrors.diagnose(asset_error, url=url, cookie_attached=cookie is not None)
del cookie # only whether there is one matters here del cookie # only whether there is one matters here
@@ -605,13 +603,34 @@ def _process_job(job_id: int) -> None:
_run_supervised(job_id, asset_id) _run_supervised(job_id, asset_id)
def _failed_link_and_cookie(
db: Session, job: DownloadJob, asset: MediaAsset
) -> tuple[str | None, str | None]:
"""The link a job's failed asset was fetched from, and the cookie that fetch would go with now.
A clip's link is its source's (a `clip` ref names the user's range, no page), asked of the
clip's own cookie gate, as its fetch was (`_clip_source`); a Plex clip fetches nothing."""
if job.job_kind == JobKind.CLIP:
spec = dict(job.edit_spec or {})
if spec.get("source") != "youtube":
return None, None
url = service.source_url("youtube", spec["ref"])
return url, credentials.usable_clip_cookie(db, job.user_id, url, asset, spec)
url = service.source_url(job.source_kind, job.source_ref)
spec = formats.normalize(job.profile_snapshot)
return url, credentials.usable_cookie(db, job.user_id, url, asset, spec)
def _finish_from_cache(job_id: int, asset_id: int) -> None: def _finish_from_cache(job_id: int, asset_id: int) -> None:
with SessionLocal() as db: with SessionLocal() as db:
job = db.get(DownloadJob, job_id) job = db.get(DownloadJob, job_id)
asset = db.get(MediaAsset, asset_id) asset = db.get(MediaAsset, asset_id)
if job and asset: if job and asset:
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None} fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
if not job.display_name and asset.title: # A clip queued under the placeholder takes the title its sibling's fetch found.
unnamed = not job.display_name or (
job.job_kind == JobKind.CLIP and job.display_name == service.clip_title(None, None)
)
if unnamed and asset.title:
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title). # display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX] fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX]
asset.last_access_at = datetime.now(timezone.utc) asset.last_access_at = datetime.now(timezone.utc)
+3 -3
View File
@@ -2,9 +2,9 @@
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with: # Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort # docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
# The prod Dockerfile + Dockerfile.test install THIS file; the gate's pip-audit (scripts/check.sh) # The prod Dockerfile + Dockerfile.test install THIS file; the gate's pip-audit (scripts/check.sh)
# scans it, blocking. A security fix may be ONE pin bumped in place instead of a full # scans it, blocking. A security fix, or a yt-dlp release YouTube's changes force, may be ONE pin
# regeneration, which would move every other package too — then `pip install -r requirements.lock && pip check` must # bumped in place instead of a full regeneration, which would move every other package too — then
# pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403). # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403).
APScheduler==3.11.3 APScheduler==3.11.3
Authlib==1.7.2 Authlib==1.7.2
+6 -4
View File
@@ -16,12 +16,14 @@ argon2-cffi>=23.1,<26
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile) # Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver # does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
# keep the floor loose and bump it when extraction breaks. # keep the floor loose and bump it when extraction breaks. 2026.8.19 marks the clients that need a
yt-dlp[default]>=2025.5.22 # stream PO token, which the shipped player clients (config.download_player_clients) rely on.
yt-dlp[default]>=2026.8.19
# Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some # Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some
# links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome # links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome
# (measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imports curl_cffi only in # (measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imported curl_cffi only in
# 0.10.x–0.15.x and silently reports every target "unavailable" outside it. Move it when bumping # 0.10.x–0.15.x and silently reported every target "unavailable" outside it (2026.08.19 takes up to
# 0.16.x; the cap stays on the measured 0.15 until a bump re-measures it). Move it when bumping
# yt-dlp; `tests/test_facebook_impersonation.py::test_the_locked_curl_cffi_can_impersonate_chrome` # yt-dlp; `tests/test_facebook_impersonation.py::test_the_locked_curl_cffi_can_impersonate_chrome`
# fails the gate if the locked pair can no longer impersonate. # fails the gate if the locked pair can no longer impersonate.
curl_cffi>=0.10,<0.16 curl_cffi>=0.10,<0.16
+15
View File
@@ -175,6 +175,21 @@ def test_the_cookie_capable_clients_match_the_installed_ytdlp():
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
def test_the_shipped_player_clients_exist_in_the_installed_ytdlp():
# yt-dlp skips a client it does not know with a warning and falls back to its own defaults, so
# a bump that renames or drops one (tv_simply, the one measured to deliver HD) would change
# what every download is fetched with, silently. Read from the Settings defaults, not from
# `settings`, which an environment may override.
from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
from app.config import Settings
fields = Settings.model_fields
for knob in ("download_player_clients", "download_player_clients_fallback"):
names = Settings._clients(fields[knob].default)
assert names and set(names) <= set(INNERTUBE_CLIENTS), (knob, names)
def test_youtube_ydl_opts_extra_carries_player_client_and_pot(): def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"] ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
assert ea["youtube"]["player_client"] == ["android_vr"] assert ea["youtube"]["player_client"] == ["android_vr"]
+139 -6
View File
@@ -11,8 +11,10 @@ from pathlib import Path
import pytest import pytest
from app import worker as workermod
from app.config import settings from app.config import settings
from app.downloads import clip as clipmod from app.downloads import clip as clipmod
from app.downloads import errors as dlerrors
from app.downloads import pipeline, quota, runner, service from app.downloads import pipeline, quota, runner, service
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.downloads.edit import INPUT_GUARD from app.downloads.edit import INPUT_GUARD
@@ -315,19 +317,20 @@ def test_a_ready_clip_asked_for_again_is_done_at_once(db, make_user, as_user, yt
@pytest.mark.parametrize( @pytest.mark.parametrize(
("body", "status", "reason"), ("body", "status", "reason", "params"),
[ [
({"end_s": 700}, 422, "too_long"), # The cap travels with its refusal, so the client never states a stale one.
({"start_s": 299.5, "end_s": 330}, 422, "past_end"), ({"end_s": 700}, 422, "too_long", {"limit_s": clipmod.MAX_CLIP_S}),
({"ref": "nope"}, 422, "bad_source"), ({"start_s": 299.5, "end_s": 330}, 422, "past_end", {}),
({"ref": "nope"}, 422, "bad_source", {}),
], ],
) )
def test_a_youtube_clip_it_cannot_cut_says_why( def test_a_youtube_clip_it_cannot_cut_says_why(
db, make_user, as_user, yt_video, body, status, reason db, make_user, as_user, yt_video, body, status, reason, params
): ):
r = _clip(as_user(make_user()), **{"source": "youtube", "ref": yt_video, **body}) r = _clip(as_user(make_user()), **{"source": "youtube", "ref": yt_video, **body})
assert r.status_code == status assert r.status_code == status
assert r.json()["detail"] == {"code": "clip", "reason": reason} assert r.json()["detail"] == {"code": "clip", "reason": reason, **params}
assert db.query(DownloadJob).count() == 0 assert db.query(DownloadJob).count() == 0
@@ -650,3 +653,133 @@ def test_a_youtube_clip_on_the_bot_wall_is_diagnosed_like_a_download(
job, asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json()) job, asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
assert (job.status, asset.status) == ("error", "error") assert (job.status, asset.status) == ("error", "error")
assert (job.error_code, job.error_detail) == ("login_required", detail) assert (job.error_code, job.error_detail) == ("login_required", detail)
# --- held to the owner's storage cap, and a clip that cannot or need not go on ---------------------
def _fetch_range(monkeypatch):
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
f = Path(staging) / "abcdefghijk.mp4"
f.write_bytes(b"range")
return {"requested_downloads": [{"filepath": str(f)}]}
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
@pytest.mark.parametrize("source", ["youtube", "plex"])
def test_a_clip_for_an_owner_whose_storage_is_full_is_refused_before_it_is_cut(
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
):
_fetch_range(monkeypatch)
ref = yt_video if source == "youtube" else "777"
job = _clip(as_user(make_user()), source=source, ref=ref).json()
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 0)
job, asset = worker["run"](job)
# Refused like a download at its cap: the job says why, the asset waits for a Retry.
assert (job.status, job.error, asset.status) == (
"error",
dlerrors.QUOTA_FULL_MESSAGE,
"pending",
)
assert "cmd" not in worker
@pytest.mark.parametrize("source", ["youtube", "plex"])
def test_a_clip_larger_than_what_its_owner_has_left_is_refused_and_not_placed(
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
):
# The range fetch's own ceiling never reaches yt-dlp's ffmpeg downloader, and a Plex clip
# fetches nothing: the cut file itself is held to the cap.
_fetch_range(monkeypatch)
ref = yt_video if source == "youtube" else "777"
job = _clip(as_user(make_user()), source=source, ref=ref).json()
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 3) # the cut is 4 bytes
job, asset = worker["run"](job)
assert (job.status, job.error, asset.status, asset.rel_path) == (
"error",
dlerrors.QUOTA_FULL_MESSAGE,
"pending",
None,
)
def test_a_plex_clip_of_a_silent_file_is_cut_silent(db, make_user, as_user, plex_episode, worker):
# The player names the track it plays 0 even when the file has none.
worker["probe"] = {"streams": [{"codec_type": "video"}]}
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
job, asset = worker["run"](job)
assert (job.status, asset.status) == ("done", "ready")
assert "0:a:0?" in worker["cmd"]
def test_a_plex_clip_of_a_file_the_probe_cannot_read_says_so(
db, make_user, as_user, plex_episode, worker
):
worker["probe"] = None
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
job, asset = worker["run"](job)
assert (job.status, asset.status) == ("error", "error")
assert job.error == "The source file could not be read."
assert "cmd" not in worker
@pytest.mark.parametrize("where", ["fetch", "encode"])
def test_a_paused_clip_frees_its_asset_for_a_later_retry(
db, make_user, as_user, yt_video, worker, monkeypatch, where
):
def stopped(*_a, **_k):
raise pipeline._Aborted() if where == "fetch" else pipeline.editmod.EditAborted()
if where == "fetch":
monkeypatch.setattr(pipeline, "_extract_with_fallback", stopped)
else:
_fetch_range(monkeypatch)
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", stopped)
job = _clip(as_user(make_user()), source="youtube", ref=yt_video).json()
job, asset = worker["run"](job)
# The route already wrote the job's own state (paused / canceled); the asset is free again.
assert (job.status, job.error, asset.status) == ("running", None, "pending")
# --- a second job on the same clip -----------------------------------------------------------------
def _second_job(db, client, **body):
"""Two jobs for one range: the second shares the first's asset."""
first = _clip(client, **body).json()
second = _clip(client, **body).json()
assert first["asset"]["id"] == second["asset"]["id"]
return db.get(DownloadJob, first["id"]), db.get(DownloadJob, second["id"])
def test_a_second_clip_job_on_a_failed_clip_gets_the_same_youtube_diagnosis(
db, make_user, as_user, yt_video
):
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref=yt_video)
asset = db.get(MediaAsset, second.asset_id)
asset.status = "error"
asset.error = f"ERROR: [youtube] {yt_video}: Sign in to confirm you’re not a bot."
second.status = "running"
db.commit()
workermod._process_job(second.id)
db.expire_all()
job = db.get(DownloadJob, second.id)
assert (job.status, job.error_code, job.error_detail) == (
"error",
"login_required",
"youtube.bot_wall",
)
def test_a_second_clip_job_takes_the_title_its_sibling_found(db, make_user, as_user):
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref="zyxwvutsrqp")
assert second.display_name == "Clip (clip)"
asset = db.get(MediaAsset, second.asset_id)
asset.status, asset.title = "ready", "Found On YouTube (clip)"
second.status = "running"
db.commit()
workermod._finish_from_cache(second.id, asset.id)
db.expire_all()
job = db.get(DownloadJob, second.id)
assert (job.status, job.display_name) == ("done", "Found On YouTube (clip)")
+34 -3
View File
@@ -451,7 +451,7 @@ def test_a_non_ascii_filename_survives_the_header_round_trip(
def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one( def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
db, as_user, make_user, _root, monkeypatch, upload_headers db, as_user, make_user, _root, monkeypatch, upload_headers
): ):
"""The sweep, held end to end rather than at one site. """The sweep, held end to end rather than at one site (the clip route joined it later).
`display_text` was introduced for the upload path and applied to three of the five writers of `display_text` was introduced for the upload path and applied to three of the five writers of
this varchar(255) column; `POST /api/downloads` and `POST /api/downloads/edit` kept the old this varchar(255) column; `POST /api/downloads` and `POST /api/downloads/edit` kept the old
@@ -503,15 +503,32 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
assert patched.json()["display_name"] == bound assert patched.json()["display_name"] == bound
assert patched.json()["display_uploader"] == bound assert patched.json()["display_uploader"] == bound
# 6. the clip route — the clip panel's name field.
clip = c.post(
"/api/downloads/clip",
json={
"source": "youtube",
"ref": "dQw4w9WgXcQ",
"start_s": 0,
"end_s": 5,
"display_name": long_name,
},
)
assert clip.status_code == 200, clip.text
assert clip.json()["display_name"] == bound
def _display_write_sites() -> dict[tuple[str, str], set[str]]: def _display_write_sites() -> dict[tuple[str, str], set[str]]:
"""Every place under `backend/app/` that puts a value into a display field, as """Every place under `backend/app/` that puts a value into a display field, as
(module relative to `app/`, enclosing function) -> the set of value expressions written there. (module relative to `app/`, enclosing function) -> the set of value expressions written there.
TWO SHAPES, because the first version of this guard walked only `ast.Assign` and was therefore THREE SHAPES, because the first version of this guard walked only `ast.Assign` and was therefore
blind to the shape the upload route itself uses — `SpooledUpload(..., display_name=...)` is an blind to the shape the upload route itself uses — `SpooledUpload(..., display_name=...)` is an
`ast.keyword`, not an assignment, so the one site with an unbounded REQUEST HEADER behind it was `ast.keyword`, not an assignment, so the one site with an unbounded REQUEST HEADER behind it was
the site the guard could not see. the site the guard could not see. The third, `fields["display_name"] = …`, is a dict a finalize
call is spread from (the worker's), which the first two missed as well. A value passed
POSITIONALLY is still unseen: a writer binds it to a name or a keyword first, as every route here
does.
ANNOTATED assignments are excluded on purpose: `display_name: Mapped[str | None] = ANNOTATED assignments are excluded on purpose: `display_name: Mapped[str | None] =
mapped_column(...)` in `models.py` and `display_name: str | None = None` in a Pydantic model are mapped_column(...)` in `models.py` and `display_name: str | None = None` in a Pydantic model are
@@ -537,6 +554,13 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
t.id if isinstance(t, ast.Name) else t.attr t.id if isinstance(t, ast.Name) else t.attr
for t in node.targets for t in node.targets
if isinstance(t, (ast.Name, ast.Attribute)) if isinstance(t, (ast.Name, ast.Attribute))
} | {
# `fields["display_name"] = …` — a dict the write is then spread from.
t.slice.value
for t in node.targets
if isinstance(t, ast.Subscript)
and isinstance(t.slice, ast.Constant)
and isinstance(t.slice.value, str)
} }
value = node.value value = node.value
elif isinstance(node, ast.keyword): elif isinstance(node, ast.keyword):
@@ -561,6 +585,9 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
# service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback # service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback
# enqueue_clip slices the source's title to the same width. # enqueue_clip slices the source's title to the same width.
# downloads.upload_download passes through `spool_upload`'s value, bounded there. # downloads.upload_download passes through `spool_upload`'s value, bounded there.
# routes enqueue_clip the `display_name=` keyword passes on the name bound above it.
# pipeline._download / _process_clip / worker._finish_from_cache
# slice a fetched or stored title to the column's width.
# #
# Held as EXACT EQUALITY rather than as a list of offenders. The offender form could only fail # Held as EXACT EQUALITY rather than as a list of offenders. The offender form could only fail
# upward: refactoring the writers into a shape the walker did not recognise left it asserting # upward: refactoring the writers into a shape the walker did not recognise left it asserting
@@ -574,11 +601,15 @@ DISPLAY_WRITE_SITES = {
}, },
("downloads/service.py", "enqueue_edit"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"}, ("downloads/service.py", "enqueue_edit"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"},
("downloads/service.py", "enqueue_clip"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"}, ("downloads/service.py", "enqueue_clip"): {"display_name or asset.title[:DISPLAY_TEXT_MAX]"},
("downloads/pipeline.py", "_download"): {"meta.title[:DISPLAY_TEXT_MAX]"},
("downloads/pipeline.py", "_process_clip"): {"named[:DISPLAY_TEXT_MAX]"},
("worker.py", "_finish_from_cache"): {"asset.title[:DISPLAY_TEXT_MAX]"},
("downloads/upload.py", "ingest_upload"): {"display_text(display_name)"}, ("downloads/upload.py", "ingest_upload"): {"display_text(display_name)"},
("routes/auth.py", "callback"): {"display_text(userinfo.get('name'))"}, ("routes/auth.py", "callback"): {"display_text(userinfo.get('name'))"},
("routes/auth.py", "_complete_link"): {"display_text(userinfo.get('name'))"}, ("routes/auth.py", "_complete_link"): {"display_text(userinfo.get('name'))"},
("routes/downloads.py", "enqueue_download"): {"display_text(body.display_name)"}, ("routes/downloads.py", "enqueue_download"): {"display_text(body.display_name)"},
("routes/downloads.py", "enqueue_edit"): {"display_text(body.display_name)"}, ("routes/downloads.py", "enqueue_edit"): {"display_text(body.display_name)"},
("routes/downloads.py", "enqueue_clip"): {"display_text(body.display_name)", "display_name"},
("routes/downloads.py", "update_download_meta"): { ("routes/downloads.py", "update_download_meta"): {
"display_text(body.display_name)", "display_text(body.display_name)",
"display_text(body.display_uploader)", "display_text(body.display_uploader)",
@@ -170,10 +170,11 @@ const youtubeReasons = [
backend("youtube/client.py").match(/^YouTubeRefusal = Literal\[([^\]]*)\]/m)?.[1] ?? "" backend("youtube/client.py").match(/^YouTubeRefusal = Literal\[([^\]]*)\]/m)?.[1] ?? ""
).matchAll(/"(\w+)"/g), ).matchAll(/"(\w+)"/g),
].map((m) => m[1]!); ].map((m) => m[1]!);
// A clip the server will not cut → `clip.errors.<reason>`: every `ClipError("<reason>")` in the backend. // A clip the server will not cut → `clip.errors.<reason>`: every `ClipError("<reason>"…)` in the backend
// (a reason may carry its numbers after it: `ClipError("too_long", limit_s=…)`).
const clipReasons = [ const clipReasons = [
...new Set( ...new Set(
appSources().flatMap((src) => [...src.matchAll(/ClipError\("(\w+)"\)/g)].map((m) => m[1]!)), appSources().flatMap((src) => [...src.matchAll(/ClipError\("(\w+)"[,)]/g)].map((m) => m[1]!)),
), ),
]; ];
+1 -1
View File
@@ -26,7 +26,7 @@
"bad_source": "This video can’t be clipped.", "bad_source": "This video can’t be clipped.",
"bad_range": "The clip’s start or end isn’t valid.", "bad_range": "The clip’s start or end isn’t valid.",
"too_short": "A clip is at least a second long.", "too_short": "A clip is at least a second long.",
"too_long": "A clip is at most 10 minutes long.", "too_long": "A clip is at most {{minutes}} minutes long.",
"bad_audio": "That audio track isn’t available.", "bad_audio": "That audio track isn’t available.",
"bad_crop": "The crop frame isn’t valid. Make it bigger, or go back to the whole picture.", "bad_crop": "The crop frame isn’t valid. Make it bigger, or go back to the whole picture.",
"not_found": "This item is no longer in the library.", "not_found": "This item is no longer in the library.",
+1 -1
View File
@@ -26,7 +26,7 @@
"bad_source": "Ebből a videóból nem lehet klipet vágni.", "bad_source": "Ebből a videóból nem lehet klipet vágni.",
"bad_range": "A klip eleje vagy vége érvénytelen.", "bad_range": "A klip eleje vagy vége érvénytelen.",
"too_short": "A klip legalább egy másodperc.", "too_short": "A klip legalább egy másodperc.",
"too_long": "A klip legfeljebb 10 perc lehet.", "too_long": "A klip legfeljebb {{minutes}} perc lehet.",
"bad_audio": "Ez a hangsáv nem érhető el.", "bad_audio": "Ez a hangsáv nem érhető el.",
"bad_crop": "A kivágókeret érvénytelen. Nagyítsd meg, vagy térj vissza a teljes képre.", "bad_crop": "A kivágókeret érvénytelen. Nagyítsd meg, vagy térj vissza a teljes képre.",
"not_found": "Ez az elem már nincs a könyvtárban.", "not_found": "Ez az elem már nincs a könyvtárban.",
+18
View File
@@ -261,6 +261,24 @@ describe("req — a structured refusal reads in the user's language", () => {
).rejects.toMatchObject({ detail: localized }); ).rejects.toMatchObject({ detail: localized });
}); });
it("states the clip cap the server refused with, not a copy of it", async () => {
// The cap is the server's (downloads/clip.py MAX_CLIP_S): a message with "10 minutes" baked in
// goes stale the day it moves.
const i18n = (await import("../../i18n")).default;
const t = vi.spyOn(i18n, "t");
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response(JSON.stringify({ detail: { code: "clip", reason: "too_long", limit_s: 300 } }), {
status: 422,
headers: { "content-type": "application/json" },
}),
);
await req("/api/downloads/clip", { method: "POST" }, { quiet: true }).catch(() => undefined);
expect(t).toHaveBeenCalledWith(
"clip.errors.too_long",
expect.objectContaining({ minutes: "5" }),
);
});
it.each(["quota", "reconnect", "unavailable"])( it.each(["quota", "reconnect", "unavailable"])(
"says what to do about an action YouTube refused (%s)", "says what to do about an action YouTube refused (%s)",
async (reason) => { async (reason) => {
+7 -3
View File
@@ -81,7 +81,7 @@ function localizeDetail(d: unknown): string {
// "check your input" message rather than leaking them into a translated sentence. // "check your input" message rather than leaking them into a translated sentence.
if (Array.isArray(d)) return t("errors.validation"); if (Array.isArray(d)) return t("errors.validation");
const o = d as const o = d as
| { code?: string; reason?: string; limit?: number; min?: number } | { code?: string; reason?: string; limit?: number; limit_s?: number; min?: number }
| null | null
| undefined; | undefined;
if (o?.code === "quota") { if (o?.code === "quota") {
@@ -95,8 +95,12 @@ function localizeDetail(d: unknown): string {
if (o?.code === "edit") if (o?.code === "edit")
return t(`downloads.errors.edit.${o.reason}`, t("downloads.errors.edit.generic")); return t(`downloads.errors.edit.${o.reason}`, t("downloads.errors.edit.generic"));
// A clip the server will not cut (downloads/clip): the range, the frame, the source. An unknown // A clip the server will not cut (downloads/clip): the range, the frame, the source. An unknown
// reason reads as the plain failure rather than a key. // reason reads as the plain failure rather than a key; a cap comes with the refusal (`limit_s`).
if (o?.code === "clip") return t(`clip.errors.${o.reason}`, t("clip.failed")); if (o?.code === "clip")
return t(`clip.errors.${o.reason}`, {
defaultValue: t("clip.failed"),
minutes: o.limit_s == null ? undefined : formatDecimal(o.limit_s / 60, i18n.language, 1, 0),
});
if (o?.code === "cookies") if (o?.code === "cookies")
return t(`downloads.errors.cookies.${o.reason}`, t("downloads.errors.cookies.generic")); return t(`downloads.errors.cookies.${o.reason}`, t("downloads.errors.cookies.generic"));
// Password sign-in refusals. The card renders the account-shaped ones as a full panel // Password sign-in refusals. The card renders the account-shaped ones as a full panel
+5 -3
View File
@@ -49,8 +49,7 @@ export type DownloadStatus = "queued" | "running" | "paused" | "done" | "error"
// from the source). Only a download has a re-fetchable source; re-download refuses the rest. // from the source). Only a download has a re-fetchable source; re-download refuses the rest.
export type DownloadJobKind = "download" | "edit" | "upload" | "clip"; export type DownloadJobKind = "download" | "edit" | "upload" | "clip";
// Phase-2 editor recipe. A single `trim` (one output file) or a `segments` cut-list joined into // An ad-hoc clip (backend downloads/clip): a marked range of a YouTube video or a Plex item.
// one file. `accurate` re-encodes for a frame-accurate cut (a crop always re-encodes).
export interface ClipRequest { export interface ClipRequest {
source: "youtube" | "plex"; source: "youtube" | "plex";
ref: string; ref: string;
@@ -62,6 +61,8 @@ export interface ClipRequest {
display_name?: string; display_name?: string;
} }
// Phase-2 editor recipe. A single `trim` (one output file) or a `segments` cut-list joined into
// one file. `accurate` re-encodes for a frame-accurate cut (a crop always re-encodes).
export interface EditSpec { export interface EditSpec {
trim?: { start_s: number; end_s?: number }; trim?: { start_s: number; end_s?: number };
segments?: { start_s: number; end_s: number }[]; segments?: { start_s: number; end_s: number }[];
@@ -102,7 +103,8 @@ export interface DownloadJob {
extra_links: string[]; // extra reference URLs (beyond source_url) extra_links: string[]; // extra reference URLs (beyond source_url)
job_kind?: DownloadJobKind; // see DownloadJobKind; absent reads as "download" job_kind?: DownloadJobKind; // see DownloadJobKind; absent reads as "download"
source_job_id?: number | null; // parent download an edit was cut from source_job_id?: number | null; // parent download an edit was cut from
edit_spec?: EditSpec | null; // An edit's recipe, or a clip's spec (its request without the name) — by job_kind.
edit_spec?: EditSpec | Omit<ClipRequest, "display_name"> | null;
can_download: boolean; can_download: boolean;
expired: boolean; expired: boolean;
asset: DownloadAsset | null; asset: DownloadAsset | null;
@@ -0,0 +1,35 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
// The limits a clip is marked to, held in two languages: the clip panel's handles, fields and keys
// keep a range the server takes, and a crop frame no smaller than it cuts. Neither side can see the
// other's copy, so the gate reads both (as `api/downloads.contract.test.ts` does for the name bound).
// A cap moved on one side alone makes the panel offer what the server refuses — or clamp below
// what it would take — with every other test green.
//
// Read as TEXT: the modules are plain constants, but reading the source is what makes a renamed or
// moved constant fail here instead of comparing two undefineds.
const src = (u: string) => readFileSync(new URL(u, import.meta.url), "utf-8");
const backend = src("../../../backend/app/downloads/clip.py");
function pyConst(name: string): number {
const m = backend.match(new RegExp(`^${name}\\s*=\\s*([\\d.]+)`, "m"));
expect(m, `backend/app/downloads/clip.py no longer defines ${name}`).toBeTruthy();
return Number(m![1]);
}
function tsConst(file: string, name: string): number {
const m = src(file).match(new RegExp(`^(?:export )?const ${name} = ([\\d.]+);`, "m"));
expect(m, `${file} no longer defines ${name}`).toBeTruthy();
return Number(m![1]);
}
describe("clip limits", () => {
it.each([
["./clipRange.ts", "MIN_CLIP_S"],
["./clipRange.ts", "MAX_CLIP_S"],
["./cropGeometry.ts", "MIN_CROP"],
])("%s %s is the server's", (file, name) => {
expect(tsConst(file, name)).toBe(pyConst(name));
});
});
+12 -1
View File
@@ -1,6 +1,7 @@
// The range an ad-hoc clip is cut from, marked in a player: the rules the clip panel's handles, its // The range an ad-hoc clip is cut from, marked in a player: the rules the clip panel's handles, its
// In / Out fields and the players' `[` / `]` keys all go by, so every way of marking keeps a range the // In / Out fields and the players' `[` / `]` keys all go by, so every way of marking keeps a range the
// server takes (backend downloads/clip.py: at least MIN_CLIP_S, at most MAX_CLIP_S — the owner's cap). // server takes (backend downloads/clip.py: at least MIN_CLIP_S, at most MAX_CLIP_S — the owner's cap;
// clipLimits.contract.test.ts holds the two copies equal).
const MIN_CLIP_S = 1; const MIN_CLIP_S = 1;
export const MAX_CLIP_S = 600; export const MAX_CLIP_S = 600;
@@ -53,3 +54,13 @@ export function zoomWindow(r: ClipRange, duration: number): { from: number; to:
const pad = Math.max(5, (r.end - r.start) * 0.25); const pad = Math.max(5, (r.end - r.start) * 0.25);
return { from: Math.max(0, r.start - pad), to: Math.min(duration, r.end + pad) }; return { from: Math.max(0, r.start - pad), to: Math.min(duration, r.end + pad) };
} }
/** The `[` key: In at the playhead `t`, opening a range there when none is marked yet. */
export function markInAt(r: ClipRange | null, t: number, duration: number): ClipRange {
return markIn(r ?? openAt(t, duration), t, duration);
}
/** The `]` key: Out at the playhead `t`; with no range yet, one that ENDS there. */
export function markOutAt(r: ClipRange | null, t: number, duration: number): ClipRange {
return markOut(r ?? openAt(Math.max(0, t - DEFAULT_CLIP_S), duration), t, duration);
}
+2 -1
View File
@@ -53,7 +53,8 @@ export function cropToSourcePx(crop: Frac, srcW: number, srcH: number): CropPx |
// picture is shown at. A shape lock is the frame's width / height IN THOSE FRACTIONS: a 9:16 output // picture is shown at. A shape lock is the frame's width / height IN THOSE FRACTIONS: a 9:16 output
// on a 16:9 picture is (9/16) / (16/9) — `fracRatio` turns one into the other. // on a 16:9 picture is (9/16) / (16/9) — `fracRatio` turns one into the other.
/** The smallest a frame gets on either side (the server holds the same floor, downloads/clip). */ /** The smallest a frame gets on either side: the server's floor (downloads/clip), held equal by
* clipLimits.contract.test.ts. */
export const MIN_CROP = 0.1; export const MIN_CROP = 0.1;
export const WHOLE: Frac = { x: 0, y: 0, w: 1, h: 1 }; export const WHOLE: Frac = { x: 0, y: 0, w: 1, h: 1 };