feat(downloads): ad-hoc clips cut straight from a YouTube video or a Plex item
POST /api/downloads/clip takes a source (a YouTube id or a Plex rating key), a range of at most ten minutes and, for Plex, the audio track being heard. The clip becomes a per-user asset in the library like an editor clip, shareable as soon as it is ready. A YouTube clip fetches only its range (yt-dlp download_ranges: 3-7 s whatever the video's length); a Plex clip reads the library file in place. Both then go through one re-encode: frame-exact at the marks, H.264 at most 1080p with HDR tonemapped to SDR, stereo AAC, no chapter track, .mp4. Job kinds are a JobKind StrEnum now, and Re-download refuses every kind but a download.
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
"""Ad-hoc clips: a marked range of a YouTube video or a Plex item, cut into a new per-user asset.
|
||||
|
||||
A clip never needs the whole source first. A YouTube clip fetches only its range (yt-dlp's
|
||||
`download_ranges`; measured: 3-7 s whether the video runs four minutes or an hour, and the fetched
|
||||
range starts on the marked frame), and a Plex clip reads the library file in place. Either way the
|
||||
range then goes through ONE re-encode built here, so every clip has the same shape whatever its
|
||||
source: frame-exact at both ends, H.264 at most 1080p (an HDR source tonemapped to SDR — kept as
|
||||
10-bit PQ it played washed out), stereo AAC, in an .mp4 that plays on any phone.
|
||||
|
||||
The spec is stored on the job (`DownloadJob.edit_spec`) and is the clip's whole identity: its
|
||||
signature is the asset's cache key, so marking the same range again is an instant cache hit, and a
|
||||
different range or audio track is a different clip.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads.edit import AAC_ARGS, EVEN_FRAME, INPUT_GUARD, X264_ARGS
|
||||
|
||||
SOURCES = ("youtube", "plex")
|
||||
# Owner's cap: a clip is a moment, not a way to export a whole film to a public link.
|
||||
MAX_CLIP_S = 600
|
||||
MIN_CLIP_S = 1
|
||||
MAX_HEIGHT = 1080
|
||||
# What yt-dlp fetches the range in (formats.normalize fills the rest): the clip's own ceiling, so the
|
||||
# re-encode never upscales and never throws away a better source than it keeps.
|
||||
YOUTUBE_FETCH_SPEC = {"mode": "av", "max_height": MAX_HEIGHT, "container": "mp4"}
|
||||
# Bump when the spec → output-bytes mapping changes, so a cached clip re-derives.
|
||||
_SIG_VERSION = 1
|
||||
|
||||
_YOUTUBE_ID = re.compile(r"[A-Za-z0-9_-]{11}")
|
||||
_RATING_KEY = re.compile(r"[0-9]{1,32}")
|
||||
_MAX_AUDIO_TRACK = 63
|
||||
# The transfer characteristics of HDR video (PQ, HLG): an 8-bit SDR encode must tonemap them.
|
||||
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
||||
# HDR → SDR as measured on the owner's 4K HDR files (2026-10-10): linearise, tonemap (hable),
|
||||
# convert the primaries and transfer to BT.709 and back to 8-bit 4:2:0.
|
||||
_TONEMAP = (
|
||||
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,"
|
||||
"zscale=t=bt709:m=bt709:r=tv,format=yuv420p"
|
||||
)
|
||||
_SCALE = f"scale=-2:'min({MAX_HEIGHT},ih)'"
|
||||
|
||||
|
||||
class ClipError(Exception):
|
||||
"""A clip request that cannot be cut; `reason` is an i18n-mappable key."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
def normalize(
|
||||
source: str, ref: str, start_s: float, end_s: float, audio_track: int | None = None
|
||||
) -> dict:
|
||||
"""The validated, canonical clip spec. Times are rounded to the millisecond so the same marks
|
||||
always give the same cache key. Raises ClipError."""
|
||||
if source not in SOURCES:
|
||||
raise ClipError("bad_source")
|
||||
pattern = _YOUTUBE_ID if source == "youtube" else _RATING_KEY
|
||||
if not isinstance(ref, str) or not pattern.fullmatch(ref):
|
||||
raise ClipError("bad_source")
|
||||
if not (math.isfinite(start_s) and math.isfinite(end_s)) or start_s < 0:
|
||||
raise ClipError("bad_range")
|
||||
start, end = round(start_s, 3), round(end_s, 3)
|
||||
if end - start < MIN_CLIP_S:
|
||||
raise ClipError("too_short")
|
||||
if end - start > MAX_CLIP_S:
|
||||
raise ClipError("too_long")
|
||||
spec: dict = {"source": source, "ref": ref, "start_s": start, "end_s": end}
|
||||
if source == "plex" and audio_track is not None:
|
||||
if not 0 <= audio_track <= _MAX_AUDIO_TRACK:
|
||||
raise ClipError("bad_audio")
|
||||
spec["audio_track"] = audio_track
|
||||
return spec
|
||||
|
||||
|
||||
def duration(spec: dict) -> float:
|
||||
return spec["end_s"] - spec["start_s"]
|
||||
|
||||
|
||||
def sig(spec: dict) -> str:
|
||||
"""The clip's cache signature: the spec itself, versioned."""
|
||||
body = json.dumps({**spec, "_v": _SIG_VERSION}, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(body.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def asset_ref(user_id: int, spec: dict) -> str:
|
||||
"""The asset's `source_ref`. The user id makes a clip private to whoever cut it, like an edit:
|
||||
never dedup-shared, never cache-served to someone else."""
|
||||
return f"{user_id}:{spec['source']}:{spec['ref']}"
|
||||
|
||||
|
||||
def reference_url(spec: dict) -> str | None:
|
||||
"""Where a YouTube clip came from: its video at the clip's start. A Plex clip has no public page."""
|
||||
if spec.get("source") != "youtube" or not spec.get("ref"):
|
||||
return None
|
||||
return f"https://www.youtube.com/watch?v={spec['ref']}&t={int(spec.get('start_s') or 0)}s"
|
||||
|
||||
|
||||
def is_hdr(probe: dict) -> bool:
|
||||
"""Whether the first video stream of an ffprobe `-show_streams` JSON carries an HDR transfer."""
|
||||
for s in probe.get("streams") or []:
|
||||
if s.get("codec_type") == "video":
|
||||
return s.get("color_transfer") in _HDR_TRANSFERS
|
||||
return False
|
||||
|
||||
|
||||
def audio_track_count(probe: dict) -> int:
|
||||
return sum(1 for s in probe.get("streams") or [] if s.get("codec_type") == "audio")
|
||||
|
||||
|
||||
def build_cmd(
|
||||
src: Path,
|
||||
dest: Path,
|
||||
start_s: float,
|
||||
dur_s: float,
|
||||
*,
|
||||
audio_track: int | None,
|
||||
hdr: bool,
|
||||
threads: int,
|
||||
) -> list[str]:
|
||||
"""ffmpeg for one clip: an input seek (decodes from the keyframe before `start_s` and drops the
|
||||
frames up to it, so the clip starts on the marked frame), the first video stream, the chosen audio
|
||||
track (the first when none was chosen; none at all is fine), no subtitles, data or chapters."""
|
||||
vf = ",".join([*([_TONEMAP] if hdr else []), _SCALE, EVEN_FRAME])
|
||||
audio = f"0:a:{audio_track}" if audio_track is not None else "0:a:0?"
|
||||
return [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
"-ss",
|
||||
f"{start_s:.3f}",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-t",
|
||||
f"{dur_s:.3f}",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
audio,
|
||||
"-sn",
|
||||
"-dn",
|
||||
# A source's chapters would ride into the .mp4 as a data track of their own (an mkv's
|
||||
# chapter list came out as `bin_data`); a clip of a moment has no chapters to keep.
|
||||
"-map_chapters",
|
||||
"-1",
|
||||
"-vf",
|
||||
vf,
|
||||
*X264_ARGS,
|
||||
"-threads",
|
||||
str(threads),
|
||||
*AAC_ARGS,
|
||||
"-ac",
|
||||
"2",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(dest),
|
||||
]
|
||||
@@ -7,10 +7,13 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
from yt_dlp.utils import download_range_func
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota
|
||||
@@ -26,7 +29,8 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
||||
from app.plex import paths as plex_paths
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
@@ -333,7 +337,7 @@ def _media_info(info: dict) -> dict:
|
||||
|
||||
|
||||
def _extract_with_fallback(
|
||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None
|
||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None, section=None
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
@@ -349,7 +353,11 @@ def _extract_with_fallback(
|
||||
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt.
|
||||
|
||||
`section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the
|
||||
range is stream-copied, and its first frame is the marked one (measured), so the clip's own
|
||||
re-encode starts at 0 instead of re-fetching keyframes at the cut."""
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
@@ -375,6 +383,9 @@ def _extract_with_fallback(
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
if section is not None:
|
||||
opts["download_ranges"] = download_range_func([], [section])
|
||||
opts["force_keyframes_at_cuts"] = False
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
||||
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
||||
@@ -896,6 +907,159 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _clip_source(job_id: int, asset_id: int, spec: dict, staging: Path) -> tuple[Path, float]:
|
||||
"""The file a clip is cut from and where in it the clip starts. A YouTube clip fetches ONLY its
|
||||
range into `staging` (so it starts at 0 there); a Plex clip reads the library file in place."""
|
||||
if spec["source"] == "youtube":
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
remaining = quota.remaining_bytes(db, job.user_id) if job else None
|
||||
if remaining == 0: # see `_download`: zero is a refusal, not "no limit"
|
||||
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
# Signed out, like a download without a cookie: the cookie gate is written for a download's
|
||||
# salted asset identity (`credentials.resolve_cookiefile`), not for a clip's.
|
||||
info = _extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
service.source_url("youtube", spec["ref"]),
|
||||
formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
|
||||
staging,
|
||||
adapters.resolve_adapter("youtube", spec["ref"]),
|
||||
max_bytes=remaining,
|
||||
section=(spec["start_s"], spec["end_s"]),
|
||||
)
|
||||
return _produced_file(_media_info(dict(info)), staging), 0.0
|
||||
with SessionLocal() as db:
|
||||
item = db.execute(
|
||||
select(PlexItem).where(PlexItem.rating_key == spec["ref"])
|
||||
).scalar_one_or_none()
|
||||
src = plex_paths.local_media_path(db, item.file_path) if item else None
|
||||
if src is None:
|
||||
raise StatedFailure("The library file is not reachable from here.")
|
||||
return src, spec["start_s"]
|
||||
|
||||
|
||||
def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
"""Cut an ad-hoc clip (downloads/clip): fetch or open its source, ONE re-encode to the clip
|
||||
shape, then into the user's library beside their edits."""
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
spec = dict(job.edit_spec or {})
|
||||
user_id = job.user_id
|
||||
staging = _edit_staging_dir(asset_id)
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
src, start = _clip_source(job_id, asset_id, spec, staging)
|
||||
data = probe.probe_json(src) or {}
|
||||
audio_track = spec.get("audio_track")
|
||||
if audio_track is not None and audio_track >= clipmod.audio_track_count(data):
|
||||
raise StatedFailure("The chosen audio track is not in the file.")
|
||||
total_s = clipmod.duration(spec)
|
||||
out = staging / "clip.mp4"
|
||||
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||
editmod.run_ffmpeg(
|
||||
clipmod.build_cmd(
|
||||
src,
|
||||
out,
|
||||
start,
|
||||
total_s,
|
||||
audio_track=audio_track,
|
||||
hdr=clipmod.is_hdr(data),
|
||||
threads=editmod.encode_thread_cap(),
|
||||
),
|
||||
total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not out.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
# The phone-safe shape by construction; still held to the rule every ready file meets.
|
||||
out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s))
|
||||
ext = out.suffix.lstrip(".").lower() or "mp4"
|
||||
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
||||
storage.place_file(out, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = final.stat().st_size if final.exists() else None
|
||||
asset.container = ext
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.duration_s = round(total_s)
|
||||
asset.media_kind = mediakind.derive(vcodec, acodec)
|
||||
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
||||
# item has no public thumbnail at all.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
title = asset.title if asset else None
|
||||
finalized = service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||
)
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, user_id, title, job_id, kind="edit")
|
||||
log.info(
|
||||
"clip job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event(
|
||||
"clip.job.done", job_id=job_id, asset_id=asset_id, source=spec["source"]
|
||||
),
|
||||
)
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-fetch or mid-encode: free the asset for a later retry.
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"clip job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("clip.job.aborted", job_id=job_id),
|
||||
)
|
||||
except _OwnerRefused as exc:
|
||||
# Quota full: the job fails, the asset goes back to pending for a Retry once there is room
|
||||
# (the download's handling; see `_download`).
|
||||
msg = str(exc)[:500]
|
||||
with SessionLocal() as db:
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"clip job %s refused for its owner: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("clip.job.refused", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip")
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
||||
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
||||
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
||||
|
||||
@@ -24,7 +24,7 @@ from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobStatus
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
@@ -86,7 +86,7 @@ def footprint(db: Session, user_id: int) -> int:
|
||||
DownloadJob.user_id == user_id,
|
||||
or_(
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
and_(DownloadJob.job_kind == "upload", DownloadJob.status == JobStatus.ERROR),
|
||||
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
|
||||
),
|
||||
DownloadJob.asset_id.is_not(None),
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.status import JobKind
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
@@ -95,14 +96,16 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _convert_upload, _download, _process_edit
|
||||
from app.downloads.pipeline import _convert_upload, _download, _process_clip, _process_edit
|
||||
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
if job_kind == JobKind.EDIT:
|
||||
_process_edit(job_id, asset_id)
|
||||
elif job_kind == "upload":
|
||||
elif job_kind == JobKind.UPLOAD:
|
||||
_convert_upload(job_id, asset_id)
|
||||
elif job_kind == JobKind.CLIP:
|
||||
_process_clip(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
|
||||
@@ -16,11 +16,21 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, DownloadJob, MediaAsset, Video
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
Channel,
|
||||
DownloadJob,
|
||||
MediaAsset,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
Video,
|
||||
)
|
||||
from app.plex import paths as plex_paths
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -40,10 +50,13 @@ def source_url(source_kind: str, source_ref: str) -> str:
|
||||
def reference_url(job, asset) -> str | None:
|
||||
"""The clean "downloaded from" URL for a job — shared by the private Downloads page and the public
|
||||
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
|
||||
own earlier download, not a web page); else the canonical page URL yt-dlp recorded; else one derived
|
||||
from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
||||
if job is None or job.job_kind == "edit":
|
||||
own earlier download, not a web page) and for a Plex clip (the library is not a public page); a
|
||||
YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else
|
||||
one derived from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
||||
if job is None or job.job_kind == JobKind.EDIT:
|
||||
return None
|
||||
if job.job_kind == JobKind.CLIP:
|
||||
return clipmod.reference_url(job.edit_spec or {})
|
||||
if asset is not None and asset.source_webpage_url:
|
||||
return asset.source_webpage_url
|
||||
if job.source_kind in ("youtube", "url"):
|
||||
@@ -251,7 +264,7 @@ def enqueue_edit(
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="edit",
|
||||
job_kind=JobKind.EDIT,
|
||||
source_kind="edit",
|
||||
source_ref=ref,
|
||||
source_job_id=source_job.id,
|
||||
@@ -275,6 +288,95 @@ def enqueue_edit(
|
||||
return job
|
||||
|
||||
|
||||
def _clip_source_facts(db: Session, spec: dict) -> dict:
|
||||
"""Title, uploader, thumbnail and length of what a clip is cut from, read from our own catalog.
|
||||
Raises ClipError when a Plex item is unknown or its file cannot be read here — said at enqueue,
|
||||
rather than as a job that fails in the queue a minute later."""
|
||||
if spec["source"] == "youtube":
|
||||
video = db.get(Video, spec["ref"])
|
||||
if video is None: # not in the catalog: the worker fills the title from yt-dlp's answer
|
||||
return {"title": None, "uploader": None, "thumbnail_url": None, "length_s": None}
|
||||
channel = db.get(Channel, video.channel_id)
|
||||
return {
|
||||
"title": video.title,
|
||||
"uploader": channel.title if channel else None,
|
||||
"thumbnail_url": video.thumbnail_url,
|
||||
"length_s": video.duration_seconds,
|
||||
}
|
||||
item = db.execute(
|
||||
select(PlexItem).where(PlexItem.rating_key == spec["ref"])
|
||||
).scalar_one_or_none()
|
||||
if item is None:
|
||||
raise clipmod.ClipError("not_found")
|
||||
if plex_paths.local_media_path(db, item.file_path) is None:
|
||||
raise clipmod.ClipError("media_missing")
|
||||
title = item.title
|
||||
show = db.get(PlexShow, item.show_id) if item.show_id else None
|
||||
if show is not None:
|
||||
episode = (
|
||||
f"S{item.season_number}·E{item.episode_number}"
|
||||
if item.season_number is not None and item.episode_number is not None
|
||||
else None
|
||||
)
|
||||
title = " — ".join(p for p in (show.title, episode, item.title) if p)
|
||||
return {
|
||||
"title": title,
|
||||
"uploader": None,
|
||||
# Plex art is served through our own authenticated routes, never a public URL; the worker
|
||||
# cuts the clip's poster from the clip itself.
|
||||
"thumbnail_url": None,
|
||||
"length_s": item.duration_s,
|
||||
}
|
||||
|
||||
|
||||
def enqueue_clip(
|
||||
db: Session, user_id: int, spec: dict, display_name: str | None = None
|
||||
) -> DownloadJob:
|
||||
"""Queue an ad-hoc clip (`clipmod.normalize`d spec) for `user_id`: a NEW per-user asset cut
|
||||
straight from the source — a YouTube range or a Plex library file (see downloads/clip).
|
||||
|
||||
Raises ClipError (unknown or unreachable source, a range past its end) or quota.QuotaExceeded."""
|
||||
facts = _clip_source_facts(db, spec)
|
||||
# One second of grace: a mark set on the last frame may round past the catalog's whole seconds.
|
||||
if facts["length_s"] and spec["end_s"] > facts["length_s"] + 1:
|
||||
raise clipmod.ClipError("past_end")
|
||||
|
||||
quota.check_enqueue(db, user_id)
|
||||
|
||||
ref = clipmod.asset_ref(user_id, spec)[:512]
|
||||
asset = get_or_create_asset(db, "clip", ref, clipmod.sig(spec))
|
||||
if not asset.title:
|
||||
asset.title = _clip_title(facts["title"], display_name)
|
||||
asset.uploader = facts["uploader"]
|
||||
asset.thumbnail_url = facts["thumbnail_url"]
|
||||
asset.duration_s = round(clipmod.duration(spec))
|
||||
asset.media_kind = mediakind.VIDEO
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind=JobKind.CLIP,
|
||||
source_kind="clip",
|
||||
source_ref=ref,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
if asset.status == "ready": # the same range was already cut for this user → instant
|
||||
job.status = "done"
|
||||
job.progress = 100
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
||||
"""Write terminal `fields` to a job in ONE atomic ``UPDATE ... WHERE status='running'`` and report
|
||||
whether it fired. This is the single owner of the invariant "a finishing worker child must never
|
||||
|
||||
@@ -33,6 +33,23 @@ HOLDING = frozenset(
|
||||
{JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE, JobStatus.ERROR}
|
||||
)
|
||||
|
||||
|
||||
class JobKind(StrEnum):
|
||||
"""What a job produces, which decides the worker path that runs it and what the library may do
|
||||
with the result (only a DOWNLOAD has a web source to fetch again)."""
|
||||
|
||||
DOWNLOAD = "download" # yt-dlp fetches a source into a shared, deduplicated asset
|
||||
EDIT = "edit" # a per-user trim/crop of a file already in the library
|
||||
UPLOAD = "upload" # a file the user sent; the worker only converts it
|
||||
CLIP = (
|
||||
"clip" # a per-user range of a YouTube video or a Plex item, cut straight from the source
|
||||
)
|
||||
|
||||
|
||||
# A job kind with no web source to fetch again: Re-download refuses these.
|
||||
NOT_REFETCHABLE = frozenset({JobKind.EDIT, JobKind.UPLOAD, JobKind.CLIP})
|
||||
|
||||
|
||||
# States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free
|
||||
# the asset, leave the route's status intact) when it observes one of these on a job it is running.
|
||||
STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED})
|
||||
|
||||
@@ -34,7 +34,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
@@ -113,7 +113,7 @@ def ingest_upload(
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
job_kind=JobKind.UPLOAD,
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
|
||||
@@ -8,6 +8,7 @@ state, serves finished files (range-aware, with the user's custom display name),
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Literal
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -16,7 +17,7 @@ from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -27,13 +28,14 @@ from app import audit, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.config import settings
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.status import JobStatus
|
||||
from app.downloads.status import NOT_REFETCHABLE, JobStatus
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
@@ -576,6 +578,34 @@ def enqueue_edit(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
class EnqueueClipIn(BaseModel):
|
||||
source: Literal["youtube", "plex"]
|
||||
# The YouTube video id or the Plex rating key.
|
||||
ref: str = Field(max_length=32)
|
||||
start_s: float
|
||||
end_s: float
|
||||
# Plex only: which of the file's audio tracks (0-based, in file order) — the one being heard.
|
||||
audio_track: int | None = None
|
||||
display_name: str | None = None
|
||||
|
||||
|
||||
@router.post("/clip")
|
||||
def enqueue_clip(
|
||||
body: EnqueueClipIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Queue an ad-hoc clip: the marked range of a YouTube video or a Plex item, cut straight from
|
||||
the source into a new asset in this user's library (downloads/clip)."""
|
||||
try:
|
||||
spec = clipmod.normalize(body.source, body.ref, body.start_s, body.end_s, body.audio_track)
|
||||
job = service.enqueue_clip(db, user.id, spec, display_text(body.display_name))
|
||||
except clipmod.ClipError as e:
|
||||
status = 404 if e.reason == "not_found" else 422
|
||||
raise HTTPException(status_code=status, detail={"code": "clip", "reason": e.reason}) from e
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
def _quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
|
||||
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
|
||||
@@ -768,9 +798,9 @@ def redownload_download(
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
if job.job_kind in NOT_REFETCHABLE:
|
||||
# None has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source, an ad-hoc clip is a range of a source the clip itself does not re-fetch.
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
|
||||
raise refused(409, "still_downloading")
|
||||
|
||||
Reference in New Issue
Block a user