peter
533571f572
PlexLibrary, PlexShow, PlexSeason, PlexItem, PlexPerson, PlexCollection, PlexState, PlexLink, PlexPlaylist and PlexPlaylistItem leave app/models.py for app/plex/models.py; app.tables registers them. Every import is re-pointed; no schema change.
1336 lines
63 KiB
Python
1336 lines
63 KiB
Python
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
|
|
|
import re
|
|
import shutil
|
|
import time
|
|
from datetime import date, datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import yt_dlp
|
|
from yt_dlp.utils import download_range_func
|
|
|
|
from app import logs, sysconfig
|
|
from app.config import settings
|
|
from app.db import SessionLocal
|
|
from app.downloads import clip as clipmod
|
|
from app.downloads import edit as editmod
|
|
from app.downloads import errors as dlerrors
|
|
from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota
|
|
from app.downloads import service, storage
|
|
from app.downloads import urlguard
|
|
from app.downloads.jobdb import (
|
|
_edit_staging_dir,
|
|
_job_status,
|
|
_reset_asset_pending,
|
|
_set_job,
|
|
_staging_dir,
|
|
_touch_job,
|
|
)
|
|
from app.downloads.stated import StatedFailure
|
|
from app.downloads.status import JobStatus
|
|
from app.models import DISPLAY_TEXT_MAX
|
|
from app.plex.models import PlexItem
|
|
from app.downloads.models import DownloadJob, MediaAsset
|
|
from app.plex import lookups as plex_lookups
|
|
from app.plex import paths as plex_paths
|
|
from app.notifications import create_notification
|
|
from app.titles import normalize_title
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
|
|
class _Aborted(Exception):
|
|
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
|
|
|
|
|
class _ClipFetchFailed(Exception):
|
|
"""A clip's YouTube range fetch failed: its message and the download catalog's diagnosis, so a
|
|
clip on YouTube's wall says what a download of the same video would."""
|
|
|
|
def __init__(self, msg: str, diagnosis: dlerrors.Diagnosis):
|
|
super().__init__(msg)
|
|
self.diagnosis = diagnosis
|
|
|
|
|
|
class _OwnerRefused(Exception):
|
|
"""A refusal that belongs to THIS job's owner, not to the source.
|
|
|
|
MediaAssets are shared: two users asking for the same video and format get one row (see
|
|
`service.get_or_create_asset`), and the generic failure handler writes the message onto it —
|
|
correctly, because an upstream failure really is everyone's. A per-user refusal is not. Written
|
|
to the shared row it would fail a second user's queued job for the same video with the first
|
|
user's private reason (`worker._process_job` reads `asset.error` verbatim), telling someone with
|
|
an empty library to free up space. So this is caught separately: the job fails, the asset goes
|
|
back to `pending` for whoever else is holding it."""
|
|
|
|
|
|
# Both progress sources call back FAR faster than the database should be touched: yt-dlp fires its
|
|
# hook per chunk, and ffmpeg's `-progress pipe:1` emits a block of ~12 lines every ~0.5s while
|
|
# run_ffmpeg calls its callbacks once per LINE. Every callback here is a round-trip (`_job_status`
|
|
# opens a session and SELECTs; `_set_job` opens one and UPDATE+COMMITs), so unthrottled they cost
|
|
# ~10^5 queries on a long job. ONE interval, ONE implementation, used by both paths — three
|
|
# hand-rolled copies is how the ffmpeg side ended up unthrottled while the download side was fine.
|
|
_DB_WRITE_INTERVAL_S = 1.0
|
|
_CANCEL_POLL_INTERVAL_S = 1.0
|
|
|
|
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
|
# The job phase while a file is brought into the phone-safe shape (`_ensure_phone_safe`) for a
|
|
# download or an upload. Its own word, not yt-dlp's "processing": it reports ffmpeg's real progress,
|
|
# so the queue row shows a % for it, where yt-dlp's post-steps only get an indeterminate pulse.
|
|
CONVERTING = "converting"
|
|
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
|
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
|
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
|
# stream as a ready asset.
|
|
_YTDLP_TEMP_RE = re.compile(
|
|
r"\.part(-Frag\d+)?$" # in-flight download / fragment
|
|
r"|\.ytdl$" # resume state
|
|
r"|\.temp\.[^.]+$" # merge/postprocess temp
|
|
r"|\.f\d+\.[^.]+$", # per-format stream kept before the merge
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
|
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
|
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
|
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
|
scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the
|
|
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
|
caller BEFORE its commit; only the folder is read here, once, for the notice's "Open"."""
|
|
if user_id is None:
|
|
return
|
|
try:
|
|
# The notice's "Open" lands on the file in its folder: both ride along (a folder the file
|
|
# has left since is the page's to notice — it then looks among every file).
|
|
job = db.get(DownloadJob, job_id)
|
|
create_notification(
|
|
db,
|
|
user_id=user_id,
|
|
type="download_done",
|
|
title=f"Ready to watch: {title}" if title else "Ready to watch",
|
|
data={
|
|
"title": title,
|
|
"kind": kind,
|
|
"job_id": job_id,
|
|
"folder_id": job.folder_id if job else None,
|
|
},
|
|
)
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("failed to write download-done notification for job %s", job_id)
|
|
|
|
|
|
def _parse_upload_date(raw) -> date | None:
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return datetime.strptime(str(raw), "%Y%m%d").date()
|
|
# silent-ok: a date we cannot read becomes a date the card does not show, which is what the
|
|
# card shows for every source that never supplies one. Absent is the honest rendering.
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
|
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
|
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
|
real title + thumbnail while it downloads, not just the URL. Zero extra network cost."""
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset is None or asset.title:
|
|
return
|
|
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
|
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
|
# the final title with its own source_ref fallback.
|
|
adapter = adapters.resolve_adapter(None, None, info)
|
|
asset.title = normalize_title(adapter.title(info)) or info.get("id")
|
|
asset.uploader = info.get("uploader") or info.get("channel")
|
|
asset.uploader_url = adapter.uploader_url(info)
|
|
asset.thumbnail_url = info.get("thumbnail")
|
|
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
|
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
|
asset.source_webpage_url = info.get("webpage_url")
|
|
db.commit()
|
|
|
|
|
|
def _make_progress_hook(job_id: int, asset_id: int):
|
|
state = {"meta_done": False}
|
|
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
|
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
|
|
|
|
def hook(d: dict) -> None:
|
|
# Cooperative pause/cancel: if the job is no longer 'running', abort the download.
|
|
if cancelled():
|
|
raise _Aborted
|
|
if not state["meta_done"] and d.get("info_dict"):
|
|
state["meta_done"] = True
|
|
try:
|
|
_fill_asset_meta_early(asset_id, d["info_dict"])
|
|
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
|
|
# Best-effort is about the DOWNLOAD, not about knowing. What this swallows is
|
|
# anything from a shape change in yt-dlp's info dict to the database being down,
|
|
# and all the user sees is a card with an id on it until the job finishes.
|
|
log.info(
|
|
"early metadata could not be filled in for this download",
|
|
extra=logs.event("download.meta.early_fill_failed", asset_id=asset_id),
|
|
exc_info=True,
|
|
)
|
|
if d.get("status") != "downloading":
|
|
return
|
|
if not allow_write():
|
|
return
|
|
# yt-dlp downloads the video and audio streams SEPARATELY (each 0→100%), then merges.
|
|
# Label the phase so the user understands the bar resetting between streams instead of
|
|
# seeing a mysterious 95%→5% jump.
|
|
info = d.get("info_dict") or {}
|
|
vcodec = info.get("vcodec") or "none"
|
|
acodec = info.get("acodec") or "none"
|
|
if vcodec != "none" and acodec == "none":
|
|
phase = "video"
|
|
elif acodec != "none" and vcodec == "none":
|
|
phase = "audio"
|
|
else:
|
|
phase = "media"
|
|
total = d.get("total_bytes") or d.get("total_bytes_estimate") or 0
|
|
done = d.get("downloaded_bytes") or 0
|
|
pct = int(done * 100 / total) if total else 0
|
|
_set_job(
|
|
job_id,
|
|
progress=min(pct, 99),
|
|
speed_bps=int(d["speed"]) if d.get("speed") else None,
|
|
eta_s=int(d["eta"]) if d.get("eta") else None,
|
|
phase=phase,
|
|
)
|
|
|
|
return hook
|
|
|
|
|
|
def _rate_limiter(interval_s: float, *, now=time.monotonic):
|
|
"""`should_run()` → True at most once per `interval_s` (the first call always passes).
|
|
`now` is injectable so tests drive a clock instead of patching the global one."""
|
|
# `None`, not 0.0: "never ran" has to be a VALUE, not a falsy timestamp. A truthiness test here
|
|
# reads a legitimate `now() == 0.0` as "never ran" and lets a second call straight through —
|
|
# unreachable with real `time.monotonic()`, but `now` exists to be injected, and a test clock
|
|
# starting at 0 is the obvious choice. That would let a throttle test encode the wrong
|
|
# behaviour and still pass.
|
|
state: dict[str, float | None] = {"at": None}
|
|
|
|
def should_run() -> bool:
|
|
t = now()
|
|
last = state["at"]
|
|
if last is not None and t - last < interval_s:
|
|
return False
|
|
state["at"] = t
|
|
return True
|
|
|
|
return should_run
|
|
|
|
|
|
def _cancel_poll(job_id: int, interval_s: float = _CANCEL_POLL_INTERVAL_S, *, now=time.monotonic):
|
|
"""Cooperative "has this job been paused/cancelled?" poll, rate-limited and sticky: once it
|
|
says yes it never queries again. Shared by the yt-dlp hook and run_ffmpeg so cancel latency is
|
|
one policy, not one per call site."""
|
|
allow = _rate_limiter(interval_s, now=now)
|
|
state = {"cancelled": False}
|
|
|
|
def cancelled() -> bool:
|
|
if not state["cancelled"] and allow():
|
|
state["cancelled"] = _job_status(job_id) not in (JobStatus.RUNNING, None)
|
|
return state["cancelled"]
|
|
|
|
return cancelled
|
|
|
|
|
|
def _ffmpeg_progress_writer(job_id: int, phase: str, *, now=time.monotonic):
|
|
"""`on_progress` for run_ffmpeg: persist a progress row write at most once per interval and only
|
|
when the whole-percent value changed (the bar shows integers), plus a separate once-per-interval
|
|
heartbeat while the percent is stuck (so a slow re-encode isn't mistaken for a stall)."""
|
|
# Progress writes have their OWN 1/interval budget so a heartbeat can never delay a real write (the
|
|
# original concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
|
# so a progress write (which already refreshes updated_at) suppresses a redundant heartbeat that
|
|
# same interval, keeping active downloads at ~1 write/interval, not two.
|
|
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S, now=now)
|
|
state = {"pct": -1}
|
|
last_touch: dict = {"t": None} # monotonic time of the last DB write/heartbeat
|
|
|
|
def write(pct: float) -> None:
|
|
whole = int(pct)
|
|
if whole != state["pct"]:
|
|
if allow_write():
|
|
state["pct"] = whole
|
|
_set_job(job_id, progress=whole, phase=phase)
|
|
last_touch["t"] = now()
|
|
return
|
|
# Same integer percent, but ffmpeg IS still working — a slow re-encode (AV1/VP9 -> H.264 or a
|
|
# crop on the CPU-only host) can sit on one percent for minutes. Heartbeat updated_at so the
|
|
# stall watchdog doesn't kill a live job (B-sub-4) — but only if no write/beat already touched
|
|
# the row this interval, so a progressing download doesn't pay a redundant write.
|
|
t = now()
|
|
if last_touch["t"] is None or t - last_touch["t"] >= _DB_WRITE_INTERVAL_S:
|
|
_touch_job(job_id)
|
|
last_touch["t"] = t
|
|
|
|
return write
|
|
|
|
|
|
def _pp_phase(pp: str) -> str:
|
|
"""Map a yt-dlp postprocessor class name to a user-facing phase label. ffmpeg post-steps
|
|
aren't byte-progress operations (no %), so the UI shows the step name + an indeterminate
|
|
pulse — this makes the long merge/finalize legible instead of a silent 'Processing'."""
|
|
if "Merg" in pp:
|
|
return "merging"
|
|
if "ExtractAudio" in pp:
|
|
return "audio_extract"
|
|
if "Thumbnail" in pp:
|
|
return "thumbnail"
|
|
if "SponsorBlock" in pp or "ModifyChapters" in pp:
|
|
return "sponsorblock"
|
|
if "Metadata" in pp:
|
|
return "metadata"
|
|
return "processing"
|
|
|
|
|
|
def _make_pp_hook(job_id: int):
|
|
"""Postprocessor hook: surface the current ffmpeg step so the bar shows a concrete label
|
|
(Merging / Embedding thumbnail / …) instead of freezing at 100% after the streams finish."""
|
|
|
|
def hook(d: dict) -> None:
|
|
if d.get("status") in ("started", "processing"):
|
|
_set_job(
|
|
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
|
|
)
|
|
|
|
return hook
|
|
|
|
|
|
def _find_thumbnail(staging: Path, media: Path) -> Path | None:
|
|
for p in staging.iterdir():
|
|
if p != media and p.suffix.lower() in _THUMB_SUFFIXES:
|
|
return p
|
|
return None
|
|
|
|
|
|
def _produced_file(info: dict, staging: Path) -> Path:
|
|
"""The media file yt-dlp actually wrote.
|
|
|
|
`requested_downloads[0].filepath` is the authoritative answer on YouTube, but an arbitrary-URL
|
|
extractor may not fill it in at all — and `Path("")` is `Path(".")`, which *passes* `.exists()`,
|
|
so a missing value used to sail past the fallback and blow up two steps later (`with_name` on an
|
|
empty name: the BBC-news failure). Anything that isn't a real file counts as absent; then fall
|
|
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
|
|
working files (see `_YTDLP_TEMP_RE`)."""
|
|
req = (info.get("requested_downloads") or [{}])[0]
|
|
fp = (
|
|
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
|
)
|
|
if fp:
|
|
produced = Path(str(fp))
|
|
if produced.is_file():
|
|
return produced
|
|
cands = [
|
|
p
|
|
for p in staging.iterdir()
|
|
if p.is_file()
|
|
and p.suffix.lower() not in _THUMB_SUFFIXES
|
|
and not _YTDLP_TEMP_RE.search(p.name)
|
|
]
|
|
if not cands:
|
|
raise StatedFailure("yt-dlp produced no output file")
|
|
return max(cands, key=lambda p: p.stat().st_size)
|
|
|
|
|
|
def _media_info(info: dict) -> dict:
|
|
"""Unwrap a playlist/multi_video extraction to the entry that carries the download.
|
|
|
|
A single non-YouTube page (a news article's video, a multi-rendition embed) can extract as a
|
|
wrapper whose own dict has no `requested_downloads` — the media lives on an entry. Pick the
|
|
FIRST entry that actually downloaded something: on a page with a skipped teaser in front of the
|
|
real video, entries[0] carries no file, and taking it blindly would label the produced file
|
|
with the wrong id/title/duration."""
|
|
entries = [e for e in (info.get("entries") or []) if e]
|
|
if not entries or info.get("requested_downloads"):
|
|
return info
|
|
chosen = next((e for e in entries if e.get("requested_downloads")), entries[0])
|
|
return {**info, **chosen}
|
|
|
|
|
|
def _extract_with_fallback(
|
|
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None, section=None
|
|
):
|
|
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
|
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
|
error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other
|
|
sources make a single attempt. Returns the info dict of the successful attempt.
|
|
|
|
Before the first attempt the adapter may resolve `url` to the link that really holds the video (a
|
|
Facebook group post → its video's watch link); when the page shows no single video there, or is not
|
|
the post at all (a refused cookie's login page), the adapter raises its own verdict
|
|
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
|
download as itself.
|
|
|
|
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
|
`credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so
|
|
the fallback stays authenticated.
|
|
|
|
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. A
|
|
`section` fetch goes through yt-dlp's ffmpeg downloader, which does not read it: a clip holds
|
|
its own output to the cap (`_process_clip`).
|
|
|
|
`section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the
|
|
range is stream-copied, and its first frame is the marked one (measured), so the clip's own
|
|
re-encode starts at 0 instead of re-fetching keyframes at the cut."""
|
|
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
|
outtmpl = str(staging / "%(id)s.%(ext)s")
|
|
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
|
_touch_job(job_id)
|
|
target = adapter.resolve_url(
|
|
url,
|
|
formats.build_ydl_opts(
|
|
spec,
|
|
outtmpl,
|
|
_make_progress_hook(job_id, asset_id),
|
|
cookiefile=cookiefile,
|
|
max_bytes=max_bytes,
|
|
),
|
|
)
|
|
last_exc: Exception | None = None
|
|
for i, clients in enumerate(client_sets):
|
|
opts = formats.build_ydl_opts(
|
|
spec,
|
|
outtmpl,
|
|
_make_progress_hook(job_id, asset_id),
|
|
extra=adapter.ydl_opts_extra(clients),
|
|
cookiefile=cookiefile,
|
|
max_bytes=max_bytes,
|
|
)
|
|
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
|
if section is not None:
|
|
opts["download_ranges"] = download_range_func([], [section])
|
|
opts["force_keyframes_at_cuts"] = False
|
|
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
|
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
|
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
|
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
|
try:
|
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
|
return ydl.extract_info(target, download=True)
|
|
except yt_dlp.utils.DownloadError as exc:
|
|
last_exc = exc
|
|
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
|
log.info(
|
|
"job %s: clients %s failed (%s); retrying with fallback",
|
|
job_id,
|
|
clients,
|
|
formats.log_safe(exc)[:70],
|
|
)
|
|
for p in list(staging.iterdir()): # drop partials before the retry
|
|
try:
|
|
p.unlink()
|
|
except OSError as exc:
|
|
# A partial left behind is not inert: the next client set resumes into it,
|
|
# so the retry this loop exists to make clean starts dirty instead.
|
|
log.info(
|
|
"a partial file could not be removed before the retry",
|
|
extra=logs.event(
|
|
"download.retry.partial_unlink_failed",
|
|
job_id=job_id,
|
|
reason=type(exc).__name__,
|
|
),
|
|
)
|
|
_set_job(job_id, progress=0, phase="downloading")
|
|
continue
|
|
raise
|
|
raise last_exc # unreachable (loop either returns or raises)
|
|
|
|
|
|
def _download_layout() -> str:
|
|
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
|
|
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
|
|
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
|
|
with SessionLocal() as db:
|
|
return sysconfig.get_str(db, "download_layout")
|
|
|
|
|
|
def _ensure_phone_safe(
|
|
job_id: int,
|
|
produced: Path,
|
|
out_dir: Path,
|
|
phase: str,
|
|
total_s: int | None = None,
|
|
*,
|
|
keep_source: bool = False,
|
|
) -> Path:
|
|
"""Bring a file that is about to become a ready library asset into the phone-safe shape
|
|
(`playable`), writing any converted copy into `out_dir`. Returns the file to store — `produced`
|
|
itself when it already is that shape, or isn't media at all — and removes `produced` once a
|
|
converted copy exists, unless `keep_source` (an upload's original, which must outlive the
|
|
conversion until the database points at the new file). The job shows `phase` with real ffmpeg
|
|
progress while it converts.
|
|
|
|
Every ready path calls this — a download, an upload's worker conversion and an editor clip — so
|
|
a share link can only ever serve the one shape every phone and browser plays."""
|
|
p = playable.plan(probe.probe_json(produced), produced.suffix)
|
|
if p is None or not p.needs_change:
|
|
return produced
|
|
dest = out_dir / f"{produced.stem}.safe.{p.ext}"
|
|
action = "re-encode" if (p.reencode_video or p.reencode_audio) else "remux"
|
|
log.info(
|
|
"job %s: %s (video=%s audio=%s) -> phone-safe .%s",
|
|
job_id,
|
|
action,
|
|
p.reencode_video,
|
|
p.reencode_audio,
|
|
p.ext,
|
|
extra=logs.event("download.phone_safe", job_id=job_id, action=action, ext=p.ext),
|
|
)
|
|
_set_job(job_id, phase=phase, progress=0, speed_bps=None, eta_s=None)
|
|
# A re-encode with no known length would hold a frozen 0%; probe the file for a denominator.
|
|
editmod.run_ffmpeg(
|
|
playable.build_cmd(produced, dest, p, threads=editmod.encode_thread_cap()),
|
|
total_s=total_s or editmod.probe_duration(produced),
|
|
on_progress=_ffmpeg_progress_writer(job_id, phase),
|
|
should_cancel=_cancel_poll(job_id),
|
|
)
|
|
if not dest.exists():
|
|
raise StatedFailure("ffmpeg produced no output file")
|
|
if not keep_source:
|
|
produced.unlink(missing_ok=True)
|
|
return dest
|
|
|
|
|
|
def _download(
|
|
job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int
|
|
) -> None:
|
|
staging = _staging_dir(asset_id)
|
|
cookiefile: Path | None = None
|
|
url: str | None = None # read by the failure branch, which can run before it is resolved
|
|
try:
|
|
if staging.exists():
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
|
|
_set_job(job_id, phase="downloading", progress=0)
|
|
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
|
url = service.source_url(source_kind, source_ref)
|
|
# SEC-12, second half: the enqueue guard ran when the job was created, which may have been
|
|
# hours ago and against a different DNS answer. This is where the socket actually gets
|
|
# opened, so the check is repeated here and the job fails rather than fetching.
|
|
if source_kind == "url":
|
|
try:
|
|
urlguard.check_url(url)
|
|
except urlguard.BlockedUrl as e:
|
|
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
|
# Attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
|
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
|
remaining_bytes: int | None = None
|
|
with SessionLocal() as cdb:
|
|
target = cdb.get(MediaAsset, asset_id)
|
|
if target is not None:
|
|
cookiefile = credentials.resolve_cookiefile(
|
|
cdb, user_id, url, target, spec, staging
|
|
)
|
|
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
|
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
|
# single job could previously blow through the cap by any amount. Handing the remainder
|
|
# to yt-dlp as `max_filesize` makes it refuse an oversized file (its HTTP downloader
|
|
# checks the announced size) instead.
|
|
remaining_bytes = quota.remaining_bytes(cdb, user_id)
|
|
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
|
|
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
|
|
# passing it down would leave the ceiling unset — the one user already at their cap would be
|
|
# the only one downloading unbounded. Fail the job here instead. Not retryable until they
|
|
# free space, which is what the `quota_exceeded` category tells them.
|
|
if remaining_bytes == 0:
|
|
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
|
with formats.watching_size_cap() as over_cap:
|
|
fetched = _extract_with_fallback(
|
|
job_id,
|
|
asset_id,
|
|
url,
|
|
spec,
|
|
staging,
|
|
adapter,
|
|
cookiefile=str(cookiefile) if cookiefile else None,
|
|
max_bytes=remaining_bytes,
|
|
)
|
|
info = _media_info(dict(fetched))
|
|
# yt-dlp refused a file larger than what is left: it returns with no file, which would
|
|
# otherwise read as a processing fault. The owner's storage, so the owner's refusal.
|
|
if over_cap[0]:
|
|
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
|
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
|
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
|
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
|
|
|
produced = _produced_file(info, staging)
|
|
produced = _ensure_phone_safe(
|
|
job_id,
|
|
produced,
|
|
staging,
|
|
CONVERTING,
|
|
int(info["duration"]) if info.get("duration") else None,
|
|
)
|
|
# Held to the cap by its real size, as a clip is: yt-dlp's `max_filesize` only stops a plain
|
|
# HTTP file that announces its size, and YouTube's fragmented formats never meet it
|
|
# (measured: 718 MB fetched with 1 MB left). Read again — other jobs may have landed.
|
|
with SessionLocal() as qdb:
|
|
room = quota.remaining_bytes(qdb, user_id)
|
|
if room is not None and produced.stat().st_size > room:
|
|
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
|
ext = produced.suffix.lstrip(".").lower()
|
|
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
|
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
|
# media_kind from those nulls misclassified a real H.264 video as a download-only 'file', and a
|
|
# missing duration hid the trim action. Probed before the sidecars so the .nfo runtime has it
|
|
# too. Same probe the upload ingest uses.
|
|
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
|
meta = storage.MediaMeta(
|
|
video_id=info.get("id") or source_ref,
|
|
title=normalize_title(adapter.title(info)) or source_ref,
|
|
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
|
upload_date=_parse_upload_date(info.get("upload_date")),
|
|
uniqueid_type=adapter.nfo_uniqueid_type(),
|
|
duration_s=duration_s,
|
|
description=info.get("description"),
|
|
thumbnail_url=info.get("thumbnail"),
|
|
)
|
|
|
|
layout = _download_layout()
|
|
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
|
thumb = _find_thumbnail(staging, produced)
|
|
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
|
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
|
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
|
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
|
storage.place_file(produced, settings.files_root, rel)
|
|
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
|
|
|
|
final = storage.abs_path(settings.files_root, rel)
|
|
size = final.stat().st_size if final.exists() else None
|
|
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "ready"
|
|
asset.rel_path = rel
|
|
asset.size_bytes = size
|
|
asset.duration_s = meta.duration_s
|
|
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
|
vcodec,
|
|
acodec,
|
|
width,
|
|
height,
|
|
)
|
|
asset.container = ext
|
|
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
|
asset.title = meta.title
|
|
asset.uploader = meta.uploader
|
|
asset.uploader_url = adapter.uploader_url(info)
|
|
asset.upload_date = meta.upload_date
|
|
asset.thumbnail_url = meta.thumbnail_url
|
|
# Record a self-hosted poster for every download: it's the reliable link-preview
|
|
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
|
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
|
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
|
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
|
asset.source_webpage_url = info.get("webpage_url")
|
|
asset.nfo_written = nfo_ok
|
|
asset.error = None
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
job = db.get(DownloadJob, job_id)
|
|
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
|
owner_id = job.user_id if job else None
|
|
fields = {
|
|
"status": JobStatus.DONE,
|
|
"progress": 100,
|
|
"phase": None,
|
|
"speed_bps": None,
|
|
"eta_s": None,
|
|
}
|
|
if job and not job.display_name and meta.title:
|
|
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
|
fields["display_name"] = meta.title[:DISPLAY_TEXT_MAX]
|
|
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
|
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
|
finalized = service.finalize_if_running(db, job_id, **fields)
|
|
db.commit()
|
|
if finalized:
|
|
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
|
# The asset id, not `rel`: the path is built from the channel and the title.
|
|
log.info(
|
|
"job %s done: asset %s",
|
|
job_id,
|
|
asset_id,
|
|
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
|
|
)
|
|
|
|
except (_Aborted, editmod.EditAborted):
|
|
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
|
# job's status (the route set it to paused/canceled). ref_count is adjusted by the route.
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"job %s aborted (pause/cancel)",
|
|
job_id,
|
|
extra=logs.event("download.job.aborted", job_id=job_id),
|
|
)
|
|
except _OwnerRefused as exc:
|
|
# Fail this job only. The asset is never marked `error`; it goes back to `pending` (below),
|
|
# so another user's job for the same video is claimed and downloaded normally instead of
|
|
# inheriting a reason that is about someone else's account.
|
|
msg = str(exc)[:500]
|
|
with SessionLocal() as db:
|
|
service.finalize_if_running(
|
|
db,
|
|
job_id,
|
|
status=JobStatus.ERROR,
|
|
error=msg,
|
|
error_code=dlerrors.classify(msg),
|
|
error_detail=None,
|
|
phase=None,
|
|
)
|
|
db.commit()
|
|
# Released AFTER the job is terminal: the moment the asset is `pending` another job can claim
|
|
# it, and this one must already be out of the way. (The staging tree is still torn down in
|
|
# `finally` below — the same narrow window the pause/cancel path above has always had.)
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"job %s refused for its owner: %s",
|
|
job_id,
|
|
formats.log_safe(msg),
|
|
extra=logs.event("download.job.refused", job_id=job_id),
|
|
)
|
|
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
|
msg = str(exc)[:500]
|
|
# The catalog needs the link and whether the user's cookie went with it: the same yt-dlp
|
|
# text is Facebook's login wall without one and a refused cookie with one.
|
|
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
|
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
|
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "error"
|
|
asset.error = msg
|
|
# Atomic fail: only while still running, so a route-set cancel/pause is never
|
|
# overwritten with `error` (error also holds the asset → would re-open the
|
|
# double-release class).
|
|
service.finalize_if_running(
|
|
db,
|
|
job_id,
|
|
status=JobStatus.ERROR,
|
|
error=msg,
|
|
error_code=diagnosis.code,
|
|
error_detail=diagnosis.detail,
|
|
phase=None,
|
|
)
|
|
db.commit()
|
|
# After the write: a job the write could not fail is not a failed job. When it raises,
|
|
# the runner reports this failure with it (`runner._run`) and the worker fails the job.
|
|
# silent-ok: not silent — `_log_download_failed` is the report.
|
|
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
|
finally:
|
|
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
|
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
|
if cookiefile is not None:
|
|
cookiefile.unlink(missing_ok=True)
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
|
|
|
|
def _failure_stack(exc: BaseException) -> dict[str, str]:
|
|
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
|
|
|
|
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
|
|
their message is the whole story, and their stack is only the `except` that caught them.
|
|
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
|
|
a database error: that is our code or our platform failing, and the stack is the lead.
|
|
|
|
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
|
|
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
|
|
unscrubbed text as its context."""
|
|
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
|
|
return {}
|
|
return logs.scrubbed_stack(exc, formats.log_safe)
|
|
|
|
|
|
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
|
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
|
|
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
|
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
|
place the failure: a recognised one is fully described by its code."""
|
|
unplaced = code in (None, "", "unknown")
|
|
log.warning(
|
|
"job %s failed: %s",
|
|
job_id,
|
|
formats.log_safe(msg),
|
|
extra={
|
|
**logs.event(
|
|
"download.job.failed",
|
|
job_id=job_id,
|
|
error_code=code,
|
|
error_type=type(exc).__name__,
|
|
),
|
|
**(_failure_stack(exc) if unplaced else {}),
|
|
},
|
|
)
|
|
|
|
|
|
def _fail_edit(
|
|
job_id: int,
|
|
asset_id: int,
|
|
msg: str,
|
|
exc: BaseException | None = None,
|
|
*,
|
|
what: str = "edit",
|
|
diagnosis: dlerrors.Diagnosis | None = None,
|
|
) -> None:
|
|
"""Fail a local ffmpeg job — an edit, a clip, or an upload's conversion (`what="upload"`) — and
|
|
say so ONCE: every way one fails comes through here, so this is where its line and its event code
|
|
(`<what>.job.failed`) live. `exc` is the exception when there was one; `diagnosis` replaces the
|
|
plain category when the failure was a fetch the download catalog can name (a clip's).
|
|
|
|
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
|
Logged after the write, as on the download error path."""
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "error"
|
|
asset.error = msg
|
|
# Atomic fail: only while still running (see the download error path).
|
|
service.finalize_if_running(
|
|
db,
|
|
job_id,
|
|
status=JobStatus.ERROR,
|
|
error=msg,
|
|
error_code=diagnosis.code
|
|
if diagnosis
|
|
else dlerrors.classify_conversion(msg)
|
|
if what == "upload"
|
|
else dlerrors.classify(msg),
|
|
error_detail=diagnosis.detail if diagnosis else None,
|
|
phase=None,
|
|
)
|
|
db.commit()
|
|
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
|
# built from the channel and the title.
|
|
log.warning(
|
|
"%s job %s failed: %s",
|
|
what,
|
|
job_id,
|
|
formats.log_safe(msg),
|
|
extra={
|
|
**logs.event(
|
|
f"{what}.job.failed",
|
|
job_id=job_id,
|
|
asset_id=asset_id,
|
|
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
|
),
|
|
**(_failure_stack(exc) if exc is not None else {}),
|
|
},
|
|
)
|
|
|
|
|
|
def _process_edit(job_id: int, asset_id: int) -> None:
|
|
"""Produce a per-user trim/crop clip from the source asset's downloaded file via ffmpeg."""
|
|
with SessionLocal() as db:
|
|
job = db.get(DownloadJob, job_id)
|
|
src = db.get(MediaAsset, job.source_asset_id) if job and job.source_asset_id else None
|
|
edit_spec = dict(job.edit_spec or {}) if job else {}
|
|
user_id = job.user_id if job else None
|
|
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
|
|
src_rel = src.rel_path if src else None
|
|
src_w, src_h, src_dur = (
|
|
(src.width, src.height, src.duration_s) if src else (None, None, None)
|
|
)
|
|
|
|
if not src_ready or not src_rel:
|
|
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
|
return
|
|
src_path = storage.abs_path(settings.files_root, src_rel)
|
|
if not src_path.exists():
|
|
_fail_edit(job_id, asset_id, "The source file is missing.")
|
|
return
|
|
|
|
src_ext = src_path.suffix.lstrip(".").lower() or "mp4"
|
|
crop = editmod.has_crop(edit_spec)
|
|
out_ext = "mp4" if editmod.needs_reencode(edit_spec) else src_ext
|
|
total_s = editmod.clip_duration(edit_spec, src_dur) or src_dur or 0
|
|
staging = _edit_staging_dir(asset_id)
|
|
|
|
try:
|
|
if staging.exists():
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
dest_staging = staging / f"out.{out_ext}"
|
|
|
|
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
|
if edit_spec.get("segments"):
|
|
cmd, prep = editmod.build_concat_plan(
|
|
src_path, dest_staging, edit_spec, out_ext, staging
|
|
)
|
|
for p, content in prep.items():
|
|
p.write_text(content, encoding="utf-8")
|
|
else:
|
|
cmd = editmod.build_edit_cmd(src_path, dest_staging, edit_spec, out_ext)
|
|
editmod.run_ffmpeg(
|
|
cmd,
|
|
total_s,
|
|
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
|
should_cancel=_cancel_poll(job_id),
|
|
)
|
|
if not dest_staging.exists():
|
|
raise StatedFailure("ffmpeg produced no output file")
|
|
# A fast trim stream-copies the source's codecs and container, so the clip is only as
|
|
# playable as its source: hold it to the same rule as every other ready file.
|
|
dest_staging = _ensure_phone_safe(job_id, dest_staging, staging, "editing", total_s)
|
|
out_ext = dest_staging.suffix.lstrip(".").lower() or out_ext
|
|
# New storage on the owner's footprint, held to what is left by its real size, as a
|
|
# download and a player clip are.
|
|
if user_id is not None:
|
|
with SessionLocal() as qdb:
|
|
room = quota.remaining_bytes(qdb, user_id)
|
|
if room is not None and dest_staging.stat().st_size > room:
|
|
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
|
|
|
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
|
storage.place_file(dest_staging, settings.files_root, rel)
|
|
final = storage.abs_path(settings.files_root, rel)
|
|
size = final.stat().st_size if final.exists() else None
|
|
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
|
_, _, out_w, out_h, _ = probe.probe_media(final)
|
|
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "ready"
|
|
asset.rel_path = rel
|
|
asset.size_bytes = size
|
|
asset.container = out_ext
|
|
asset.duration_s = total_s or asset.duration_s
|
|
if out_w and out_h:
|
|
asset.width, asset.height = out_w, out_h
|
|
elif crop:
|
|
asset.width = edit_spec["crop"]["w"]
|
|
asset.height = edit_spec["crop"]["h"]
|
|
else:
|
|
asset.width, asset.height = src_w, src_h
|
|
asset.error = None
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
job = db.get(DownloadJob, job_id)
|
|
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
|
owner_id = job.user_id if job else None
|
|
dl_title = asset.title if asset else None
|
|
# Atomic terminal write (see the download path): don't resurrect a route-stopped job.
|
|
finalized = service.finalize_if_running(
|
|
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
|
)
|
|
db.commit()
|
|
if finalized:
|
|
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
|
# The asset id, not `rel`: the path is built from the channel and the title.
|
|
log.info(
|
|
"edit job %s done: asset %s",
|
|
job_id,
|
|
asset_id,
|
|
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
|
|
)
|
|
|
|
except editmod.EditAborted:
|
|
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"edit job %s aborted (pause/cancel)",
|
|
job_id,
|
|
extra=logs.event("edit.job.aborted", job_id=job_id),
|
|
)
|
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
|
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
|
finally:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
|
|
|
|
def _clip_room(user_id: int) -> int | None:
|
|
"""What the clip's owner may still store (`quota.remaining_bytes`), refusing at zero — see
|
|
`_download`: zero is a refusal, not "no limit"."""
|
|
with SessionLocal() as db:
|
|
remaining = quota.remaining_bytes(db, user_id)
|
|
if remaining == 0:
|
|
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
|
return remaining
|
|
|
|
|
|
def _clip_source(
|
|
job_id: int, asset_id: int, spec: dict, staging: Path, remaining: int | None
|
|
) -> tuple[Path, float, str | None]:
|
|
"""The file a clip is cut from, where in it the clip starts, and the source's title when only
|
|
the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at
|
|
0 there); a Plex clip reads the library file in place."""
|
|
if spec["source"] == "youtube":
|
|
url = service.source_url("youtube", spec["ref"])
|
|
cookiefile: Path | None = None
|
|
with SessionLocal() as db:
|
|
job = db.get(DownloadJob, job_id)
|
|
asset = db.get(MediaAsset, asset_id)
|
|
# Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates
|
|
# single videos signed out ("not a bot"), and the clip is theirs alone.
|
|
if job and asset:
|
|
cookiefile = credentials.resolve_clip_cookiefile(
|
|
db, job.user_id, url, asset, spec, staging
|
|
)
|
|
_set_job(job_id, phase="downloading", progress=0)
|
|
try:
|
|
with formats.watching_size_cap() as over_cap:
|
|
info = _extract_with_fallback(
|
|
job_id,
|
|
asset_id,
|
|
url,
|
|
formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
|
|
staging,
|
|
adapters.resolve_adapter("youtube", spec["ref"]),
|
|
cookiefile=str(cookiefile) if cookiefile else None,
|
|
max_bytes=remaining,
|
|
section=(spec["start_s"], spec["end_s"]),
|
|
)
|
|
# Should yt-dlp's own cap ever refuse the range (see `_download`): the owner's storage.
|
|
if over_cap[0]:
|
|
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
|
except (_Aborted, _OwnerRefused):
|
|
raise
|
|
except Exception as exc: # re-raised below with the download catalog's diagnosis
|
|
msg = str(exc)[:500]
|
|
raise _ClipFetchFailed(
|
|
msg, dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
|
) from exc
|
|
finally:
|
|
# Never leave a plaintext session cookie on disk past the fetch (see `_download`).
|
|
if cookiefile is not None:
|
|
cookiefile.unlink(missing_ok=True)
|
|
info = _media_info(dict(info))
|
|
title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info))
|
|
return _produced_file(info, staging), 0.0, title
|
|
with SessionLocal() as db:
|
|
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
|
src = plex_paths.local_media_path(db, item.file_path) if item else None
|
|
if src is None:
|
|
raise StatedFailure("The library file is not reachable from here.")
|
|
return src, spec["start_s"], None
|
|
|
|
|
|
def _clip_crop(spec: dict, data: dict) -> str:
|
|
"""The clip's crop filter, from its frame and the picture the file really holds ("" for none)."""
|
|
frame = spec.get("crop")
|
|
if not frame:
|
|
return ""
|
|
dar = clipmod.display_aspect(data)
|
|
if dar is None:
|
|
raise StatedFailure("The picture's size could not be read, so the crop cannot be placed.")
|
|
crop = clipmod.crop_filter(frame, dar)
|
|
if crop is None:
|
|
raise StatedFailure("The crop frame lies outside the picture.")
|
|
return crop
|
|
|
|
|
|
def _process_clip(job_id: int, asset_id: int) -> None:
|
|
"""Cut an ad-hoc clip (downloads/clip): fetch or open its source, ONE re-encode to the clip
|
|
shape, then into the user's library beside their edits."""
|
|
with SessionLocal() as db:
|
|
job = db.get(DownloadJob, job_id)
|
|
if job is None:
|
|
return
|
|
spec = dict(job.edit_spec or {})
|
|
user_id = job.user_id
|
|
staging = _edit_staging_dir(asset_id)
|
|
try:
|
|
if staging.exists():
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
# Refused before the fetch and the encode when the owner's storage is already full.
|
|
remaining = _clip_room(user_id)
|
|
src, start, fetched_title = _clip_source(job_id, asset_id, spec, staging, remaining)
|
|
data = probe.probe_json(src)
|
|
if not data:
|
|
raise StatedFailure("The source file could not be read.")
|
|
audio_track = spec.get("audio_track")
|
|
tracks = clipmod.audio_track_count(data)
|
|
if tracks == 0:
|
|
# A silent source: the player still names its track 0, and the clip is silent too.
|
|
audio_track = None
|
|
elif audio_track is not None and audio_track >= tracks:
|
|
raise StatedFailure("The chosen audio track is not in the file.")
|
|
crop = _clip_crop(spec, data)
|
|
total_s = clipmod.duration(spec)
|
|
out = staging / "clip.mp4"
|
|
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
|
editmod.run_ffmpeg(
|
|
clipmod.build_cmd(
|
|
src,
|
|
out,
|
|
start,
|
|
total_s,
|
|
audio_track=audio_track,
|
|
hdr=clipmod.is_hdr(data),
|
|
threads=editmod.encode_thread_cap(),
|
|
crop=crop,
|
|
),
|
|
total_s,
|
|
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
|
should_cancel=_cancel_poll(job_id),
|
|
)
|
|
if not out.exists():
|
|
raise StatedFailure("ffmpeg produced no output file")
|
|
# The phone-safe shape by construction; still held to the rule every ready file meets.
|
|
out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s))
|
|
# Held to the cap by its real size: a range fetch passes yt-dlp's `max_filesize` to its
|
|
# ffmpeg downloader, which never reads it, and a Plex clip fetches nothing at all. Read
|
|
# again, since other jobs of this owner may have landed during the encode.
|
|
remaining = _clip_room(user_id)
|
|
if remaining is not None and out.stat().st_size > remaining:
|
|
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
|
ext = out.suffix.lstrip(".").lower() or "mp4"
|
|
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
|
storage.place_file(out, settings.files_root, rel)
|
|
final = storage.abs_path(settings.files_root, rel)
|
|
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
|
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "ready"
|
|
asset.rel_path = rel
|
|
asset.size_bytes = final.stat().st_size if final.exists() else None
|
|
asset.container = ext
|
|
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
|
vcodec,
|
|
acodec,
|
|
width,
|
|
height,
|
|
)
|
|
asset.duration_s = round(total_s)
|
|
asset.media_kind = mediakind.derive(vcodec, acodec)
|
|
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
|
# item has no public thumbnail at all.
|
|
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
|
asset.error = None
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
|
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
|
|
placeholder = service.clip_title(None, None)
|
|
if asset and fetched_title and asset.title == placeholder:
|
|
named = service.clip_title(fetched_title, None)
|
|
asset.title = named
|
|
if job_row := db.get(DownloadJob, job_id):
|
|
if job_row.display_name == placeholder:
|
|
fields["display_name"] = named[:DISPLAY_TEXT_MAX]
|
|
title = asset.title if asset else None
|
|
finalized = service.finalize_if_running(db, job_id, **fields)
|
|
db.commit()
|
|
if finalized:
|
|
_notify_download_done(db, user_id, title, job_id, kind="edit")
|
|
log.info(
|
|
"clip job %s done: asset %s",
|
|
job_id,
|
|
asset_id,
|
|
extra=logs.event(
|
|
"clip.job.done", job_id=job_id, asset_id=asset_id, source=spec["source"]
|
|
),
|
|
)
|
|
except (_Aborted, editmod.EditAborted):
|
|
# Paused/canceled mid-fetch or mid-encode: free the asset for a later retry.
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"clip job %s aborted (pause/cancel)",
|
|
job_id,
|
|
extra=logs.event("clip.job.aborted", job_id=job_id),
|
|
)
|
|
except _OwnerRefused as exc:
|
|
# Quota full: the job fails, the asset goes back to pending for a Retry once there is room
|
|
# (the download's handling; see `_download`).
|
|
msg = str(exc)[:500]
|
|
with SessionLocal() as db:
|
|
service.finalize_if_running(
|
|
db,
|
|
job_id,
|
|
status=JobStatus.ERROR,
|
|
error=msg,
|
|
error_code=dlerrors.classify(msg),
|
|
error_detail=None,
|
|
phase=None,
|
|
)
|
|
db.commit()
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"clip job %s refused for its owner: %s",
|
|
job_id,
|
|
formats.log_safe(msg),
|
|
extra=logs.event("clip.job.refused", job_id=job_id),
|
|
)
|
|
except _ClipFetchFailed as exc:
|
|
# silent-ok: not silent — `_fail_edit` is the report, with the fetch's diagnosis.
|
|
_fail_edit(job_id, asset_id, str(exc), exc, what="clip", diagnosis=exc.diagnosis)
|
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
|
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip")
|
|
finally:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
|
|
|
|
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
|
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
|
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
|
for the owner to delete or keep."""
|
|
if owner_id is None:
|
|
return
|
|
with SessionLocal() as db:
|
|
if not quota.fits_after_conversion(db, owner_id, old_size, new_size):
|
|
raise StatedFailure(dlerrors.CONVERTED_TOO_BIG_MESSAGE)
|
|
|
|
|
|
def _unplace(placed: Path | None) -> None:
|
|
"""Remove a converted upload that was placed at its new rel but never committed: the row still
|
|
names the original, so nothing else would ever find this file."""
|
|
if placed is not None:
|
|
placed.unlink(missing_ok=True)
|
|
|
|
|
|
def _conversion_landed(asset_id: int, new_rel: str) -> bool | None:
|
|
"""After the commit that makes a converted upload ready RAISED: did it land anyway? A commit can
|
|
reach the server and lose its answer (the connection drops after COMMIT). True/False by what the
|
|
row now says; None when the row cannot be read either."""
|
|
try:
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
return asset is not None and asset.status == "ready" and asset.rel_path == new_rel
|
|
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
|
# silent-ok: the caller re-raises the commit's own error, which `_fail_edit` reports.
|
|
return None
|
|
|
|
|
|
def _drop_original(job_id: int, src: Path) -> None:
|
|
"""Remove an upload's original once the converted file is committed in its place. Best effort:
|
|
the job is done and its file is ready, so a failure here must not fail it — the original is
|
|
named by no row now, and the GC's `.uploads` sweep takes it."""
|
|
try:
|
|
src.unlink(missing_ok=True)
|
|
except OSError as exc:
|
|
log.warning(
|
|
"upload job %s: original not removed, left to the GC sweep: %s",
|
|
job_id,
|
|
exc,
|
|
extra=logs.event("upload.original.unlink_failed", job_id=job_id),
|
|
)
|
|
|
|
|
|
def _free_unheld_upload(asset_id: int) -> bool:
|
|
"""Free an upload no job holds any more — cancelled or deleted while its conversion ran, when
|
|
`release_asset` had to leave it because the conversion still read its file. Once that has
|
|
stopped, free it as the release would have; left alone, its file would sit on disk counted by no
|
|
owner's footprint until the GC's upload pass. False when a job still holds it (paused)."""
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset is None or asset.ref_count != 0:
|
|
return False
|
|
if asset.rel_path:
|
|
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
|
db.delete(asset)
|
|
db.commit()
|
|
return True
|
|
|
|
|
|
def _convert_upload(job_id: int, asset_id: int) -> None:
|
|
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
|
|
|
|
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
|
|
under the new extension, and only then is the original removed — so a failed or stopped
|
|
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
|
|
A converted file placed but never committed is removed again on the way out. When the commit's
|
|
own outcome is unknown neither file is removed: the row names one of them, and the GC's `.uploads`
|
|
sweep takes the one it does not — as it does an original whose unlink after the commit failed."""
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
old_rel = asset.rel_path if asset else None
|
|
old_ext = asset.container if asset else None
|
|
old_size = (asset.size_bytes or 0) if asset else 0
|
|
duration = asset.duration_s if asset else None
|
|
job = db.get(DownloadJob, job_id)
|
|
owner_id = job.user_id if job else None
|
|
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
|
|
if old_rel is None or src is None or not src.exists():
|
|
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
|
return
|
|
staging = _staging_dir(asset_id)
|
|
placed: Path | None = None # the converted file while it is at its new rel, uncommitted
|
|
try:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
safe = _ensure_phone_safe(job_id, src, staging, CONVERTING, duration, keep_source=True)
|
|
ext = safe.suffix.lstrip(".").lower()
|
|
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
|
if safe != src:
|
|
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
|
storage.place_file(safe, settings.files_root, new_rel)
|
|
if new_rel != old_rel:
|
|
placed = storage.abs_path(settings.files_root, new_rel)
|
|
final = storage.abs_path(settings.files_root, new_rel)
|
|
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
|
with SessionLocal() as db:
|
|
asset = db.get(MediaAsset, asset_id)
|
|
if asset:
|
|
asset.status = "ready"
|
|
asset.rel_path = new_rel
|
|
asset.container = ext
|
|
asset.size_bytes = final.stat().st_size
|
|
asset.vcodec, asset.acodec = vcodec, acodec
|
|
asset.width, asset.height = width or asset.width, height or asset.height
|
|
asset.duration_s = dur or asset.duration_s
|
|
asset.error = None
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
title = asset.title if asset else None
|
|
finalized = service.finalize_if_running(
|
|
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
|
)
|
|
try:
|
|
db.commit()
|
|
except Exception:
|
|
landed = _conversion_landed(asset_id, new_rel)
|
|
if landed is False:
|
|
raise # rolled back: the original is still the asset's file — fail as any step
|
|
# Landed, or cannot tell: the converted file may be the asset's file now, so it is
|
|
# never removed blind (the sweep takes whichever file no row names).
|
|
placed = None
|
|
if landed is None:
|
|
raise
|
|
db.rollback()
|
|
placed = None
|
|
if finalized:
|
|
_notify_download_done(db, owner_id, title, job_id, kind="upload")
|
|
if new_rel != old_rel:
|
|
_drop_original(job_id, src)
|
|
if not finalized:
|
|
_free_unheld_upload(asset_id) # finished after its job was cancelled or deleted
|
|
if finalized:
|
|
log.info(
|
|
"upload job %s converted: asset %s",
|
|
job_id,
|
|
asset_id,
|
|
extra=logs.event("upload.job.done", job_id=job_id, asset_id=asset_id, ext=ext),
|
|
)
|
|
except editmod.EditAborted:
|
|
# Paused/canceled mid-encode: the original is untouched. A paused job keeps it for its
|
|
# resume; one cancelled or deleted holds it no more, so it is freed now.
|
|
_unplace(placed)
|
|
if not _free_unheld_upload(asset_id):
|
|
_reset_asset_pending(asset_id)
|
|
log.info(
|
|
"upload job %s conversion aborted (pause/cancel)",
|
|
job_id,
|
|
extra=logs.event("upload.job.aborted", job_id=job_id),
|
|
)
|
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
|
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
|
_unplace(placed)
|
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="upload")
|
|
|
|
finally:
|
|
shutil.rmtree(staging, ignore_errors=True)
|