style: apply ruff format across backend + scripts (line-length 100)

Mechanical, whole-tree `ruff format` pass — no behaviour change. Recorded in
.git-blame-ignore-revs (next commit) so git blame skips it.
This commit is contained in:
2026-08-23 07:43:06 +02:00
parent 434a98d6ed
commit b1832afd18
217 changed files with 3149 additions and 1744 deletions
+5 -2
View File
@@ -10,6 +10,7 @@ Adapters are stateless singletons; get one with `resolve_adapter(source_kind, so
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""
from urllib.parse import urlparse
from app.config import settings
@@ -140,8 +141,10 @@ def resolve_adapter(
def host_is(*domains: str) -> bool:
return any(_host_is(h, *domains) for h in hosts)
if source_kind == "youtube" or "youtube" in extractor or host_is(
"youtube.com", "youtu.be", "youtube-nocookie.com"
if (
source_kind == "youtube"
or "youtube" in extractor
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
):
return _YOUTUBE
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
+1
View File
@@ -10,6 +10,7 @@ URL may have expired — those just get a locally-generated poster instead).
Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
"""
import logging
import yt_dlp
+151 -29
View File
@@ -20,6 +20,7 @@ Also builds the editor **filmstrip** — a single tiled sprite of the source vid
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
for a player hover-preview later.
"""
import hashlib
import json
import logging
@@ -40,6 +41,7 @@ class EditAborted(Exception):
# --- edit spec ------------------------------------------------------------------------------
def normalize_edit_spec(spec: dict | None) -> dict:
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
@@ -140,6 +142,7 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
# --- ffmpeg: trim / crop --------------------------------------------------------------------
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
@@ -153,8 +156,15 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
reencode = needs_reencode(spec)
cmd = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats",
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
]
if start:
cmd += ["-ss", f"{start:.3f}"]
@@ -170,8 +180,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
if crop:
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
cmd += [
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
"-c:a", "aac", "-b:a", "192k",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"20",
"-c:a",
"aac",
"-b:a",
"192k",
]
else:
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
@@ -204,8 +222,15 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
crop = spec.get("crop")
prep: dict[Path, str] = {}
head = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats",
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
]
if needs_reencode(spec):
@@ -218,9 +243,24 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
labels.append(f"[v{i}][a{i}]")
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
cmd = head + [
"-i", str(src), "-filter_complex", ";".join(parts),
"-map", "[v]", "-map", "[a]",
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "192k",
"-i",
str(src),
"-filter_complex",
";".join(parts),
"-map",
"[v]",
"-map",
"[a]",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"20",
"-c:a",
"aac",
"-b:a",
"192k",
]
else:
listp = staging / "concat.txt"
@@ -314,7 +354,9 @@ def _stop(proc: subprocess.Popen) -> None:
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
def run_ffmpeg(
cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S
) -> None:
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
@@ -411,9 +453,20 @@ def probe_codecs(path: Path) -> tuple[str | None, str | None]:
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
"-of", "json", str(path)],
capture_output=True, text=True, timeout=30, check=True,
[
"ffprobe",
"-v",
"error",
"-show_entries",
"stream=codec_type,codec_name",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
return None, None
@@ -425,16 +478,29 @@ def probe_duration(path: Path) -> int:
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=nw=1:nk=1", str(path)],
capture_output=True, text=True, timeout=30, check=True,
[
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=nw=1:nk=1",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout.strip()
return int(float(out)) if out else 0
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
return 0
def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]:
def browser_transcode_flags(
vcodec: str | None, acodec: str | None, is_mp4: bool
) -> tuple[bool, bool, bool]:
"""Decide what (if anything) makes a file universally browser-playable. Returns
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
@@ -443,15 +509,30 @@ def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool
return (rv or ra or not is_mp4), rv, ra
def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]:
def build_browser_safe_cmd(
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
) -> list[str]:
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
pipe:1` streams machine-readable progress for run_ffmpeg."""
cmd = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats", "-i", str(src),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
"-i",
str(src),
]
cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"]
cmd += (
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
if reencode_video
else ["-c:v", "copy"]
)
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
cmd += ["-movflags", "+faststart", str(dest)]
return cmd
@@ -486,25 +567,63 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
fps = geom["count"] / dur
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-i", str(src), "-frames:v", "1", "-q:v", "5", "-vf", vf, str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-i",
str(src),
"-frames:v",
"1",
"-q:v",
"5",
"-vf",
vf,
str(dest),
]
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-ss",
f"{max(0.0, at_s):.3f}",
"-i",
str(src),
"-frames:v",
"1",
"-q:v",
"3",
"-vf",
"scale=640:-2",
str(dest),
]
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-i", str(src), "-frames:v", "1", "-update", "1", str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-i",
str(src),
"-frames:v",
"1",
"-update",
"1",
str(dest),
]
@@ -550,7 +669,10 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
return None
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
if err is not None:
log.info("thumbnail->jpg conversion failed (%s); poster falls back to a cut frame", str(err)[:120])
log.info(
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
str(err)[:120],
)
return None
return dest if dest.exists() else None
+2 -3
View File
@@ -13,6 +13,7 @@ spec shape (all keys optional; defaults applied by `normalize`):
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
"""
import hashlib
import json
@@ -215,8 +216,6 @@ def build_ydl_opts(
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
# path), so it never needed the convertor and does not reintroduce the crash.
if s["embed_thumbnail"]:
opts["postprocessors"].append(
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
)
opts["postprocessors"].append({"key": "EmbedThumbnail", "already_have_thumbnail": True})
return opts
+59 -35
View File
@@ -15,6 +15,7 @@ Five passes each run:
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
@@ -40,9 +41,7 @@ _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight do
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
db.execute(
select(DownloadJob.user_id)
.where(DownloadJob.asset_id == asset_id)
.distinct()
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
).scalars()
)
@@ -101,7 +100,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
pass
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
for d in sorted(
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
(p for p in entry.rglob("*") if p.is_dir()),
key=lambda p: len(p.parts),
reverse=True,
):
try:
d.rmdir()
@@ -121,15 +122,19 @@ def run_download_gc(db: Session) -> dict:
# 1. Pre-expiry warnings.
warn_before = now + timedelta(days=grace_days)
soon = db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
soon = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
)
)
).scalars().all()
.scalars()
.all()
)
for asset in soon:
_notify_holders(db, asset, "download_expiring")
asset.gc_notified = True
@@ -138,13 +143,17 @@ def run_download_gc(db: Session) -> dict:
db.commit()
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
dead = db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
dead = (
db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
)
)
).scalars().all()
.scalars()
.all()
)
for asset in dead:
_delete_asset(db, asset, "download_expired")
expired += 1
@@ -154,17 +163,24 @@ def run_download_gc(db: Session) -> dict:
# 3. LRU eviction over the total cache cap.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar() or 0
total = (
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar()
or 0
)
if total > cap:
candidates = db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
).scalars().all()
candidates = (
db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
)
.scalars()
.all()
)
for asset in candidates:
if total <= cap:
break
@@ -176,13 +192,17 @@ def run_download_gc(db: Session) -> dict:
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
# referenced errored row to pending on re-enqueue, so these only accumulate once every
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
orphans = db.execute(
select(MediaAsset).where(
MediaAsset.status == "error",
MediaAsset.ref_count == 0,
MediaAsset.rel_path.is_(None),
orphans = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "error",
MediaAsset.ref_count == 0,
MediaAsset.rel_path.is_(None),
)
)
).scalars().all()
.scalars()
.all()
)
for asset in orphans:
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
reaped += 1
@@ -194,7 +214,11 @@ def run_download_gc(db: Session) -> dict:
swept = _sweep_orphan_files(db, settings.download_root)
result = {
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
"warned": warned,
"expired": expired,
"evicted": evicted,
"reaped": reaped,
"swept": swept,
}
log.info("download_gc %s", result)
return result
+2
View File
@@ -1,4 +1,5 @@
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
from pathlib import Path
from sqlalchemy import text
@@ -10,6 +11,7 @@ from app.models import DownloadJob, MediaAsset
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
def _set_job(job_id: int, **fields) -> None:
with SessionLocal() as db:
job = db.get(DownloadJob, job_id)
+7 -4
View File
@@ -6,6 +6,7 @@ require a short-lived signed **grant** on the file request — because a `<video
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
so the raw password never rides in the media URL (or server logs).
"""
import hashlib
import hmac
import secrets
@@ -27,10 +28,9 @@ def is_expired(link: DownloadLink) -> bool:
# --- signed grants (password-protected file access) ----------------------------------------
def _sign(msg: str) -> str:
return hmac.new(
settings.secret_key.encode(), msg.encode(), hashlib.sha256
).hexdigest()[:32]
return hmac.new(settings.secret_key.encode(), msg.encode(), hashlib.sha256).hexdigest()[:32]
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
@@ -58,6 +58,7 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
# --- serialization -------------------------------------------------------------------------
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
"""What the file owner sees when managing their links."""
return {
@@ -72,7 +73,9 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
}
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
def public_meta(
link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None
) -> dict:
"""Metadata for the public watch page (only ever returned once access is authorized).
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
+2 -3
View File
@@ -6,6 +6,7 @@ This injects per-video OG/Twitter tags into the served HTML so a shared link unf
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
"""
import html
import re
from pathlib import Path
@@ -71,9 +72,7 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
def _watch_tags(token: str, db: Session) -> str | None:
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
link = db.execute(
select(DownloadLink).where(DownloadLink.token == token)
).scalar_one_or_none()
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
if link is None or linksmod.is_expired(link):
return None
url = f"{settings.app_base}/watch/{token}"
+52 -19
View File
@@ -1,4 +1,5 @@
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
import logging
import re
import shutil
@@ -57,9 +58,7 @@ _YTDLP_TEMP_RE = re.compile(
)
def _notify_download_done(
db, user_id: int | None, title: str | None, job_id, *, kind: str
) -> None:
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
@@ -113,7 +112,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
def _make_progress_hook(job_id: int, asset_id: int):
state = {"meta_done": False}
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
def hook(d: dict) -> None:
@@ -247,7 +246,9 @@ def _make_pp_hook(job_id: int):
def hook(d: dict) -> None:
if d.get("status") in ("started", "processing"):
_set_job(job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None)
_set_job(
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
)
return hook
@@ -269,7 +270,9 @@ def _produced_file(info: dict, staging: Path) -> Path:
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
working files (see `_YTDLP_TEMP_RE`)."""
req = (info.get("requested_downloads") or [{}])[0]
fp = req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
fp = (
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
)
if fp:
produced = Path(str(fp))
if produced.is_file():
@@ -310,7 +313,9 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
spec,
outtmpl,
_make_progress_hook(job_id, asset_id),
extra=adapter.ydl_opts_extra(clients),
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
@@ -324,7 +329,12 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
log.info(
"job %s: clients %s failed (%s); retrying with fallback",
job_id,
clients,
str(exc)[:70],
)
for p in list(staging.iterdir()): # drop partials before the retry
try:
p.unlink()
@@ -363,7 +373,9 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
dest = produced.with_name(produced.stem + ".browser.mp4")
action = "re-encode" if (reenc_v or reenc_a) else "remux"
log.info("job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4)
log.info(
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
)
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
# re-encode still shows real progress instead of a frozen 0%.
@@ -436,8 +448,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.width = int(info["width"]) if info.get("width") else None
asset.height = int(info["height"]) if info.get("height") else None
asset.container = ext
asset.vcodec = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.vcodec = (
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
)
asset.acodec = (
info.get("acodec") if info.get("acodec") not in (None, "none") else None
)
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = adapter.uploader_url(info)
@@ -456,8 +472,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
job = db.get(DownloadJob, job_id)
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
owner_id = job.user_id if job else None
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None, "speed_bps": None,
"eta_s": None}
fields = {
"status": JobStatus.DONE,
"progress": 100,
"phase": None,
"speed_bps": None,
"eta_s": None,
}
if job and not job.display_name and meta.title:
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
fields["display_name"] = meta.title[:255]
@@ -484,8 +505,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
# with `error` (error also holds the asset → would re-open the double-release class).
service.finalize_if_running(
db, job_id, status=JobStatus.ERROR, error=msg,
error_code=dlerrors.classify(msg), phase=None,
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
)
db.commit()
log.warning("job %s failed: %s", job_id, msg)
@@ -501,8 +526,12 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
asset.error = msg
# Atomic fail: only while still running (see the download error path).
service.finalize_if_running(
db, job_id, status=JobStatus.ERROR, error=msg,
error_code=dlerrors.classify(msg), phase=None,
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
)
db.commit()
log.warning("edit job %s failed: %s", job_id, msg)
@@ -517,7 +546,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
user_id = job.user_id if job else None
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
src_rel = src.rel_path if src else None
src_w, src_h, src_dur = (src.width, src.height, src.duration_s) if src else (None, None, None)
src_w, src_h, src_dur = (
(src.width, src.height, src.duration_s) if src else (None, None, None)
)
if not src_ready or not src_rel:
_fail_edit(job_id, asset_id, "The source download is no longer available.")
@@ -541,7 +572,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
if edit_spec.get("segments"):
cmd, prep = editmod.build_concat_plan(src_path, dest_staging, edit_spec, out_ext, staging)
cmd, prep = editmod.build_concat_plan(
src_path, dest_staging, edit_spec, out_ext, staging
)
for p, content in prep.items():
p.write_text(content, encoding="utf-8")
else:
+1
View File
@@ -8,6 +8,7 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
"""
from dataclasses import dataclass
from sqlalchemy import func, select
+1
View File
@@ -10,6 +10,7 @@ Contract: the asset is already claimed ('downloading') by the parent. This proce
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
"""
import logging
import sys
+3 -1
View File
@@ -6,6 +6,7 @@ cache hit (asset already ready) — marks the job done instantly with no re-down
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
"""
from datetime import datetime, timezone
from sqlalchemy import func, select, update
@@ -166,7 +167,8 @@ def enqueue(
source_ref=source_ref,
profile_id=profile_id,
profile_snapshot=spec,
display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None),
display_name=display_name
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
status="queued",
queue_pos=next_queue_pos(db),
)
+24 -7
View File
@@ -10,6 +10,7 @@ Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg`
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
date-based — the community-standard mapping for YouTube-in-Plex.
"""
import re
import shutil
import unicodedata
@@ -36,9 +37,27 @@ _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
# that accented names show in non-UTF-8 tools.
_FOLD_MAP = str.maketrans(
{
"ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D",
"ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE",
"ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-",
"ß": "ss",
"ẞ": "SS",
"ø": "o",
"Ø": "O",
"ł": "l",
"Ł": "L",
"đ": "d",
"Đ": "D",
"ð": "d",
"Ð": "D",
"þ": "th",
"Þ": "Th",
"æ": "ae",
"Æ": "AE",
"œ": "oe",
"Œ": "OE",
"ı": "i",
"İ": "I",
"·": " ",
"–": "-",
"—": "-",
}
)
@@ -174,7 +193,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
f" <premiered>{aired}</premiered>",
f" <year>{year}</year>",
f" <plot>{escape(meta.description or '')}</plot>",
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
f' <uniqueid type="{escape(meta.uniqueid_type)}" default="true">{escape(meta.video_id)}</uniqueid>',
]
if meta.duration_s:
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
@@ -190,9 +209,7 @@ def _sidecar(media: Path, ext: str) -> Path:
return media.with_suffix(ext)
def write_sidecars(
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
) -> bool:
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
media = abs_path(download_root, rel)
try: