style: apply ruff format across backend + scripts (line-length 100)
Mechanical, whole-tree `ruff format` pass — no behaviour change. Recorded in .git-blame-ignore-revs (next commit) so git blame skips it.
This commit is contained in:
@@ -10,6 +10,7 @@ Adapters are stateless singletons; get one with `resolve_adapter(source_kind, so
|
||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import settings
|
||||
@@ -140,8 +141,10 @@ def resolve_adapter(
|
||||
def host_is(*domains: str) -> bool:
|
||||
return any(_host_is(h, *domains) for h in hosts)
|
||||
|
||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
||||
if (
|
||||
source_kind == "youtube"
|
||||
or "youtube" in extractor
|
||||
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
|
||||
):
|
||||
return _YOUTUBE
|
||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||
|
||||
@@ -10,6 +10,7 @@ URL may have expired — those just get a locally-generated poster instead).
|
||||
Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import yt_dlp
|
||||
|
||||
+151
-29
@@ -20,6 +20,7 @@ Also builds the editor **filmstrip** — a single tiled sprite of the source vid
|
||||
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
|
||||
for a player hover-preview later.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
@@ -40,6 +41,7 @@ class EditAborted(Exception):
|
||||
|
||||
# --- edit spec ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
||||
|
||||
@@ -140,6 +142,7 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
||||
|
||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||
|
||||
@@ -153,8 +156,15 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
reencode = needs_reencode(spec)
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-progress", "pipe:1", "-nostats",
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
@@ -170,8 +180,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
if crop:
|
||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||
cmd += [
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
]
|
||||
else:
|
||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||
@@ -204,8 +222,15 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
crop = spec.get("crop")
|
||||
prep: dict[Path, str] = {}
|
||||
head = [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-progress", "pipe:1", "-nostats",
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
]
|
||||
|
||||
if needs_reencode(spec):
|
||||
@@ -218,9 +243,24 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
labels.append(f"[v{i}][a{i}]")
|
||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||
cmd = head + [
|
||||
"-i", str(src), "-filter_complex", ";".join(parts),
|
||||
"-map", "[v]", "-map", "[a]",
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "192k",
|
||||
"-i",
|
||||
str(src),
|
||||
"-filter_complex",
|
||||
";".join(parts),
|
||||
"-map",
|
||||
"[v]",
|
||||
"-map",
|
||||
"[a]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
]
|
||||
else:
|
||||
listp = staging / "concat.txt"
|
||||
@@ -314,7 +354,9 @@ def _stop(proc: subprocess.Popen) -> None:
|
||||
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
||||
|
||||
|
||||
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
|
||||
def run_ffmpeg(
|
||||
cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S
|
||||
) -> None:
|
||||
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
|
||||
|
||||
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
|
||||
@@ -411,9 +453,20 @@ def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
|
||||
"-of", "json", str(path)],
|
||||
capture_output=True, text=True, timeout=30, check=True,
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=codec_type,codec_name",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=True,
|
||||
).stdout
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
|
||||
return None, None
|
||||
@@ -425,16 +478,29 @@ def probe_duration(path: Path) -> int:
|
||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "default=nw=1:nk=1", str(path)],
|
||||
capture_output=True, text=True, timeout=30, check=True,
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=nw=1:nk=1",
|
||||
str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return int(float(out)) if out else 0
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]:
|
||||
def browser_transcode_flags(
|
||||
vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||
) -> tuple[bool, bool, bool]:
|
||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
||||
@@ -443,15 +509,30 @@ def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||
return (rv or ra or not is_mp4), rv, ra
|
||||
|
||||
|
||||
def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]:
|
||||
def build_browser_safe_cmd(
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
|
||||
) -> list[str]:
|
||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-progress", "pipe:1", "-nostats", "-i", str(src),
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
"-i",
|
||||
str(src),
|
||||
]
|
||||
cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"]
|
||||
cmd += (
|
||||
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
||||
if reencode_video
|
||||
else ["-c:v", "copy"]
|
||||
)
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||
cmd += ["-movflags", "+faststart", str(dest)]
|
||||
return cmd
|
||||
@@ -486,25 +567,63 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
||||
fps = geom["count"] / dur
|
||||
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
|
||||
return [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-i", str(src), "-frames:v", "1", "-q:v", "5", "-vf", vf, str(dest),
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-q:v",
|
||||
"5",
|
||||
"-vf",
|
||||
vf,
|
||||
str(dest),
|
||||
]
|
||||
|
||||
|
||||
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
||||
return [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
|
||||
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-ss",
|
||||
f"{max(0.0, at_s):.3f}",
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-vf",
|
||||
"scale=640:-2",
|
||||
str(dest),
|
||||
]
|
||||
|
||||
|
||||
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
|
||||
return [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-i", str(src), "-frames:v", "1", "-update", "1", str(dest),
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-update",
|
||||
"1",
|
||||
str(dest),
|
||||
]
|
||||
|
||||
|
||||
@@ -550,7 +669,10 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
return None
|
||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||
if err is not None:
|
||||
log.info("thumbnail->jpg conversion failed (%s); poster falls back to a cut frame", str(err)[:120])
|
||||
log.info(
|
||||
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||
str(err)[:120],
|
||||
)
|
||||
return None
|
||||
return dest if dest.exists() else None
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ spec shape (all keys optional; defaults applied by `normalize`):
|
||||
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
@@ -215,8 +216,6 @@ def build_ydl_opts(
|
||||
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
|
||||
# path), so it never needed the convertor and does not reintroduce the crash.
|
||||
if s["embed_thumbnail"]:
|
||||
opts["postprocessors"].append(
|
||||
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
|
||||
)
|
||||
opts["postprocessors"].append({"key": "EmbedThumbnail", "already_have_thumbnail": True})
|
||||
|
||||
return opts
|
||||
|
||||
+59
-35
@@ -15,6 +15,7 @@ Five passes each run:
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -40,9 +41,7 @@ _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight do
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.distinct()
|
||||
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
@@ -101,7 +100,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
pass
|
||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||
for d in sorted(
|
||||
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
|
||||
(p for p in entry.rglob("*") if p.is_dir()),
|
||||
key=lambda p: len(p.parts),
|
||||
reverse=True,
|
||||
):
|
||||
try:
|
||||
d.rmdir()
|
||||
@@ -121,15 +122,19 @@ def run_download_gc(db: Session) -> dict:
|
||||
|
||||
# 1. Pre-expiry warnings.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
soon = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in soon:
|
||||
_notify_holders(db, asset, "download_expiring")
|
||||
asset.gc_notified = True
|
||||
@@ -138,13 +143,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||
dead = db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
dead = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in dead:
|
||||
_delete_asset(db, asset, "download_expired")
|
||||
expired += 1
|
||||
@@ -154,17 +163,24 @@ def run_download_gc(db: Session) -> dict:
|
||||
# 3. LRU eviction over the total cache cap.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
).scalar() or 0
|
||||
total = (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
candidates = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
).scalars().all()
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in candidates:
|
||||
if total <= cap:
|
||||
break
|
||||
@@ -176,13 +192,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
||||
orphans = db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "error",
|
||||
MediaAsset.ref_count == 0,
|
||||
MediaAsset.rel_path.is_(None),
|
||||
orphans = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "error",
|
||||
MediaAsset.ref_count == 0,
|
||||
MediaAsset.rel_path.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in orphans:
|
||||
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
||||
reaped += 1
|
||||
@@ -194,7 +214,11 @@ def run_download_gc(db: Session) -> dict:
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
|
||||
result = {
|
||||
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
|
||||
"warned": warned,
|
||||
"expired": expired,
|
||||
"evicted": evicted,
|
||||
"reaped": reaped,
|
||||
"swept": swept,
|
||||
}
|
||||
log.info("download_gc %s", result)
|
||||
return result
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
@@ -10,6 +11,7 @@ from app.models import DownloadJob, MediaAsset
|
||||
|
||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||
|
||||
|
||||
def _set_job(job_id: int, **fields) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
|
||||
@@ -6,6 +6,7 @@ require a short-lived signed **grant** on the file request — because a `<video
|
||||
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
||||
so the raw password never rides in the media URL (or server logs).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
@@ -27,10 +28,9 @@ def is_expired(link: DownloadLink) -> bool:
|
||||
|
||||
# --- signed grants (password-protected file access) ----------------------------------------
|
||||
|
||||
|
||||
def _sign(msg: str) -> str:
|
||||
return hmac.new(
|
||||
settings.secret_key.encode(), msg.encode(), hashlib.sha256
|
||||
).hexdigest()[:32]
|
||||
return hmac.new(settings.secret_key.encode(), msg.encode(), hashlib.sha256).hexdigest()[:32]
|
||||
|
||||
|
||||
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
|
||||
@@ -58,6 +58,7 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
||||
|
||||
# --- serialization -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
||||
"""What the file owner sees when managing their links."""
|
||||
return {
|
||||
@@ -72,7 +73,9 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
||||
}
|
||||
|
||||
|
||||
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
|
||||
def public_meta(
|
||||
link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None
|
||||
) -> dict:
|
||||
"""Metadata for the public watch page (only ever returned once access is authorized).
|
||||
|
||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||
|
||||
@@ -6,6 +6,7 @@ This injects per-video OG/Twitter tags into the served HTML so a shared link unf
|
||||
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
|
||||
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -71,9 +72,7 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
||||
|
||||
def _watch_tags(token: str, db: Session) -> str | None:
|
||||
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
||||
link = db.execute(
|
||||
select(DownloadLink).where(DownloadLink.token == token)
|
||||
).scalar_one_or_none()
|
||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||
if link is None or linksmod.is_expired(link):
|
||||
return None
|
||||
url = f"{settings.app_base}/watch/{token}"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
@@ -57,9 +58,7 @@ _YTDLP_TEMP_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _notify_download_done(
|
||||
db, user_id: int | None, title: str | None, job_id, *, kind: str
|
||||
) -> None:
|
||||
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||
@@ -113,7 +112,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
|
||||
def _make_progress_hook(job_id: int, asset_id: int):
|
||||
state = {"meta_done": False}
|
||||
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
||||
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
||||
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
|
||||
|
||||
def hook(d: dict) -> None:
|
||||
@@ -247,7 +246,9 @@ def _make_pp_hook(job_id: int):
|
||||
|
||||
def hook(d: dict) -> None:
|
||||
if d.get("status") in ("started", "processing"):
|
||||
_set_job(job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None)
|
||||
_set_job(
|
||||
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
|
||||
)
|
||||
|
||||
return hook
|
||||
|
||||
@@ -269,7 +270,9 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
||||
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
|
||||
working files (see `_YTDLP_TEMP_RE`)."""
|
||||
req = (info.get("requested_downloads") or [{}])[0]
|
||||
fp = req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
||||
fp = (
|
||||
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
||||
)
|
||||
if fp:
|
||||
produced = Path(str(fp))
|
||||
if produced.is_file():
|
||||
@@ -310,7 +313,9 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
@@ -324,7 +329,12 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
||||
log.info(
|
||||
"job %s: clients %s failed (%s); retrying with fallback",
|
||||
job_id,
|
||||
clients,
|
||||
str(exc)[:70],
|
||||
)
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
p.unlink()
|
||||
@@ -363,7 +373,9 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
|
||||
dest = produced.with_name(produced.stem + ".browser.mp4")
|
||||
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
||||
log.info("job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4)
|
||||
log.info(
|
||||
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
||||
)
|
||||
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
||||
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
||||
# re-encode still shows real progress instead of a frozen 0%.
|
||||
@@ -436,8 +448,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
asset.width = int(info["width"]) if info.get("width") else None
|
||||
asset.height = int(info["height"]) if info.get("height") else None
|
||||
asset.container = ext
|
||||
asset.vcodec = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
asset.vcodec = (
|
||||
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
)
|
||||
asset.acodec = (
|
||||
info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
)
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
@@ -456,8 +472,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||
owner_id = job.user_id if job else None
|
||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None, "speed_bps": None,
|
||||
"eta_s": None}
|
||||
fields = {
|
||||
"status": JobStatus.DONE,
|
||||
"progress": 100,
|
||||
"phase": None,
|
||||
"speed_bps": None,
|
||||
"eta_s": None,
|
||||
}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
@@ -484,8 +505,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
||||
error_code=dlerrors.classify(msg), phase=None,
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
@@ -501,8 +526,12 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running (see the download error path).
|
||||
service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
||||
error_code=dlerrors.classify(msg), phase=None,
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
@@ -517,7 +546,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
user_id = job.user_id if job else None
|
||||
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
|
||||
src_rel = src.rel_path if src else None
|
||||
src_w, src_h, src_dur = (src.width, src.height, src.duration_s) if src else (None, None, None)
|
||||
src_w, src_h, src_dur = (
|
||||
(src.width, src.height, src.duration_s) if src else (None, None, None)
|
||||
)
|
||||
|
||||
if not src_ready or not src_rel:
|
||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||
@@ -541,7 +572,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
|
||||
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||
if edit_spec.get("segments"):
|
||||
cmd, prep = editmod.build_concat_plan(src_path, dest_staging, edit_spec, out_ext, staging)
|
||||
cmd, prep = editmod.build_concat_plan(
|
||||
src_path, dest_staging, edit_spec, out_ext, staging
|
||||
)
|
||||
for p, content in prep.items():
|
||||
p.write_text(content, encoding="utf-8")
|
||||
else:
|
||||
|
||||
@@ -8,6 +8,7 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
|
||||
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
||||
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
@@ -10,6 +10,7 @@ Contract: the asset is already claimed ('downloading') by the parent. This proce
|
||||
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
|
||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ cache hit (asset already ready) — marks the job done instantly with no re-down
|
||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -166,7 +167,8 @@ def enqueue(
|
||||
source_ref=source_ref,
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
display_name=display_name
|
||||
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg`
|
||||
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
@@ -36,9 +37,27 @@ _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
|
||||
# that accented names show in non-UTF-8 tools.
|
||||
_FOLD_MAP = str.maketrans(
|
||||
{
|
||||
"ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D",
|
||||
"ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE",
|
||||
"ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-",
|
||||
"ß": "ss",
|
||||
"ẞ": "SS",
|
||||
"ø": "o",
|
||||
"Ø": "O",
|
||||
"ł": "l",
|
||||
"Ł": "L",
|
||||
"đ": "d",
|
||||
"Đ": "D",
|
||||
"ð": "d",
|
||||
"Ð": "D",
|
||||
"þ": "th",
|
||||
"Þ": "Th",
|
||||
"æ": "ae",
|
||||
"Æ": "AE",
|
||||
"œ": "oe",
|
||||
"Œ": "OE",
|
||||
"ı": "i",
|
||||
"İ": "I",
|
||||
"·": " ",
|
||||
"–": "-",
|
||||
"—": "-",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -174,7 +193,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
f" <premiered>{aired}</premiered>",
|
||||
f" <year>{year}</year>",
|
||||
f" <plot>{escape(meta.description or '')}</plot>",
|
||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
f' <uniqueid type="{escape(meta.uniqueid_type)}" default="true">{escape(meta.video_id)}</uniqueid>',
|
||||
]
|
||||
if meta.duration_s:
|
||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||
@@ -190,9 +209,7 @@ def _sidecar(media: Path, ext: str) -> Path:
|
||||
return media.with_suffix(ext)
|
||||
|
||||
|
||||
def write_sidecars(
|
||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
||||
) -> bool:
|
||||
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user