diff --git a/VERSION b/VERSION index 84767f21..fdbbf177 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.50.0 \ No newline at end of file +0.51.0 \ No newline at end of file diff --git a/backend/alembic/versions/0058_dl_default_720p.py b/backend/alembic/versions/0058_dl_default_720p.py new file mode 100644 index 00000000..6d8e1a91 --- /dev/null +++ b/backend/alembic/versions/0058_dl_default_720p.py @@ -0,0 +1,71 @@ +"""reset builtin download profiles: 720p is now the default (first) preset + +Revision ID: 0058_dl_default_720p +Revises: 0057_plex_ppl_enriched +Create Date: 2026-07-22 + +The download dialog preselects the first builtin (lowest sort_order). Make 720p that default — +it plays everywhere, downloads faster, and is plenty for the shared /watch player — with 1080p and +Best still one click away. Re-seeds the builtin rows so fresh installs and existing DBs converge +(same mechanism as 0040). Existing jobs are unaffected (they snapshot their spec; a profile_id that +pointed at a replaced row just SET NULLs). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0058_dl_default_720p" +down_revision: Union[str, None] = "0057_plex_ppl_enriched" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _spec(mode, height=None, container=None, audio_format=None): + return { + "mode": mode, + "max_height": height, + "container": container, + "audio_format": audio_format, + "vcodec": None, + "embed_subs": False, + "embed_chapters": False, + "embed_thumbnail": False, + "sponsorblock": False, + } + + +# 720p first (the default preselection), then 1080p, Best, then the rest. +_BUILTINS = [ + ("720p (MP4)", _spec("av", 720, "mp4"), 10), + ("1080p (MP4)", _spec("av", 1080, "mp4"), 20), + ("Best (MP4)", _spec("av", None, "mp4"), 30), + ("Audio (M4A)", _spec("a", None, None, "m4a"), 40), + ("Audio (MP3)", _spec("a", None, None, "mp3"), 50), + ("Video only (MP4)", _spec("v", None, "mp4"), 60), +] + +_profiles = sa.table( + "download_profiles", + sa.column("user_id", sa.Integer), + sa.column("name", sa.String), + sa.column("spec", sa.JSON), + sa.column("is_builtin", sa.Boolean), + sa.column("sort_order", sa.Integer), +) + + +def upgrade() -> None: + op.execute("DELETE FROM download_profiles WHERE is_builtin = true") + op.bulk_insert( + _profiles, + [ + {"user_id": None, "name": n, "spec": s, "is_builtin": True, "sort_order": o} + for n, s, o in _BUILTINS + ], + ) + + +def downgrade() -> None: + # No-op: the previous builtin set (1080p first) was seeded by 0040; not worth reconstructing. + pass diff --git a/backend/app/downloads/edit.py b/backend/app/downloads/edit.py index cff449a5..0d4944d7 100644 --- a/backend/app/downloads/edit.py +++ b/backend/app/downloads/edit.py @@ -268,6 +268,78 @@ def run_ffmpeg(cmd, total_s, on_progress, should_cancel) -> None: raise RuntimeError(f"ffmpeg failed ({ret}): {err[:300]}") +# --- ffmpeg: universal browser playability -------------------------------------------------- +# A downloaded file must play in EVERY browser, whatever the source shipped. Universal = H.264 +# video + AAC/MP3 audio in an mp4 container: H.264 is the only video codec all browsers decode +# (VP9/AV1 fail on Safari/iOS; HEVC fails on Chrome/Firefox), and mp4/AAC is the safe pairing. +# Anything else is fixed post-download: re-encode the offending stream(s), or just remux when the +# codecs are already fine but the container isn't. +_BROWSER_SAFE_VCODEC = ("h264",) +_BROWSER_SAFE_ACODEC = ("aac", "mp3") + + +def probe_codecs(path: Path) -> tuple[str | None, str | None]: + """Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing + stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream).""" + try: + out = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name", + "-of", "default=nw=1", str(path)], + capture_output=True, text=True, timeout=30, check=True, + ).stdout.lower() + except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError): + return None, None + vcodec = acodec = None + ctype = None + for line in out.splitlines(): + if line.startswith("codec_type="): + ctype = line.split("=", 1)[1].strip() + elif line.startswith("codec_name="): + name = line.split("=", 1)[1].strip() or None + if ctype == "video" and vcodec is None: + vcodec = name + elif ctype == "audio" and acodec is None: + acodec = name + return vcodec, acodec + + +def probe_duration(path: Path) -> int: + """Media duration in whole seconds via ffprobe, or 0 if unknown — a progress denominator for a + re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions).""" + try: + out = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=nw=1:nk=1", str(path)], + capture_output=True, text=True, timeout=30, check=True, + ).stdout.strip() + return int(float(out)) if out else 0 + except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError): + return 0 + + +def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]: + """Decide what (if anything) makes a file universally browser-playable. Returns + (needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3 + in mp4. A None codec (no such stream, or a failed probe) is never re-encoded.""" + rv = bool(vcodec) and vcodec not in _BROWSER_SAFE_VCODEC + ra = bool(acodec) and acodec not in _BROWSER_SAFE_ACODEC + return (rv or ra or not is_mp4), rv, ra + + +def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]: + """ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the + stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress + pipe:1` streams machine-readable progress for run_ffmpeg.""" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin", + "-progress", "pipe:1", "-nostats", "-i", str(src), + ] + cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"] + cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"] + cmd += ["-movflags", "+faststart", str(dest)] + return cmd + + # --- filmstrip (editor scrub track) --------------------------------------------------------- _SB_COLS = 12 diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 28b7f453..5170b39e 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -47,7 +47,9 @@ _DEFAULTS = { # Bump when the spec→output-bytes mapping changes so an identical spec gets a fresh cache # identity instead of hitting a stale asset produced by the old rule. v2: mp4 downloads now # prefer H.264+AAC (browser/iOS-compatible) rather than yt-dlp's default VP9/Opus best. -_SIG_VERSION = 2 +# v3: mp4 compat now *selects* H.264 (avc1) with a progressive fallback + re-encodes the rare +# AV1/VP9-only result to H.264, so cached AV1 assets must re-derive (fixes iOS broken-player). +_SIG_VERSION = 3 _SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"] _VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"} @@ -76,11 +78,16 @@ def normalize(spec: dict) -> dict: return s -def format_sig(spec: dict) -> str: - """Stable short hash of the output-affecting spec — the MediaAsset cache key component.""" +def format_sig(spec: dict, salt: str | None = None) -> str: + """Stable short hash of the output-affecting spec — the MediaAsset cache key component. + + `salt` forks a distinct, non-dedup identity for the same spec (used by force-redownload so it + always re-fetches into its own asset instead of sharing a cached file with other jobs).""" s = normalize(spec) data = {k: s[k] for k in _SIG_KEYS} data["_v"] = _SIG_VERSION + if salt is not None: + data["_salt"] = salt payload = json.dumps(data, sort_keys=True, separators=(",", ":")) return hashlib.sha256(payload.encode()).hexdigest()[:24] @@ -142,22 +149,44 @@ def build_ydl_opts( ) else: hcap = f"[height<=?{h}]" if h else "" + # Compatibility-first for mp4 (the universal container): produce H.264 video + AAC audio + # so shared/downloaded files play in EVERY browser, including iOS/Safari. WebKit (which + # every iOS browser, Brave included, is forced to use) decodes only H.264/H.265 + AAC + # inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there, even though Chromium + # plays it fine. A custom profile that explicitly sets `vcodec` opts out. + compat = s["container"] == "mp4" and s["vcodec"] is None if s["mode"] == "v": - opts["format"] = f"bestvideo{hcap}/best{hcap}" + if compat: + # Video-only: keep it audio-free. Prefer an H.264 *video-only* stream, then any + # video-only, and only fall to a progressive (audio-bearing) `best` as a last + # resort — same audio-free-when-possible guarantee as the old `bestvideo/best`. + opts["format"] = ( + f"bestvideo[vcodec^=avc1]{hcap}/bestvideo{hcap}" + f"/best[vcodec^=avc1]{hcap}/best{hcap}" + ) + else: + opts["format"] = f"bestvideo{hcap}/best{hcap}" else: # av - opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}" + if compat: + # Prefer H.264 at the SELECTION level, not just via format_sort: a sort key is only + # a *preference* and can't conjure an H.264 stream a source doesn't expose as + # video-only. A non-YouTube source (e.g. Facebook) may ship AV1 as the best + # video-only stream and H.264 only as a *progressive* (a+v) format — which + # `bestvideo` skips — so `bestvideo+bestaudio` picked AV1 and iOS broke. This chain + # tries avc1 video-only, then the avc1 progressive, then anything. + opts["format"] = ( + f"bestvideo[vcodec^=avc1]{hcap}+bestaudio[acodec^=mp4a]" + f"/best[vcodec^=avc1]{hcap}" + f"/bestvideo{hcap}+bestaudio/best{hcap}" + ) + else: + opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}" if s["container"]: opts["merge_output_format"] = s["container"] sort = [] - # Compatibility-first for mp4 (the universal container): prefer H.264 video + AAC - # audio so shared/downloaded files play in EVERY browser, including iOS/Safari. - # WebKit (which every iOS browser, Brave included, is forced to use) decodes only - # H.264/H.265 + AAC inside mp4 — a VP9/Opus-in-mp4 file shows a broken player there, - # even though Chromium plays it fine. These sort keys rank codec ABOVE resolution, so - # a 720p H.264 stream is chosen over a 1080p-VP9-only one; VP9/AV1 remain a graceful - # fallback only when no H.264 stream exists. A custom profile that explicitly sets - # `vcodec` opts out (its codec choice wins over the compatibility default). - if s["container"] == "mp4" and s["vcodec"] is None: + # Keep the codec preference as a tiebreaker (ranks H.264/AAC above resolution among the + # selected candidates); the selector above is the hard guarantee, this is belt-and-braces. + if compat: sort += ["vcodec:h264", "acodec:aac"] if h: sort.append(f"res:{h}") diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index e2bf16f4..1e4514de 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -106,7 +106,7 @@ def populate_from_catalog(db: Session, asset: MediaAsset) -> bool: return True -def _next_queue_pos(db: Session) -> int: +def next_queue_pos(db: Session) -> int: return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1 @@ -138,7 +138,7 @@ def enqueue( profile_snapshot=spec, display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None), status="queued", - queue_pos=_next_queue_pos(db), + queue_pos=next_queue_pos(db), ) db.add(job) asset.ref_count = (asset.ref_count or 0) + 1 @@ -207,7 +207,7 @@ def enqueue_edit( profile_snapshot={}, display_name=display_name or asset.title, status="queued", - queue_pos=_next_queue_pos(db), + queue_pos=next_queue_pos(db), ) db.add(job) asset.ref_count = (asset.ref_count or 0) + 1 diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py index 665e107c..b98eab64 100644 --- a/backend/app/downloads/storage.py +++ b/backend/app/downloads/storage.py @@ -20,13 +20,35 @@ from xml.sax.saxutils import escape # Reserve room for the " [id].ext" suffix + parent dirs within the 255-byte component limit. _MAX_TITLE = 120 +# User-facing (Content-Disposition) names can be longer than an on-disk component. +_MAX_DISPLAY = 150 # Filesystem-illegal on Windows/Plex + control chars. _ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]') # Unicode categories we drop entirely: emoji/pictographs (So), modifier symbols/skin-tones -# (Sk), and every control/format/private/surrogate/unassigned char (C*). Letters (incl. -# accented á/ö/ü/ß), digits, and ordinary punctuation are KEPT — Siftlode is trilingual, so -# ASCII-folding would mangle Hungarian/German titles. +# (Sk), and every control/format/private/surrogate/unassigned char (C*). _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"} +# Latin letters NFKD can't decompose to an ASCII base — fold them explicitly (trilingual HU/DE +# plus common European). Everything else with a diacritic (á/é/ö/ő/ü/ű, ç, ñ…) NFKD-decomposes +# into base + a combining mark we then drop, and any remaining non-ASCII is stripped, so every +# generated name is pure ASCII: no spaces, no accents — portable and free of the `?` mojibake +# that accented names show in non-UTF-8 tools. +_FOLD_MAP = str.maketrans( + { + "ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D", + "ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE", + "ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-", + } +) + + +def _ascii_fold(text: str) -> str: + """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop + the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have + no ASCII base).""" + text = text.translate(_FOLD_MAP) + text = unicodedata.normalize("NFKD", text) + text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn") + return text.encode("ascii", "ignore").decode("ascii") @dataclass @@ -41,16 +63,17 @@ class MediaMeta: def sanitize(name: str, limit: int = _MAX_TITLE) -> str: - """Turn an arbitrary (emoji-laden, clickbait) title into a clean, space-free path token. + """Turn an arbitrary (emoji-laden, clickbait) title into a clean, space-free ASCII path token. Steps: NFKC-normalize (fold fancy width/compat forms) → drop emoji/symbols/control chars → - strip filesystem-illegal chars → collapse whitespace and runs of the same punctuation → - join words with underscores → trim junk punctuation from the ends → length-cap. Accented - letters and digits survive; the result never contains spaces or path separators.""" + ASCII-fold accents (á→a, ő→o, ß→ss) → strip filesystem-illegal chars → collapse whitespace and + runs of the same punctuation → join words with underscores → trim junk punctuation from the ends + → length-cap. The result is pure ASCII with no spaces or path separators (digits survive).""" text = unicodedata.normalize("NFKC", name or "") text = "".join( ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES ) + text = _ascii_fold(text) text = _ILLEGAL.sub("", text) # Collapse repeated punctuation ("!!!", "???", "-----") to a single mark. text = re.sub(r"([^\w\s])\1{1,}", r"\1", text) @@ -68,13 +91,19 @@ def sanitize(name: str, limit: int = _MAX_TITLE) -> str: def display_filename(name: str) -> str: - """User-facing download filename (Content-Disposition): strip emoji/symbols/control + - filesystem-illegal chars, but KEEP spaces, accents, and ordinary punctuation — unlike the - underscore-joined on-disk name, this is the readable name the user saves to their device.""" + """User-facing download filename (Content-Disposition) base: the same ASCII-slug policy as the + on-disk name — emoji/symbols/control + illegal chars dropped, accents folded, whitespace joined + with underscores — so a saved file is portable (no spaces, no accents). Only the length cap + differs (a served name may be longer than an on-disk path component).""" text = unicodedata.normalize("NFKC", name or "") text = "".join(ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES) + text = _ascii_fold(text) text = _ILLEGAL.sub("", text) - text = re.sub(r"\s+", " ", text).strip(" .") + text = re.sub(r"([^\w\s])\1{1,}", r"\1", text) + text = re.sub(r"\s+", "_", text.strip()) + text = re.sub(r"_{2,}", "_", text).strip("_.-·—–|,;:!?ّ ") + if len(text) > _MAX_DISPLAY: + text = text[:_MAX_DISPLAY].rstrip("_.-") return text or "download" diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 33171302..048e1db1 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -20,7 +20,7 @@ from app.config import settings from app.db import get_db from app.downloads import edit as editmod from app.downloads import links as linksmod -from app.downloads import quota, service, storage +from app.downloads import formats, quota, service, storage from app.models import ( DownloadJob, DownloadLink, @@ -448,6 +448,52 @@ def resume_download( return _serialize_job(db, job) +@router.post("/{job_id}/redownload") +def redownload_download( + job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db) +) -> dict: + """Force a fresh re-download of a finished (or failed) job, rebuilding the file + Plex tree. + + The SAME job row is kept, so any public share link (DownloadLink → job_id) keeps working and + resolves to the newly-downloaded file — a re-download never breaks a link you already sent out + (only a *failed* re-download leaves the job in `error`, which is the intended broken-link case). + + The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job + id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds + under the current format/naming rules — never silently reuses a shared cached file — and (b) + never deletes a file another job/user still shares: the old asset is only removed if THIS job + was its last holder.""" + job = _own_job(db, user, job_id) + if job.job_kind == "edit": + raise HTTPException(status_code=409, detail="Editor clips can't be re-downloaded.") + if job.status in ("queued", "running", "paused"): + raise HTTPException(status_code=409, detail="This download is still in progress.") + + # Resolve the fresh private asset FIRST (before touching any file). Salting with the current + # asset id makes it distinct from both the dedup pool and this job's current asset, so the + # insert never collides — no IntegrityError/rollback can strand us after a file was deleted. + spec = formats.normalize(job.profile_snapshot) + sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}") + asset = service.get_or_create_asset(db, job.source_kind, job.source_ref, sig) + service.populate_from_catalog(db, asset) + + # Now drop the old hold — `_release_asset` deletes the old file + row only if this job was its + # last holder (a co-held asset is left intact for its other holders). + _release_asset(db, job) + + asset.ref_count = (asset.ref_count or 0) + 1 + job.asset_id = asset.id + job.status = "queued" + job.progress = 0 + job.phase = None + job.error = None + job.speed_bps = None + job.eta_s = None + job.queue_pos = service.next_queue_pos(db) # tail of the queue, not its stale original slot + db.commit() + return _serialize_job(db, job) + + @router.post("/{job_id}/cancel") def cancel_download( job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db) diff --git a/backend/app/titles.py b/backend/app/titles.py index e609f58f..83746636 100644 --- a/backend/app/titles.py +++ b/backend/app/titles.py @@ -5,6 +5,8 @@ yt-dlp, so the feed, search, channel pages, and the download center all show tid raw title is preserved in `Video.original_title` so this is reversible / re-derivable. Rules (user-approved "option b"): + 0. Strip a leading social engagement prefix ("1.6M views · 66K reactions | …") some sources + (Facebook) prepend, up to and including the pipe. 1. Drop emoji / pictographs / symbols / control chars (keep accents — HU/DE/…). 2. Strip trailing SEO hashtag clusters (word hashtags at the end); a numeric "#3" episode marker survives. @@ -21,6 +23,16 @@ import re import unicodedata _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"} +# Social engagement prefix some sources (Facebook) prepend, e.g. "1.6M views · 66K reactions | ". +# Match one-or-more "