diff --git a/VERSION b/VERSION index 57dc14fa..0c55b9cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.74.0 \ No newline at end of file +0.75.0 \ No newline at end of file diff --git a/backend/alembic/versions/0074_download_credentials.py b/backend/alembic/versions/0074_download_credentials.py new file mode 100644 index 00000000..b6d7d70c --- /dev/null +++ b/backend/alembic/versions/0074_download_credentials.py @@ -0,0 +1,46 @@ +"""download_credentials (S6: per-user, per-host yt-dlp cookie/auth store) + +Each user stores their OWN login cookies (Netscape cookies.txt) for an auth-walled download host +(Facebook, Instagram, …), Fernet-encrypted at rest. A download of a yt-dlp-supported, auth-walled +link on that host is retried with the acting user's cookie. Unique per (user_id, host_key) so a +user holds at most one credential per host. Empty for every existing install; forward-only in +practice (a downgrade just drops the table + its secrets). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0074_download_credentials" +down_revision: Union[str, None] = "0073_saved_view_source" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "download_credentials", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("host_key", sa.String(length=64), nullable=False), + sa.Column("cookies_enc", sa.Text(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "host_key", name="uq_download_credential"), + ) + op.create_index("ix_download_credentials_user_id", "download_credentials", ["user_id"]) + + +def downgrade() -> None: + op.drop_index("ix_download_credentials_user_id", table_name="download_credentials") + op.drop_table("download_credentials") diff --git a/backend/alembic/versions/0075_playlist_sync_baseline.py b/backend/alembic/versions/0075_playlist_sync_baseline.py new file mode 100644 index 00000000..b786fd60 --- /dev/null +++ b/backend/alembic/versions/0075_playlist_sync_baseline.py @@ -0,0 +1,26 @@ +"""playlist synced_video_ids (S7: 3-way merge baseline for two-way YouTube sync) + +The one-directional "local wins" push is replaced by a 3-way merge that classifies each item +against the last-synced state. That needs the raw ordered id set as a baseline, not just the +`synced_fingerprint` hash. Add a nullable `synced_video_ids` (JSON list) column; NULL on every +existing row means "no baseline yet" — the first merge then unions losslessly and records one. +Forward-only in practice (a downgrade just drops the column). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0075_playlist_sync_baseline" +down_revision: Union[str, None] = "0074_download_credentials" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("playlists", sa.Column("synced_video_ids", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("playlists", "synced_video_ids") diff --git a/backend/app/audit.py b/backend/app/audit.py index cba72f66..1578a656 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -44,6 +44,9 @@ class AuditAction: SYNC_RESUME = "sync_resume" # Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic) JOB_RUN = "job_run" + # Per-user download cookie/auth credential set or removed (the value is NEVER recorded) + DOWNLOAD_CREDENTIAL_SET = "download_credential_set" + DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete" # The log itself was cleared (a single row survives the clear as tamper-evidence) AUDIT_CLEAR = "audit_clear" diff --git a/backend/app/config.py b/backend/app/config.py index edb511eb..d48d6de9 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -186,6 +186,21 @@ class Settings(BaseSettings): download_supervisor_poll_seconds: int = 3 # Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation). download_kill_grace_seconds: int = 5 + # --- worker-child resource de-prioritization --- + # The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex + # playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web + # requests + playback until it finishes. The child lowers its OWN priority at startup — inherited + # by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort + # and independently disable-able with 0. + download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice + download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched + # Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 = + # reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many. + download_ffmpeg_threads: int = 0 + # Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO + # in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables. + download_gc_sweep_pause_ms: int = 20 + download_gc_sweep_batch: int = 400 # Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the # least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit). download_total_max_bytes: int = 0 diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py new file mode 100644 index 00000000..af5b79fe --- /dev/null +++ b/backend/app/downloads/credentials.py @@ -0,0 +1,258 @@ +"""Per-user, per-host download cookies — the single source of truth for "does the acting user's +cookie apply to this download, and how does that keep the result private". + +Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not +in prose): + * `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the + authed download becomes a PRIVATE asset that is never cache-served to another user. + * the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it + is downloading into is that private, salted identity — so a cookie can never poison a shared + asset (the load-bearing cross-user privacy fix). + +Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift. + +Security controls enforced here: + * **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated + registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it + belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no + cookie. + * **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted + in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route, + logged, or written to a job's error/snapshot. +""" + +import logging +import os +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.downloads import formats +from app.downloads.adapters import _host_is +from app.models import DownloadCredential +from app.security import decrypt_optional, encrypt + +log = logging.getLogger("siftlode.downloads.credentials") + + +class HostSpec: + """A supported cookie host: a short canonical `key` (stored on the credential + folded into the + cache salt), a human `label` for the UI, and the netloc `domains` it matches.""" + + __slots__ = ("key", "label", "domains") + + def __init__(self, key: str, label: str, domains: tuple[str, ...]): + self.key = key + self.label = label + self.domains = domains + + +# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several +# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt +# all derive from this, so there is nothing else to wire up. +SUPPORTED_HOSTS: tuple[HostSpec, ...] = ( + HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")), + HostSpec("instagram", "Instagram", ("instagram.com",)), + HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")), + HostSpec("tiktok", "TikTok", ("tiktok.com",)), + HostSpec("reddit", "Reddit", ("reddit.com",)), +) + +_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS} + + +def is_supported_host(host_key: str) -> bool: + return host_key in _BY_KEY + + +def host_label(host_key: str) -> str: + spec = _BY_KEY.get(host_key) + return spec.label if spec else host_key + + +def host_key_for_url(url: str | None) -> str | None: + """The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via + `_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When + several specs could match, the most-specific (longest matched domain) wins so the result is + deterministic across the enqueue and worker call sites.""" + if not url: + return None + from urllib.parse import urlparse + + host = urlparse(url).netloc.lower() + if not host: + return None + best_key: str | None = None + best_len = -1 + for spec in SUPPORTED_HOSTS: + for dom in spec.domains: + if _host_is(host, dom) and len(dom) > best_len: + best_key, best_len = spec.key, len(dom) + return best_key + + +def cookie_salt(user_id: int, host_key: str) -> str: + """The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed + asset is private to that user (never dedup-shared) and stable across that user's own re-adds.""" + return f"cookie:{user_id}:{host_key}" + + +def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str: + """The salted `format_sig` an authed download for (user, host) resolves to — the private asset + identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE.""" + return formats.format_sig(spec, salt=cookie_salt(user_id, host_key)) + + +def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None: + """At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a + normal shared download). Returns the salt string so the caller passes it straight to + `format_sig(spec, salt=...)`.""" + host_key = host_key_for_url(url) + if host_key is None: + return None + if get_credential(db, user_id, host_key) is None: + return None + return cookie_salt(user_id, host_key) + + +# --- storage (CRUD) --------------------------------------------------------------------------- + + +def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None: + return db.execute( + select(DownloadCredential).where( + DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key + ) + ).scalar_one_or_none() + + +class InvalidCookies(ValueError): + """The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route.""" + + +def validate_cookies(raw: str) -> str: + """Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads + the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path, + secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty, + or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than + silently producing a cookie file yt-dlp ignores.""" + text = (raw or "").strip() + if not text: + raise InvalidCookies("empty") + has_data_line = any( + line.strip() and not line.lstrip().startswith("#") and "\t" in line + for line in text.splitlines() + ) + if not has_data_line: + raise InvalidCookies("format") + # Netscape cookie files must end with a newline for some parsers; normalise. + return text if text.endswith("\n") else text + "\n" + + +def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential: + """Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is + supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit + row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call.""" + cred = get_credential(db, user_id, host_key) + enc = encrypt(raw) + if cred is None: + cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc) + db.add(cred) + else: + cred.cookies_enc = enc + return cred + + +def delete_credential(db: Session, user_id: int, host_key: str) -> bool: + """Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit.""" + cred = get_credential(db, user_id, host_key) + if cred is None: + return False + db.delete(cred) + return True + + +def status_list(db: Session, user_id: int) -> list[dict]: + """The supported hosts with this user's configured/updated state — the write-only Settings view. + Never includes the blob.""" + have = { + c.host_key: c + for c in db.execute( + select(DownloadCredential).where(DownloadCredential.user_id == user_id) + ).scalars() + } + out = [] + for spec in SUPPORTED_HOSTS: + cred = have.get(spec.key) + out.append( + { + "host_key": spec.key, + "label": spec.label, + "configured": cred is not None, + "updated_at": cred.updated_at.isoformat() if cred is not None else None, + } + ) + return out + + +# --- worker: resolve + write the cookie file -------------------------------------------------- + + +def resolve_cookiefile( + db: Session, + user_id: int, + url: str | None, + asset_format_sig: str, + spec: dict, + staging: Path, +) -> Path | None: + """In the worker child: the cookies.txt to hand yt-dlp for this download, or None. + + The double gate that prevents cross-user leakage: + 1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND + 2. the asset being downloaded into must BE this user's private, salted identity + (`asset_format_sig == authed_format_sig(...)`). + + If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the + global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key → + InvalidToken) also yields None; the download then fails honestly on the login wall (S3's + `login_required`).""" + host_key = host_key_for_url(url) + if host_key is None: + return None + cred = get_credential(db, user_id, host_key) + if cred is None: + return None + if asset_format_sig != authed_format_sig(spec, user_id, host_key): + return None # shared asset — never attach a cookie to a non-private identity + raw = decrypt_optional(cred.cookies_enc) + if not raw: + log.warning( + "download cookie for user %s host %s could not be decrypted (rotated key?) — " + "proceeding without it", + user_id, + host_key, + ) + return None + return _write_cookiefile(raw, staging) + + +def _write_cookiefile(raw: str, staging: Path) -> Path: + """Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path. + + Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there, + and `_produced_file` picks the largest file there as the output — either would corrupt a + cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still + removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a + `finally` on the normal/exception paths.""" + auth_dir = staging / ".auth" + auth_dir.mkdir(mode=0o700, exist_ok=True) + path = auth_dir / "cookies.txt" + # Open with 0600 from the start (don't create world-readable then chmod). + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + os.write(fd, raw.encode("utf-8")) + finally: + os.close(fd) + return path diff --git a/backend/app/downloads/edit.py b/backend/app/downloads/edit.py index 7f4d6a7a..a4a37b8d 100644 --- a/backend/app/downloads/edit.py +++ b/backend/app/downloads/edit.py @@ -25,6 +25,7 @@ import hashlib import json import logging import math +import os import queue import shutil import subprocess @@ -32,9 +33,21 @@ import threading import time from pathlib import Path +from app.config import settings + log = logging.getLogger("siftlode.edit") +def encode_thread_cap() -> int: + """How many encoder threads a browser-safe re-encode may use. `download_ffmpeg_threads` when the + admin pinned one (>0), else reserve a core for foreground work (max(1, cpu_count - 1)). Shared by + the ffmpeg `-threads` flag and the child's `OMP_NUM_THREADS` so both honour the same budget.""" + n = settings.download_ffmpeg_threads + if n > 0: + return n + return max(1, (os.cpu_count() or 2) - 1) + + class EditAborted(Exception): """Raised by run_ffmpeg when the job was paused/canceled mid-encode.""" @@ -510,11 +523,12 @@ def browser_transcode_flags( def build_browser_safe_cmd( - src: Path, dest: Path, reencode_video: bool, reencode_audio: bool + src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None ) -> list[str]: """ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress - pipe:1` streams machine-readable progress for run_ffmpeg.""" + pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a + re-encode can't claim every core on the shared box (see the worker-child de-prioritization).""" cmd = [ "ffmpeg", "-y", @@ -534,6 +548,8 @@ def build_browser_safe_cmd( else ["-c:v", "copy"] ) cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"] + if threads and threads > 0: + cmd += ["-threads", str(threads)] cmd += ["-movflags", "+faststart", str(dest)] return cmd diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 68947a8a..0c883bfa 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -24,6 +24,16 @@ _RULES: tuple[tuple[str, str], ...] = ( # Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account # authentication is required." — an auth wall we deliberately don't cross, not a transient error. ("authentication is required", "login_required"), + # Closed-group / private-post auth walls (Facebook groups, Instagram private, …). yt-dlp phrases + # these many ways; the common thread is "you must be signed in" or its own suggestion to supply + # cookies. Treating the cookie suggestion as an auth signal is what makes a login-walled link read + # honestly (and stay non-retryable) when the acting user has no usable cookie for that host. + ("log in to", "login_required"), + ("login required", "login_required"), + ("you must be logged in", "login_required"), + ("only available for registered users", "login_required"), + ("use --cookies", "login_required"), + ("--cookies-from-browser", "login_required"), # Geo-restricted. ("in your country", "geo_blocked"), ("available in your location", "geo_blocked"), @@ -42,6 +52,10 @@ _RULES: tuple[tuple[str, str], ...] = ( ("account associated with this video has been terminated", "unavailable"), ("this video is not available", "unavailable"), ("this video has been removed", "unavailable"), + # yt-dlp doesn't support this site at all ("Unsupported URL: …") — distinct from an auth wall on a + # supported site. Non-retryable: no cookie or retry can make an unsupported host downloadable. + ("unsupported url", "unsupported_host"), + ("no video could be found in this url", "unsupported_host"), # DRM. ("drm protected", "drm"), ("drm-protected", "drm"), diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 47a5090b..5ad7dce6 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -98,12 +98,17 @@ def build_ydl_opts( outtmpl: str, progress_hook, extra: dict | None = None, + cookiefile: str | None = None, ) -> dict: """Assemble yt-dlp options for a single download to `outtmpl` (a full path template). `extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's `extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the - generic options below. Everything here is source-agnostic.""" + generic options below. Everything here is source-agnostic. + + `cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker + after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra` + merge so a source adapter can never clobber it.""" s = normalize(spec) h = s["max_height"] opts: dict = { @@ -132,6 +137,11 @@ def build_ydl_opts( if extra: opts.update(extra) + # The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not + # be able to override the authenticated session the worker resolved for this specific user. + if cookiefile: + opts["cookiefile"] = cookiefile + if s["mode"] == "a": opts["format"] = "bestaudio/best" opts["postprocessors"].append( diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 9ce499af..a640c22e 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -86,11 +86,25 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int: cutoff = time.time() - _ORPHAN_MIN_AGE_S removed = 0 + # Throttle the whole-tree walk: this runs in the API process, and an unpaced rglob+stat over a + # large download root monopolizes disk IO long enough to make web requests + playback stutter. + # Yield the IO channel after every `batch` files touched so foreground work interleaves. + pause_s = max(0, settings.download_gc_sweep_pause_ms) / 1000.0 + batch = max(1, settings.download_gc_sweep_batch) + touched = 0 + + def _throttle() -> None: + nonlocal touched + touched += 1 + if pause_s and touched % batch == 0: + time.sleep(pause_s) + for entry in root.iterdir(): if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile continue files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()] for f in files: + _throttle() try: if f.resolve() in keep or f.stat().st_mtime > cutoff: continue diff --git a/backend/app/downloads/links.py b/backend/app/downloads/links.py index 38dc2038..0344bf17 100644 --- a/backend/app/downloads/links.py +++ b/backend/app/downloads/links.py @@ -83,6 +83,10 @@ def public_meta( from app.downloads import service source_url = service.reference_url(job, asset) + # Audio-only assets (mp3 rips, extracted audio) carry a codec but no video stream. WebKit (all + # iOS browsers) refuses to play them inside a