diff --git a/backend/alembic/versions/0074_download_credentials.py b/backend/alembic/versions/0074_download_credentials.py new file mode 100644 index 00000000..b6d7d70c --- /dev/null +++ b/backend/alembic/versions/0074_download_credentials.py @@ -0,0 +1,46 @@ +"""download_credentials (S6: per-user, per-host yt-dlp cookie/auth store) + +Each user stores their OWN login cookies (Netscape cookies.txt) for an auth-walled download host +(Facebook, Instagram, …), Fernet-encrypted at rest. A download of a yt-dlp-supported, auth-walled +link on that host is retried with the acting user's cookie. Unique per (user_id, host_key) so a +user holds at most one credential per host. Empty for every existing install; forward-only in +practice (a downgrade just drops the table + its secrets). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0074_download_credentials" +down_revision: Union[str, None] = "0073_saved_view_source" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "download_credentials", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("host_key", sa.String(length=64), nullable=False), + sa.Column("cookies_enc", sa.Text(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "host_key", name="uq_download_credential"), + ) + op.create_index("ix_download_credentials_user_id", "download_credentials", ["user_id"]) + + +def downgrade() -> None: + op.drop_index("ix_download_credentials_user_id", table_name="download_credentials") + op.drop_table("download_credentials") diff --git a/backend/app/audit.py b/backend/app/audit.py index cba72f66..1578a656 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -44,6 +44,9 @@ class AuditAction: SYNC_RESUME = "sync_resume" # Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic) JOB_RUN = "job_run" + # Per-user download cookie/auth credential set or removed (the value is NEVER recorded) + DOWNLOAD_CREDENTIAL_SET = "download_credential_set" + DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete" # The log itself was cleared (a single row survives the clear as tamper-evidence) AUDIT_CLEAR = "audit_clear" diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py new file mode 100644 index 00000000..af5b79fe --- /dev/null +++ b/backend/app/downloads/credentials.py @@ -0,0 +1,258 @@ +"""Per-user, per-host download cookies — the single source of truth for "does the acting user's +cookie apply to this download, and how does that keep the result private". + +Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not +in prose): + * `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the + authed download becomes a PRIVATE asset that is never cache-served to another user. + * the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it + is downloading into is that private, salted identity — so a cookie can never poison a shared + asset (the load-bearing cross-user privacy fix). + +Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift. + +Security controls enforced here: + * **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated + registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it + belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no + cookie. + * **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted + in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route, + logged, or written to a job's error/snapshot. +""" + +import logging +import os +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.downloads import formats +from app.downloads.adapters import _host_is +from app.models import DownloadCredential +from app.security import decrypt_optional, encrypt + +log = logging.getLogger("siftlode.downloads.credentials") + + +class HostSpec: + """A supported cookie host: a short canonical `key` (stored on the credential + folded into the + cache salt), a human `label` for the UI, and the netloc `domains` it matches.""" + + __slots__ = ("key", "label", "domains") + + def __init__(self, key: str, label: str, domains: tuple[str, ...]): + self.key = key + self.label = label + self.domains = domains + + +# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several +# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt +# all derive from this, so there is nothing else to wire up. +SUPPORTED_HOSTS: tuple[HostSpec, ...] = ( + HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")), + HostSpec("instagram", "Instagram", ("instagram.com",)), + HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")), + HostSpec("tiktok", "TikTok", ("tiktok.com",)), + HostSpec("reddit", "Reddit", ("reddit.com",)), +) + +_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS} + + +def is_supported_host(host_key: str) -> bool: + return host_key in _BY_KEY + + +def host_label(host_key: str) -> str: + spec = _BY_KEY.get(host_key) + return spec.label if spec else host_key + + +def host_key_for_url(url: str | None) -> str | None: + """The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via + `_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When + several specs could match, the most-specific (longest matched domain) wins so the result is + deterministic across the enqueue and worker call sites.""" + if not url: + return None + from urllib.parse import urlparse + + host = urlparse(url).netloc.lower() + if not host: + return None + best_key: str | None = None + best_len = -1 + for spec in SUPPORTED_HOSTS: + for dom in spec.domains: + if _host_is(host, dom) and len(dom) > best_len: + best_key, best_len = spec.key, len(dom) + return best_key + + +def cookie_salt(user_id: int, host_key: str) -> str: + """The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed + asset is private to that user (never dedup-shared) and stable across that user's own re-adds.""" + return f"cookie:{user_id}:{host_key}" + + +def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str: + """The salted `format_sig` an authed download for (user, host) resolves to — the private asset + identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE.""" + return formats.format_sig(spec, salt=cookie_salt(user_id, host_key)) + + +def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None: + """At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a + normal shared download). Returns the salt string so the caller passes it straight to + `format_sig(spec, salt=...)`.""" + host_key = host_key_for_url(url) + if host_key is None: + return None + if get_credential(db, user_id, host_key) is None: + return None + return cookie_salt(user_id, host_key) + + +# --- storage (CRUD) --------------------------------------------------------------------------- + + +def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None: + return db.execute( + select(DownloadCredential).where( + DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key + ) + ).scalar_one_or_none() + + +class InvalidCookies(ValueError): + """The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route.""" + + +def validate_cookies(raw: str) -> str: + """Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads + the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path, + secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty, + or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than + silently producing a cookie file yt-dlp ignores.""" + text = (raw or "").strip() + if not text: + raise InvalidCookies("empty") + has_data_line = any( + line.strip() and not line.lstrip().startswith("#") and "\t" in line + for line in text.splitlines() + ) + if not has_data_line: + raise InvalidCookies("format") + # Netscape cookie files must end with a newline for some parsers; normalise. + return text if text.endswith("\n") else text + "\n" + + +def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential: + """Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is + supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit + row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call.""" + cred = get_credential(db, user_id, host_key) + enc = encrypt(raw) + if cred is None: + cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc) + db.add(cred) + else: + cred.cookies_enc = enc + return cred + + +def delete_credential(db: Session, user_id: int, host_key: str) -> bool: + """Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit.""" + cred = get_credential(db, user_id, host_key) + if cred is None: + return False + db.delete(cred) + return True + + +def status_list(db: Session, user_id: int) -> list[dict]: + """The supported hosts with this user's configured/updated state — the write-only Settings view. + Never includes the blob.""" + have = { + c.host_key: c + for c in db.execute( + select(DownloadCredential).where(DownloadCredential.user_id == user_id) + ).scalars() + } + out = [] + for spec in SUPPORTED_HOSTS: + cred = have.get(spec.key) + out.append( + { + "host_key": spec.key, + "label": spec.label, + "configured": cred is not None, + "updated_at": cred.updated_at.isoformat() if cred is not None else None, + } + ) + return out + + +# --- worker: resolve + write the cookie file -------------------------------------------------- + + +def resolve_cookiefile( + db: Session, + user_id: int, + url: str | None, + asset_format_sig: str, + spec: dict, + staging: Path, +) -> Path | None: + """In the worker child: the cookies.txt to hand yt-dlp for this download, or None. + + The double gate that prevents cross-user leakage: + 1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND + 2. the asset being downloaded into must BE this user's private, salted identity + (`asset_format_sig == authed_format_sig(...)`). + + If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the + global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key → + InvalidToken) also yields None; the download then fails honestly on the login wall (S3's + `login_required`).""" + host_key = host_key_for_url(url) + if host_key is None: + return None + cred = get_credential(db, user_id, host_key) + if cred is None: + return None + if asset_format_sig != authed_format_sig(spec, user_id, host_key): + return None # shared asset — never attach a cookie to a non-private identity + raw = decrypt_optional(cred.cookies_enc) + if not raw: + log.warning( + "download cookie for user %s host %s could not be decrypted (rotated key?) — " + "proceeding without it", + user_id, + host_key, + ) + return None + return _write_cookiefile(raw, staging) + + +def _write_cookiefile(raw: str, staging: Path) -> Path: + """Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path. + + Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there, + and `_produced_file` picks the largest file there as the output — either would corrupt a + cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still + removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a + `finally` on the normal/exception paths.""" + auth_dir = staging / ".auth" + auth_dir.mkdir(mode=0o700, exist_ok=True) + path = auth_dir / "cookies.txt" + # Open with 0600 from the start (don't create world-readable then chmod). + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + os.write(fd, raw.encode("utf-8")) + finally: + os.close(fd) + return path diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 47a5090b..5ad7dce6 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -98,12 +98,17 @@ def build_ydl_opts( outtmpl: str, progress_hook, extra: dict | None = None, + cookiefile: str | None = None, ) -> dict: """Assemble yt-dlp options for a single download to `outtmpl` (a full path template). `extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's `extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the - generic options below. Everything here is source-agnostic.""" + generic options below. Everything here is source-agnostic. + + `cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker + after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra` + merge so a source adapter can never clobber it.""" s = normalize(spec) h = s["max_height"] opts: dict = { @@ -132,6 +137,11 @@ def build_ydl_opts( if extra: opts.update(extra) + # The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not + # be able to override the authenticated session the worker resolved for this specific user. + if cookiefile: + opts["cookiefile"] = cookiefile + if s["mode"] == "a": opts["format"] = "bestaudio/best" opts["postprocessors"].append( diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 6b8088c6..be55c786 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -8,13 +8,14 @@ from datetime import date, datetime, timedelta, timezone from pathlib import Path import yt_dlp +from sqlalchemy import select from app import sysconfig from app.config import settings from app.db import SessionLocal from app.downloads import edit as editmod from app.downloads import errors as dlerrors -from app.downloads import adapters, formats, service, storage +from app.downloads import adapters, credentials, formats, service, storage from app.downloads.jobdb import ( _edit_staging_dir, _job_status, @@ -304,10 +305,13 @@ def _media_info(info: dict) -> dict: return {**info, **chosen} -def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter): +def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter, cookiefile=None): """Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format - error); other sources make a single attempt. Returns the info dict of the successful attempt.""" + error); other sources make a single attempt. Returns the info dict of the successful attempt. + + `cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by + `credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.""" client_sets = adapter.client_sets() outtmpl = str(staging / "%(id)s.%(ext)s") last_exc: Exception | None = None @@ -317,6 +321,7 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter): outtmpl, _make_progress_hook(job_id, asset_id), extra=adapter.ydl_opts_extra(clients), + cookiefile=cookiefile, ) opts["postprocessor_hooks"] = [_make_pp_hook(job_id)] _touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow / @@ -397,8 +402,11 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict return dest -def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None: +def _download( + job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int +) -> None: staging = _staging_dir(asset_id) + cookiefile: Path | None = None try: if staging.exists(): shutil.rmtree(staging, ignore_errors=True) @@ -407,7 +415,27 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe _set_job(job_id, phase="downloading", progress=0) adapter = adapters.resolve_adapter(source_kind, source_ref) url = service.source_url(source_kind, source_ref) - info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter)) + # S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND + # this asset is that user's private, salted identity (the double gate lives in credentials). + with SessionLocal() as cdb: + asset_sig = cdb.execute( + select(MediaAsset.format_sig).where(MediaAsset.id == asset_id) + ).scalar_one_or_none() + if asset_sig is not None: + cookiefile = credentials.resolve_cookiefile( + cdb, user_id, url, asset_sig, spec, staging + ) + info = _media_info( + _extract_with_fallback( + job_id, + asset_id, + url, + spec, + staging, + adapter, + cookiefile=str(cookiefile) if cookiefile else None, + ) + ) # Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only # identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source. adapter = adapters.resolve_adapter(source_kind, source_ref, info) @@ -517,6 +545,10 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe db.commit() log.warning("job %s failed: %s", job_id, msg) finally: + # Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging + # tree too, but never leave a plaintext session cookie on disk a moment longer than needed). + if cookiefile is not None: + cookiefile.unlink(missing_ok=True) shutil.rmtree(staging, ignore_errors=True) diff --git a/backend/app/downloads/runner.py b/backend/app/downloads/runner.py index f3ff581c..8b1d777a 100644 --- a/backend/app/downloads/runner.py +++ b/backend/app/downloads/runner.py @@ -67,6 +67,7 @@ def main() -> None: job_kind = job.job_kind spec = dict(job.profile_snapshot or {}) source_kind, source_ref = job.source_kind, job.source_ref + user_id = job.user_id if asset_id is None: return @@ -77,7 +78,7 @@ def main() -> None: if job_kind == "edit": _process_edit(job_id, asset_id) else: - _download(job_id, asset_id, source_kind, source_ref, spec) + _download(job_id, asset_id, source_kind, source_ref, spec, user_id) if __name__ == "__main__": diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index 71139fef..92a39535 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -15,6 +15,7 @@ from sqlalchemy.orm import Session from app.config import settings from app.downloads import adapters +from app.downloads import credentials from app.downloads import edit as editmod from app.downloads import formats, quota, storage from app.downloads.status import HOLDING, JobStatus @@ -155,7 +156,12 @@ def enqueue( Raises quota.QuotaExceeded if the user is at their job-count or footprint cap.""" quota.check_enqueue(db, user_id) spec = formats.normalize(spec) - sig = formats.format_sig(spec) + # If this user has a stored cookie for the download's host, salt the cache signature with their + # id so the cookie-authed result is a PRIVATE asset — never dedup-shared with, or cache-served + # to, another user (the load-bearing privacy fix; the worker attaches the cookie to exactly this + # salted identity). A normal download (no matching cookie) gets salt=None and shares as before. + salt = credentials.salt_for_enqueue(db, user_id, source_url(source_kind, source_ref)) + sig = formats.format_sig(spec, salt=salt) asset = get_or_create_asset(db, source_kind, source_ref, sig) # Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos). populate_from_catalog(db, asset) diff --git a/backend/app/models.py b/backend/app/models.py index 27e19064..df47f972 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -867,6 +867,35 @@ class DownloadLink(Base, TimestampMixin): view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") +class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): + """A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …). + + Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host + where they're logged in (e.g. a closed Facebook group they belong to), and a download of a + yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The + blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit — + the routes are write-only (host + updated_at only). See `downloads.credentials`. + + Two load-bearing security controls live around this row, NOT on it: + * **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the + user id, so the authed asset is private to that user and can never be cache-served to + another user (`downloads.credentials.cookie_salt`). + * **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches + this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=` + can't exfiltrate the user's session to a look-alike host. + + `host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the + raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch).""" + + __tablename__ = "download_credentials" + __table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + host_key: Mapped[str] = mapped_column(String(64)) + cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged + + class MessageKey(Base, TimestampMixin): """A user's end-to-end-encryption key material for direct messaging (phase 2). diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 3722f72a..950639c0 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -16,10 +16,11 @@ from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.orm import Session -from app import sysconfig +from app import audit, sysconfig from app.auth import admin_user, require_human from app.config import settings from app.db import get_db +from app.downloads import credentials as dlcreds from app.downloads import edit as editmod from app.downloads import links as linksmod from app.downloads import quota, service, storage @@ -247,6 +248,65 @@ def delete_profile( return {"deleted": profile_id} +# --- per-user download auth (cookies) ------------------------------------------------------ +# Write-only surface: the stored blob is NEVER returned — only the supported hosts + whether each +# is configured + when. See downloads.credentials for the host-scope + cache-privacy guarantees. + + +@router.get("/credentials") +def list_credentials(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict: + return {"hosts": dlcreds.status_list(db, user.id)} + + +class SetCredentialIn(BaseModel): + cookies: str + + +@router.put("/credentials/{host_key}") +def set_credential( + host_key: str, + body: SetCredentialIn, + user: User = Depends(require_human), + db: Session = Depends(get_db), +) -> dict: + """Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is + never echoed back, logged, or audited — only the fact that it changed.""" + if not dlcreds.is_supported_host(host_key): + raise HTTPException(status_code=404, detail="Unknown host") + try: + cleaned = dlcreds.validate_cookies(body.cookies) + except dlcreds.InvalidCookies as e: + raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)}) + dlcreds.upsert_credential(db, user.id, host_key, cleaned) + audit.record( + db, + user.id, + audit.AuditAction.DOWNLOAD_CREDENTIAL_SET, + target_type="download_credential", + target_id=host_key, + ) + db.commit() + return {"ok": True, "hosts": dlcreds.status_list(db, user.id)} + + +@router.delete("/credentials/{host_key}") +def delete_credential( + host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db) +) -> dict: + if not dlcreds.is_supported_host(host_key): + raise HTTPException(status_code=404, detail="Unknown host") + if dlcreds.delete_credential(db, user.id, host_key): + audit.record( + db, + user.id, + audit.AuditAction.DOWNLOAD_CREDENTIAL_DELETE, + target_type="download_credential", + target_id=host_key, + ) + db.commit() + return {"ok": True, "hosts": dlcreds.status_list(db, user.id)} + + def _resolve_spec( db: Session, user: User, profile_id: int | None, spec: dict | None ) -> tuple[dict, int | None]: diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py new file mode 100644 index 00000000..48e4f42c --- /dev/null +++ b/backend/tests/test_download_credentials.py @@ -0,0 +1,257 @@ +"""S6 — per-user, per-host download cookies: host-scope, cache privacy, encryption, write-only API. + +The two load-bearing security properties are covered as behaviour, not prose: + * **cache privacy** — a cookie-authed download is a PRIVATE, salted asset; a second user without a + cookie enqueuing the same URL gets the shared (unsalted) asset and can never be cache-served the + authed one, and the worker attaches a cookie ONLY to that private identity. + * **host scope (SEC-12)** — a cookie is resolved ONLY for a URL whose netloc the credential owns; a + look-alike or unrelated host resolves to no host_key and gets no cookie. + +Pure-logic parts run with no DB; the CRUD / enqueue-salting / API parts use the `db`/`as_user` lane. +""" + +import os +from pathlib import Path + +import pytest +from cryptography.fernet import Fernet + +from app import security +from app.downloads import credentials, formats, service +from app.models import DownloadCredential, MediaAsset, User + + +@pytest.fixture(autouse=True) +def _fernet(monkeypatch): + # A throwaway key so encrypt/decrypt work without TOKEN_ENCRYPTION_KEY in the env (mirrors + # test_security). Autouse: every credential write in this module encrypts. + monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key())) + + +COOKIES = "# Netscape HTTP Cookie File\n.facebook.com\tTRUE\t/\tTRUE\t0\tc_user\t123\n" + + +# --- host scope (pure logic) ------------------------------------------------------------------ + + +class TestHostKeyForUrl: + def test_matches_a_supported_host(self): + assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook" + assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook" + assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter" + + def test_look_alike_host_never_matches(self): + assert credentials.host_key_for_url("https://myfacebook.com/x") is None + assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None + + def test_unrelated_or_empty_url_has_no_host_key(self): + assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None + assert credentials.host_key_for_url("") is None + assert credentials.host_key_for_url(None) is None + + +class TestValidateCookies: + def test_accepts_a_netscape_file(self): + assert credentials.validate_cookies(COOKIES).endswith("\n") + + def test_rejects_empty(self): + with pytest.raises(credentials.InvalidCookies): + credentials.validate_cookies(" ") + + def test_rejects_non_cookie_paste(self): + # Comments only, no tab-separated data line -> not a usable cookies.txt. + with pytest.raises(credentials.InvalidCookies): + credentials.validate_cookies("just some pasted text\n# a comment") + + +def test_authed_sig_differs_per_user_and_from_the_shared_sig(): + spec = formats.normalize({}) + shared = formats.format_sig(spec) + a = credentials.authed_format_sig(spec, 1, "facebook") + b = credentials.authed_format_sig(spec, 2, "facebook") + assert a != shared and b != shared and a != b + + +def test_write_cookiefile_is_private_and_survives_the_retry_wipe(tmp_path): + staging = tmp_path / "staging" + staging.mkdir() + path = credentials._write_cookiefile(COOKIES, staging) + assert path.read_text() == COOKIES + # 0600 (owner-only) — POSIX perms only (the worker is Linux-only; Windows has no mode bits). + if os.name == "posix": + assert (path.stat().st_mode & 0o777) == 0o600 + # It lives in a SUBDIR so the download retry loop (which unlinks files in the staging ROOT) and + # _produced_file (which scans the root) never see it. + assert path.parent == staging / ".auth" + assert not any(p.is_file() for p in staging.iterdir()) + + +# --- CRUD + encryption at rest (DB) ----------------------------------------------------------- + + +def _user(db, email="cred@example.com") -> User: + u = User(email=email) + db.add(u) + db.flush() + return u + + +class TestCrud: + def test_upsert_stores_ciphertext_not_plaintext(self, db): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + row = db.get(DownloadCredential, 1) + assert row.cookies_enc != COOKIES # encrypted at rest + assert COOKIES not in row.cookies_enc + assert security.decrypt(row.cookies_enc) == COOKIES + + def test_upsert_replaces_in_place(self, db): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + credentials.upsert_credential(db, u.id, "facebook", COOKIES.replace("123", "999")) + db.commit() + rows = db.query(DownloadCredential).filter_by(user_id=u.id, host_key="facebook").all() + assert len(rows) == 1 + assert "999" in security.decrypt(rows[0].cookies_enc) + + def test_status_list_never_leaks_the_blob(self, db): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + status = credentials.status_list(db, u.id) + fb = next(h for h in status if h["host_key"] == "facebook") + assert fb["configured"] is True and fb["updated_at"] is not None + ig = next(h for h in status if h["host_key"] == "instagram") + assert ig["configured"] is False and ig["updated_at"] is None + # No field anywhere carries the cookie value. + assert all(COOKIES not in str(v) for h in status for v in h.values()) + + def test_delete(self, db): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + assert credentials.delete_credential(db, u.id, "facebook") is True + db.commit() + assert credentials.get_credential(db, u.id, "facebook") is None + assert credentials.delete_credential(db, u.id, "facebook") is False + + +# --- the crux: enqueue salts privately, no cross-user cache serve (DB) ------------------------- + +FB_URL = "https://www.facebook.com/watch?v=100" + + +class TestCachePrivacy: + def test_enqueue_with_a_cookie_creates_a_private_salted_asset(self, db): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + job = service.enqueue(db, u.id, "url", FB_URL, {}) + asset = db.get(MediaAsset, job.asset_id) + spec = formats.normalize({}) + assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "facebook") + assert asset.format_sig != formats.format_sig(spec) + + def test_enqueue_without_a_cookie_uses_the_shared_sig(self, db): + u = _user(db) + job = service.enqueue(db, u.id, "url", FB_URL, {}) + asset = db.get(MediaAsset, job.asset_id) + assert asset.format_sig == formats.format_sig(formats.normalize({})) + + def test_two_users_never_share_the_authed_asset(self, db): + a = _user(db, "a@example.com") + b = _user(db, "b@example.com") + credentials.upsert_credential(db, a.id, "facebook", COOKIES) + db.commit() + ja = service.enqueue(db, a.id, "url", FB_URL, {}) # A: authed, private + jb = service.enqueue(db, b.id, "url", FB_URL, {}) # B: no cookie, shared + assert ja.asset_id != jb.asset_id + # B's asset is the shared identity — A's private group video can never be served to B. + assert db.get(MediaAsset, jb.asset_id).format_sig == formats.format_sig( + formats.normalize({}) + ) + + +# --- the worker gate: attach only to the private identity, host-scoped (DB) -------------------- + + +class TestResolveCookiefile: + def _asset_sig(self, sig: str) -> MediaAsset: + return MediaAsset( + source_kind="url", source_ref=FB_URL, format_sig=sig, status="downloading" + ) + + def test_attaches_to_the_private_salted_asset(self, db, tmp_path): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + spec = formats.normalize({}) + salted = credentials.authed_format_sig(spec, u.id, "facebook") + path = credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path) + assert path is not None and Path(path).read_text() == COOKIES + + def test_never_attaches_to_a_shared_asset(self, db, tmp_path): + # Cookie exists, but the asset is the UNSALTED shared identity -> attaching would poison the + # global cache with a private-group video. Must return None. + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + spec = formats.normalize({}) + shared = formats.format_sig(spec) + assert credentials.resolve_cookiefile(db, u.id, FB_URL, shared, spec, tmp_path) is None + + def test_non_matching_host_gets_no_cookie(self, db, tmp_path): + u = _user(db) + credentials.upsert_credential(db, u.id, "facebook", COOKIES) + db.commit() + spec = formats.normalize({}) + # A YouTube URL has no supported host_key -> no cookie regardless of what's stored. + yt = "https://www.youtube.com/watch?v=abc" + salted = credentials.authed_format_sig(spec, u.id, "facebook") + assert credentials.resolve_cookiefile(db, u.id, yt, salted, spec, tmp_path) is None + + def test_no_credential_gets_no_cookie(self, db, tmp_path): + u = _user(db) + spec = formats.normalize({}) + salted = credentials.authed_format_sig(spec, u.id, "facebook") + assert credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path) is None + + +# --- API: write-only, never echoes the blob (DB) ---------------------------------------------- + + +class TestApi: + def test_put_get_delete_roundtrip_never_returns_the_blob(self, db, as_user): + u = _user(db) + db.commit() + client = as_user(u) + + r = client.put("/api/downloads/credentials/facebook", json={"cookies": COOKIES}) + assert r.status_code == 200 + body = r.json() + assert body["ok"] is True + assert COOKIES not in r.text # the response must never carry the value + fb = next(h for h in body["hosts"] if h["host_key"] == "facebook") + assert fb["configured"] is True + + r = client.get("/api/downloads/credentials") + assert r.status_code == 200 and COOKIES not in r.text + + r = client.delete("/api/downloads/credentials/facebook") + assert r.status_code == 200 + assert credentials.get_credential(db, u.id, "facebook") is None + + def test_unknown_host_is_404(self, db, as_user): + u = _user(db) + db.commit() + r = as_user(u).put("/api/downloads/credentials/nope", json={"cookies": COOKIES}) + assert r.status_code == 404 + + def test_bad_cookies_are_rejected(self, db, as_user): + u = _user(db) + db.commit() + r = as_user(u).put("/api/downloads/credentials/facebook", json={"cookies": " "}) + assert r.status_code == 400 + assert r.json()["detail"]["code"] == "cookies" diff --git a/frontend/src/components/SettingsPanel.downloadAuth.dom.test.tsx b/frontend/src/components/SettingsPanel.downloadAuth.dom.test.tsx new file mode 100644 index 00000000..b8a777aa --- /dev/null +++ b/frontend/src/components/SettingsPanel.downloadAuth.dom.test.tsx @@ -0,0 +1,93 @@ +// @vitest-environment jsdom +// +// S6 — the per-user download-cookies section is WRITE-ONLY: the stored blob is never fetched or +// shown back, so opening "Set"/"Update" always reveals an EMPTY field, and saving posts exactly the +// pasted value to the matching host. This drives the real DownloadAuth against a mocked api so we can +// assert the field never carries a prior value and the right host is written. +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen, cleanup, fireEvent, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +const h = vi.hoisted(() => ({ + downloadCredentials: vi.fn(), + setDownloadCredential: vi.fn(), + deleteDownloadCredential: vi.fn(), +})); + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), + }; +}); +vi.mock("../lib/api", () => ({ api: h })); +// Partial mock: SettingsPanel's module graph (PrefsProvider) also pulls getNotifSettings from here. +vi.mock("../lib/notifications", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, notify: vi.fn() }; +}); +vi.mock("./ConfirmProvider", () => ({ useConfirm: () => vi.fn(async () => true) })); + +import { DownloadAuth } from "./SettingsPanel"; + +const HOSTS = [ + { host_key: "facebook", label: "Facebook", configured: false, updated_at: null }, + { host_key: "instagram", label: "Instagram", configured: true, updated_at: "2026-08-01T00:00:00Z" }, +]; + +function renderPanel() { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +} + +describe("DownloadAuth (write-only cookies)", () => { + beforeEach(() => { + h.downloadCredentials.mockResolvedValue({ hosts: HOSTS }); + h.setDownloadCredential.mockResolvedValue({ ok: true, hosts: HOSTS }); + h.deleteDownloadCredential.mockResolvedValue({ ok: true, hosts: HOSTS }); + }); + afterEach(cleanup); + + it("lists every supported host with its configured state", async () => { + renderPanel(); + await screen.findByTestId("cookie-host-facebook"); + expect(screen.getByTestId("cookie-status-facebook").textContent).toContain( + "settings.downloadAuth.notSet", + ); + expect(screen.getByTestId("cookie-status-instagram").textContent).toContain( + "settings.downloadAuth.configured", + ); + }); + + it("opens an EMPTY field on Set and posts the pasted value to that host", async () => { + renderPanel(); + await screen.findByTestId("cookie-host-facebook"); + // The Set button is the outline action on the facebook row. + const setBtn = screen.getByText("settings.downloadAuth.set"); + fireEvent.click(setBtn); + const field = (await screen.findByTestId("cookie-input-facebook")) as HTMLTextAreaElement; + expect(field.value).toBe(""); // write-only: never prefilled + + fireEvent.change(field, { target: { value: "# Netscape\n.facebook.com\tTRUE\t/\tT\t0\tc\t1\n" } }); + fireEvent.click(screen.getByText("settings.downloadAuth.save")); + await waitFor(() => + expect(h.setDownloadCredential).toHaveBeenCalledWith( + "facebook", + "# Netscape\n.facebook.com\tTRUE\t/\tT\t0\tc\t1\n", + ), + ); + }); + + it("re-opening Update on a configured host still shows an empty field", async () => { + renderPanel(); + await screen.findByTestId("cookie-host-instagram"); + fireEvent.click(screen.getByText("settings.downloadAuth.update")); + const field = (await screen.findByTestId("cookie-input-instagram")) as HTMLTextAreaElement; + expect(field.value).toBe(""); // the stored blob is never fetched into the field + }); +}); diff --git a/frontend/src/components/SettingsPanel.tsx b/frontend/src/components/SettingsPanel.tsx index f97ee5ab..49b378c5 100644 --- a/frontend/src/components/SettingsPanel.tsx +++ b/frontend/src/components/SettingsPanel.tsx @@ -583,6 +583,145 @@ function PlexWatchSync() { ); } +// Per-user download authentication (S6): store your own login cookies for an auth-walled host +// (e.g. a private Facebook group you belong to) so downloads from it use YOUR login. Write-only — +// the stored blob is never shown back; each row shows only the host + whether it's set + when. +// A cookie-authed download is kept private to your account (never cache-shared with another user). +// Exported for a focused DOM test (write-only behaviour); rendered inside Account below. +export function DownloadAuth() { + const { t, i18n } = useTranslation(); + const qc = useQueryClient(); + const confirm = useConfirm(); + const q = useQuery({ + queryKey: qk.downloadCredentials(), + queryFn: () => api.downloadCredentials(), + }); + const [openHost, setOpenHost] = useState(null); + const [draft, setDraft] = useState(""); + + const save = useMutation({ + mutationFn: ({ host, cookies }: { host: string; cookies: string }) => + api.setDownloadCredential(host, cookies), + onSuccess: (res) => { + qc.setQueryData(qk.downloadCredentials(), res); + setOpenHost(null); + setDraft(""); + notify({ level: "success", message: t("settings.downloadAuth.saved") }); + }, + // Validation errors (empty / not-a-cookies.txt) surface via the global error modal, which + // localizes the structured { code: "cookies" } detail (see lib/api core localizeDetail). + }); + const remove = useMutation({ + mutationFn: (host: string) => api.deleteDownloadCredential(host), + onSuccess: (res) => { + qc.setQueryData(qk.downloadCredentials(), res); + notify({ level: "success", message: t("settings.downloadAuth.removed") }); + }, + }); + + const hosts = q.data?.hosts ?? []; + const onRemove = async (host: string) => { + const ok = await confirm({ + title: t("settings.downloadAuth.removeConfirmTitle"), + message: t("settings.downloadAuth.removeConfirmBody"), + confirmLabel: t("settings.downloadAuth.removeConfirmAction"), + danger: true, + }); + if (ok) remove.mutate(host); + }; + + return ( +
+

{t("settings.downloadAuth.intro")}

+
+ {hosts.map((h) => { + const when = h.updated_at ? new Date(h.updated_at).toLocaleDateString(i18n.language) : ""; + const open = openHost === h.host_key; + return ( +
+
+
+
{h.label}
+

+ {h.configured + ? t("settings.downloadAuth.configured", { when }) + : t("settings.downloadAuth.notSet")} +

+
+
+ {h.configured && ( + + )} + +
+
+ {open && ( +
+ {/* Write-only: the field is ALWAYS empty on open — the stored blob is never + fetched or prefilled. */} +