peter
f221c1962a
A clip fetched its range signed out, so a video behind YouTube's per-video "not a bot" wall could never be clipped while its download went through with the user's cookie - and the failure copy told the user to add a cookie the clip would not use. The clip asset is private by construction (`clip.asset_ref` names its user), so `usable_clip_cookie` attaches the user's YouTube cookie on that identity, and a failed range fetch now carries the download catalog's diagnosis (`youtube.bot_wall` / `youtube.cookie_rejected`) instead of a bare category.
336 lines
15 KiB
Python
336 lines
15 KiB
Python
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
|
cookie apply to this download, and how does that keep the result private".
|
|
|
|
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
|
in prose):
|
|
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
|
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
|
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
|
cookie the user added after a failure goes with the next attempt.
|
|
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
|
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
|
asset (the load-bearing cross-user privacy fix).
|
|
|
|
Both key the private identity on `cookie_salt` / `host_key_for_url` here, so they can't drift.
|
|
|
|
Security controls enforced here:
|
|
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
|
registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it
|
|
belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no
|
|
cookie.
|
|
* **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted
|
|
in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route,
|
|
logged, or written to a job's error/snapshot.
|
|
"""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs
|
|
from app.downloads import clip as clipmod
|
|
from app.downloads import formats
|
|
from app.downloads.adapters import _host_is
|
|
from app.models import DownloadCredential, MediaAsset
|
|
from app.security import decrypt_optional, encrypt
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
|
|
class HostSpec:
|
|
"""A supported cookie host: a short canonical `key` (stored on the credential + folded into the
|
|
cache salt), a human `label` for the UI, and the netloc `domains` it matches."""
|
|
|
|
__slots__ = ("key", "label", "domains")
|
|
|
|
def __init__(self, key: str, label: str, domains: tuple[str, ...]):
|
|
self.key = key
|
|
self.label = label
|
|
self.domains = domains
|
|
|
|
|
|
# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several
|
|
# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt
|
|
# all derive from this, so there is nothing else to wire up.
|
|
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
|
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
|
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
|
# job 273: every player client fails on that one video while its neighbours download fine, and
|
|
# the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
|
|
# row here the `login_required` copy sent the user to a dropdown that could not help them.
|
|
HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
|
|
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
|
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
|
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
|
HostSpec("reddit", "Reddit", ("reddit.com",)),
|
|
)
|
|
|
|
_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS}
|
|
|
|
|
|
def is_supported_host(host_key: str) -> bool:
|
|
return host_key in _BY_KEY
|
|
|
|
|
|
def host_label(host_key: str) -> str:
|
|
spec = _BY_KEY.get(host_key)
|
|
return spec.label if spec else host_key
|
|
|
|
|
|
def host_key_for_url(url: str | None) -> str | None:
|
|
"""The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via
|
|
`_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When
|
|
several specs could match, the most-specific (longest matched domain) wins so the result is
|
|
deterministic across the enqueue and worker call sites."""
|
|
if not url:
|
|
return None
|
|
from urllib.parse import urlparse
|
|
|
|
host = urlparse(url).netloc.lower()
|
|
if not host:
|
|
return None
|
|
best_key: str | None = None
|
|
best_len = -1
|
|
for spec in SUPPORTED_HOSTS:
|
|
for dom in spec.domains:
|
|
if _host_is(host, dom) and len(dom) > best_len:
|
|
best_key, best_len = spec.key, len(dom)
|
|
return best_key
|
|
|
|
|
|
def cookie_salt(user_id: int, host_key: str) -> str:
|
|
"""The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed
|
|
asset is private to that user (never dedup-shared) and stable across that user's own re-adds."""
|
|
return f"cookie:{user_id}:{host_key}"
|
|
|
|
|
|
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
|
"""The salted `format_sig` an authed download for (user, host) resolves to under the current
|
|
signature version — the private asset enqueue creates, and one of the identities `is_authed_asset`
|
|
recognises. Neither calls it; the tests use it to hold the two to each other."""
|
|
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
|
|
|
|
|
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
|
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
|
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
|
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
|
one, where a salt built from the job's previous asset id could not be."""
|
|
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
|
|
|
|
|
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
|
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
|
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
|
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
|
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie).
|
|
Either signature version counts (`formats.format_sigs`): a job queued before a deploy that bumped
|
|
the version still downloads into the asset it was given."""
|
|
host_key = host_key_for_url(url)
|
|
if host_key is None:
|
|
return False
|
|
salt = cookie_salt(user_id, host_key)
|
|
if asset.format_sig in formats.format_sigs(spec, salt=salt):
|
|
return True
|
|
return asset.id is not None and asset.format_sig in formats.format_sigs(
|
|
spec, salt=rebuild_salt(asset.id, salt)
|
|
)
|
|
|
|
|
|
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
|
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
|
normal shared download). Returns the salt string so the caller passes it straight to
|
|
`format_sig(spec, salt=...)`."""
|
|
host_key = host_key_for_url(url)
|
|
if host_key is None:
|
|
return None
|
|
if get_credential(db, user_id, host_key) is None:
|
|
return None
|
|
return cookie_salt(user_id, host_key)
|
|
|
|
|
|
# --- storage (CRUD) ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
|
return db.execute(
|
|
select(DownloadCredential).where(
|
|
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
|
)
|
|
).scalar_one_or_none()
|
|
|
|
|
|
class InvalidCookies(ValueError):
|
|
"""The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route."""
|
|
|
|
|
|
def validate_cookies(raw: str) -> str:
|
|
"""Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads
|
|
the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path,
|
|
secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty,
|
|
or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than
|
|
silently producing a cookie file yt-dlp ignores."""
|
|
text = (raw or "").strip()
|
|
if not text:
|
|
raise InvalidCookies("empty")
|
|
has_data_line = any(
|
|
line.strip() and not line.lstrip().startswith("#") and "\t" in line
|
|
for line in text.splitlines()
|
|
)
|
|
if not has_data_line:
|
|
raise InvalidCookies("format")
|
|
# Netscape cookie files must end with a newline for some parsers; normalise.
|
|
return text if text.endswith("\n") else text + "\n"
|
|
|
|
|
|
def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential:
|
|
"""Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is
|
|
supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit
|
|
row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call."""
|
|
cred = get_credential(db, user_id, host_key)
|
|
enc = encrypt(raw)
|
|
if cred is None:
|
|
cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc)
|
|
db.add(cred)
|
|
else:
|
|
cred.cookies_enc = enc
|
|
return cred
|
|
|
|
|
|
def delete_credential(db: Session, user_id: int, host_key: str) -> bool:
|
|
"""Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit."""
|
|
cred = get_credential(db, user_id, host_key)
|
|
if cred is None:
|
|
return False
|
|
db.delete(cred)
|
|
return True
|
|
|
|
|
|
def status_list(db: Session, user_id: int) -> list[dict]:
|
|
"""The supported hosts with this user's configured/updated state — the write-only Settings view.
|
|
Never includes the blob."""
|
|
have = {
|
|
c.host_key: c
|
|
for c in db.execute(
|
|
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
|
).scalars()
|
|
}
|
|
out = []
|
|
for spec in SUPPORTED_HOSTS:
|
|
cred = have.get(spec.key)
|
|
out.append(
|
|
{
|
|
"host_key": spec.key,
|
|
"label": spec.label,
|
|
"configured": cred is not None,
|
|
"updated_at": cred.updated_at.isoformat() if cred is not None else None,
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
# --- worker: resolve + write the cookie file --------------------------------------------------
|
|
|
|
|
|
def usable_cookie(
|
|
db: Session, user_id: int, url: str | None, asset: MediaAsset, spec: dict
|
|
) -> str | None:
|
|
"""The user's cookie text a download of `url` into `asset` goes with, or None — the ONE decision
|
|
whether a download is signed in. The worker child writes it out for yt-dlp (`resolve_cookiefile`);
|
|
the worker asks it again for a job that inherits a failed asset. That second ask sees the
|
|
credential as it is when the job is claimed: a cookie the user removed or added since the failed
|
|
download changes this job's explanation to fit the cookie its Retry would go with.
|
|
|
|
The double gate that prevents cross-user leakage:
|
|
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
|
2. the asset being downloaded into must BE this user's private, salted identity
|
|
(`is_authed_asset`).
|
|
|
|
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
|
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
|
InvalidToken) also yields None; the download then fails honestly on the login wall (the
|
|
`login_required` category)."""
|
|
host_key = host_key_for_url(url)
|
|
if host_key is None:
|
|
return None
|
|
cred = get_credential(db, user_id, host_key)
|
|
if cred is None:
|
|
return None
|
|
if not is_authed_asset(asset, spec, user_id, url):
|
|
return None # shared asset — never attach a cookie to a non-private identity
|
|
return _readable(cred, user_id, host_key)
|
|
|
|
|
|
def usable_clip_cookie(
|
|
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict
|
|
) -> str | None:
|
|
"""`usable_cookie` for an ad-hoc clip's range fetch. The second gate is the clip's own identity:
|
|
a clip asset is private by construction (`clip.asset_ref` names its user, see downloads/clip),
|
|
so a cookie goes with it exactly when that asset is THIS user's clip of `clip_spec`. Without it a
|
|
video YouTube gates signed out could never be clipped, while its download goes through."""
|
|
host_key = host_key_for_url(url)
|
|
if host_key is None:
|
|
return None
|
|
cred = get_credential(db, user_id, host_key)
|
|
if cred is None:
|
|
return None
|
|
if asset.source_kind != "clip" or asset.source_ref != clipmod.asset_ref(user_id, clip_spec):
|
|
return None
|
|
return _readable(cred, user_id, host_key)
|
|
|
|
|
|
def _readable(cred: DownloadCredential, user_id: int, host_key: str) -> str | None:
|
|
raw = decrypt_optional(cred.cookies_enc)
|
|
if not raw:
|
|
log.warning(
|
|
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
|
"treated as no cookie",
|
|
user_id,
|
|
host_key,
|
|
)
|
|
return None
|
|
return raw
|
|
|
|
|
|
def resolve_cookiefile(
|
|
db: Session,
|
|
user_id: int,
|
|
url: str | None,
|
|
asset: MediaAsset,
|
|
spec: dict,
|
|
staging: Path,
|
|
) -> Path | None:
|
|
"""In the worker child: the cookies.txt to hand yt-dlp for this download (`usable_cookie`), or None."""
|
|
raw = usable_cookie(db, user_id, url, asset, spec)
|
|
return _write_cookiefile(raw, staging) if raw else None
|
|
|
|
|
|
def resolve_clip_cookiefile(
|
|
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict, staging: Path
|
|
) -> Path | None:
|
|
"""`resolve_cookiefile` for an ad-hoc clip (`usable_clip_cookie`)."""
|
|
raw = usable_clip_cookie(db, user_id, url, asset, clip_spec)
|
|
return _write_cookiefile(raw, staging) if raw else None
|
|
|
|
|
|
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
|
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
|
|
|
Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there,
|
|
and `_produced_file` picks the largest file there as the output — either would corrupt a
|
|
cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still
|
|
removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a
|
|
`finally` on the normal/exception paths."""
|
|
auth_dir = staging / ".auth"
|
|
auth_dir.mkdir(mode=0o700, exist_ok=True)
|
|
path = auth_dir / "cookies.txt"
|
|
# Open with 0600 from the start (don't create world-readable then chmod).
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
try:
|
|
os.write(fd, raw.encode("utf-8"))
|
|
finally:
|
|
os.close(fd)
|
|
return path
|