feat(downloads): per-user, per-host yt-dlp cookie/auth support (S6)
Each user stores their own login cookies (Netscape cookies.txt) for an auth-walled download host (Facebook, Instagram, X, TikTok, Reddit) in Settings; a download of an auth-walled link on that host is retried with the acting user's cookie. Two load-bearing security controls, both in downloads.credentials as the single source of truth so enqueue and the worker can't drift: - cache privacy: a cookie-authed download salts its MediaAsset format_sig with the user id, so the authed asset is private and never cache-served to another user. The worker attaches a cookie ONLY when the asset it is downloading into IS that private, salted identity. - host scope (SEC-12): a cookie is resolved only for a URL whose netloc the credential owns (netloc-scoped _host_is); a look-alike/unrelated host gets none. cookies_enc is Fernet-encrypted at rest, only decrypted in-worker into a 0600 file under staging/.auth/ (survives the retry wipe + _produced_file scan) and unlinked in finally. The blob is never returned by a route, logged, audited, or written to a job's error/snapshot. cookiefile is set after the adapter's extra merge so an adapter can't clobber it. Settings UI is write-only (host + set/updated only, never echoed). New download_credentials table + migration 0074; per-user routes; HU+EN i18n. Tests: host-scope, cache-privacy (no cross-user serve), encrypt-at-rest, write-only API, worker attach-gate; FE dom test for the write-only field.
This commit is contained in:
@@ -8,13 +8,14 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, formats, service, storage
|
||||
from app.downloads import adapters, credentials, formats, service, storage
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
_job_status,
|
||||
@@ -304,10 +305,13 @@ def _media_info(info: dict) -> dict:
|
||||
return {**info, **chosen}
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter, cookiefile=None):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated."""
|
||||
client_sets = adapter.client_sets()
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
last_exc: Exception | None = None
|
||||
@@ -317,6 +321,7 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
cookiefile=cookiefile,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
@@ -397,8 +402,11 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
return dest
|
||||
|
||||
|
||||
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
||||
def _download(
|
||||
job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -407,7 +415,27 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
)
|
||||
info = _media_info(
|
||||
_extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
url,
|
||||
spec,
|
||||
staging,
|
||||
adapter,
|
||||
cookiefile=str(cookiefile) if cookiefile else None,
|
||||
)
|
||||
)
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
@@ -517,6 +545,10 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
if cookiefile is not None:
|
||||
cookiefile.unlink(missing_ok=True)
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user