fix(downloads): Retry and Re-download take a cookie added after the failure
The login-wall copy tells the user to add their cookies and press Retry, but Retry kept the job on the shared asset the worker never attaches a cookie to, so it failed on the same wall. Resume now moves a pending/errored job onto the asset enqueue would pick with the cookie (never off a file or a download in flight). Re-download salted its fresh asset with the job's previous asset id, which the cookie gate cannot recompute, so even a cookie download rebuilt without the cookie. The rebuild salt is now the new asset's own id (plus the user's cookie salt when one applies), and is_authed_asset recognises it.
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
||||
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
||||
cookie the user added after a failure goes with the next attempt.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
@@ -30,7 +32,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
@@ -104,12 +106,27 @@ def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def is_authed_asset(asset_format_sig: str, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
||||
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
||||
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
||||
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
||||
one, where a salt built from the job's previous asset id could not be."""
|
||||
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
||||
|
||||
|
||||
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
||||
kind of asset a cookie is ever attached to. The worker's cookie gate asks it before a download,
|
||||
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
||||
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
||||
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie)."""
|
||||
host_key = host_key_for_url(url)
|
||||
return host_key is not None and asset_format_sig == authed_format_sig(spec, user_id, host_key)
|
||||
if host_key is None:
|
||||
return False
|
||||
if asset.format_sig == authed_format_sig(spec, user_id, host_key):
|
||||
return True
|
||||
return asset.id is not None and asset.format_sig == formats.format_sig(
|
||||
spec, salt=rebuild_salt(asset.id, cookie_salt(user_id, host_key))
|
||||
)
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
@@ -211,7 +228,7 @@ def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset_format_sig: str,
|
||||
asset: MediaAsset,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
@@ -220,7 +237,7 @@ def resolve_cookiefile(
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`asset_format_sig == authed_format_sig(...)`).
|
||||
(`is_authed_asset`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
@@ -232,7 +249,7 @@ def resolve_cookiefile(
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if not is_authed_asset(asset_format_sig, spec, user_id, url):
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
|
||||
@@ -8,7 +8,6 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
@@ -446,12 +445,10 @@ def _download(
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
target = cdb.get(MediaAsset, asset_id)
|
||||
if target is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
cdb, user_id, url, target, spec, staging
|
||||
)
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
|
||||
@@ -7,6 +7,7 @@ handles everything else. State transitions used by the routes (pause/resume/canc
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -341,7 +342,11 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset).
|
||||
|
||||
A cookie the user added AFTER the job was queued moves the job onto their private asset first
|
||||
(`_follow_the_cookie`): the worker only ever attaches a cookie to that identity, so without the move
|
||||
the Retry a login-wall message asks for would fail on the same wall."""
|
||||
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
@@ -349,6 +354,8 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
job.error_detail = None
|
||||
if job.asset_id:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is not None:
|
||||
asset = _follow_the_cookie(db, job, asset)
|
||||
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
@@ -356,6 +363,32 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
"""Re-point a requeued job at the asset enqueue would pick NOW, when the user has a cookie for the
|
||||
link's host but the job still holds an asset the worker won't attach it to (queued before the
|
||||
cookie existed). Returns the asset the job holds afterwards.
|
||||
|
||||
Only a `pending` / `error` asset moves — neither has a file nor a download in flight. A `ready` one
|
||||
already has the file, and a `downloading` one may be this job's own child still being stopped by
|
||||
the pause; the next Retry after it settles moves the job then. Removing a cookie never moves a job
|
||||
back: the worker attaches nothing without a credential, so the private asset is as good as any."""
|
||||
if asset.status not in ("pending", "error"):
|
||||
return asset
|
||||
url = source_url(job.source_kind, job.source_ref)
|
||||
salt = credentials.salt_for_enqueue(db, job.user_id, url)
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
if salt is None or credentials.is_authed_asset(asset, spec, job.user_id, url):
|
||||
return asset
|
||||
target = get_or_create_asset(
|
||||
db, job.source_kind, job.source_ref, formats.format_sig(spec, salt=salt)
|
||||
)
|
||||
populate_from_catalog(db, target)
|
||||
release_asset(db, job) # drop the old hold BEFORE re-pointing — it keys off job.asset_id
|
||||
incr_ref_count(db, target.id)
|
||||
job.asset_id = target.id
|
||||
return target
|
||||
|
||||
|
||||
def cancel(db: Session, job: DownloadJob) -> None:
|
||||
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||
@@ -374,13 +407,31 @@ def delete(db: Session, job: DownloadJob) -> None:
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
the new asset's own id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409).
|
||||
|
||||
When the user has a cookie for the link's host, the salt carries it (`credentials.rebuild_salt`),
|
||||
so the worker recognises the rebuild as their private identity and attaches the cookie — a cookie
|
||||
download, or one the user added after a failure, rebuilds with the cookie."""
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||
cookie = credentials.salt_for_enqueue(
|
||||
db, job.user_id, source_url(job.source_kind, job.source_ref)
|
||||
)
|
||||
# The salt names the row's own id, which exists only once the row does: insert it under a unique
|
||||
# placeholder, then write the real signature in the same transaction (nobody sees the placeholder).
|
||||
asset = MediaAsset(
|
||||
source_kind=job.source_kind,
|
||||
source_ref=job.source_ref,
|
||||
format_sig=f"rebuild:{uuid.uuid4().hex}",
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
asset.format_sig = formats.format_sig(spec, salt=credentials.rebuild_salt(asset.id, cookie))
|
||||
db.flush()
|
||||
populate_from_catalog(db, asset)
|
||||
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||
release_asset(db, job)
|
||||
|
||||
@@ -630,8 +630,9 @@ def redownload_download(
|
||||
resolves to the newly-downloaded file — a re-download never breaks a link you already sent out
|
||||
(only a *failed* re-download leaves the job in `error`, which is the intended broken-link case).
|
||||
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job
|
||||
id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the new
|
||||
asset's own id, and the user's cookie salt when one applies — `credentials.rebuild_salt`) rather
|
||||
than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
under the current format/naming rules — never silently reuses a shared cached file — and (b)
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
|
||||
@@ -410,7 +410,7 @@ def _process_job(job_id: int) -> None:
|
||||
asset_error,
|
||||
url=url,
|
||||
cookie_attached=credentials.is_authed_asset(
|
||||
asset.format_sig, formats.normalize(job.profile_snapshot), job.user_id, url
|
||||
asset, formats.normalize(job.profile_snapshot), job.user_id, url
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user