fix(downloads): a YouTube clip goes with the user's own cookie and is diagnosed like a download

A clip fetched its range signed out, so a video behind YouTube's per-video "not a bot" wall could
never be clipped while its download went through with the user's cookie - and the failure copy told
the user to add a cookie the clip would not use. The clip asset is private by construction
(`clip.asset_ref` names its user), so `usable_clip_cookie` attaches the user's YouTube cookie on that
identity, and a failed range fetch now carries the download catalog's diagnosis
(`youtube.bot_wall` / `youtube.cookie_rejected`) instead of a bare category.
This commit is contained in:
2026-10-10 03:58:27 +02:00
parent d466ae915b
commit f221c1962a
3 changed files with 170 additions and 18 deletions
+31
View File
@@ -30,6 +30,7 @@ from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import logs from app import logs
from app.downloads import clip as clipmod
from app.downloads import formats from app.downloads import formats
from app.downloads.adapters import _host_is from app.downloads.adapters import _host_is
from app.models import DownloadCredential, MediaAsset from app.models import DownloadCredential, MediaAsset
@@ -259,6 +260,28 @@ def usable_cookie(
return None return None
if not is_authed_asset(asset, spec, user_id, url): if not is_authed_asset(asset, spec, user_id, url):
return None # shared asset — never attach a cookie to a non-private identity return None # shared asset — never attach a cookie to a non-private identity
return _readable(cred, user_id, host_key)
def usable_clip_cookie(
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict
) -> str | None:
"""`usable_cookie` for an ad-hoc clip's range fetch. The second gate is the clip's own identity:
a clip asset is private by construction (`clip.asset_ref` names its user, see downloads/clip),
so a cookie goes with it exactly when that asset is THIS user's clip of `clip_spec`. Without it a
video YouTube gates signed out could never be clipped, while its download goes through."""
host_key = host_key_for_url(url)
if host_key is None:
return None
cred = get_credential(db, user_id, host_key)
if cred is None:
return None
if asset.source_kind != "clip" or asset.source_ref != clipmod.asset_ref(user_id, clip_spec):
return None
return _readable(cred, user_id, host_key)
def _readable(cred: DownloadCredential, user_id: int, host_key: str) -> str | None:
raw = decrypt_optional(cred.cookies_enc) raw = decrypt_optional(cred.cookies_enc)
if not raw: if not raw:
log.warning( log.warning(
@@ -284,6 +307,14 @@ def resolve_cookiefile(
return _write_cookiefile(raw, staging) if raw else None return _write_cookiefile(raw, staging) if raw else None
def resolve_clip_cookiefile(
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict, staging: Path
) -> Path | None:
"""`resolve_cookiefile` for an ad-hoc clip (`usable_clip_cookie`)."""
raw = usable_clip_cookie(db, user_id, url, asset, clip_spec)
return _write_cookiefile(raw, staging) if raw else None
def _write_cookiefile(raw: str, staging: Path) -> Path: def _write_cookiefile(raw: str, staging: Path) -> Path:
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path. """Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
+61 -18
View File
@@ -41,6 +41,15 @@ class _Aborted(Exception):
"""Raised inside the progress hook when the job was paused/canceled mid-download.""" """Raised inside the progress hook when the job was paused/canceled mid-download."""
class _ClipFetchFailed(Exception):
"""A clip's YouTube range fetch failed: its message and the download catalog's diagnosis, so a
clip on YouTube's wall says what a download of the same video would."""
def __init__(self, msg: str, diagnosis: dlerrors.Diagnosis):
super().__init__(msg)
self.diagnosis = diagnosis
class _OwnerRefused(Exception): class _OwnerRefused(Exception):
"""A refusal that belongs to THIS job's owner, not to the source. """A refusal that belongs to THIS job's owner, not to the source.
@@ -351,7 +360,8 @@ def _extract_with_fallback(
download as itself. download as itself.
`cookiefile` is the acting user's per-host cookies.txt, already gated by `cookiefile` is the acting user's per-host cookies.txt, already gated by
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated. `credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so
the fallback stays authenticated.
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. `max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt.
@@ -747,11 +757,18 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
def _fail_edit( def _fail_edit(
job_id: int, asset_id: int, msg: str, exc: BaseException | None = None, *, what: str = "edit" job_id: int,
asset_id: int,
msg: str,
exc: BaseException | None = None,
*,
what: str = "edit",
diagnosis: dlerrors.Diagnosis | None = None,
) -> None: ) -> None:
"""Fail a local ffmpeg job — an edit, or an upload's conversion (`what="upload"`) — and say so """Fail a local ffmpeg job — an edit, a clip, or an upload's conversion (`what="upload"`) — and
ONCE: every way one fails comes through here, so this is where its line and its event code say so ONCE: every way one fails comes through here, so this is where its line and its event code
(`<what>.job.failed`) live. `exc` is the exception when there was one. (`<what>.job.failed`) live. `exc` is the exception when there was one; `diagnosis` replaces the
plain category when the failure was a fetch the download catalog can name (a clip's).
Its traceback goes with the line unless the message already tells it all (`_failure_stack`). Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
Logged after the write, as on the download error path.""" Logged after the write, as on the download error path."""
@@ -766,9 +783,12 @@ def _fail_edit(
job_id, job_id,
status=JobStatus.ERROR, status=JobStatus.ERROR,
error=msg, error=msg,
error_code=dlerrors.classify_conversion(msg) error_code=diagnosis.code
if diagnosis
else dlerrors.classify_conversion(msg)
if what == "upload" if what == "upload"
else dlerrors.classify(msg), else dlerrors.classify(msg),
error_detail=diagnosis.detail if diagnosis else None,
phase=None, phase=None,
) )
db.commit() db.commit()
@@ -914,24 +934,44 @@ def _clip_source(
the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at
0 there); a Plex clip reads the library file in place.""" 0 there); a Plex clip reads the library file in place."""
if spec["source"] == "youtube": if spec["source"] == "youtube":
url = service.source_url("youtube", spec["ref"])
cookiefile: Path | None = None
with SessionLocal() as db: with SessionLocal() as db:
job = db.get(DownloadJob, job_id) job = db.get(DownloadJob, job_id)
asset = db.get(MediaAsset, asset_id)
remaining = quota.remaining_bytes(db, job.user_id) if job else None remaining = quota.remaining_bytes(db, job.user_id) if job else None
# Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates
# single videos signed out ("not a bot"), and the clip is theirs alone.
if job and asset:
cookiefile = credentials.resolve_clip_cookiefile(
db, job.user_id, url, asset, spec, staging
)
if remaining == 0: # see `_download`: zero is a refusal, not "no limit" if remaining == 0: # see `_download`: zero is a refusal, not "no limit"
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE) raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
_set_job(job_id, phase="downloading", progress=0) _set_job(job_id, phase="downloading", progress=0)
# Signed out, like a download without a cookie: the cookie gate is written for a download's try:
# salted asset identity (`credentials.resolve_cookiefile`), not for a clip's. info = _extract_with_fallback(
info = _extract_with_fallback( job_id,
job_id, asset_id,
asset_id, url,
service.source_url("youtube", spec["ref"]), formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
formats.normalize(clipmod.YOUTUBE_FETCH_SPEC), staging,
staging, adapters.resolve_adapter("youtube", spec["ref"]),
adapters.resolve_adapter("youtube", spec["ref"]), cookiefile=str(cookiefile) if cookiefile else None,
max_bytes=remaining, max_bytes=remaining,
section=(spec["start_s"], spec["end_s"]), section=(spec["start_s"], spec["end_s"]),
) )
except (_Aborted, _OwnerRefused):
raise
except Exception as exc: # re-raised below with the download catalog's diagnosis
msg = str(exc)[:500]
raise _ClipFetchFailed(
msg, dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
) from exc
finally:
# Never leave a plaintext session cookie on disk past the fetch (see `_download`).
if cookiefile is not None:
cookiefile.unlink(missing_ok=True)
info = _media_info(dict(info)) info = _media_info(dict(info))
title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info)) title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info))
return _produced_file(info, staging), 0.0, title return _produced_file(info, staging), 0.0, title
@@ -1065,6 +1105,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
formats.log_safe(msg), formats.log_safe(msg),
extra=logs.event("clip.job.refused", job_id=job_id), extra=logs.event("clip.job.refused", job_id=job_id),
) )
except _ClipFetchFailed as exc:
# silent-ok: not silent — `_fail_edit` is the report, with the fetch's diagnosis.
_fail_edit(job_id, asset_id, str(exc), exc, what="clip", diagnosis=exc.diagnosis)
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack. # silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip") _fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip")
+78
View File
@@ -440,3 +440,81 @@ def test_a_youtube_clip_outside_the_catalog_takes_its_title_from_the_fetch(
"Found On YouTube (clip)", "Found On YouTube (clip)",
"Found On YouTube (clip)", "Found On YouTube (clip)",
) )
# --- a YouTube clip goes with the user's own cookie, and its failure is diagnosed -----------------
_YT_COOKIES = "# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tabc\n"
@pytest.fixture
def fernet(monkeypatch):
from cryptography.fernet import Fernet
from app import security
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
def _capture_extract(monkeypatch, asked: dict):
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
cookiefile = kw.get("cookiefile")
asked["cookiefile"] = cookiefile
asked["cookie_text"] = Path(cookiefile).read_text() if cookiefile else None
f = Path(staging) / "abcdefghijk.mp4"
f.write_bytes(b"range")
return {"requested_downloads": [{"filepath": str(f)}]}
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
def test_a_youtube_clip_goes_signed_in_with_the_users_own_cookie_and_leaves_none_behind(
db, make_user, as_user, yt_video, worker, monkeypatch, fernet
):
from app.downloads import credentials
user = make_user()
credentials.upsert_credential(db, user.id, "youtube", _YT_COOKIES)
db.commit()
asked: dict = {}
_capture_extract(monkeypatch, asked)
job, _asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
assert job.status == "done"
assert asked["cookie_text"] == _YT_COOKIES
assert not Path(asked["cookiefile"]).exists()
def test_a_youtube_clip_never_goes_with_another_users_cookie(
db, make_user, as_user, yt_video, worker, monkeypatch, fernet
):
from app.downloads import credentials
other = make_user()
credentials.upsert_credential(db, other.id, "youtube", _YT_COOKIES)
db.commit()
asked: dict = {}
_capture_extract(monkeypatch, asked)
job, _asset = worker["run"](_clip(as_user(make_user()), source="youtube", ref=yt_video).json())
assert job.status == "done" and asked["cookiefile"] is None
@pytest.mark.parametrize(
("with_cookie", "detail"), [(False, "youtube.bot_wall"), (True, "youtube.cookie_rejected")]
)
def test_a_youtube_clip_on_the_bot_wall_is_diagnosed_like_a_download(
db, make_user, as_user, yt_video, worker, monkeypatch, fernet, with_cookie, detail
):
from app.downloads import credentials
user = make_user()
if with_cookie:
credentials.upsert_credential(db, user.id, "youtube", _YT_COOKIES)
db.commit()
def walled(*_a, **_k):
raise RuntimeError(f"ERROR: [youtube] {yt_video}: Sign in to confirm you’re not a bot.")
monkeypatch.setattr(pipeline, "_extract_with_fallback", walled)
job, asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
assert (job.status, asset.status) == ("error", "error")
assert (job.error_code, job.error_detail) == ("login_required", detail)