From 864f673ef59470d0736bee973b53eb28f0c500f4 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 16 Sep 2026 02:03:00 +0200 Subject: [PATCH 1/4] fix: a YouTube video behind the bot wall is downloadable with cookies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A YouTube link can fail with "Sign in to confirm you're not a bot" while every other YouTube download works — YouTube gates INDIVIDUAL videos behind a signed-in session. Measured on prod job 273: all twelve of yt-dlp's player clients fail on that one video while its neighbours download fine, the bgutil PO-token sidecar is live and answering, and the newest yt-dlp fails identically. So the download is not broken by our stack or a stale yt-dlp, and no retry, client switch or upgrade gets past it. A cookie is the only way in. The failure copy already sends the user to Settings -> Account -> Download authentication, but YouTube was not in the cookie host registry, so that dropdown could not help them: the app asked for something it did not offer. credentials: YouTube joins SUPPORTED_HOSTS (youtube.com, youtu.be, youtube-nocookie.com). The Settings dropdown, the SEC-12 host scope and the private-asset cache salt all derive from the row, so that is the whole wiring. adapters: a signed-in YouTube download runs only cookie-capable player clients. Measured against yt-dlp's own INNERTUBE_CLIENTS table: with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES is False, and a set it empties that way raises "No player clients have been requested" — which is not a per-client flake it retries past. Our primary set is android_vr, one of those, so without this the cookie would have failed the download on attempt 1 and never reached the POT-backed web fallback it exists for. A test pins the client list against the installed yt-dlp so a version bump turns the gate red instead of silently un-authing every YouTube download, and the empty-set message joins the retriable markers as the second line of defence. errors: catalog entries for the wall with and without a cookie, so a user who has already added cookies is told they were rejected (YouTube rotates exported ones) rather than to add some. The marker is quote-agnostic: YouTube writes the apostrophe as U+2019, so "you're not a bot" with a straight quote never matches the real text. EN + HU copy. --- backend/app/config.py | 3 ++ backend/app/downloads/adapters.py | 35 +++++++++++++++-- backend/app/downloads/credentials.py | 5 +++ backend/app/downloads/errors.py | 24 ++++++++++++ backend/tests/test_adapters.py | 35 +++++++++++++++++ backend/tests/test_download_credentials.py | 23 +++++++++-- backend/tests/test_download_errors.py | 43 +++++++++++++++++++++ frontend/src/i18n/locales/en/downloads.json | 10 +++++ frontend/src/i18n/locales/hu/downloads.json | 10 +++++ 9 files changed, 182 insertions(+), 6 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 0483011c..334e85e5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -155,6 +155,9 @@ class Settings(BaseSettings): # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token + # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works. + # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients + # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there. + # See adapters.YouTubeAdapter._COOKIE_CAPABLE. download_player_clients: str = "android_vr" download_player_clients_fallback: str = "web_safari,web,android" diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 16c2994d..45682446 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -93,12 +93,41 @@ class YouTubeAdapter(SourceAdapter): "Requested format is not available", "Only images are available", "Sign in to confirm", + # A set yt-dlp emptied because none of its clients take cookies. `client_sets` filters those + # out below, so this is the safety net for a yt-dlp that renames a client or an admin who + # configures both knobs with cookie-incapable clients only. + "No player clients have been requested", ) + # Measured against yt-dlp's own INNERTUBE_CLIENTS table (2026-09-16, on 2026.07.04 and + # 2026.08.19): with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES + # is False, and a set left empty raises "No player clients have been requested". Our primary set + # is android_vr — one of those — so a signed-in download would fail on the first attempt and + # never reach the POT-backed web fallback that the cookie exists for. Pinned by test_adapters + # against the installed yt-dlp, so a version bump that moves the table turns the gate red + # instead of silently un-authing every YouTube download. + _COOKIE_CAPABLE = frozenset( + { + "mweb", + "tv", + "tv_downgraded", + "web", + "web_creator", + "web_embedded", + "web_music", + "web_safari", + } + ) + # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all. + # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for + # the per-client flakiness that the two-tier ladder gives a signed-out download. + _AUTHED_DEFAULT = ("web_safari", "web") def client_sets(self, authed: bool = False) -> list: - return [ - c for c in (settings.player_client_list, settings.player_client_fallback_list) if c - ] or [None] + sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c] + if authed: + kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets] + return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)] + return sets or [None] def ydl_opts_extra(self, clients) -> dict: # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values). diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 7e87388a..c3e20ebb 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -55,6 +55,11 @@ class HostSpec: # all derive from this, so there is nothing else to wire up. SUPPORTED_HOSTS: tuple[HostSpec, ...] = ( HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273: every player client fails on that one video while its neighbours download fine, and + # the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a + # row here the `login_required` copy sent the user to a dropdown that could not help them. + HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")), HostSpec("instagram", "Instagram", ("instagram.com",)), HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")), HostSpec("tiktok", "TikTok", ("tiktok.com",)), diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 47810a8a..48cae607 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -212,6 +212,30 @@ _CATALOG: tuple[_Entry, ...] = ( cookie=False, link=facebook_group_post_id, ), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its + # neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails + # identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp). + # The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so + # "you're not a bot" with a straight quote never matches the real text. + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.bot_wall", + cookie=False, + ), + # The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first + # one is being added now) — but "add cookies" is provably the wrong advice for someone who has, + # and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no + # longer valid. They have likely been rotated in the browser as a security measure." + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.cookie_rejected", + cookie=True, + ), ) diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index ef5ac453..88880062 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -1,6 +1,7 @@ """Per-source adapter selection + the source-specific policy each one carries. These lock the behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch.""" +from app.config import settings from app.titles import normalize_title from app.downloads.adapters import ( FacebookAdapter, @@ -111,6 +112,9 @@ def test_client_sets_and_retriable_are_per_source(): assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True assert YouTubeAdapter().retriable("some unrelated network blip") is False + # The empty-set error of a cookie-incapable client list (see _COOKIE_CAPABLE): retriable, so an + # admin who configures one still gets the next set tried instead of a hard failure. + assert YouTubeAdapter().retriable("ERROR: No player clients have been requested") is True # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind # the parse wall (without a cookie that wall is the login wall — no disguise gets past it). fb = FacebookAdapter() @@ -121,6 +125,37 @@ def test_client_sets_and_retriable_are_per_source(): assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False +def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch): + yt = YouTubeAdapter() + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android") + # Signed out: the admin's ladder exactly as configured. + assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]] + # Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way + # raises "No player clients have been requested" — which would fail the download on attempt 1, + # before the POT-backed web fallback the cookie exists for. So the adapter never asks for one: + # the android_vr-only set is gone and `android` is dropped out of the fallback. + assert yt.client_sets(authed=True) == [["web_safari", "web"]] + + +def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch): + # Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all. + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios") + assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]] + + +def test_the_cookie_capable_clients_match_the_installed_ytdlp(): + # The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a + # yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending + # a signed-in download to a client that drops the cookie. + from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS + + live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")} + assert live == set(YouTubeAdapter._COOKIE_CAPABLE) + assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live + + def test_youtube_ydl_opts_extra_carries_player_client_and_pot(): ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"] assert ea["youtube"]["player_client"] == ["android_vr"] diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 1dcdace6..1b072dc0 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -39,13 +39,23 @@ class TestHostKeyForUrl: assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook" assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook" assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter" + # YouTube gates individual videos behind its bot check; the cookie is the only way past it. + for url in ( + "https://www.youtube.com/watch?v=abc", + "https://youtu.be/abc", + "https://music.youtube.com/watch?v=abc", + "https://www.youtube-nocookie.com/embed/abc", + ): + assert credentials.host_key_for_url(url) == "youtube", url def test_look_alike_host_never_matches(self): assert credentials.host_key_for_url("https://myfacebook.com/x") is None assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None + assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None def test_unrelated_or_empty_url_has_no_host_key(self): - assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://vimeo.com/123456") is None assert credentials.host_key_for_url("") is None assert credentials.host_key_for_url(None) is None @@ -229,9 +239,16 @@ class TestResolveCookiefile: credentials.upsert_credential(db, u.id, "facebook", COOKIES) db.commit() spec = formats.normalize({}) - # A YouTube URL has no supported host_key -> no cookie regardless of what's stored. - yt = "https://www.youtube.com/watch?v=abc" + # A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is + # a supported host in its own right now, so the case needs a host that is not. + other = "https://vimeo.com/123456" salted = credentials.authed_format_sig(spec, u.id, "facebook") + assert ( + credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path) + is None + ) + # ...and a YouTube link gets nothing either: the stored credential is Facebook's. + yt = "https://www.youtube.com/watch?v=abc" assert ( credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path) is None diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index 0745e898..e120c765 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -84,6 +84,49 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" _GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?" +# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why +# neither the rule nor the catalog entry may spell out "you’re" with a straight quote. +_YT_BOT_WALL = ( + "ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use " + "--cookies-from-browser or --cookies for the authentication. See " + "https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually " + "pass cookies." +) +_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI" + + +def test_youtube_bot_wall_is_that_video_own_signed_out_gate(): + # Measured on prod: all 12 player clients fail on this video while its neighbours download, the + # POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in. + assert classify(_YT_BOT_WALL) == "login_required" + for url in ( + _YT, + "https://youtu.be/7JPmS_bWjhI", + "https://www.youtube-nocookie.com/embed/7JPmS_bWjhI", + "https://music.youtube.com/watch?v=7JPmS_bWjhI", + ): + assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == ( + "login_required", + "youtube.bot_wall", + ), url + + +def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused(): + # Telling a user who already gave a cookie to add one is what this entry exists to avoid. + assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == ( + "login_required", + "youtube.cookie_rejected", + ) + + +def test_the_youtube_entries_never_match_another_site(): + # Host-scoped: the same words about a Facebook link stay the bare category. + assert dlerrors.diagnose( + "ERROR: [facebook] 1: Sign in to confirm you’re not a bot", + url="https://www.facebook.com/reel/1/", + ) == ("login_required", None) + + def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie(): for url in ( _REEL, diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 55c0d9e4..d566d29c 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -304,6 +304,16 @@ "short": "No video found in this post", "body": "We couldn't find a single downloadable video in this group post. If it does have one, open the video on Facebook and paste its link." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube sign-in required", + "body": "YouTube only serves this video to a signed-in viewer — it asks the downloader to confirm it isn't a bot. Other YouTube videos are unaffected. Add your YouTube cookies under Settings → Account → Download authentication, then retry the download from the list." + }, + "cookie_rejected": { + "short": "YouTube didn't accept the cookies", + "body": "YouTube rotates exported cookies as a security measure, so yours have probably expired. Export them again from a private browsing window, close that window without browsing on, paste them in under Settings → Account → Download authentication, then retry the download from the list." + } } } }, diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index 1bfecac3..9c03a6b7 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -304,6 +304,16 @@ "short": "Ebben a posztban nem található videó", "body": "Nem találtunk egyértelműen letölthető videót ebben a csoportposztban. Ha mégis van benne, nyisd meg a videót a Facebookon, és annak a linkjét másold be." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube-bejelentkezés szükséges", + "body": "A YouTube ezt a videót csak bejelentkezett nézőnek adja ki — a letöltőtől azt kéri, igazolja, hogy nem robot. A többi YouTube-videót ez nem érinti. Illeszd be a YouTube-cookie-kat a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + }, + "cookie_rejected": { + "short": "A YouTube nem fogadta el a cookie-kat", + "body": "A YouTube biztonsági okból cserélgeti az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + } } } }, From c06a456a9aa919a289ed8f39ac82b061f3f4da42 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 16 Sep 2026 03:43:29 +0200 Subject: [PATCH 2/4] fix: a YouTube stream refused after extraction is the gate, not a network fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on the isolated test stack the same night, on the very same video ~90 minutes after it was answering with the bot message: yt-dlp got the player response and picked its formats, and the CDN then refused the stream itself — "unable to download video data: HTTP Error 403: Forbidden". Signed out it did that on every attempt; with the owner's cookie attached the video downloaded in full. So the wall moves between the player response and the stream, and the text it wears there was costing the download twice. retriable: the format URLs belong to the client that minted them, so the next client set is a real second chance — but this text classifies as `network` and nothing in the YouTube marker list matched it, so the ladder ended the job on attempt 1 and never tried the fallback. catalog: `youtube.media_forbidden` for the cookie-less case, so the UI stops calling it a network error when nothing was wrong with the network, and names what actually gets past it. Scoped exactly as narrowly as the measurement: another host keeps the generic `network` category, and so does a download that already carried a cookie — telling a signed-in user to sign in is the failure the catalog exists to avoid. EN + HU copy. --- backend/app/downloads/adapters.py | 7 +++++++ backend/app/downloads/errors.py | 13 ++++++++++++ backend/tests/test_adapters.py | 11 ++++++++++ backend/tests/test_download_errors.py | 23 +++++++++++++++++++++ frontend/src/i18n/locales/en/downloads.json | 4 ++++ frontend/src/i18n/locales/hu/downloads.json | 4 ++++ 6 files changed, 62 insertions(+) diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 45682446..5aead80d 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -97,6 +97,13 @@ class YouTubeAdapter(SourceAdapter): # out below, so this is the safety net for a yt-dlp that renames a client or an admin who # configures both knobs with cookie-incapable clients only. "No player clients have been requested", + # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then + # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves + # between the player response and the stream, so this is the same per-client flakiness one + # step later — and it used to end the job on attempt 1, because the text classifies as + # `network` and nothing here matched it. The format URLs belong to the client that minted + # them, so the next client set is a real second chance. + "unable to download video data", ) # Measured against yt-dlp's own INNERTUBE_CLIENTS table (2026-09-16, on 2026.07.04 and # 2026.08.19): with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 48cae607..eaa57424 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -236,6 +236,19 @@ _CATALOG: tuple[_Entry, ...] = ( detail="youtube.cookie_rejected", cookie=True, ), + # The same gate one step later: the player response comes back, and then the CDN refuses the + # stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the + # very same video, ~90 minutes after it was answering with the bot message — signed out it + # 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So + # the category this text carries generically, `network`, is a lie here: nothing was wrong with + # the network. Scoped to a cookie-less YouTube download, where that is what we measured. + _Entry( + host="youtube", + markers=("unable to download video data", "403"), + code="login_required", + detail="youtube.media_forbidden", + cookie=False, + ), ) diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index 88880062..a437d453 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -115,6 +115,17 @@ def test_client_sets_and_retriable_are_per_source(): # The empty-set error of a cookie-incapable client list (see _COOKIE_CAPABLE): retriable, so an # admin who configures one still gets the next set tried instead of a hard failure. assert YouTubeAdapter().retriable("ERROR: No player clients have been requested") is True + # The media fetch refused after a successful extraction: the format URLs belong to the client + # that minted them, so the next client set is a real second chance (it used to end the job). + assert ( + YouTubeAdapter().retriable( + "ERROR: unable to download video data: HTTP Error 403: Forbidden" + ) + is True + ) + assert ( + SourceAdapter().retriable("ERROR: unable to download video data: HTTP Error 403") is False + ) # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind # the parse wall (without a cookie that wall is the login wall — no disguise gets past it). fb = FacebookAdapter() diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index e120c765..c97b55f2 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -119,6 +119,29 @@ def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused(): ) +# Verbatim from the isolated test stack (2026-09-16), the SAME video ~90 minutes later: yt-dlp got +# the player response and picked formats, and the CDN then refused the stream. +_YT_MEDIA_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden" + + +def test_youtube_media_403_is_the_wall_one_step_later_not_a_network_fault(): + # Generically this text is a network failure — and for every other host it stays one. + assert classify(_YT_MEDIA_403) == "network" + assert dlerrors.diagnose(_YT_MEDIA_403, url="https://vimeo.com/1") == ("network", None) + # On a signed-out YouTube download it is the gate: measured, the same video downloaded in full + # once the owner's cookie was attached. Calling that "network error" is the honest-states bug. + assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT) == ( + "login_required", + "youtube.media_forbidden", + ) + + +def test_youtube_media_403_with_a_cookie_is_not_claimed_as_the_gate(): + # Only the cookie-less case was measured. With a cookie the same text keeps its bare category + # rather than telling a signed-in user to sign in. + assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT, cookie_attached=True) == ("network", None) + + def test_the_youtube_entries_never_match_another_site(): # Host-scoped: the same words about a Facebook link stay the bare category. assert dlerrors.diagnose( diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index d566d29c..8a517eba 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -313,6 +313,10 @@ "cookie_rejected": { "short": "YouTube didn't accept the cookies", "body": "YouTube rotates exported cookies as a security measure, so yours have probably expired. Export them again from a private browsing window, close that window without browsing on, paste them in under Settings → Account → Download authentication, then retry the download from the list." + }, + "media_forbidden": { + "short": "YouTube refused the video stream", + "body": "YouTube described the video but then refused to hand over the file to a signed-out download. It comes and goes, so a retry from the list is worth one try — and adding your YouTube cookies under Settings → Account → Download authentication is what gets a stubborn one through." } } } diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index 9c03a6b7..b4ae4080 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -313,6 +313,10 @@ "cookie_rejected": { "short": "A YouTube nem fogadta el a cookie-kat", "body": "A YouTube biztonsági okból cserélgeti az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + }, + "media_forbidden": { + "short": "A YouTube megtagadta a videó adatfolyamát", + "body": "A YouTube leírta a videót, de a fájlt már nem adta ki egy kijelentkezett letöltésnek. Ez jön-megy, ezért egy újrapróbálkozás a listából megér egy kísérletet — a makacs eseteken pedig a YouTube-cookie-k segítenek, a Beállítások → Fiók → Letöltés-hitelesítés alatt." } } } From 2b56c5c2eb8ec56544f3ccdf8f4fec93ac20a188 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 16 Sep 2026 04:29:13 +0200 Subject: [PATCH 3/4] fix: name the error yt-dlp actually raises for a cookie-emptied client set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review round read yt-dlp's source and found the comment behind the previous commit's safety-net marker did not match it. Measured in the worker with a real account cookie and `player_client=android_vr`: yt-dlp logs `Skipping client "android_vr" since it does not support cookies`, drops it, and the request left with no client fails with "Requested format is not available" — not "No player clients have been requested". That raise sits EARLIER in yt-dlp, before the cookie-capability removal, so the cookie path can never reach it; and an unknown or renamed client is skipped with a warning and replaced by yt-dlp's defaults, so the other scenario the comment named cannot reach it either. So the marker bought nothing: the symptom that does occur was already retriable here before this branch. It is dropped, the comment now states what was measured, and the test asserts the real symptom instead of a message the code cannot produce. The cookie-capability filter itself is vindicated by the same measurement — without it every signed-in download would burn its first attempt on a client yt-dlp discards. Also from the round: an enqueue test for a `source_kind="youtube"` job, the one join nothing else made (a YouTube job stores a bare 11-char ref and the URL the cookie gate keys on is rebuilt by the adapter — both ends were pinned apart, the join was not, and a break there would stop every YouTube cookie silently while signed-out downloads kept working). And copy corrections: the group's only unhedged claim is hedged, "described the video" loses the pipeline vocabulary, and four Hungarian phrasings that read as translated English are rewritten. --- backend/app/downloads/adapters.py | 20 +++++++++----------- backend/tests/test_adapters.py | 14 +++++++++++--- backend/tests/test_download_credentials.py | 16 ++++++++++++++++ frontend/src/i18n/locales/en/downloads.json | 4 ++-- frontend/src/i18n/locales/hu/downloads.json | 8 ++++---- 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 5aead80d..3f0c4ca5 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -93,10 +93,6 @@ class YouTubeAdapter(SourceAdapter): "Requested format is not available", "Only images are available", "Sign in to confirm", - # A set yt-dlp emptied because none of its clients take cookies. `client_sets` filters those - # out below, so this is the safety net for a yt-dlp that renames a client or an admin who - # configures both knobs with cookie-incapable clients only. - "No player clients have been requested", # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves # between the player response and the stream, so this is the same per-client flakiness one @@ -105,13 +101,15 @@ class YouTubeAdapter(SourceAdapter): # them, so the next client set is a real second chance. "unable to download video data", ) - # Measured against yt-dlp's own INNERTUBE_CLIENTS table (2026-09-16, on 2026.07.04 and - # 2026.08.19): with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES - # is False, and a set left empty raises "No player clients have been requested". Our primary set - # is android_vr — one of those — so a signed-in download would fail on the first attempt and - # never reach the POT-backed web fallback that the cookie exists for. Pinned by test_adapters - # against the installed yt-dlp, so a version bump that moves the table turns the gate red - # instead of silently un-authing every YouTube download. + # Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with + # `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support + # cookies`, drops it, and the request that is left with no client fails with "Requested format is + # not available" — which is already a retriable marker above, so the cost is a wasted attempt + # rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every + # signed-in download would burn its first attempt that way before reaching the POT-backed web + # fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...] + # ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump + # that moves the table turns the gate red instead of silently un-authing every YouTube download. _COOKIE_CAPABLE = frozenset( { "mweb", diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index a437d453..4aa6ca58 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -112,9 +112,17 @@ def test_client_sets_and_retriable_are_per_source(): assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True assert YouTubeAdapter().retriable("some unrelated network blip") is False - # The empty-set error of a cookie-incapable client list (see _COOKIE_CAPABLE): retriable, so an - # admin who configures one still gets the next set tried instead of a hard failure. - assert YouTubeAdapter().retriable("ERROR: No player clients have been requested") is True + # A client list yt-dlp empties because none of its clients take cookies does NOT raise "No + # player clients have been requested" (that raise sits earlier in yt-dlp, before the + # cookie-capability removal). Measured with a real cookie: the emptied request fails with + # "Requested format is not available", which is retriable here already — so the wasted attempt + # is recoverable and `_COOKIE_CAPABLE` exists to avoid spending it at all. + assert ( + YouTubeAdapter().retriable( + "ERROR: [youtube] abc: Requested format is not available. Use --list-formats" + ) + is True + ) # The media fetch refused after a successful extraction: the format URLs belong to the client # that minted them, so the next client set is a real second chance (it used to end the job). assert ( diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 1b072dc0..3cc8af39 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -185,6 +185,22 @@ class TestCachePrivacy: asset = db.get(MediaAsset, job.asset_id) assert asset.format_sig == formats.format_sig(formats.normalize({})) + def test_a_youtube_job_reaches_the_gate_through_its_rebuilt_watch_url(self, db): + # The one join nothing else makes. A YouTube job stores a bare 11-char ref, and the URL the + # cookie gate keys on is REBUILT by the adapter (service.source_url -> YouTubeAdapter). Both + # ends are pinned apart — the watch-URL shape in test_adapters, the host_key in + # TestHostKeyForUrl — so this welds them: an enqueue that ever handed the gate the raw ref + # would silently stop salting, every YouTube cookie would quietly stop being attached, and + # signed-out downloads would keep working, so nothing else here would go red. + u = _user(db) + credentials.upsert_credential(db, u.id, "youtube", COOKIES) + db.commit() + job = service.enqueue(db, u.id, "youtube", "VIDID12345_", {}) + asset = db.get(MediaAsset, job.asset_id) + spec = formats.normalize({}) + assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "youtube") + assert asset.format_sig != formats.format_sig(spec) + def test_two_users_never_share_the_authed_asset(self, db): a = _user(db, "a@example.com") b = _user(db, "b@example.com") diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 8a517eba..1e750cba 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -308,7 +308,7 @@ "youtube": { "bot_wall": { "short": "YouTube sign-in required", - "body": "YouTube only serves this video to a signed-in viewer — it asks the downloader to confirm it isn't a bot. Other YouTube videos are unaffected. Add your YouTube cookies under Settings → Account → Download authentication, then retry the download from the list." + "body": "YouTube only serves this video to a signed-in viewer — it asks the downloader to confirm it isn't a bot. Other YouTube videos are usually unaffected. Add your YouTube cookies under Settings → Account → Download authentication, then retry the download from the list." }, "cookie_rejected": { "short": "YouTube didn't accept the cookies", @@ -316,7 +316,7 @@ }, "media_forbidden": { "short": "YouTube refused the video stream", - "body": "YouTube described the video but then refused to hand over the file to a signed-out download. It comes and goes, so a retry from the list is worth one try — and adding your YouTube cookies under Settings → Account → Download authentication is what gets a stubborn one through." + "body": "YouTube found the video but refused to send the file to a signed-out download. It comes and goes, so a retry from the list is worth one try — and adding your YouTube cookies under Settings → Account → Download authentication is what gets a stubborn one through." } } } diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index b4ae4080..7c1ec57e 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -308,15 +308,15 @@ "youtube": { "bot_wall": { "short": "YouTube-bejelentkezés szükséges", - "body": "A YouTube ezt a videót csak bejelentkezett nézőnek adja ki — a letöltőtől azt kéri, igazolja, hogy nem robot. A többi YouTube-videót ez nem érinti. Illeszd be a YouTube-cookie-kat a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + "body": "A YouTube ezt a videót csak bejelentkezett nézőnek adja ki — a letöltőprogramtól azt kéri, igazolja, hogy nem robot. A többi YouTube-videót ez általában nem érinti. Illeszd be a YouTube-cookie-kat a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." }, "cookie_rejected": { "short": "A YouTube nem fogadta el a cookie-kat", - "body": "A YouTube biztonsági okból cserélgeti az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + "body": "A YouTube biztonsági okból lecseréli az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." }, "media_forbidden": { - "short": "A YouTube megtagadta a videó adatfolyamát", - "body": "A YouTube leírta a videót, de a fájlt már nem adta ki egy kijelentkezett letöltésnek. Ez jön-megy, ezért egy újrapróbálkozás a listából megér egy kísérletet — a makacs eseteken pedig a YouTube-cookie-k segítenek, a Beállítások → Fiók → Letöltés-hitelesítés alatt." + "short": "A YouTube nem adta ki a videófájlt", + "body": "A YouTube visszaadta a videó adatait, de a fájlt már nem adta ki bejelentkezés nélkül. Ez hol van, hol nincs, ezért érdemes újrapróbálni a listából — a makacs eseteken pedig a YouTube-cookie-k segítenek, a Beállítások → Fiók → Letöltés-hitelesítés alatt." } } } From 3cf8f04d733e973aa6d88cd4b3ea4b50b98a755e Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 16 Sep 2026 04:41:10 +0200 Subject: [PATCH 4/4] chore: 0.87.1 release notes --- CHANGELOG.md | 17 +++++++++++++++++ VERSION | 2 +- frontend/src/lib/releaseNotes.ts | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8ab6c1b..4e66f937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.87.1 — 2026-09-16 + +A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it. + +### Features + +- YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login. + +### Fixes + +- A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again. +- A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network. + +### Internal + +- The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not. + ## 0.87.0 — 2026-09-15 A shared or library video keeps playing when you switch apps on an iPad, and a new tab opens as the account it belongs to without flashing another page first. Facebook reels keep their titles and an mp3 stays an mp3. diff --git a/VERSION b/VERSION index 359ee08a..7921aa12 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.87.0 +0.87.1 diff --git a/frontend/src/lib/releaseNotes.ts b/frontend/src/lib/releaseNotes.ts index af68b8c5..71672ff2 100644 --- a/frontend/src/lib/releaseNotes.ts +++ b/frontend/src/lib/releaseNotes.ts @@ -14,6 +14,22 @@ export interface ReleaseEntry { } export const RELEASE_NOTES: ReleaseEntry[] = [ + { + version: "0.87.1", + date: "2026-09-16", + summary: + "A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it.", + features: [ + "YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login.", + ], + fixes: [ + "A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again.", + "A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network.", + ], + chores: [ + "The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not.", + ], + }, { version: "0.87.0", date: "2026-09-15",