From 864f673ef59470d0736bee973b53eb28f0c500f4 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 16 Sep 2026 02:03:00 +0200 Subject: [PATCH] fix: a YouTube video behind the bot wall is downloadable with cookies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A YouTube link can fail with "Sign in to confirm you're not a bot" while every other YouTube download works — YouTube gates INDIVIDUAL videos behind a signed-in session. Measured on prod job 273: all twelve of yt-dlp's player clients fail on that one video while its neighbours download fine, the bgutil PO-token sidecar is live and answering, and the newest yt-dlp fails identically. So the download is not broken by our stack or a stale yt-dlp, and no retry, client switch or upgrade gets past it. A cookie is the only way in. The failure copy already sends the user to Settings -> Account -> Download authentication, but YouTube was not in the cookie host registry, so that dropdown could not help them: the app asked for something it did not offer. credentials: YouTube joins SUPPORTED_HOSTS (youtube.com, youtu.be, youtube-nocookie.com). The Settings dropdown, the SEC-12 host scope and the private-asset cache salt all derive from the row, so that is the whole wiring. adapters: a signed-in YouTube download runs only cookie-capable player clients. Measured against yt-dlp's own INNERTUBE_CLIENTS table: with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES is False, and a set it empties that way raises "No player clients have been requested" — which is not a per-client flake it retries past. Our primary set is android_vr, one of those, so without this the cookie would have failed the download on attempt 1 and never reached the POT-backed web fallback it exists for. A test pins the client list against the installed yt-dlp so a version bump turns the gate red instead of silently un-authing every YouTube download, and the empty-set message joins the retriable markers as the second line of defence. errors: catalog entries for the wall with and without a cookie, so a user who has already added cookies is told they were rejected (YouTube rotates exported ones) rather than to add some. The marker is quote-agnostic: YouTube writes the apostrophe as U+2019, so "you're not a bot" with a straight quote never matches the real text. EN + HU copy. --- backend/app/config.py | 3 ++ backend/app/downloads/adapters.py | 35 +++++++++++++++-- backend/app/downloads/credentials.py | 5 +++ backend/app/downloads/errors.py | 24 ++++++++++++ backend/tests/test_adapters.py | 35 +++++++++++++++++ backend/tests/test_download_credentials.py | 23 +++++++++-- backend/tests/test_download_errors.py | 43 +++++++++++++++++++++ frontend/src/i18n/locales/en/downloads.json | 10 +++++ frontend/src/i18n/locales/hu/downloads.json | 10 +++++ 9 files changed, 182 insertions(+), 6 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 0483011c..334e85e5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -155,6 +155,9 @@ class Settings(BaseSettings): # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token + # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works. + # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients + # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there. + # See adapters.YouTubeAdapter._COOKIE_CAPABLE. download_player_clients: str = "android_vr" download_player_clients_fallback: str = "web_safari,web,android" diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 16c2994d..45682446 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -93,12 +93,41 @@ class YouTubeAdapter(SourceAdapter): "Requested format is not available", "Only images are available", "Sign in to confirm", + # A set yt-dlp emptied because none of its clients take cookies. `client_sets` filters those + # out below, so this is the safety net for a yt-dlp that renames a client or an admin who + # configures both knobs with cookie-incapable clients only. + "No player clients have been requested", ) + # Measured against yt-dlp's own INNERTUBE_CLIENTS table (2026-09-16, on 2026.07.04 and + # 2026.08.19): with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES + # is False, and a set left empty raises "No player clients have been requested". Our primary set + # is android_vr — one of those — so a signed-in download would fail on the first attempt and + # never reach the POT-backed web fallback that the cookie exists for. Pinned by test_adapters + # against the installed yt-dlp, so a version bump that moves the table turns the gate red + # instead of silently un-authing every YouTube download. + _COOKIE_CAPABLE = frozenset( + { + "mweb", + "tv", + "tv_downgraded", + "web", + "web_creator", + "web_embedded", + "web_music", + "web_safari", + } + ) + # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all. + # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for + # the per-client flakiness that the two-tier ladder gives a signed-out download. + _AUTHED_DEFAULT = ("web_safari", "web") def client_sets(self, authed: bool = False) -> list: - return [ - c for c in (settings.player_client_list, settings.player_client_fallback_list) if c - ] or [None] + sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c] + if authed: + kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets] + return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)] + return sets or [None] def ydl_opts_extra(self, clients) -> dict: # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values). diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 7e87388a..c3e20ebb 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -55,6 +55,11 @@ class HostSpec: # all derive from this, so there is nothing else to wire up. SUPPORTED_HOSTS: tuple[HostSpec, ...] = ( HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273: every player client fails on that one video while its neighbours download fine, and + # the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a + # row here the `login_required` copy sent the user to a dropdown that could not help them. + HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")), HostSpec("instagram", "Instagram", ("instagram.com",)), HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")), HostSpec("tiktok", "TikTok", ("tiktok.com",)), diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 47810a8a..48cae607 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -212,6 +212,30 @@ _CATALOG: tuple[_Entry, ...] = ( cookie=False, link=facebook_group_post_id, ), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its + # neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails + # identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp). + # The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so + # "you're not a bot" with a straight quote never matches the real text. + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.bot_wall", + cookie=False, + ), + # The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first + # one is being added now) — but "add cookies" is provably the wrong advice for someone who has, + # and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no + # longer valid. They have likely been rotated in the browser as a security measure." + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.cookie_rejected", + cookie=True, + ), ) diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index ef5ac453..88880062 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -1,6 +1,7 @@ """Per-source adapter selection + the source-specific policy each one carries. These lock the behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch.""" +from app.config import settings from app.titles import normalize_title from app.downloads.adapters import ( FacebookAdapter, @@ -111,6 +112,9 @@ def test_client_sets_and_retriable_are_per_source(): assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True assert YouTubeAdapter().retriable("some unrelated network blip") is False + # The empty-set error of a cookie-incapable client list (see _COOKIE_CAPABLE): retriable, so an + # admin who configures one still gets the next set tried instead of a hard failure. + assert YouTubeAdapter().retriable("ERROR: No player clients have been requested") is True # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind # the parse wall (without a cookie that wall is the login wall — no disguise gets past it). fb = FacebookAdapter() @@ -121,6 +125,37 @@ def test_client_sets_and_retriable_are_per_source(): assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False +def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch): + yt = YouTubeAdapter() + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android") + # Signed out: the admin's ladder exactly as configured. + assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]] + # Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way + # raises "No player clients have been requested" — which would fail the download on attempt 1, + # before the POT-backed web fallback the cookie exists for. So the adapter never asks for one: + # the android_vr-only set is gone and `android` is dropped out of the fallback. + assert yt.client_sets(authed=True) == [["web_safari", "web"]] + + +def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch): + # Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all. + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios") + assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]] + + +def test_the_cookie_capable_clients_match_the_installed_ytdlp(): + # The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a + # yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending + # a signed-in download to a client that drops the cookie. + from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS + + live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")} + assert live == set(YouTubeAdapter._COOKIE_CAPABLE) + assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live + + def test_youtube_ydl_opts_extra_carries_player_client_and_pot(): ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"] assert ea["youtube"]["player_client"] == ["android_vr"] diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 1dcdace6..1b072dc0 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -39,13 +39,23 @@ class TestHostKeyForUrl: assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook" assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook" assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter" + # YouTube gates individual videos behind its bot check; the cookie is the only way past it. + for url in ( + "https://www.youtube.com/watch?v=abc", + "https://youtu.be/abc", + "https://music.youtube.com/watch?v=abc", + "https://www.youtube-nocookie.com/embed/abc", + ): + assert credentials.host_key_for_url(url) == "youtube", url def test_look_alike_host_never_matches(self): assert credentials.host_key_for_url("https://myfacebook.com/x") is None assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None + assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None def test_unrelated_or_empty_url_has_no_host_key(self): - assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://vimeo.com/123456") is None assert credentials.host_key_for_url("") is None assert credentials.host_key_for_url(None) is None @@ -229,9 +239,16 @@ class TestResolveCookiefile: credentials.upsert_credential(db, u.id, "facebook", COOKIES) db.commit() spec = formats.normalize({}) - # A YouTube URL has no supported host_key -> no cookie regardless of what's stored. - yt = "https://www.youtube.com/watch?v=abc" + # A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is + # a supported host in its own right now, so the case needs a host that is not. + other = "https://vimeo.com/123456" salted = credentials.authed_format_sig(spec, u.id, "facebook") + assert ( + credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path) + is None + ) + # ...and a YouTube link gets nothing either: the stored credential is Facebook's. + yt = "https://www.youtube.com/watch?v=abc" assert ( credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path) is None diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index 0745e898..e120c765 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -84,6 +84,49 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" _GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?" +# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why +# neither the rule nor the catalog entry may spell out "you’re" with a straight quote. +_YT_BOT_WALL = ( + "ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use " + "--cookies-from-browser or --cookies for the authentication. See " + "https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually " + "pass cookies." +) +_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI" + + +def test_youtube_bot_wall_is_that_video_own_signed_out_gate(): + # Measured on prod: all 12 player clients fail on this video while its neighbours download, the + # POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in. + assert classify(_YT_BOT_WALL) == "login_required" + for url in ( + _YT, + "https://youtu.be/7JPmS_bWjhI", + "https://www.youtube-nocookie.com/embed/7JPmS_bWjhI", + "https://music.youtube.com/watch?v=7JPmS_bWjhI", + ): + assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == ( + "login_required", + "youtube.bot_wall", + ), url + + +def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused(): + # Telling a user who already gave a cookie to add one is what this entry exists to avoid. + assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == ( + "login_required", + "youtube.cookie_rejected", + ) + + +def test_the_youtube_entries_never_match_another_site(): + # Host-scoped: the same words about a Facebook link stay the bare category. + assert dlerrors.diagnose( + "ERROR: [facebook] 1: Sign in to confirm you’re not a bot", + url="https://www.facebook.com/reel/1/", + ) == ("login_required", None) + + def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie(): for url in ( _REEL, diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 55c0d9e4..d566d29c 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -304,6 +304,16 @@ "short": "No video found in this post", "body": "We couldn't find a single downloadable video in this group post. If it does have one, open the video on Facebook and paste its link." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube sign-in required", + "body": "YouTube only serves this video to a signed-in viewer — it asks the downloader to confirm it isn't a bot. Other YouTube videos are unaffected. Add your YouTube cookies under Settings → Account → Download authentication, then retry the download from the list." + }, + "cookie_rejected": { + "short": "YouTube didn't accept the cookies", + "body": "YouTube rotates exported cookies as a security measure, so yours have probably expired. Export them again from a private browsing window, close that window without browsing on, paste them in under Settings → Account → Download authentication, then retry the download from the list." + } } } }, diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index 1bfecac3..9c03a6b7 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -304,6 +304,16 @@ "short": "Ebben a posztban nem található videó", "body": "Nem találtunk egyértelműen letölthető videót ebben a csoportposztban. Ha mégis van benne, nyisd meg a videót a Facebookon, és annak a linkjét másold be." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube-bejelentkezés szükséges", + "body": "A YouTube ezt a videót csak bejelentkezett nézőnek adja ki — a letöltőtől azt kéri, igazolja, hogy nem robot. A többi YouTube-videót ez nem érinti. Illeszd be a YouTube-cookie-kat a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + }, + "cookie_rejected": { + "short": "A YouTube nem fogadta el a cookie-kat", + "body": "A YouTube biztonsági okból cserélgeti az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + } } } },