diff --git a/CHANGELOG.md b/CHANGELOG.md index a8ab6c1b..4e66f937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.87.1 — 2026-09-16 + +A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it. + +### Features + +- YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login. + +### Fixes + +- A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again. +- A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network. + +### Internal + +- The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not. + ## 0.87.0 — 2026-09-15 A shared or library video keeps playing when you switch apps on an iPad, and a new tab opens as the account it belongs to without flashing another page first. Facebook reels keep their titles and an mp3 stays an mp3. diff --git a/VERSION b/VERSION index 359ee08a..7921aa12 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.87.0 +0.87.1 diff --git a/backend/app/config.py b/backend/app/config.py index d8782e2f..eaa9976e 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -183,6 +183,9 @@ class Settings(BaseSettings): # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token + # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works. + # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients + # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there. + # See adapters.YouTubeAdapter._COOKIE_CAPABLE. download_player_clients: str = "android_vr" download_player_clients_fallback: str = "web_safari,web,android" diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 16c2994d..3f0c4ca5 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -93,12 +93,46 @@ class YouTubeAdapter(SourceAdapter): "Requested format is not available", "Only images are available", "Sign in to confirm", + # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then + # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves + # between the player response and the stream, so this is the same per-client flakiness one + # step later — and it used to end the job on attempt 1, because the text classifies as + # `network` and nothing here matched it. The format URLs belong to the client that minted + # them, so the next client set is a real second chance. + "unable to download video data", ) + # Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with + # `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support + # cookies`, drops it, and the request that is left with no client fails with "Requested format is + # not available" — which is already a retriable marker above, so the cost is a wasted attempt + # rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every + # signed-in download would burn its first attempt that way before reaching the POT-backed web + # fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...] + # ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump + # that moves the table turns the gate red instead of silently un-authing every YouTube download. + _COOKIE_CAPABLE = frozenset( + { + "mweb", + "tv", + "tv_downgraded", + "web", + "web_creator", + "web_embedded", + "web_music", + "web_safari", + } + ) + # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all. + # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for + # the per-client flakiness that the two-tier ladder gives a signed-out download. + _AUTHED_DEFAULT = ("web_safari", "web") def client_sets(self, authed: bool = False) -> list: - return [ - c for c in (settings.player_client_list, settings.player_client_fallback_list) if c - ] or [None] + sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c] + if authed: + kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets] + return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)] + return sets or [None] def ydl_opts_extra(self, clients) -> dict: # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values). diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 7e87388a..c3e20ebb 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -55,6 +55,11 @@ class HostSpec: # all derive from this, so there is nothing else to wire up. SUPPORTED_HOSTS: tuple[HostSpec, ...] = ( HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273: every player client fails on that one video while its neighbours download fine, and + # the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a + # row here the `login_required` copy sent the user to a dropdown that could not help them. + HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")), HostSpec("instagram", "Instagram", ("instagram.com",)), HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")), HostSpec("tiktok", "TikTok", ("tiktok.com",)), diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 47810a8a..eaa57424 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -212,6 +212,43 @@ _CATALOG: tuple[_Entry, ...] = ( cookie=False, link=facebook_group_post_id, ), + # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod + # job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its + # neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails + # identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp). + # The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so + # "you're not a bot" with a straight quote never matches the real text. + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.bot_wall", + cookie=False, + ), + # The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first + # one is being added now) — but "add cookies" is provably the wrong advice for someone who has, + # and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no + # longer valid. They have likely been rotated in the browser as a security measure." + _Entry( + host="youtube", + markers=("[youtube]", "not a bot"), + code="login_required", + detail="youtube.cookie_rejected", + cookie=True, + ), + # The same gate one step later: the player response comes back, and then the CDN refuses the + # stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the + # very same video, ~90 minutes after it was answering with the bot message — signed out it + # 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So + # the category this text carries generically, `network`, is a lie here: nothing was wrong with + # the network. Scoped to a cookie-less YouTube download, where that is what we measured. + _Entry( + host="youtube", + markers=("unable to download video data", "403"), + code="login_required", + detail="youtube.media_forbidden", + cookie=False, + ), ) diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index ef5ac453..4aa6ca58 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -1,6 +1,7 @@ """Per-source adapter selection + the source-specific policy each one carries. These lock the behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch.""" +from app.config import settings from app.titles import normalize_title from app.downloads.adapters import ( FacebookAdapter, @@ -111,6 +112,28 @@ def test_client_sets_and_retriable_are_per_source(): assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True assert YouTubeAdapter().retriable("some unrelated network blip") is False + # A client list yt-dlp empties because none of its clients take cookies does NOT raise "No + # player clients have been requested" (that raise sits earlier in yt-dlp, before the + # cookie-capability removal). Measured with a real cookie: the emptied request fails with + # "Requested format is not available", which is retriable here already — so the wasted attempt + # is recoverable and `_COOKIE_CAPABLE` exists to avoid spending it at all. + assert ( + YouTubeAdapter().retriable( + "ERROR: [youtube] abc: Requested format is not available. Use --list-formats" + ) + is True + ) + # The media fetch refused after a successful extraction: the format URLs belong to the client + # that minted them, so the next client set is a real second chance (it used to end the job). + assert ( + YouTubeAdapter().retriable( + "ERROR: unable to download video data: HTTP Error 403: Forbidden" + ) + is True + ) + assert ( + SourceAdapter().retriable("ERROR: unable to download video data: HTTP Error 403") is False + ) # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind # the parse wall (without a cookie that wall is the login wall — no disguise gets past it). fb = FacebookAdapter() @@ -121,6 +144,37 @@ def test_client_sets_and_retriable_are_per_source(): assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False +def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch): + yt = YouTubeAdapter() + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android") + # Signed out: the admin's ladder exactly as configured. + assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]] + # Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way + # raises "No player clients have been requested" — which would fail the download on attempt 1, + # before the POT-backed web fallback the cookie exists for. So the adapter never asks for one: + # the android_vr-only set is gone and `android` is dropped out of the fallback. + assert yt.client_sets(authed=True) == [["web_safari", "web"]] + + +def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch): + # Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all. + monkeypatch.setattr(settings, "download_player_clients", "android_vr") + monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios") + assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]] + + +def test_the_cookie_capable_clients_match_the_installed_ytdlp(): + # The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a + # yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending + # a signed-in download to a client that drops the cookie. + from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS + + live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")} + assert live == set(YouTubeAdapter._COOKIE_CAPABLE) + assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live + + def test_youtube_ydl_opts_extra_carries_player_client_and_pot(): ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"] assert ea["youtube"]["player_client"] == ["android_vr"] diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 1dcdace6..3cc8af39 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -39,13 +39,23 @@ class TestHostKeyForUrl: assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook" assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook" assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter" + # YouTube gates individual videos behind its bot check; the cookie is the only way past it. + for url in ( + "https://www.youtube.com/watch?v=abc", + "https://youtu.be/abc", + "https://music.youtube.com/watch?v=abc", + "https://www.youtube-nocookie.com/embed/abc", + ): + assert credentials.host_key_for_url(url) == "youtube", url def test_look_alike_host_never_matches(self): assert credentials.host_key_for_url("https://myfacebook.com/x") is None assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None + assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None def test_unrelated_or_empty_url_has_no_host_key(self): - assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None + assert credentials.host_key_for_url("https://vimeo.com/123456") is None assert credentials.host_key_for_url("") is None assert credentials.host_key_for_url(None) is None @@ -175,6 +185,22 @@ class TestCachePrivacy: asset = db.get(MediaAsset, job.asset_id) assert asset.format_sig == formats.format_sig(formats.normalize({})) + def test_a_youtube_job_reaches_the_gate_through_its_rebuilt_watch_url(self, db): + # The one join nothing else makes. A YouTube job stores a bare 11-char ref, and the URL the + # cookie gate keys on is REBUILT by the adapter (service.source_url -> YouTubeAdapter). Both + # ends are pinned apart — the watch-URL shape in test_adapters, the host_key in + # TestHostKeyForUrl — so this welds them: an enqueue that ever handed the gate the raw ref + # would silently stop salting, every YouTube cookie would quietly stop being attached, and + # signed-out downloads would keep working, so nothing else here would go red. + u = _user(db) + credentials.upsert_credential(db, u.id, "youtube", COOKIES) + db.commit() + job = service.enqueue(db, u.id, "youtube", "VIDID12345_", {}) + asset = db.get(MediaAsset, job.asset_id) + spec = formats.normalize({}) + assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "youtube") + assert asset.format_sig != formats.format_sig(spec) + def test_two_users_never_share_the_authed_asset(self, db): a = _user(db, "a@example.com") b = _user(db, "b@example.com") @@ -229,9 +255,16 @@ class TestResolveCookiefile: credentials.upsert_credential(db, u.id, "facebook", COOKIES) db.commit() spec = formats.normalize({}) - # A YouTube URL has no supported host_key -> no cookie regardless of what's stored. - yt = "https://www.youtube.com/watch?v=abc" + # A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is + # a supported host in its own right now, so the case needs a host that is not. + other = "https://vimeo.com/123456" salted = credentials.authed_format_sig(spec, u.id, "facebook") + assert ( + credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path) + is None + ) + # ...and a YouTube link gets nothing either: the stored credential is Facebook's. + yt = "https://www.youtube.com/watch?v=abc" assert ( credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path) is None diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index 0745e898..c97b55f2 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -84,6 +84,72 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" _GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?" +# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why +# neither the rule nor the catalog entry may spell out "you’re" with a straight quote. +_YT_BOT_WALL = ( + "ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use " + "--cookies-from-browser or --cookies for the authentication. See " + "https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually " + "pass cookies." +) +_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI" + + +def test_youtube_bot_wall_is_that_video_own_signed_out_gate(): + # Measured on prod: all 12 player clients fail on this video while its neighbours download, the + # POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in. + assert classify(_YT_BOT_WALL) == "login_required" + for url in ( + _YT, + "https://youtu.be/7JPmS_bWjhI", + "https://www.youtube-nocookie.com/embed/7JPmS_bWjhI", + "https://music.youtube.com/watch?v=7JPmS_bWjhI", + ): + assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == ( + "login_required", + "youtube.bot_wall", + ), url + + +def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused(): + # Telling a user who already gave a cookie to add one is what this entry exists to avoid. + assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == ( + "login_required", + "youtube.cookie_rejected", + ) + + +# Verbatim from the isolated test stack (2026-09-16), the SAME video ~90 minutes later: yt-dlp got +# the player response and picked formats, and the CDN then refused the stream. +_YT_MEDIA_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden" + + +def test_youtube_media_403_is_the_wall_one_step_later_not_a_network_fault(): + # Generically this text is a network failure — and for every other host it stays one. + assert classify(_YT_MEDIA_403) == "network" + assert dlerrors.diagnose(_YT_MEDIA_403, url="https://vimeo.com/1") == ("network", None) + # On a signed-out YouTube download it is the gate: measured, the same video downloaded in full + # once the owner's cookie was attached. Calling that "network error" is the honest-states bug. + assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT) == ( + "login_required", + "youtube.media_forbidden", + ) + + +def test_youtube_media_403_with_a_cookie_is_not_claimed_as_the_gate(): + # Only the cookie-less case was measured. With a cookie the same text keeps its bare category + # rather than telling a signed-in user to sign in. + assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT, cookie_attached=True) == ("network", None) + + +def test_the_youtube_entries_never_match_another_site(): + # Host-scoped: the same words about a Facebook link stay the bare category. + assert dlerrors.diagnose( + "ERROR: [facebook] 1: Sign in to confirm you’re not a bot", + url="https://www.facebook.com/reel/1/", + ) == ("login_required", None) + + def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie(): for url in ( _REEL, diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 55c0d9e4..1e750cba 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -304,6 +304,20 @@ "short": "No video found in this post", "body": "We couldn't find a single downloadable video in this group post. If it does have one, open the video on Facebook and paste its link." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube sign-in required", + "body": "YouTube only serves this video to a signed-in viewer — it asks the downloader to confirm it isn't a bot. Other YouTube videos are usually unaffected. Add your YouTube cookies under Settings → Account → Download authentication, then retry the download from the list." + }, + "cookie_rejected": { + "short": "YouTube didn't accept the cookies", + "body": "YouTube rotates exported cookies as a security measure, so yours have probably expired. Export them again from a private browsing window, close that window without browsing on, paste them in under Settings → Account → Download authentication, then retry the download from the list." + }, + "media_forbidden": { + "short": "YouTube refused the video stream", + "body": "YouTube found the video but refused to send the file to a signed-out download. It comes and goes, so a retry from the list is worth one try — and adding your YouTube cookies under Settings → Account → Download authentication is what gets a stubborn one through." + } } } }, diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index 1bfecac3..7c1ec57e 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -304,6 +304,20 @@ "short": "Ebben a posztban nem található videó", "body": "Nem találtunk egyértelműen letölthető videót ebben a csoportposztban. Ha mégis van benne, nyisd meg a videót a Facebookon, és annak a linkjét másold be." } + }, + "youtube": { + "bot_wall": { + "short": "YouTube-bejelentkezés szükséges", + "body": "A YouTube ezt a videót csak bejelentkezett nézőnek adja ki — a letöltőprogramtól azt kéri, igazolja, hogy nem robot. A többi YouTube-videót ez általában nem érinti. Illeszd be a YouTube-cookie-kat a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + }, + "cookie_rejected": { + "short": "A YouTube nem fogadta el a cookie-kat", + "body": "A YouTube biztonsági okból lecseréli az exportált cookie-kat, ezért a tiéid valószínűleg lejártak. Exportáld őket újra egy privát böngészőablakból, majd zárd be azt az ablakot további böngészés nélkül, illeszd be a Beállítások → Fiók → Letöltés-hitelesítés alatt, majd a listában indítsd újra a letöltést." + }, + "media_forbidden": { + "short": "A YouTube nem adta ki a videófájlt", + "body": "A YouTube visszaadta a videó adatait, de a fájlt már nem adta ki bejelentkezés nélkül. Ez hol van, hol nincs, ezért érdemes újrapróbálni a listából — a makacs eseteken pedig a YouTube-cookie-k segítenek, a Beállítások → Fiók → Letöltés-hitelesítés alatt." + } } } }, diff --git a/frontend/src/lib/releaseNotes.ts b/frontend/src/lib/releaseNotes.ts index af68b8c5..71672ff2 100644 --- a/frontend/src/lib/releaseNotes.ts +++ b/frontend/src/lib/releaseNotes.ts @@ -14,6 +14,22 @@ export interface ReleaseEntry { } export const RELEASE_NOTES: ReleaseEntry[] = [ + { + version: "0.87.1", + date: "2026-09-16", + summary: + "A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it.", + features: [ + "YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login.", + ], + fixes: [ + "A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again.", + "A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network.", + ], + chores: [ + "The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not.", + ], + }, { version: "0.87.0", date: "2026-09-15",