fix: a YouTube video behind the bot wall is downloadable with cookies
A YouTube link can fail with "Sign in to confirm you're not a bot" while every other YouTube download works — YouTube gates INDIVIDUAL videos behind a signed-in session. Measured on prod job 273: all twelve of yt-dlp's player clients fail on that one video while its neighbours download fine, the bgutil PO-token sidecar is live and answering, and the newest yt-dlp fails identically. So the download is not broken by our stack or a stale yt-dlp, and no retry, client switch or upgrade gets past it. A cookie is the only way in. The failure copy already sends the user to Settings -> Account -> Download authentication, but YouTube was not in the cookie host registry, so that dropdown could not help them: the app asked for something it did not offer. credentials: YouTube joins SUPPORTED_HOSTS (youtube.com, youtu.be, youtube-nocookie.com). The Settings dropdown, the SEC-12 host scope and the private-asset cache salt all derive from the row, so that is the whole wiring. adapters: a signed-in YouTube download runs only cookie-capable player clients. Measured against yt-dlp's own INNERTUBE_CLIENTS table: with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES is False, and a set it empties that way raises "No player clients have been requested" — which is not a per-client flake it retries past. Our primary set is android_vr, one of those, so without this the cookie would have failed the download on attempt 1 and never reached the POT-backed web fallback it exists for. A test pins the client list against the installed yt-dlp so a version bump turns the gate red instead of silently un-authing every YouTube download, and the empty-set message joins the retriable markers as the second line of defence. errors: catalog entries for the wall with and without a cookie, so a user who has already added cookies is told they were rejected (YouTube rotates exported ones) rather than to add some. The marker is quote-agnostic: YouTube writes the apostrophe as U+2019, so "you're not a bot" with a straight quote never matches the real text. EN + HU copy.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
|
||||
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
|
||||
|
||||
from app.config import settings
|
||||
from app.titles import normalize_title
|
||||
from app.downloads.adapters import (
|
||||
FacebookAdapter,
|
||||
@@ -111,6 +112,9 @@ def test_client_sets_and_retriable_are_per_source():
|
||||
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
|
||||
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
|
||||
assert YouTubeAdapter().retriable("some unrelated network blip") is False
|
||||
# The empty-set error of a cookie-incapable client list (see _COOKIE_CAPABLE): retriable, so an
|
||||
# admin who configures one still gets the next set tried instead of a hard failure.
|
||||
assert YouTubeAdapter().retriable("ERROR: No player clients have been requested") is True
|
||||
# Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind
|
||||
# the parse wall (without a cookie that wall is the login wall — no disguise gets past it).
|
||||
fb = FacebookAdapter()
|
||||
@@ -121,6 +125,37 @@ def test_client_sets_and_retriable_are_per_source():
|
||||
assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False
|
||||
|
||||
|
||||
def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch):
|
||||
yt = YouTubeAdapter()
|
||||
monkeypatch.setattr(settings, "download_player_clients", "android_vr")
|
||||
monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android")
|
||||
# Signed out: the admin's ladder exactly as configured.
|
||||
assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]]
|
||||
# Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way
|
||||
# raises "No player clients have been requested" — which would fail the download on attempt 1,
|
||||
# before the POT-backed web fallback the cookie exists for. So the adapter never asks for one:
|
||||
# the android_vr-only set is gone and `android` is dropped out of the fallback.
|
||||
assert yt.client_sets(authed=True) == [["web_safari", "web"]]
|
||||
|
||||
|
||||
def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch):
|
||||
# Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all.
|
||||
monkeypatch.setattr(settings, "download_player_clients", "android_vr")
|
||||
monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios")
|
||||
assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]]
|
||||
|
||||
|
||||
def test_the_cookie_capable_clients_match_the_installed_ytdlp():
|
||||
# The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a
|
||||
# yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending
|
||||
# a signed-in download to a client that drops the cookie.
|
||||
from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
|
||||
|
||||
live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")}
|
||||
assert live == set(YouTubeAdapter._COOKIE_CAPABLE)
|
||||
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
|
||||
|
||||
|
||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||
|
||||
@@ -39,13 +39,23 @@ class TestHostKeyForUrl:
|
||||
assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook"
|
||||
assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook"
|
||||
assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter"
|
||||
# YouTube gates individual videos behind its bot check; the cookie is the only way past it.
|
||||
for url in (
|
||||
"https://www.youtube.com/watch?v=abc",
|
||||
"https://youtu.be/abc",
|
||||
"https://music.youtube.com/watch?v=abc",
|
||||
"https://www.youtube-nocookie.com/embed/abc",
|
||||
):
|
||||
assert credentials.host_key_for_url(url) == "youtube", url
|
||||
|
||||
def test_look_alike_host_never_matches(self):
|
||||
assert credentials.host_key_for_url("https://myfacebook.com/x") is None
|
||||
assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None
|
||||
assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None
|
||||
assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None
|
||||
|
||||
def test_unrelated_or_empty_url_has_no_host_key(self):
|
||||
assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None
|
||||
assert credentials.host_key_for_url("https://vimeo.com/123456") is None
|
||||
assert credentials.host_key_for_url("") is None
|
||||
assert credentials.host_key_for_url(None) is None
|
||||
|
||||
@@ -229,9 +239,16 @@ class TestResolveCookiefile:
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
# A YouTube URL has no supported host_key -> no cookie regardless of what's stored.
|
||||
yt = "https://www.youtube.com/watch?v=abc"
|
||||
# A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is
|
||||
# a supported host in its own right now, so the case needs a host that is not.
|
||||
other = "https://vimeo.com/123456"
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert (
|
||||
credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path)
|
||||
is None
|
||||
)
|
||||
# ...and a YouTube link gets nothing either: the stored credential is Facebook's.
|
||||
yt = "https://www.youtube.com/watch?v=abc"
|
||||
assert (
|
||||
credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path)
|
||||
is None
|
||||
|
||||
@@ -84,6 +84,49 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||
_GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?"
|
||||
|
||||
|
||||
# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why
|
||||
# neither the rule nor the catalog entry may spell out "you’re" with a straight quote.
|
||||
_YT_BOT_WALL = (
|
||||
"ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use "
|
||||
"--cookies-from-browser or --cookies for the authentication. See "
|
||||
"https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually "
|
||||
"pass cookies."
|
||||
)
|
||||
_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI"
|
||||
|
||||
|
||||
def test_youtube_bot_wall_is_that_video_own_signed_out_gate():
|
||||
# Measured on prod: all 12 player clients fail on this video while its neighbours download, the
|
||||
# POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in.
|
||||
assert classify(_YT_BOT_WALL) == "login_required"
|
||||
for url in (
|
||||
_YT,
|
||||
"https://youtu.be/7JPmS_bWjhI",
|
||||
"https://www.youtube-nocookie.com/embed/7JPmS_bWjhI",
|
||||
"https://music.youtube.com/watch?v=7JPmS_bWjhI",
|
||||
):
|
||||
assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == (
|
||||
"login_required",
|
||||
"youtube.bot_wall",
|
||||
), url
|
||||
|
||||
|
||||
def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused():
|
||||
# Telling a user who already gave a cookie to add one is what this entry exists to avoid.
|
||||
assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == (
|
||||
"login_required",
|
||||
"youtube.cookie_rejected",
|
||||
)
|
||||
|
||||
|
||||
def test_the_youtube_entries_never_match_another_site():
|
||||
# Host-scoped: the same words about a Facebook link stay the bare category.
|
||||
assert dlerrors.diagnose(
|
||||
"ERROR: [facebook] 1: Sign in to confirm you’re not a bot",
|
||||
url="https://www.facebook.com/reel/1/",
|
||||
) == ("login_required", None)
|
||||
|
||||
|
||||
def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie():
|
||||
for url in (
|
||||
_REEL,
|
||||
|
||||
Reference in New Issue
Block a user