Merge branch 'main' into feature/e22-activity-trail

backend/app/config.py

@@ -183,6 +183,9 @@ class Settings(BaseSettings):
     # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
     # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
     # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
+    # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
+    # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
+    # See adapters.YouTubeAdapter._COOKIE_CAPABLE.
     download_player_clients: str = "android_vr"
     download_player_clients_fallback: str = "web_safari,web,android"
 

backend/app/downloads/adapters.py

@@ -93,12 +93,46 @@ class YouTubeAdapter(SourceAdapter):
         "Requested format is not available",
         "Only images are available",
         "Sign in to confirm",
+        # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
+        # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
+        # between the player response and the stream, so this is the same per-client flakiness one
+        # step later — and it used to end the job on attempt 1, because the text classifies as
+        # `network` and nothing here matched it. The format URLs belong to the client that minted
+        # them, so the next client set is a real second chance.
+        "unable to download video data",
     )
+    # Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
+    # `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
+    # cookies`, drops it, and the request that is left with no client fails with "Requested format is
+    # not available" — which is already a retriable marker above, so the cost is a wasted attempt
+    # rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
+    # signed-in download would burn its first attempt that way before reaching the POT-backed web
+    # fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
+    # ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
+    # that moves the table turns the gate red instead of silently un-authing every YouTube download.
+    _COOKIE_CAPABLE = frozenset(
+        {
+            "mweb",
+            "tv",
+            "tv_downgraded",
+            "web",
+            "web_creator",
+            "web_embedded",
+            "web_music",
+            "web_safari",
+        }
+    )
+    # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
+    # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
+    # the per-client flakiness that the two-tier ladder gives a signed-out download.
+    _AUTHED_DEFAULT = ("web_safari", "web")
 
     def client_sets(self, authed: bool = False) -> list:
-        return [
-            c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
-        ] or [None]
+        sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
+        if authed:
+            kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
+            return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
+        return sets or [None]
 
     def ydl_opts_extra(self, clients) -> dict:
         # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).

backend/app/downloads/credentials.py

@@ -55,6 +55,11 @@ class HostSpec:
 # all derive from this, so there is nothing else to wire up.
 SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
     HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
+    # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
+    # job 273: every player client fails on that one video while its neighbours download fine, and
+    # the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
+    # row here the `login_required` copy sent the user to a dropdown that could not help them.
+    HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
     HostSpec("instagram", "Instagram", ("instagram.com",)),
     HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
     HostSpec("tiktok", "TikTok", ("tiktok.com",)),

backend/app/downloads/errors.py

@@ -212,6 +212,43 @@ _CATALOG: tuple[_Entry, ...] = (
         cookie=False,
         link=facebook_group_post_id,
     ),
+    # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
+    # job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
+    # neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
+    # identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
+    # The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
+    # "you're not a bot" with a straight quote never matches the real text.
+    _Entry(
+        host="youtube",
+        markers=("[youtube]", "not a bot"),
+        code="login_required",
+        detail="youtube.bot_wall",
+        cookie=False,
+    ),
+    # The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
+    # one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
+    # and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
+    # longer valid. They have likely been rotated in the browser as a security measure."
+    _Entry(
+        host="youtube",
+        markers=("[youtube]", "not a bot"),
+        code="login_required",
+        detail="youtube.cookie_rejected",
+        cookie=True,
+    ),
+    # The same gate one step later: the player response comes back, and then the CDN refuses the
+    # stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
+    # very same video, ~90 minutes after it was answering with the bot message — signed out it
+    # 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
+    # the category this text carries generically, `network`, is a lie here: nothing was wrong with
+    # the network. Scoped to a cookie-less YouTube download, where that is what we measured.
+    _Entry(
+        host="youtube",
+        markers=("unable to download video data", "403"),
+        code="login_required",
+        detail="youtube.media_forbidden",
+        cookie=False,
+    ),
 )
 
 

backend/tests/test_adapters.py

@@ -1,6 +1,7 @@
 """Per-source adapter selection + the source-specific policy each one carries. These lock the
 behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
 
+from app.config import settings
 from app.titles import normalize_title
 from app.downloads.adapters import (
     FacebookAdapter,
@@ -111,6 +112,28 @@ def test_client_sets_and_retriable_are_per_source():
     assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
     assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
     assert YouTubeAdapter().retriable("some unrelated network blip") is False
+    # A client list yt-dlp empties because none of its clients take cookies does NOT raise "No
+    # player clients have been requested" (that raise sits earlier in yt-dlp, before the
+    # cookie-capability removal). Measured with a real cookie: the emptied request fails with
+    # "Requested format is not available", which is retriable here already — so the wasted attempt
+    # is recoverable and `_COOKIE_CAPABLE` exists to avoid spending it at all.
+    assert (
+        YouTubeAdapter().retriable(
+            "ERROR: [youtube] abc: Requested format is not available. Use --list-formats"
+        )
+        is True
+    )
+    # The media fetch refused after a successful extraction: the format URLs belong to the client
+    # that minted them, so the next client set is a real second chance (it used to end the job).
+    assert (
+        YouTubeAdapter().retriable(
+            "ERROR: unable to download video data: HTTP Error 403: Forbidden"
+        )
+        is True
+    )
+    assert (
+        SourceAdapter().retriable("ERROR: unable to download video data: HTTP Error 403") is False
+    )
     # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind
     # the parse wall (without a cookie that wall is the login wall — no disguise gets past it).
     fb = FacebookAdapter()
@@ -121,6 +144,37 @@ def test_client_sets_and_retriable_are_per_source():
     assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False
 
 
+def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch):
+    yt = YouTubeAdapter()
+    monkeypatch.setattr(settings, "download_player_clients", "android_vr")
+    monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android")
+    # Signed out: the admin's ladder exactly as configured.
+    assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]]
+    # Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way
+    # raises "No player clients have been requested" — which would fail the download on attempt 1,
+    # before the POT-backed web fallback the cookie exists for. So the adapter never asks for one:
+    # the android_vr-only set is gone and `android` is dropped out of the fallback.
+    assert yt.client_sets(authed=True) == [["web_safari", "web"]]
+
+
+def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch):
+    # Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all.
+    monkeypatch.setattr(settings, "download_player_clients", "android_vr")
+    monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios")
+    assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]]
+
+
+def test_the_cookie_capable_clients_match_the_installed_ytdlp():
+    # The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a
+    # yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending
+    # a signed-in download to a client that drops the cookie.
+    from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
+
+    live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")}
+    assert live == set(YouTubeAdapter._COOKIE_CAPABLE)
+    assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
+
+
 def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
     ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
     assert ea["youtube"]["player_client"] == ["android_vr"]

backend/tests/test_download_credentials.py

@@ -39,13 +39,23 @@ class TestHostKeyForUrl:
         assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook"
         assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook"
         assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter"
+        # YouTube gates individual videos behind its bot check; the cookie is the only way past it.
+        for url in (
+            "https://www.youtube.com/watch?v=abc",
+            "https://youtu.be/abc",
+            "https://music.youtube.com/watch?v=abc",
+            "https://www.youtube-nocookie.com/embed/abc",
+        ):
+            assert credentials.host_key_for_url(url) == "youtube", url
 
     def test_look_alike_host_never_matches(self):
         assert credentials.host_key_for_url("https://myfacebook.com/x") is None
         assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None
+        assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None
+        assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None
 
     def test_unrelated_or_empty_url_has_no_host_key(self):
-        assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None
+        assert credentials.host_key_for_url("https://vimeo.com/123456") is None
         assert credentials.host_key_for_url("") is None
         assert credentials.host_key_for_url(None) is None
 
@@ -175,6 +185,22 @@ class TestCachePrivacy:
         asset = db.get(MediaAsset, job.asset_id)
         assert asset.format_sig == formats.format_sig(formats.normalize({}))
 
+    def test_a_youtube_job_reaches_the_gate_through_its_rebuilt_watch_url(self, db):
+        # The one join nothing else makes. A YouTube job stores a bare 11-char ref, and the URL the
+        # cookie gate keys on is REBUILT by the adapter (service.source_url -> YouTubeAdapter). Both
+        # ends are pinned apart — the watch-URL shape in test_adapters, the host_key in
+        # TestHostKeyForUrl — so this welds them: an enqueue that ever handed the gate the raw ref
+        # would silently stop salting, every YouTube cookie would quietly stop being attached, and
+        # signed-out downloads would keep working, so nothing else here would go red.
+        u = _user(db)
+        credentials.upsert_credential(db, u.id, "youtube", COOKIES)
+        db.commit()
+        job = service.enqueue(db, u.id, "youtube", "VIDID12345_", {})
+        asset = db.get(MediaAsset, job.asset_id)
+        spec = formats.normalize({})
+        assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "youtube")
+        assert asset.format_sig != formats.format_sig(spec)
+
     def test_two_users_never_share_the_authed_asset(self, db):
         a = _user(db, "a@example.com")
         b = _user(db, "b@example.com")
@@ -229,9 +255,16 @@ class TestResolveCookiefile:
         credentials.upsert_credential(db, u.id, "facebook", COOKIES)
         db.commit()
         spec = formats.normalize({})
-        # A YouTube URL has no supported host_key -> no cookie regardless of what's stored.
-        yt = "https://www.youtube.com/watch?v=abc"
+        # A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is
+        # a supported host in its own right now, so the case needs a host that is not.
+        other = "https://vimeo.com/123456"
         salted = credentials.authed_format_sig(spec, u.id, "facebook")
+        assert (
+            credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path)
+            is None
+        )
+        # ...and a YouTube link gets nothing either: the stored credential is Facebook's.
+        yt = "https://www.youtube.com/watch?v=abc"
         assert (
             credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path)
             is None

backend/tests/test_download_errors.py

@@ -84,6 +84,72 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
 _GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?"
 
 
+# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why
+# neither the rule nor the catalog entry may spell out "you’re" with a straight quote.
+_YT_BOT_WALL = (
+    "ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use "
+    "--cookies-from-browser or --cookies for the authentication. See  "
+    "https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp  for how to manually "
+    "pass cookies."
+)
+_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI"
+
+
+def test_youtube_bot_wall_is_that_video_own_signed_out_gate():
+    # Measured on prod: all 12 player clients fail on this video while its neighbours download, the
+    # POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in.
+    assert classify(_YT_BOT_WALL) == "login_required"
+    for url in (
+        _YT,
+        "https://youtu.be/7JPmS_bWjhI",
+        "https://www.youtube-nocookie.com/embed/7JPmS_bWjhI",
+        "https://music.youtube.com/watch?v=7JPmS_bWjhI",
+    ):
+        assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == (
+            "login_required",
+            "youtube.bot_wall",
+        ), url
+
+
+def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused():
+    # Telling a user who already gave a cookie to add one is what this entry exists to avoid.
+    assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == (
+        "login_required",
+        "youtube.cookie_rejected",
+    )
+
+
+# Verbatim from the isolated test stack (2026-09-16), the SAME video ~90 minutes later: yt-dlp got
+# the player response and picked formats, and the CDN then refused the stream.
+_YT_MEDIA_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden"
+
+
+def test_youtube_media_403_is_the_wall_one_step_later_not_a_network_fault():
+    # Generically this text is a network failure — and for every other host it stays one.
+    assert classify(_YT_MEDIA_403) == "network"
+    assert dlerrors.diagnose(_YT_MEDIA_403, url="https://vimeo.com/1") == ("network", None)
+    # On a signed-out YouTube download it is the gate: measured, the same video downloaded in full
+    # once the owner's cookie was attached. Calling that "network error" is the honest-states bug.
+    assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT) == (
+        "login_required",
+        "youtube.media_forbidden",
+    )
+
+
+def test_youtube_media_403_with_a_cookie_is_not_claimed_as_the_gate():
+    # Only the cookie-less case was measured. With a cookie the same text keeps its bare category
+    # rather than telling a signed-in user to sign in.
+    assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT, cookie_attached=True) == ("network", None)
+
+
+def test_the_youtube_entries_never_match_another_site():
+    # Host-scoped: the same words about a Facebook link stay the bare category.
+    assert dlerrors.diagnose(
+        "ERROR: [facebook] 1: Sign in to confirm you’re not a bot",
+        url="https://www.facebook.com/reel/1/",
+    ) == ("login_required", None)
+
+
 def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie():
     for url in (
         _REEL,