Merge branch 'main' into feature/e22-activity-trail

backend/app/config.py

@@ -183,6 +183,9 @@ class Settings(BaseSettings):
     # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
     # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
     # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
+    # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
+    # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
+    # See adapters.YouTubeAdapter._COOKIE_CAPABLE.
     download_player_clients: str = "android_vr"
     download_player_clients_fallback: str = "web_safari,web,android"
 

backend/app/downloads/adapters.py

@@ -93,12 +93,46 @@ class YouTubeAdapter(SourceAdapter):
         "Requested format is not available",
         "Only images are available",
         "Sign in to confirm",
+        # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
+        # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
+        # between the player response and the stream, so this is the same per-client flakiness one
+        # step later — and it used to end the job on attempt 1, because the text classifies as
+        # `network` and nothing here matched it. The format URLs belong to the client that minted
+        # them, so the next client set is a real second chance.
+        "unable to download video data",
     )
+    # Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
+    # `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
+    # cookies`, drops it, and the request that is left with no client fails with "Requested format is
+    # not available" — which is already a retriable marker above, so the cost is a wasted attempt
+    # rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
+    # signed-in download would burn its first attempt that way before reaching the POT-backed web
+    # fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
+    # ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
+    # that moves the table turns the gate red instead of silently un-authing every YouTube download.
+    _COOKIE_CAPABLE = frozenset(
+        {
+            "mweb",
+            "tv",
+            "tv_downgraded",
+            "web",
+            "web_creator",
+            "web_embedded",
+            "web_music",
+            "web_safari",
+        }
+    )
+    # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
+    # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
+    # the per-client flakiness that the two-tier ladder gives a signed-out download.
+    _AUTHED_DEFAULT = ("web_safari", "web")
 
     def client_sets(self, authed: bool = False) -> list:
-        return [
-            c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
-        ] or [None]
+        sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
+        if authed:
+            kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
+            return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
+        return sets or [None]
 
     def ydl_opts_extra(self, clients) -> dict:
         # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).

backend/app/downloads/credentials.py

@@ -55,6 +55,11 @@ class HostSpec:
 # all derive from this, so there is nothing else to wire up.
 SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
     HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
+    # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
+    # job 273: every player client fails on that one video while its neighbours download fine, and
+    # the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
+    # row here the `login_required` copy sent the user to a dropdown that could not help them.
+    HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
     HostSpec("instagram", "Instagram", ("instagram.com",)),
     HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
     HostSpec("tiktok", "TikTok", ("tiktok.com",)),

backend/app/downloads/errors.py

@@ -212,6 +212,43 @@ _CATALOG: tuple[_Entry, ...] = (
         cookie=False,
         link=facebook_group_post_id,
     ),
+    # YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
+    # job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
+    # neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
+    # identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
+    # The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
+    # "you're not a bot" with a straight quote never matches the real text.
+    _Entry(
+        host="youtube",
+        markers=("[youtube]", "not a bot"),
+        code="login_required",
+        detail="youtube.bot_wall",
+        cookie=False,
+    ),
+    # The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
+    # one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
+    # and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
+    # longer valid. They have likely been rotated in the browser as a security measure."
+    _Entry(
+        host="youtube",
+        markers=("[youtube]", "not a bot"),
+        code="login_required",
+        detail="youtube.cookie_rejected",
+        cookie=True,
+    ),
+    # The same gate one step later: the player response comes back, and then the CDN refuses the
+    # stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
+    # very same video, ~90 minutes after it was answering with the bot message — signed out it
+    # 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
+    # the category this text carries generically, `network`, is a lie here: nothing was wrong with
+    # the network. Scoped to a cookie-less YouTube download, where that is what we measured.
+    _Entry(
+        host="youtube",
+        markers=("unable to download video data", "403"),
+        code="login_required",
+        detail="youtube.media_forbidden",
+        cookie=False,
+    ),
 )