Merge branch 'main' into feature/e22-activity-trail

backend/tests/test_adapters.py

@@ -1,6 +1,7 @@
 """Per-source adapter selection + the source-specific policy each one carries. These lock the
 behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
 
+from app.config import settings
 from app.titles import normalize_title
 from app.downloads.adapters import (
     FacebookAdapter,
@@ -111,6 +112,28 @@ def test_client_sets_and_retriable_are_per_source():
     assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
     assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
     assert YouTubeAdapter().retriable("some unrelated network blip") is False
+    # A client list yt-dlp empties because none of its clients take cookies does NOT raise "No
+    # player clients have been requested" (that raise sits earlier in yt-dlp, before the
+    # cookie-capability removal). Measured with a real cookie: the emptied request fails with
+    # "Requested format is not available", which is retriable here already — so the wasted attempt
+    # is recoverable and `_COOKIE_CAPABLE` exists to avoid spending it at all.
+    assert (
+        YouTubeAdapter().retriable(
+            "ERROR: [youtube] abc: Requested format is not available. Use --list-formats"
+        )
+        is True
+    )
+    # The media fetch refused after a successful extraction: the format URLs belong to the client
+    # that minted them, so the next client set is a real second chance (it used to end the job).
+    assert (
+        YouTubeAdapter().retriable(
+            "ERROR: unable to download video data: HTTP Error 403: Forbidden"
+        )
+        is True
+    )
+    assert (
+        SourceAdapter().retriable("ERROR: unable to download video data: HTTP Error 403") is False
+    )
     # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind
     # the parse wall (without a cookie that wall is the login wall — no disguise gets past it).
     fb = FacebookAdapter()
@@ -121,6 +144,37 @@ def test_client_sets_and_retriable_are_per_source():
     assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False
 
 
+def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch):
+    yt = YouTubeAdapter()
+    monkeypatch.setattr(settings, "download_player_clients", "android_vr")
+    monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android")
+    # Signed out: the admin's ladder exactly as configured.
+    assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]]
+    # Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way
+    # raises "No player clients have been requested" — which would fail the download on attempt 1,
+    # before the POT-backed web fallback the cookie exists for. So the adapter never asks for one:
+    # the android_vr-only set is gone and `android` is dropped out of the fallback.
+    assert yt.client_sets(authed=True) == [["web_safari", "web"]]
+
+
+def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch):
+    # Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all.
+    monkeypatch.setattr(settings, "download_player_clients", "android_vr")
+    monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios")
+    assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]]
+
+
+def test_the_cookie_capable_clients_match_the_installed_ytdlp():
+    # The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a
+    # yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending
+    # a signed-in download to a client that drops the cookie.
+    from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
+
+    live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")}
+    assert live == set(YouTubeAdapter._COOKIE_CAPABLE)
+    assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
+
+
 def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
     ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
     assert ea["youtube"]["player_client"] == ["android_vr"]

backend/tests/test_download_credentials.py

@@ -39,13 +39,23 @@ class TestHostKeyForUrl:
         assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook"
         assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook"
         assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter"
+        # YouTube gates individual videos behind its bot check; the cookie is the only way past it.
+        for url in (
+            "https://www.youtube.com/watch?v=abc",
+            "https://youtu.be/abc",
+            "https://music.youtube.com/watch?v=abc",
+            "https://www.youtube-nocookie.com/embed/abc",
+        ):
+            assert credentials.host_key_for_url(url) == "youtube", url
 
     def test_look_alike_host_never_matches(self):
         assert credentials.host_key_for_url("https://myfacebook.com/x") is None
         assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None
+        assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None
+        assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None
 
     def test_unrelated_or_empty_url_has_no_host_key(self):
-        assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None
+        assert credentials.host_key_for_url("https://vimeo.com/123456") is None
         assert credentials.host_key_for_url("") is None
         assert credentials.host_key_for_url(None) is None
 
@@ -175,6 +185,22 @@ class TestCachePrivacy:
         asset = db.get(MediaAsset, job.asset_id)
         assert asset.format_sig == formats.format_sig(formats.normalize({}))
 
+    def test_a_youtube_job_reaches_the_gate_through_its_rebuilt_watch_url(self, db):
+        # The one join nothing else makes. A YouTube job stores a bare 11-char ref, and the URL the
+        # cookie gate keys on is REBUILT by the adapter (service.source_url -> YouTubeAdapter). Both
+        # ends are pinned apart — the watch-URL shape in test_adapters, the host_key in
+        # TestHostKeyForUrl — so this welds them: an enqueue that ever handed the gate the raw ref
+        # would silently stop salting, every YouTube cookie would quietly stop being attached, and
+        # signed-out downloads would keep working, so nothing else here would go red.
+        u = _user(db)
+        credentials.upsert_credential(db, u.id, "youtube", COOKIES)
+        db.commit()
+        job = service.enqueue(db, u.id, "youtube", "VIDID12345_", {})
+        asset = db.get(MediaAsset, job.asset_id)
+        spec = formats.normalize({})
+        assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "youtube")
+        assert asset.format_sig != formats.format_sig(spec)
+
     def test_two_users_never_share_the_authed_asset(self, db):
         a = _user(db, "a@example.com")
         b = _user(db, "b@example.com")
@@ -229,9 +255,16 @@ class TestResolveCookiefile:
         credentials.upsert_credential(db, u.id, "facebook", COOKIES)
         db.commit()
         spec = formats.normalize({})
-        # A YouTube URL has no supported host_key -> no cookie regardless of what's stored.
-        yt = "https://www.youtube.com/watch?v=abc"
+        # A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is
+        # a supported host in its own right now, so the case needs a host that is not.
+        other = "https://vimeo.com/123456"
         salted = credentials.authed_format_sig(spec, u.id, "facebook")
+        assert (
+            credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path)
+            is None
+        )
+        # ...and a YouTube link gets nothing either: the stored credential is Facebook's.
+        yt = "https://www.youtube.com/watch?v=abc"
         assert (
             credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path)
             is None

backend/tests/test_download_errors.py

@@ -84,6 +84,72 @@ _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
 _GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?"
 
 
+# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why
+# neither the rule nor the catalog entry may spell out "you’re" with a straight quote.
+_YT_BOT_WALL = (
+    "ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use "
+    "--cookies-from-browser or --cookies for the authentication. See  "
+    "https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp  for how to manually "
+    "pass cookies."
+)
+_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI"
+
+
+def test_youtube_bot_wall_is_that_video_own_signed_out_gate():
+    # Measured on prod: all 12 player clients fail on this video while its neighbours download, the
+    # POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in.
+    assert classify(_YT_BOT_WALL) == "login_required"
+    for url in (
+        _YT,
+        "https://youtu.be/7JPmS_bWjhI",
+        "https://www.youtube-nocookie.com/embed/7JPmS_bWjhI",
+        "https://music.youtube.com/watch?v=7JPmS_bWjhI",
+    ):
+        assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == (
+            "login_required",
+            "youtube.bot_wall",
+        ), url
+
+
+def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused():
+    # Telling a user who already gave a cookie to add one is what this entry exists to avoid.
+    assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == (
+        "login_required",
+        "youtube.cookie_rejected",
+    )
+
+
+# Verbatim from the isolated test stack (2026-09-16), the SAME video ~90 minutes later: yt-dlp got
+# the player response and picked formats, and the CDN then refused the stream.
+_YT_MEDIA_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden"
+
+
+def test_youtube_media_403_is_the_wall_one_step_later_not_a_network_fault():
+    # Generically this text is a network failure — and for every other host it stays one.
+    assert classify(_YT_MEDIA_403) == "network"
+    assert dlerrors.diagnose(_YT_MEDIA_403, url="https://vimeo.com/1") == ("network", None)
+    # On a signed-out YouTube download it is the gate: measured, the same video downloaded in full
+    # once the owner's cookie was attached. Calling that "network error" is the honest-states bug.
+    assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT) == (
+        "login_required",
+        "youtube.media_forbidden",
+    )
+
+
+def test_youtube_media_403_with_a_cookie_is_not_claimed_as_the_gate():
+    # Only the cookie-less case was measured. With a cookie the same text keeps its bare category
+    # rather than telling a signed-in user to sign in.
+    assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT, cookie_attached=True) == ("network", None)
+
+
+def test_the_youtube_entries_never_match_another_site():
+    # Host-scoped: the same words about a Facebook link stay the bare category.
+    assert dlerrors.diagnose(
+        "ERROR: [facebook] 1: Sign in to confirm you’re not a bot",
+        url="https://www.facebook.com/reel/1/",
+    ) == ("login_required", None)
+
+
 def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie():
     for url in (
         _REEL,