From 1354f5b503796a77ce74bfcad244bf6941b80154 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 13 Sep 2026 22:48:21 +0200 Subject: [PATCH] feat(downloads): a Facebook cookie download stuck at "Cannot parse data" is retried as Chrome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on dev: with the user's cookie some reels still answer "Cannot parse data" and only extract when the request impersonates Chrome. Impersonation is a second attempt behind that one error, only when a cookie is attached — links that already download with the cookie keep their plain path, and without a cookie the same error is the login wall. curl_cffi is pinned inside yt-dlp's supported range (0.10-0.15; outside it every target silently reports "unavailable"); the lock only gains the new packages. A test fails the gate if the locked pair can no longer impersonate. --- backend/app/downloads/adapters.py | 29 +++++- backend/app/downloads/pipeline.py | 5 +- backend/requirements.lock | 5 ++ backend/requirements.txt | 6 ++ backend/tests/test_adapters.py | 11 ++- backend/tests/test_facebook_impersonation.py | 92 ++++++++++++++++++++ 6 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 backend/tests/test_facebook_impersonation.py diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 30c85919..86242787 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -13,6 +13,8 @@ Resolution works both before extraction (from source_kind / URL host) and after from urllib.parse import urlparse +from yt_dlp.networking.impersonate import ImpersonateTarget + from app.config import settings @@ -22,9 +24,10 @@ class SourceAdapter: key = "generic" # --- extraction (worker._extract_with_fallback / backfill) --------------------------------- - def client_sets(self) -> list: - """Player-client sets to attempt, in order. A generic source needs no client override, so a - single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept.""" + def client_sets(self, authed: bool = False) -> list: + """Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one + is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`). + `authed` is whether the acting user's cookie file goes with the download.""" return [None] def ydl_opts_extra(self, clients) -> dict: @@ -67,7 +70,7 @@ class YouTubeAdapter(SourceAdapter): "Sign in to confirm", ) - def client_sets(self) -> list: + def client_sets(self, authed: bool = False) -> list: return [ c for c in (settings.player_client_list, settings.player_client_fallback_list) if c ] or [None] @@ -93,6 +96,24 @@ class YouTubeAdapter(SourceAdapter): class FacebookAdapter(SourceAdapter): key = "facebook" + # Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request + # with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a + # SECOND attempt behind that one error, not the default: the links that already download with the + # cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without + # a cookie the same error is the login wall, which no disguise gets past. + _IMPERSONATE = "impersonate" + _WALL = "cannot parse data" + + def client_sets(self, authed: bool = False) -> list: + return [None, self._IMPERSONATE] if authed else [None] + + def ydl_opts_extra(self, clients) -> dict: + if clients == self._IMPERSONATE: + return {"impersonate": ImpersonateTarget.from_str("chrome")} + return {} + + def retriable(self, error_text: str) -> bool: + return self._WALL in error_text.lower() def uploader_url(self, info: dict) -> str | None: # Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id` diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index e683913f..23251899 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -322,13 +322,14 @@ def _extract_with_fallback( ): """Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format - error); other sources make a single attempt. Returns the info dict of the successful attempt. + error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other + sources make a single attempt. Returns the info dict of the successful attempt. `cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by `credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated. `max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt.""" - client_sets = adapter.client_sets() + client_sets = adapter.client_sets(authed=cookiefile is not None) outtmpl = str(staging / "%(id)s.%(ext)s") last_exc: Exception | None = None for i, clients in enumerate(client_sets): diff --git a/backend/requirements.lock b/backend/requirements.lock index f1dd08e0..737de225 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -7,6 +7,7 @@ Authlib==1.7.2 Mako==1.4.1 MarkupSafe==3.0.3 PyYAML==6.0.3 +Pygments==2.21.0 SQLAlchemy==2.0.52 alembic==1.19.1 annotated-doc==0.0.5 @@ -21,6 +22,7 @@ cffi==2.1.1 charset-normalizer==3.5.1 click==8.4.2 cryptography==50.0.0 +curl_cffi==0.15.0 fastapi==0.141.1 feedparser-sgmllib==2.1.0 feedparser==6.0.14 @@ -32,6 +34,8 @@ httpx==0.28.1 idna==3.18 itsdangerous==2.2.0 joserfc==1.7.4 +markdown-it-py==4.2.0 +mdurl==0.1.2 mutagen==1.48.1 numpy==2.5.2 psycopg-binary==3.3.4 @@ -45,6 +49,7 @@ pydantic_core==2.46.4 python-dotenv==1.2.3 python-multipart==0.0.32 requests==2.34.2 +rich==15.0.0 starlette==1.6.0 typing-inspection==0.4.4 typing_extensions==4.16.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index d801c0fd..63f9a24b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -21,6 +21,12 @@ argon2-cffi>=23.1,<26 # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so # keep the floor loose and bump it when extraction breaks. yt-dlp[default]>=2025.5.22 +# Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some +# links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome (E11.5, +# measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imports curl_cffi only in +# 0.10.x–0.15.x and silently reports every target "unavailable" outside it. Move it when bumping +# yt-dlp; `test_adapters` fails the gate if the locked pair can no longer impersonate. +curl_cffi>=0.10,<0.16 # PO Token provider plugin: talks to the bgutil sidecar (brainicism/bgutil-ytdlp-pot-provider) # to mint YouTube Proof-of-Origin tokens on demand, bypassing "confirm you're not a bot" and # unlocking high-quality formats. Pin to match the sidecar image tag (versions must align). diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index 15da7d5b..d5909a44 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -103,12 +103,21 @@ def test_generic_source_url_is_passthrough_youtube_reconstructs(): ) -def test_client_sets_and_retriable_are_youtube_only(): +def test_client_sets_and_retriable_are_per_source(): # Generic sources make a single attempt and never retry on the YouTube per-client markers. assert SourceAdapter().client_sets() == [None] + assert SourceAdapter().client_sets(authed=True) == [None] assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True assert YouTubeAdapter().retriable("some unrelated network blip") is False + # Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind + # the parse wall (without a cookie that wall is the login wall — no disguise gets past it). + fb = FacebookAdapter() + assert fb.client_sets() == [None] + assert len(fb.client_sets(authed=True)) == 2 + assert fb.ydl_opts_extra(fb.client_sets(authed=True)[0]) == {} + assert fb.retriable("ERROR: [facebook] 1: Cannot parse data; please report this issue") is True + assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False def test_youtube_ydl_opts_extra_carries_player_client_and_pot(): diff --git a/backend/tests/test_facebook_impersonation.py b/backend/tests/test_facebook_impersonation.py new file mode 100644 index 00000000..04f1ec1c --- /dev/null +++ b/backend/tests/test_facebook_impersonation.py @@ -0,0 +1,92 @@ +"""E11.5 S3 — a Facebook cookie download stuck at "Cannot parse data" is retried as Chrome. + +Measured on dev (yt-dlp 2026.07.04, the owner's cookie): reel 806379462330494 fails with the cookie +alone and extracts 2560p with `--impersonate chrome`; the reels that already download with the cookie +keep their plain first attempt. These pin the attempt order through the real download loop +(`pipeline._extract_with_fallback`, with yt-dlp faked at the `YoutubeDL` boundary), and that the +LOCKED curl_cffi can actually impersonate — the pair only works inside yt-dlp's own version range, and +outside it every target silently reports "unavailable". +""" + +import pytest +import yt_dlp +from yt_dlp.networking.impersonate import ImpersonateTarget + +from app.downloads import adapters, pipeline + +_URL = "https://www.facebook.com/reel/806379462330494/?fs=e&fs=e" +_WALL = ( + "ERROR: [facebook] 806379462330494: Cannot parse data; please report this issue on " + "https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template." +) +_NO_FORMATS = "ERROR: [facebook] 2541212983053999: No video formats found!" + + +def test_the_locked_curl_cffi_can_impersonate_chrome(): + # YoutubeDL refuses to start when the target is unavailable — the same check a download makes. + with yt_dlp.YoutubeDL({"impersonate": ImpersonateTarget.from_str("chrome"), "quiet": True}): + pass + + +@pytest.fixture +def ydl(monkeypatch): + """Fake `yt_dlp.YoutubeDL`: records each attempt's options and plays back `outcomes` in order.""" + seen: list[dict] = [] + outcomes: list = [] + + class FakeYDL: + def __init__(self, opts): + seen.append(opts) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def extract_info(self, url, download): + outcome = outcomes.pop(0) + if isinstance(outcome, str): + raise yt_dlp.utils.DownloadError(outcome) + return outcome + + monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", FakeYDL) + monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None) + monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **fields: None) + return seen, outcomes + + +def _run(tmp_path, cookiefile): + return pipeline._extract_with_fallback( + 1, 1, _URL, {}, tmp_path, adapters.FacebookAdapter(), cookiefile=cookiefile + ) + + +def test_a_cookie_download_at_the_wall_is_retried_as_chrome(ydl, tmp_path): + seen, outcomes = ydl + outcomes += [_WALL, {"id": "806379462330494"}] + + assert _run(tmp_path, "/cookies.txt") == {"id": "806379462330494"} + + assert [o.get("impersonate") for o in seen] == [None, ImpersonateTarget.from_str("chrome")] + assert all(o["cookiefile"] == "/cookies.txt" for o in seen) # the retry stays signed in + + +def test_without_a_cookie_the_wall_is_the_login_wall_and_final(ydl, tmp_path): + seen, outcomes = ydl + outcomes += [_WALL] + + with pytest.raises(yt_dlp.utils.DownloadError): + _run(tmp_path, None) + + assert len(seen) == 1 and "impersonate" not in seen[0] + + +def test_other_facebook_failures_are_not_retried_as_chrome(ydl, tmp_path): + seen, outcomes = ydl + outcomes += [_NO_FORMATS] + + with pytest.raises(yt_dlp.utils.DownloadError): + _run(tmp_path, "/cookies.txt") + + assert len(seen) == 1 and "impersonate" not in seen[0]