diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 86242787..02a7f822 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -11,8 +11,10 @@ Resolution works both before extraction (from source_kind / URL host) and after `extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter. """ +import re from urllib.parse import urlparse +import yt_dlp from yt_dlp.networking.impersonate import ImpersonateTarget from app.config import settings @@ -24,6 +26,12 @@ class SourceAdapter: key = "generic" # --- extraction (worker._extract_with_fallback / backfill) --------------------------------- + def resolve_url(self, url: str, ydl_opts: dict) -> str | None: + """The link to hand yt-dlp for `url`, resolved BEFORE the first attempt, or None when the source + itself shows there is no single video to download there. `ydl_opts` are the download's own + options (cookie, timeouts, address family) for any page this needs to read. Generic: `url`.""" + return url + def client_sets(self, authed: bool = False) -> list: """Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`). @@ -103,6 +111,22 @@ class FacebookAdapter(SourceAdapter): # a cookie the same error is the login wall, which no disguise gets past. _IMPERSONATE = "impersonate" _WALL = "cannot parse data" + # A post page is ~4–5 MB of inline JSON; anything far past that is not a post page. + _POST_PAGE_MAX_BYTES = 16 * 1024 * 1024 + + def resolve_url(self, url: str, ydl_opts: dict) -> str | None: + # Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse — + # downloads ANOTHER video from the page's feed data. The post's own video is the one Video node + # in the page scripts that carry the post id; its watch link downloads (plain first, then the + # Chrome retry above). Needs the cookie: signed out, the page is the login wall. + post_id = facebook_group_post_id(url) + if post_id is None or not ydl_opts.get("cookiefile"): + return url + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES) + video_id = facebook_post_video_id(raw.decode("utf-8", "replace"), post_id) + # Never fall back to the post link: that is exactly where the wrong video comes from. + return f"https://www.facebook.com/watch/?v={video_id}" if video_id else None def client_sets(self, authed: bool = False) -> list: return [None, self._IMPERSONATE] if authed else [None] @@ -128,6 +152,34 @@ class FacebookAdapter(SourceAdapter): return True +_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$") +_SCRIPT_BLOCK = re.compile(r"]*>(.*?)", re.S) +_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"') + + +def facebook_group_post_id(url: str) -> str | None: + """The post id of a Facebook group post link (`/groups//permalink|posts//`), else None.""" + parsed = urlparse(url) + if not _host_is(parsed.netloc.lower(), "facebook.com"): + return None + match = _FB_GROUP_POST_PATH.match(parsed.path) + return match.group(1) if match else None + + +def facebook_post_video_id(page: str, post_id: str) -> str | None: + """The id of the one video attached to `post_id`, read from its page: the Video nodes of the script + blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks). + None for no video AND for more than one: guessing between two would be the wrong-video bug again.""" + quoted = f'"{post_id}"' + ids = { + vid + for block in _SCRIPT_BLOCK.findall(page) + if quoted in block + for vid in _VIDEO_NODE.findall(block) + } + return ids.pop() if len(ids) == 1 else None + + class RedditAdapter(SourceAdapter): key = "reddit" diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 0d79b480..3b68e300 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -28,7 +28,7 @@ from urllib.parse import urlparse from app.downloads.credentials import host_key_for_url -# Two worker-authored strings that this module owns because both are coupled to `_RULES`. +# Worker-authored strings that this module owns because each is coupled to `_RULES`. # # QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule # below matches a fragment of it, and test_download_errors pins the pair, because a message and its @@ -43,6 +43,10 @@ from app.downloads.credentials import host_key_for_url # classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more. QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again." NETWORK_REDACTED_MESSAGE = "The server could not reach that address." +# POST_WITHOUT_VIDEO_MESSAGE is raised by the worker when a source adapter read a post's own page and +# found no single video in it (`SourceAdapter.resolve_url` → None; a Facebook group post, E11.5 S4). +# Its rule gives the category back without a link; the catalog entry gives the site's sentence. +POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post." # (marker, category) — first match wins, so order most-specific → most-general. Matching is # case-insensitive on a substring, against the raw yt-dlp/ffmpeg message. @@ -52,6 +56,8 @@ _RULES: tuple[tuple[str, str], ...] = ( ("download storage is full", "quota_exceeded"), # Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too. (NETWORK_REDACTED_MESSAGE.lower(), "network"), + # Our own verdict on a post link whose page holds no single video — our wording too. + (POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"), # Login / age / members / bot-check — the user must be signed in (we deliberately are not). ("sign in to confirm your age", "login_required"), ("confirm your age", "login_required"), @@ -166,9 +172,17 @@ _CATALOG: tuple[_Entry, ...] = ( detail="facebook.cookie_rejected", cookie=True, ), - # A group post whose video yt-dlp does not reach from the post's own link, although the video's - # own link downloads (prod job 66, with the cookie). Only for group links: a reel with no formats - # is a different failure. + # A group post with no single video of its own. With a cookie the adapter reads the post's page + # and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and + # answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared + # clips). Only for group links: a reel with no formats is a different failure. + _Entry( + host="facebook", + markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),), + code="format_unavailable", + detail="facebook.group_post_video", + link=("/groups/",), + ), _Entry( host="facebook", markers=("[facebook]", "no video formats found"), diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 0a08fdb6..e5e5f277 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -325,12 +325,30 @@ def _extract_with_fallback( error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other sources make a single attempt. Returns the info dict of the successful attempt. + Before the first attempt the adapter may resolve `url` to the link that really holds the video (a + Facebook group post → its video's watch link); when it finds no single video there the job fails + with POST_WITHOUT_VIDEO_MESSAGE instead of letting yt-dlp guess. + `cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by `credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated. `max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt.""" client_sets = adapter.client_sets(authed=cookiefile is not None) outtmpl = str(staging / "%(id)s.%(ext)s") + # Resolving may read a page first — network time the stall watchdog must not see as a stall. + _touch_job(job_id) + target = adapter.resolve_url( + url, + formats.build_ydl_opts( + spec, + outtmpl, + _make_progress_hook(job_id, asset_id), + cookiefile=cookiefile, + max_bytes=max_bytes, + ), + ) + if target is None: + raise RuntimeError(dlerrors.POST_WITHOUT_VIDEO_MESSAGE) last_exc: Exception | None = None for i, clients in enumerate(client_sets): opts = formats.build_ydl_opts( @@ -348,7 +366,7 @@ def _extract_with_fallback( # killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.) try: with yt_dlp.YoutubeDL(opts) as ydl: - return ydl.extract_info(url, download=True) + return ydl.extract_info(target, download=True) except yt_dlp.utils.DownloadError as exc: last_exc = exc if i < len(client_sets) - 1 and adapter.retriable(str(exc)): diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index b467dee6..e85d3b93 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -113,6 +113,18 @@ def test_a_facebook_group_post_without_formats_names_the_link(): ) +def test_our_post_without_a_video_verdict_names_the_group_link(): + # The worker's own text when the post page holds no single video (S4): the message and its rule and + # catalog entry are pinned together, or an edit to the wording orphans the copy. + msg = dlerrors.POST_WITHOUT_VIDEO_MESSAGE + assert classify(msg) == "format_unavailable" + assert dlerrors.diagnose(msg, url=_GROUP_POST, cookie_attached=True) == ( + "format_unavailable", + "facebook.group_post_video", + ) + assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None) + + def test_a_catalog_entry_never_leaves_its_host(): # "Cannot parse data" is yt-dlp's wording for a broken extractor on every site. other = _FB_PARSE.replace("[facebook]", "[vimeo]") diff --git a/backend/tests/test_facebook_impersonation.py b/backend/tests/test_facebook_impersonation.py index 04f1ec1c..186fa7ea 100644 --- a/backend/tests/test_facebook_impersonation.py +++ b/backend/tests/test_facebook_impersonation.py @@ -1,11 +1,16 @@ -"""E11.5 S3 — a Facebook cookie download stuck at "Cannot parse data" is retried as Chrome. +"""E11.5 S3 + S4 — the Facebook download path with the user's cookie. -Measured on dev (yt-dlp 2026.07.04, the owner's cookie): reel 806379462330494 fails with the cookie +S3, measured on dev (yt-dlp 2026.07.04, the owner's cookie): reel 806379462330494 fails with the cookie alone and extracts 2560p with `--impersonate chrome`; the reels that already download with the cookie keep their plain first attempt. These pin the attempt order through the real download loop (`pipeline._extract_with_fallback`, with yt-dlp faked at the `YoutubeDL` boundary), and that the LOCKED curl_cffi can actually impersonate — the pair only works inside yt-dlp's own version range, and outside it every target silently reports "unavailable". + +S4, measured on dev and in the owner's Chrome: given a group post's link, yt-dlp finds no formats or +downloads another video from the page's feed. The post's own video is the single Video node in the +page scripts that carry the post id (prod posts 66, 145, 140, 143, 221; text posts have none while the +page holds ~10 other videos), and its watch link downloads. """ import pytest @@ -13,8 +18,31 @@ import yt_dlp from yt_dlp.networking.impersonate import ImpersonateTarget from app.downloads import adapters, pipeline +from app.downloads import errors as dlerrors _URL = "https://www.facebook.com/reel/806379462330494/?fs=e&fs=e" +_POST = "1755988225525547" +_GROUP_LINK = f"https://www.facebook.com/groups/983554479435596/permalink/{_POST}/?" +_POST_VIDEO = "1408015447857753" +_FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video +_CHROME = ImpersonateTarget.from_str("chrome") + + +def _video_node(video_id: str) -> str: + # The measured shape of a post's attachment in the page JSON. + return '{"media":{"__typename":"Video","id":"%s","__isNode":"Video"},"styles":{}}' % video_id + + +def _page(*post_videos: str, post: str = _POST) -> str: + attachments = ",".join(_video_node(v) for v in post_videos) + return ( + '' + '' + ) + + _WALL = ( "ERROR: [facebook] 806379462330494: Cannot parse data; please report this issue on " "https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template." @@ -30,9 +58,11 @@ def test_the_locked_curl_cffi_can_impersonate_chrome(): @pytest.fixture def ydl(monkeypatch): - """Fake `yt_dlp.YoutubeDL`: records each attempt's options and plays back `outcomes` in order.""" + """Fake `yt_dlp.YoutubeDL`: records each attempt's options and plays back `outcomes` in order; + `web["page"]` is what a page read returns, and `web` records the pages opened and links extracted.""" seen: list[dict] = [] outcomes: list = [] + web: dict = {"page": "", "opened": [], "extracted": []} class FakeYDL: def __init__(self, opts): @@ -44,7 +74,18 @@ def ydl(monkeypatch): def __exit__(self, *exc): return False + def urlopen(self, url): + web["opened"].append(url) + body = web["page"].encode() + + class Response: + def read(self, amt=None): + return body + + return Response() + def extract_info(self, url, download): + web["extracted"].append(url) outcome = outcomes.pop(0) if isinstance(outcome, str): raise yt_dlp.utils.DownloadError(outcome) @@ -53,27 +94,28 @@ def ydl(monkeypatch): monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", FakeYDL) monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None) monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **fields: None) - return seen, outcomes + return seen, outcomes, web -def _run(tmp_path, cookiefile): +def _run(tmp_path, cookiefile, url=_URL): return pipeline._extract_with_fallback( - 1, 1, _URL, {}, tmp_path, adapters.FacebookAdapter(), cookiefile=cookiefile + 1, 1, url, {}, tmp_path, adapters.FacebookAdapter(), cookiefile=cookiefile ) def test_a_cookie_download_at_the_wall_is_retried_as_chrome(ydl, tmp_path): - seen, outcomes = ydl + seen, outcomes, web = ydl outcomes += [_WALL, {"id": "806379462330494"}] assert _run(tmp_path, "/cookies.txt") == {"id": "806379462330494"} - assert [o.get("impersonate") for o in seen] == [None, ImpersonateTarget.from_str("chrome")] + assert [o.get("impersonate") for o in seen] == [None, _CHROME] assert all(o["cookiefile"] == "/cookies.txt" for o in seen) # the retry stays signed in + assert web["opened"] == [] # a reel is not a group post: no page read def test_without_a_cookie_the_wall_is_the_login_wall_and_final(ydl, tmp_path): - seen, outcomes = ydl + seen, outcomes, _ = ydl outcomes += [_WALL] with pytest.raises(yt_dlp.utils.DownloadError): @@ -83,10 +125,81 @@ def test_without_a_cookie_the_wall_is_the_login_wall_and_final(ydl, tmp_path): def test_other_facebook_failures_are_not_retried_as_chrome(ydl, tmp_path): - seen, outcomes = ydl + seen, outcomes, _ = ydl outcomes += [_NO_FORMATS] with pytest.raises(yt_dlp.utils.DownloadError): _run(tmp_path, "/cookies.txt") assert len(seen) == 1 and "impersonate" not in seen[0] + + +# --- S4: a group post downloads ITS video, or fails saying it has none ---------------------------- + + +def test_a_cookie_group_post_downloads_the_posts_own_video(ydl, tmp_path): + seen, outcomes, web = ydl + web["page"] = _page(_POST_VIDEO) + outcomes += [_WALL, {"id": _POST_VIDEO}] # the watch link needs Chrome too (measured) + + assert _run(tmp_path, "/cookies.txt", _GROUP_LINK) == {"id": _POST_VIDEO} + + assert web["opened"] == [_GROUP_LINK] + assert seen[0]["cookiefile"] == "/cookies.txt" # the post page is read signed in + watch = f"https://www.facebook.com/watch/?v={_POST_VIDEO}" + # Never the post link: that is where the feed video comes from. + assert web["extracted"] == [watch, watch] + assert [o.get("impersonate") for o in seen[1:]] == [None, _CHROME] + + +def test_a_group_post_without_a_single_video_fails_saying_so(ydl, tmp_path): + _, _, web = ydl + web["page"] = _page() # a text post: the only video on the page is the feed's + + with pytest.raises(RuntimeError) as exc: + _run(tmp_path, "/cookies.txt", _GROUP_LINK) + + assert web["extracted"] == [] # yt-dlp is not left to guess from the post link + assert dlerrors.diagnose(str(exc.value), url=_GROUP_LINK, cookie_attached=True) == ( + "format_unavailable", + "facebook.group_post_video", + ) + + +def test_without_a_cookie_the_group_post_link_goes_to_ytdlp_unchanged(ydl, tmp_path): + _, outcomes, web = ydl + outcomes += [_NO_FORMATS] + + with pytest.raises(yt_dlp.utils.DownloadError): + _run(tmp_path, None, _GROUP_LINK) + + assert web["opened"] == [] # signed out the page is the login wall: nothing to read + assert web["extracted"] == [_GROUP_LINK] + + +def test_the_group_post_link_forms_are_recognised(): + for url in ( + _GROUP_LINK, + f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/", + f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?", + f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}", + ): + assert adapters.facebook_group_post_id(url) == _POST, url + for url in ( + _URL, + f"https://www.facebook.com/watch/?v={_POST_VIDEO}", + f"https://facebook.com.evil/groups/x/permalink/{_POST}/", + "https://www.facebook.com/groups/983554479435596/", + ): + assert adapters.facebook_group_post_id(url) is None, url + assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK + + +def test_only_the_single_video_of_the_posts_own_blocks_counts(): + assert adapters.facebook_post_video_id(_page(_POST_VIDEO), _POST) == _POST_VIDEO + # No video in the post, although the page has one elsewhere. + assert adapters.facebook_post_video_id(_page(), _POST) is None + # Two videos: guessing between them would be the wrong-video bug again. + assert adapters.facebook_post_video_id(_page(_POST_VIDEO, "2063302677911049"), _POST) is None + # A longer number that merely STARTS with the post id is another object. + assert adapters.facebook_post_video_id(_page(_POST_VIDEO, post=_POST + "0"), _POST) is None diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 1c21a6c4..7f0a8242 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -299,8 +299,8 @@ "body": "The cookies have probably expired, or you signed out in that browser. Export them again, upload them, and retry. If it still fails, the content has been removed or isn't visible to you." }, "group_post_video": { - "short": "The video isn't reachable from the post link", - "body": "This is the link of a group post, not of the video. Open the video on Facebook and paste its link." + "short": "No video found in this post", + "body": "We couldn't find a single downloadable video in this group post. If it does have one, open the video on Facebook and paste its link." } } } diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index c1883b52..57c3b775 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -299,8 +299,8 @@ "body": "A cookie valószínűleg lejárt, vagy kijelentkeztél abban a böngészőben. Exportáld újra, töltsd fel, és próbáld újra. Ha így sem megy, a tartalom törölt, vagy számodra nem látható." }, "group_post_video": { - "short": "A videó nem érhető el a poszt linkjéről", - "body": "Ez a csoportposzt linkje, nem a videóé. Nyisd meg a videót a Facebookon, és annak a linkjét másold be." + "short": "Ebben a posztban nem található videó", + "body": "Nem találtunk egyértelműen letölthető videót ebben a csoportposztban. Ha mégis van benne, nyisd meg a videót a Facebookon, és annak a linkjét másold be." } } }