feat(downloads): a Facebook group post downloads its own video, or says it has none

Given a group post's link, yt-dlp either found no formats or took another
video from the page's feed data, so a "successful" retry of such a post was a
wrong download. With the user's cookie the Facebook adapter now reads the post
page, takes the single video node in the script blocks that carry the post id,
and downloads that video's watch link (the Chrome retry does the rest). No
single video in the post fails with its own message and the group-post copy,
reworded to say the post has no video; the post link is never handed to yt-dlp
as a fallback. Without a cookie nothing changes.
This commit is contained in:
2026-09-13 23:38:11 +02:00
parent 118fc70e08
commit 90ee34a1b4
7 changed files with 228 additions and 19 deletions
+52
View File
@@ -11,8 +11,10 @@ Resolution works both before extraction (from source_kind / URL host) and after
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""
import re
from urllib.parse import urlparse
import yt_dlp
from yt_dlp.networking.impersonate import ImpersonateTarget
from app.config import settings
@@ -24,6 +26,12 @@ class SourceAdapter:
key = "generic"
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
def resolve_url(self, url: str, ydl_opts: dict) -> str | None:
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt, or None when the source
itself shows there is no single video to download there. `ydl_opts` are the download's own
options (cookie, timeouts, address family) for any page this needs to read. Generic: `url`."""
return url
def client_sets(self, authed: bool = False) -> list:
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
@@ -103,6 +111,22 @@ class FacebookAdapter(SourceAdapter):
# a cookie the same error is the login wall, which no disguise gets past.
_IMPERSONATE = "impersonate"
_WALL = "cannot parse data"
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
def resolve_url(self, url: str, ydl_opts: dict) -> str | None:
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
post_id = facebook_group_post_id(url)
if post_id is None or not ydl_opts.get("cookiefile"):
return url
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
video_id = facebook_post_video_id(raw.decode("utf-8", "replace"), post_id)
# Never fall back to the post link: that is exactly where the wrong video comes from.
return f"https://www.facebook.com/watch/?v={video_id}" if video_id else None
def client_sets(self, authed: bool = False) -> list:
return [None, self._IMPERSONATE] if authed else [None]
@@ -128,6 +152,34 @@ class FacebookAdapter(SourceAdapter):
return True
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
def facebook_group_post_id(url: str) -> str | None:
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
parsed = urlparse(url)
if not _host_is(parsed.netloc.lower(), "facebook.com"):
return None
match = _FB_GROUP_POST_PATH.match(parsed.path)
return match.group(1) if match else None
def facebook_post_video_id(page: str, post_id: str) -> str | None:
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
quoted = f'"{post_id}"'
ids = {
vid
for block in _SCRIPT_BLOCK.findall(page)
if quoted in block
for vid in _VIDEO_NODE.findall(block)
}
return ids.pop() if len(ids) == 1 else None
class RedditAdapter(SourceAdapter):
key = "reddit"