peter
6169e32ca5
Since a Facebook cookie was added on prod, every reel came down titled "Video": measured on dev, a signed-in read gives yt-dlp the placeholder "Video" (an 18+ reel read as Chrome gives "Facebook") while the post's text is still in the description. A new SourceAdapter.title hook takes the whole first line of the description in that case, and both title writes (the early fill and the finished download) go through it, so the row, the file name and the share preview carry the post's text again. The line is not cut with an ellipsis: normalize_title's NFKC turns "…" into "..." and collapses it to ".", which read as a finished sentence (seen in dev UAT). The "Audio (MP3)" profile's extracted .mp3 was remuxed into an .mp4 by the browser-safe step (the audio spec is clamped to container mp4 too) and served as video/mp4, which iPad's WebKit will not play (prod job 268). Audio mode now skips that step, so an .mp3 is served as audio/mpeg and an .m4a as audio/mp4. Audio downloads version their cache identity on their own (v5), so a cached remuxed audio re-derives without re-downloading every cached video.
273 lines
13 KiB
Python
273 lines
13 KiB
Python
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||
/ reddit / generic).
|
||
|
||
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||
|
||
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||
"""
|
||
|
||
import re
|
||
from urllib.parse import urlparse
|
||
|
||
import yt_dlp
|
||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||
|
||
from app.config import settings
|
||
|
||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||
|
||
|
||
class SourceAdapter:
|
||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||
|
||
key = "generic"
|
||
|
||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||
|
||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||
download cannot go ahead: the content is there but holds no single video
|
||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||
return url
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||
`authed` is whether the acting user's cookie file goes with the download."""
|
||
return [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||
current entry from `client_sets()`. Generic: nothing."""
|
||
return {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||
return False
|
||
|
||
# --- metadata -----------------------------------------------------------------------------
|
||
def title(self, info: dict) -> str | None:
|
||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||
return info.get("title")
|
||
|
||
def uploader_url(self, info: dict) -> str | None:
|
||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||
return info.get("channel_url") or info.get("uploader_url")
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||
return False
|
||
|
||
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||
def source_url(self, source_ref: str) -> str:
|
||
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||
return source_ref
|
||
|
||
def nfo_uniqueid_type(self) -> str:
|
||
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||
return self.key
|
||
|
||
|
||
class YouTubeAdapter(SourceAdapter):
|
||
key = "youtube"
|
||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||
_RETRIABLE = (
|
||
"not a bot",
|
||
"DRM protected",
|
||
"Requested format is not available",
|
||
"Only images are available",
|
||
"Sign in to confirm",
|
||
)
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
return [
|
||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||
] or [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||
ea: dict = {}
|
||
if clients:
|
||
ea["youtube"] = {"player_client": clients}
|
||
if settings.download_pot_base_url:
|
||
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||
return {"extractor_args": ea} if ea else {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
return any(m in error_text for m in self._RETRIABLE)
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
return True
|
||
|
||
def source_url(self, source_ref: str) -> str:
|
||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||
|
||
|
||
class FacebookAdapter(SourceAdapter):
|
||
key = "facebook"
|
||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||
# a cookie the same error is the login wall, which no disguise gets past.
|
||
_IMPERSONATE = "impersonate"
|
||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||
|
||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||
post_id = facebook_group_post_id(url)
|
||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||
return url
|
||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||
page = raw.decode("utf-8", "replace")
|
||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||
if _quoted(post_id) not in page:
|
||
raise RuntimeError(POST_NOT_SHOWN_MESSAGE)
|
||
video_id = facebook_post_video_id(page, post_id)
|
||
if video_id is None:
|
||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||
raise RuntimeError(POST_WITHOUT_VIDEO_MESSAGE)
|
||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
return [None, self._IMPERSONATE] if authed else [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
if clients == self._IMPERSONATE:
|
||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||
return {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||
|
||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||
|
||
def title(self, info: dict) -> str | None:
|
||
raw = info.get("title")
|
||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||
return raw
|
||
line = next(
|
||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||
)
|
||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||
|
||
def uploader_url(self, info: dict) -> str | None:
|
||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||
# (the page id) that resolves at facebook.com/<id>.
|
||
base = super().uploader_url(info)
|
||
if base:
|
||
return base
|
||
uid = info.get("uploader_id")
|
||
return f"https://www.facebook.com/{uid}" if uid else None
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
return True
|
||
|
||
|
||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||
|
||
|
||
def _quoted(post_id: str) -> str:
|
||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||
return f'"{post_id}"'
|
||
|
||
|
||
def facebook_group_post_id(url: str) -> str | None:
|
||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||
parsed = urlparse(url)
|
||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||
return None
|
||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||
return match.group(1) if match else None
|
||
|
||
|
||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||
quoted = _quoted(post_id)
|
||
ids = {
|
||
vid
|
||
for block in _SCRIPT_BLOCK.findall(page)
|
||
if quoted in block
|
||
for vid in _VIDEO_NODE.findall(block)
|
||
}
|
||
return ids.pop() if len(ids) == 1 else None
|
||
|
||
|
||
class RedditAdapter(SourceAdapter):
|
||
key = "reddit"
|
||
|
||
|
||
_YOUTUBE = YouTubeAdapter()
|
||
_FACEBOOK = FacebookAdapter()
|
||
_REDDIT = RedditAdapter()
|
||
_GENERIC = SourceAdapter()
|
||
|
||
|
||
def _host_is(host: str, *domains: str) -> bool:
|
||
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
|
||
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
|
||
return any(host == d or host.endswith("." + d) for d in domains)
|
||
|
||
|
||
def resolve_adapter(
|
||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||
) -> SourceAdapter:
|
||
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
|
||
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
|
||
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
|
||
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
|
||
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
|
||
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||
hosts = []
|
||
if source_ref:
|
||
hosts.append(urlparse(source_ref).netloc.lower())
|
||
if info and info.get("webpage_url"):
|
||
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
|
||
|
||
def host_is(*domains: str) -> bool:
|
||
return any(_host_is(h, *domains) for h in hosts)
|
||
|
||
if (
|
||
source_kind == "youtube"
|
||
or "youtube" in extractor
|
||
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
|
||
):
|
||
return _YOUTUBE
|
||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||
return _FACEBOOK
|
||
if "reddit" in extractor or host_is("reddit.com"):
|
||
return _REDDIT
|
||
return _GENERIC
|