peter
0b672ed6e7
- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a message that is the whole story). A plain RuntimeError, RecursionError or OSError is also what a bug of ours raises, and its stack is the only lead. - The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError under a DB error no longer prints yt-dlp's text unscrubbed. - Engine hide_parameters: a DB error's message no longer carries the title or path it was writing. - The download and edit failure lines go out even when their DB write fails. - The worker's child_exit/crash/supervisor lines say download.job.failed only when the job failed there; a crash the DB will not record no longer ends the thread. Stopping a paused child is download.child.stopped, not a second abort. - log_safe also covers generic ids with spaces, playlist titles and bare ids. - The ffmpeg image steps log the failure kind, not the argv with its path. - The consent contract accepts only the body of a plain consented_trace() check, and any level that may be TRACE. - Tests for each; every one mutation-checked.
308 lines
15 KiB
Python
308 lines
15 KiB
Python
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||
/ reddit / generic).
|
||
|
||
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||
|
||
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||
"""
|
||
|
||
import re
|
||
from urllib.parse import urlparse
|
||
|
||
import yt_dlp
|
||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||
|
||
from app.config import settings
|
||
from app.downloads.stated import StatedFailure
|
||
|
||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||
|
||
|
||
class SourceAdapter:
|
||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||
|
||
key = "generic"
|
||
|
||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||
|
||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||
download cannot go ahead: the content is there but holds no single video
|
||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||
return url
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||
`authed` is whether the acting user's cookie file goes with the download."""
|
||
return [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||
current entry from `client_sets()`. Generic: nothing."""
|
||
return {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||
return False
|
||
|
||
# --- metadata -----------------------------------------------------------------------------
|
||
def title(self, info: dict) -> str | None:
|
||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||
return info.get("title")
|
||
|
||
def uploader_url(self, info: dict) -> str | None:
|
||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||
return info.get("channel_url") or info.get("uploader_url")
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||
return False
|
||
|
||
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||
def source_url(self, source_ref: str) -> str:
|
||
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||
return source_ref
|
||
|
||
def nfo_uniqueid_type(self) -> str:
|
||
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||
return self.key
|
||
|
||
|
||
class YouTubeAdapter(SourceAdapter):
|
||
key = "youtube"
|
||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||
_RETRIABLE = (
|
||
"not a bot",
|
||
"DRM protected",
|
||
"Requested format is not available",
|
||
"Only images are available",
|
||
"Sign in to confirm",
|
||
# Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
|
||
# refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
|
||
# between the player response and the stream, so this is the same per-client flakiness one
|
||
# step later — and it used to end the job on attempt 1, because the text classifies as
|
||
# `network` and nothing here matched it. The format URLs belong to the client that minted
|
||
# them, so the next client set is a real second chance.
|
||
"unable to download video data",
|
||
)
|
||
# Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
|
||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||
# that moves the table turns the gate red instead of silently un-authing every YouTube download.
|
||
_COOKIE_CAPABLE = frozenset(
|
||
{
|
||
"mweb",
|
||
"tv",
|
||
"tv_downgraded",
|
||
"web",
|
||
"web_creator",
|
||
"web_embedded",
|
||
"web_music",
|
||
"web_safari",
|
||
}
|
||
)
|
||
# Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
|
||
# Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
|
||
# the per-client flakiness that the two-tier ladder gives a signed-out download.
|
||
_AUTHED_DEFAULT = ("web_safari", "web")
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
|
||
if authed:
|
||
kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
|
||
return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
|
||
return sets or [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||
ea: dict = {}
|
||
if clients:
|
||
ea["youtube"] = {"player_client": clients}
|
||
if settings.download_pot_base_url:
|
||
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||
return {"extractor_args": ea} if ea else {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
return any(m in error_text for m in self._RETRIABLE)
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
return True
|
||
|
||
def source_url(self, source_ref: str) -> str:
|
||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||
|
||
|
||
class FacebookAdapter(SourceAdapter):
|
||
key = "facebook"
|
||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||
# a cookie the same error is the login wall, which no disguise gets past.
|
||
_IMPERSONATE = "impersonate"
|
||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||
|
||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||
post_id = facebook_group_post_id(url)
|
||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||
return url
|
||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||
page = raw.decode("utf-8", "replace")
|
||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||
if _quoted(post_id) not in page:
|
||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||
video_id = facebook_post_video_id(page, post_id)
|
||
if video_id is None:
|
||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||
raise StatedFailure(POST_WITHOUT_VIDEO_MESSAGE)
|
||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||
|
||
def client_sets(self, authed: bool = False) -> list:
|
||
return [None, self._IMPERSONATE] if authed else [None]
|
||
|
||
def ydl_opts_extra(self, clients) -> dict:
|
||
if clients == self._IMPERSONATE:
|
||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||
return {}
|
||
|
||
def retriable(self, error_text: str) -> bool:
|
||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||
|
||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||
|
||
def title(self, info: dict) -> str | None:
|
||
raw = info.get("title")
|
||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||
return raw
|
||
line = next(
|
||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||
)
|
||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||
|
||
def uploader_url(self, info: dict) -> str | None:
|
||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||
# (the page id) that resolves at facebook.com/<id>.
|
||
base = super().uploader_url(info)
|
||
if base:
|
||
return base
|
||
uid = info.get("uploader_id")
|
||
return f"https://www.facebook.com/{uid}" if uid else None
|
||
|
||
def wants_metadata_backfill(self) -> bool:
|
||
return True
|
||
|
||
|
||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||
|
||
|
||
def _quoted(post_id: str) -> str:
|
||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||
return f'"{post_id}"'
|
||
|
||
|
||
def facebook_group_post_id(url: str) -> str | None:
|
||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||
parsed = urlparse(url)
|
||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||
return None
|
||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||
return match.group(1) if match else None
|
||
|
||
|
||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||
quoted = _quoted(post_id)
|
||
ids = {
|
||
vid
|
||
for block in _SCRIPT_BLOCK.findall(page)
|
||
if quoted in block
|
||
for vid in _VIDEO_NODE.findall(block)
|
||
}
|
||
return ids.pop() if len(ids) == 1 else None
|
||
|
||
|
||
class RedditAdapter(SourceAdapter):
|
||
key = "reddit"
|
||
|
||
|
||
_YOUTUBE = YouTubeAdapter()
|
||
_FACEBOOK = FacebookAdapter()
|
||
_REDDIT = RedditAdapter()
|
||
_GENERIC = SourceAdapter()
|
||
|
||
|
||
def _host_is(host: str, *domains: str) -> bool:
|
||
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
|
||
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
|
||
return any(host == d or host.endswith("." + d) for d in domains)
|
||
|
||
|
||
def resolve_adapter(
|
||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||
) -> SourceAdapter:
|
||
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
|
||
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
|
||
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
|
||
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
|
||
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
|
||
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||
hosts = []
|
||
if source_ref:
|
||
hosts.append(urlparse(source_ref).netloc.lower())
|
||
if info and info.get("webpage_url"):
|
||
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
|
||
|
||
def host_is(*domains: str) -> bool:
|
||
return any(_host_is(h, *domains) for h in hosts)
|
||
|
||
if (
|
||
source_kind == "youtube"
|
||
or "youtube" in extractor
|
||
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
|
||
):
|
||
return _YOUTUBE
|
||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||
return _FACEBOOK
|
||
if "reddit" in extractor or host_is("reddit.com"):
|
||
return _REDDIT
|
||
return _GENERIC
|