siftlode/backend/app/downloads/adapters.py

"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
/ reddit / generic).

yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
scattered across worker.py / backfill.py / service.py with no shared source identity.

Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""

import re
from urllib.parse import urlparse

import yt_dlp
from yt_dlp.networking.impersonate import ImpersonateTarget

from app.config import settings

# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
FACEBOOK_PARSE_WALL = "cannot parse data"
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
# them from here because this module must not import errors (errors → credentials → adapters).
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."


class SourceAdapter:
    """Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""

    key = "generic"

    # --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
    def resolve_url(self, url: str, ydl_opts: dict) -> str:
        """The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
        download's own options (cookie, timeouts, address family) for any page this needs to read.

        Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
        download cannot go ahead: the content is there but holds no single video
        (`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
        refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
        is raised as itself, so it is classified like any other download error. Generic: `url`."""
        return url

    def client_sets(self, authed: bool = False) -> list:
        """Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
        is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
        `authed` is whether the acting user's cookie file goes with the download."""
        return [None]

    def ydl_opts_extra(self, clients) -> dict:
        """yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
        current entry from `client_sets()`. Generic: nothing."""
        return {}

    def retriable(self, error_text: str) -> bool:
        """Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
        return False

    # --- metadata -----------------------------------------------------------------------------
    def title(self, info: dict) -> str | None:
        """The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
        return info.get("title")

    def uploader_url(self, info: dict) -> str | None:
        """The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
        `channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
        return info.get("channel_url") or info.get("uploader_url")

    def wants_metadata_backfill(self) -> bool:
        """Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
        return False

    # --- source ref ↔ URL ---------------------------------------------------------------------
    def source_url(self, source_ref: str) -> str:
        """A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
        return source_ref

    def nfo_uniqueid_type(self) -> str:
        """The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
        return self.key


class YouTubeAdapter(SourceAdapter):
    key = "youtube"
    # Errors worth retrying with a different player-client set (YouTube per-client flakiness).
    _RETRIABLE = (
        "not a bot",
        "DRM protected",
        "Requested format is not available",
        "Only images are available",
        "Sign in to confirm",
        # A set yt-dlp emptied because none of its clients take cookies. `client_sets` filters those
        # out below, so this is the safety net for a yt-dlp that renames a client or an admin who
        # configures both knobs with cookie-incapable clients only.
        "No player clients have been requested",
        # Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
        # refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
        # between the player response and the stream, so this is the same per-client flakiness one
        # step later — and it used to end the job on attempt 1, because the text classifies as
        # `network` and nothing here matched it. The format URLs belong to the client that minted
        # them, so the next client set is a real second chance.
        "unable to download video data",
    )
    # Measured against yt-dlp's own INNERTUBE_CLIENTS table (2026-09-16, on 2026.07.04 and
    # 2026.08.19): with account cookies attached yt-dlp REMOVES every client whose SUPPORTS_COOKIES
    # is False, and a set left empty raises "No player clients have been requested". Our primary set
    # is android_vr — one of those — so a signed-in download would fail on the first attempt and
    # never reach the POT-backed web fallback that the cookie exists for. Pinned by test_adapters
    # against the installed yt-dlp, so a version bump that moves the table turns the gate red
    # instead of silently un-authing every YouTube download.
    _COOKIE_CAPABLE = frozenset(
        {
            "mweb",
            "tv",
            "tv_downgraded",
            "web",
            "web_creator",
            "web_embedded",
            "web_music",
            "web_safari",
        }
    )
    # Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
    # Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
    # the per-client flakiness that the two-tier ladder gives a signed-out download.
    _AUTHED_DEFAULT = ("web_safari", "web")

    def client_sets(self, authed: bool = False) -> list:
        sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
        if authed:
            kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
            return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
        return sets or [None]

    def ydl_opts_extra(self, clients) -> dict:
        # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
        ea: dict = {}
        if clients:
            ea["youtube"] = {"player_client": clients}
        if settings.download_pot_base_url:
            ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
        return {"extractor_args": ea} if ea else {}

    def retriable(self, error_text: str) -> bool:
        return any(m in error_text for m in self._RETRIABLE)

    def wants_metadata_backfill(self) -> bool:
        return True

    def source_url(self, source_ref: str) -> str:
        return f"https://www.youtube.com/watch?v={source_ref}"


class FacebookAdapter(SourceAdapter):
    key = "facebook"
    # Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
    # with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
    # SECOND attempt behind that one error, not the default: the links that already download with the
    # cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
    # a cookie the same error is the login wall, which no disguise gets past.
    _IMPERSONATE = "impersonate"
    # A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
    _POST_PAGE_MAX_BYTES = 16 * 1024 * 1024

    def resolve_url(self, url: str, ydl_opts: dict) -> str:
        # Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
        # downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
        # in the page scripts that carry the post id; its watch link downloads (plain first, then the
        # Chrome retry above). Needs the cookie: signed out, the page is the login wall.
        post_id = facebook_group_post_id(url)
        if post_id is None or not ydl_opts.get("cookiefile"):
            return url
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
        page = raw.decode("utf-8", "replace")
        # Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
        # form, no post id at all), which is not a post without a video — the user's cookie is the cause.
        if _quoted(post_id) not in page:
            raise RuntimeError(POST_NOT_SHOWN_MESSAGE)
        video_id = facebook_post_video_id(page, post_id)
        if video_id is None:
            # Never fall back to the post link: that is exactly where the wrong video comes from.
            raise RuntimeError(POST_WITHOUT_VIDEO_MESSAGE)
        return f"https://www.facebook.com/watch/?v={video_id}"

    def client_sets(self, authed: bool = False) -> list:
        return [None, self._IMPERSONATE] if authed else [None]

    def ydl_opts_extra(self, clients) -> dict:
        if clients == self._IMPERSONATE:
            return {"impersonate": ImpersonateTarget.from_str("chrome")}
        return {}

    def retriable(self, error_text: str) -> bool:
        return FACEBOOK_PARSE_WALL in error_text.lower()

    # Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
    # 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
    # signed-out read titles the page with that same text behind the engagement counts, cut short.
    _PLACEHOLDER_TITLES = frozenset({"video", "facebook"})

    def title(self, info: dict) -> str | None:
        raw = info.get("title")
        if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
            return raw
        line = next(
            (ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
        )
        # The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
        # "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
        # stored title is unbounded; display_name and the file name clip it themselves.
        return line or raw  # no description either (the 18+ reel): nothing better on the page

    def uploader_url(self, info: dict) -> str | None:
        # Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
        # (the page id) that resolves at facebook.com/<id>.
        base = super().uploader_url(info)
        if base:
            return base
        uid = info.get("uploader_id")
        return f"https://www.facebook.com/{uid}" if uid else None

    def wants_metadata_backfill(self) -> bool:
        return True


_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')


def _quoted(post_id: str) -> str:
    # How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
    return f'"{post_id}"'


def facebook_group_post_id(url: str) -> str | None:
    """The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
    parsed = urlparse(url)
    if not _host_is(parsed.netloc.lower(), "facebook.com"):
        return None
    match = _FB_GROUP_POST_PATH.match(parsed.path)
    return match.group(1) if match else None


def facebook_post_video_id(page: str, post_id: str) -> str | None:
    """The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
    blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
    None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
    quoted = _quoted(post_id)
    ids = {
        vid
        for block in _SCRIPT_BLOCK.findall(page)
        if quoted in block
        for vid in _VIDEO_NODE.findall(block)
    }
    return ids.pop() if len(ids) == 1 else None


class RedditAdapter(SourceAdapter):
    key = "reddit"


_YOUTUBE = YouTubeAdapter()
_FACEBOOK = FacebookAdapter()
_REDDIT = RedditAdapter()
_GENERIC = SourceAdapter()


def _host_is(host: str, *domains: str) -> bool:
    """True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
    loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
    return any(host == d or host.endswith("." + d) for d in domains)


def resolve_adapter(
    source_kind: str | None, source_ref: str | None = None, info: dict | None = None
) -> SourceAdapter:
    """Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
    (post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
    (pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
    the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
    source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
    extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
    hosts = []
    if source_ref:
        hosts.append(urlparse(source_ref).netloc.lower())
    if info and info.get("webpage_url"):
        hosts.append(urlparse(info["webpage_url"]).netloc.lower())

    def host_is(*domains: str) -> bool:
        return any(_host_is(h, *domains) for h in hosts)

    if (
        source_kind == "youtube"
        or "youtube" in extractor
        or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
    ):
        return _YOUTUBE
    if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
        return _FACEBOOK
    if "reddit" in extractor or host_is("reddit.com"):
        return _REDDIT
    return _GENERIC