refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp

Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered
across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source
identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source
overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp
extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is
policy only.

Moved into adapters (behaviour-equivalent):
- uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the
  worker/backfill duplication (build_uploader_url + _should_refetch gone).
- the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts
  now takes source-agnostic opts plus an `extra` dict from the adapter).
- source_url reconstruction (service.source_url delegates to the adapter).

Two intentional (approved) behaviour changes:
- the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction
  attempt instead of a pointless double attempt with youtube-only client args.
- the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the
  hardcoded "youtube" that mislabelled every url-source download.

Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless
across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's
YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path).

Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the
.nfo uniqueid type are correct per source.

backend/app/downloads/adapters.py

@@ -0,0 +1,136 @@
+"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
+/ reddit / generic).
+
+yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
+base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
+subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
+scattered across worker.py / backfill.py / service.py with no shared source identity.
+
+Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
+Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
+`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
+"""
+from urllib.parse import urlparse
+
+from app.config import settings
+
+
+class SourceAdapter:
+    """Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
+
+    key = "generic"
+
+    # --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
+    def client_sets(self) -> list:
+        """Player-client sets to attempt, in order. A generic source needs no client override, so a
+        single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
+        return [None]
+
+    def ydl_opts_extra(self, clients) -> dict:
+        """yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
+        current entry from `client_sets()`. Generic: nothing."""
+        return {}
+
+    def retriable(self, error_text: str) -> bool:
+        """Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
+        return False
+
+    # --- metadata -----------------------------------------------------------------------------
+    def uploader_url(self, info: dict) -> str | None:
+        """The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
+        `channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
+        return info.get("channel_url") or info.get("uploader_url")
+
+    def wants_metadata_backfill(self) -> bool:
+        """Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
+        return False
+
+    # --- source ref ↔ URL ---------------------------------------------------------------------
+    def source_url(self, source_ref: str) -> str:
+        """A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
+        return source_ref
+
+    def nfo_uniqueid_type(self) -> str:
+        """The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
+        return self.key
+
+
+class YouTubeAdapter(SourceAdapter):
+    key = "youtube"
+    # Errors worth retrying with a different player-client set (YouTube per-client flakiness).
+    _RETRIABLE = (
+        "not a bot",
+        "DRM protected",
+        "Requested format is not available",
+        "Only images are available",
+        "Sign in to confirm",
+    )
+
+    def client_sets(self) -> list:
+        return [
+            c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
+        ] or [None]
+
+    def ydl_opts_extra(self, clients) -> dict:
+        # == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
+        ea: dict = {}
+        if clients:
+            ea["youtube"] = {"player_client": clients}
+        if settings.download_pot_base_url:
+            ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
+        return {"extractor_args": ea} if ea else {}
+
+    def retriable(self, error_text: str) -> bool:
+        return any(m in error_text for m in self._RETRIABLE)
+
+    def wants_metadata_backfill(self) -> bool:
+        return True
+
+    def source_url(self, source_ref: str) -> str:
+        return f"https://www.youtube.com/watch?v={source_ref}"
+
+
+class FacebookAdapter(SourceAdapter):
+    key = "facebook"
+
+    def uploader_url(self, info: dict) -> str | None:
+        # Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
+        # (the page id) that resolves at facebook.com/<id>.
+        base = super().uploader_url(info)
+        if base:
+            return base
+        uid = info.get("uploader_id")
+        return f"https://www.facebook.com/{uid}" if uid else None
+
+    def wants_metadata_backfill(self) -> bool:
+        return True
+
+
+class RedditAdapter(SourceAdapter):
+    key = "reddit"
+
+
+_YOUTUBE = YouTubeAdapter()
+_FACEBOOK = FacebookAdapter()
+_REDDIT = RedditAdapter()
+_GENERIC = SourceAdapter()
+
+
+def resolve_adapter(
+    source_kind: str | None, source_ref: str | None = None, info: dict | None = None
+) -> SourceAdapter:
+    """Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key`
+    from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to
+    the generic adapter — so behaviour is unchanged for any source without a dedicated one."""
+    tag = ""
+    if info:
+        extractor = info.get("extractor_key") or info.get("extractor") or ""
+        tag = f"{extractor} {info.get('webpage_url') or ''}".lower()
+    host = urlparse(source_ref).netloc.lower() if source_ref else ""
+    if source_kind == "youtube" or "youtube" in tag or "youtu.be" in host or "youtube.com" in host:
+        return _YOUTUBE
+    if "facebook" in tag or "facebook.com" in host or "fb.watch" in host:
+        return _FACEBOOK
+    if "reddit" in tag or "reddit.com" in host:
+        return _REDDIT
+    return _GENERIC

backend/app/downloads/backfill.py

@@ -11,40 +11,26 @@ Run once per environment:
     docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
 """
 import logging
-from urllib.parse import urlparse
 
 import yt_dlp
 from sqlalchemy import select
 
 from app.config import settings
 from app.db import SessionLocal
+from app.downloads import adapters
 from app.downloads import edit as editmod
 from app.downloads import formats, service
 from app.models import MediaAsset
-from app.worker import build_uploader_url
 
 log = logging.getLogger("siftlode.backfill")
 
 
-def _should_refetch(asset: MediaAsset) -> bool:
-    """Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
-    if asset.source_kind == "youtube":
-        return True
-    if asset.source_kind == "url":
-        host = urlparse(asset.source_ref).netloc.lower()
-        return "facebook.com" in host or "fb.watch" in host
-    return False
-
-
-def _extract_meta(url: str) -> dict | None:
-    """Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
-    POT/player-client strategy even just to extract)."""
-    client_sets = [
-        c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
-    ] or [None]
-    for clients in client_sets:
+def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
+    """Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
+    needs the POT/player-client strategy even just to extract)."""
+    for clients in adapter.client_sets():
         opts = formats.build_ydl_opts(
-            {}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
+            {}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
         )
         opts["skip_download"] = True
         try:
@@ -70,10 +56,11 @@ def run() -> dict:
             .all()
         )
         for a in assets:
-            if a.uploader_url is None and _should_refetch(a):
+            adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
+            if a.uploader_url is None and adapter.wants_metadata_backfill():
                 url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
-                info = _extract_meta(url)
-                uu = build_uploader_url(info) if info else None
+                info = _extract_meta(url, adapter)
+                uu = adapter.uploader_url(info) if info else None
                 if uu:
                     a.uploader_url = uu
                     filled_url += 1

backend/app/downloads/formats.py

@@ -96,13 +96,13 @@ def build_ydl_opts(
     spec: dict,
     outtmpl: str,
     progress_hook,
-    pot_base_url: str | None = None,
-    player_clients: list[str] | None = None,
+    extra: dict | None = None,
 ) -> dict:
     """Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
 
-    `pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
-    `player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
+    `extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
+    `extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
+    generic options below. Everything here is source-agnostic."""
     s = normalize(spec)
     h = s["max_height"]
     opts: dict = {
@@ -127,16 +127,9 @@ def build_ydl_opts(
         "socket_timeout": 30,
     }
 
-    # Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
-    # (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
-    # are lists in the Python API.
-    ea: dict = {}
-    if player_clients:
-        ea["youtube"] = {"player_client": player_clients}
-    if pot_base_url:
-        ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
-    if ea:
-        opts["extractor_args"] = ea
+    # Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
+    if extra:
+        opts.update(extra)
 
     if s["mode"] == "a":
         opts["format"] = "bestaudio/best"

backend/app/downloads/service.py

@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from app.config import settings
+from app.downloads import adapters
 from app.downloads import edit as editmod
 from app.downloads import formats, quota, storage
 from app.models import Channel, DownloadJob, MediaAsset, Video
@@ -27,10 +28,9 @@ class EditError(Exception):
 
 
 def source_url(source_kind: str, source_ref: str) -> str:
-    """The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
-    if source_kind == "youtube":
-        return f"https://www.youtube.com/watch?v={source_ref}"
-    return source_ref
+    """The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
+    URL); ad-hoc sources store the URL verbatim."""
+    return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
 
 
 def reference_url(job, asset) -> str | None:

backend/app/downloads/storage.py

@@ -59,6 +59,9 @@ class MediaMeta:
     title: str
     uploader: str
     upload_date: date | None
+    # The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
+    # the SourceAdapter. Defaults to "youtube" only as a safety net for un-updated callers.
+    uniqueid_type: str = "youtube"
     duration_s: int | None = None
     description: str | None = None
     thumbnail_url: str | None = None
@@ -170,7 +173,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
         f"  <premiered>{aired}</premiered>",
         f"  <year>{year}</year>",
         f"  <plot>{escape(meta.description or '')}</plot>",
-        f"  <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
+        f"  <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
     ]
     if meta.duration_s:
         parts.append(f"  <runtime>{round(meta.duration_s / 60)}</runtime>")

backend/app/worker.py

@@ -35,7 +35,7 @@ from app.config import settings
 from app.db import SessionLocal
 from app.downloads import edit as editmod
 from app.downloads import errors as dlerrors
-from app.downloads import formats, quota, service, storage
+from app.downloads import adapters, formats, quota, service, storage
 from app.models import DownloadJob, MediaAsset
 from app.notifications import create_notification
 from app.titles import normalize_title
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
         db.commit()
 
 
-def build_uploader_url(info: dict) -> str | None:
-    """The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
-    `channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
-    `uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
-    no derivable channel page (e.g. reddit / a raw media URL)."""
-    url = info.get("channel_url") or info.get("uploader_url")
-    if url:
-        return url
-    uid = info.get("uploader_id")
-    tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
-    if uid and "facebook" in tag:
-        return f"https://www.facebook.com/{uid}"
-    return None
-
-
 def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
     """Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
     (only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
         # the final title with its own source_ref fallback.
         asset.title = normalize_title(info.get("title")) or info.get("id")
         asset.uploader = info.get("uploader") or info.get("channel")
-        asset.uploader_url = build_uploader_url(info)
+        asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
         asset.thumbnail_url = info.get("thumbnail")
         asset.duration_s = int(info["duration"]) if info.get("duration") else None
         asset.upload_date = _parse_upload_date(info.get("upload_date"))
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
     return {**info, **chosen}
 
 
-# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
-_RETRIABLE_MARKERS = (
-    "not a bot",
-    "DRM protected",
-    "Requested format is not available",
-    "Only images are available",
-    "Sign in to confirm",
-)
-
-
-def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
-    """Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
-    on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
-    client_sets = [
-        c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
-    ] or [None]
+def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
+    """Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
+    supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
+    error); other sources make a single attempt. Returns the info dict of the successful attempt."""
+    client_sets = adapter.client_sets()
     outtmpl = str(staging / "%(id)s.%(ext)s")
     last_exc: Exception | None = None
     for i, clients in enumerate(client_sets):
         opts = formats.build_ydl_opts(
             spec, outtmpl, _make_progress_hook(job_id, asset_id),
-            settings.download_pot_base_url, clients,
+            extra=adapter.ydl_opts_extra(clients),
         )
         opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
         _touch_job(job_id)  # stall-watchdog heartbeat: extraction writes no progress, so a slow /
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
                 return ydl.extract_info(url, download=True)
         except yt_dlp.utils.DownloadError as exc:
             last_exc = exc
-            retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
-            if i < len(client_sets) - 1 and retriable:
+            if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
                 log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
                 for p in list(staging.iterdir()):  # drop partials before the retry
                     try:
@@ -815,8 +788,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
         staging.mkdir(parents=True, exist_ok=True)
 
         _set_job(job_id, phase="downloading", progress=0)
+        adapter = adapters.resolve_adapter(source_kind, source_ref)
         url = service.source_url(source_kind, source_ref)
-        info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
+        info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
 
         produced = _produced_file(info, staging)
         produced = _ensure_browser_playable(job_id, produced, spec, info)
@@ -826,6 +800,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
             title=normalize_title(info.get("title")) or source_ref,
             uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
             upload_date=_parse_upload_date(info.get("upload_date")),
+            uniqueid_type=adapter.nfo_uniqueid_type(),
             duration_s=int(info["duration"]) if info.get("duration") else None,
             description=info.get("description"),
             thumbnail_url=info.get("thumbnail"),
@@ -859,7 +834,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
                 asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
                 asset.title = meta.title
                 asset.uploader = meta.uploader
-                asset.uploader_url = build_uploader_url(info)
+                asset.uploader_url = adapters.resolve_adapter(source_kind, source_ref, info).uploader_url(info)
                 asset.upload_date = meta.upload_date
                 asset.thumbnail_url = meta.thumbnail_url
                 # Record a self-hosted poster for every download: it's the reliable link-preview