refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp

Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered
across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source
identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source
overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp
extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is
policy only.

Moved into adapters (behaviour-equivalent):
- uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the
  worker/backfill duplication (build_uploader_url + _should_refetch gone).
- the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts
  now takes source-agnostic opts plus an `extra` dict from the adapter).
- source_url reconstruction (service.source_url delegates to the adapter).

Two intentional (approved) behaviour changes:
- the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction
  attempt instead of a pointless double attempt with youtube-only client args.
- the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the
  hardcoded "youtube" that mislabelled every url-source download.

Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless
across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's
YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path).

Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the
.nfo uniqueid type are correct per source.
This commit is contained in:
2026-08-13 00:34:42 +02:00
parent 44efdf328d
commit 8e4dee1bb6
9 changed files with 276 additions and 80 deletions
+136
View File
@@ -0,0 +1,136 @@
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
/ reddit / generic).
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
scattered across worker.py / backfill.py / service.py with no shared source identity.
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""
from urllib.parse import urlparse
from app.config import settings
class SourceAdapter:
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
key = "generic"
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
def client_sets(self) -> list:
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
return [None]
def ydl_opts_extra(self, clients) -> dict:
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
current entry from `client_sets()`. Generic: nothing."""
return {}
def retriable(self, error_text: str) -> bool:
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
return False
# --- metadata -----------------------------------------------------------------------------
def uploader_url(self, info: dict) -> str | None:
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
return info.get("channel_url") or info.get("uploader_url")
def wants_metadata_backfill(self) -> bool:
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
return False
# --- source ref ↔ URL ---------------------------------------------------------------------
def source_url(self, source_ref: str) -> str:
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
return source_ref
def nfo_uniqueid_type(self) -> str:
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
return self.key
class YouTubeAdapter(SourceAdapter):
key = "youtube"
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def client_sets(self) -> list:
return [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
def ydl_opts_extra(self, clients) -> dict:
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
ea: dict = {}
if clients:
ea["youtube"] = {"player_client": clients}
if settings.download_pot_base_url:
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
return {"extractor_args": ea} if ea else {}
def retriable(self, error_text: str) -> bool:
return any(m in error_text for m in self._RETRIABLE)
def wants_metadata_backfill(self) -> bool:
return True
def source_url(self, source_ref: str) -> str:
return f"https://www.youtube.com/watch?v={source_ref}"
class FacebookAdapter(SourceAdapter):
key = "facebook"
def uploader_url(self, info: dict) -> str | None:
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
# (the page id) that resolves at facebook.com/<id>.
base = super().uploader_url(info)
if base:
return base
uid = info.get("uploader_id")
return f"https://www.facebook.com/{uid}" if uid else None
def wants_metadata_backfill(self) -> bool:
return True
class RedditAdapter(SourceAdapter):
key = "reddit"
_YOUTUBE = YouTubeAdapter()
_FACEBOOK = FacebookAdapter()
_REDDIT = RedditAdapter()
_GENERIC = SourceAdapter()
def resolve_adapter(
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
) -> SourceAdapter:
"""Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key`
from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to
the generic adapter — so behaviour is unchanged for any source without a dedicated one."""
tag = ""
if info:
extractor = info.get("extractor_key") or info.get("extractor") or ""
tag = f"{extractor} {info.get('webpage_url') or ''}".lower()
host = urlparse(source_ref).netloc.lower() if source_ref else ""
if source_kind == "youtube" or "youtube" in tag or "youtu.be" in host or "youtube.com" in host:
return _YOUTUBE
if "facebook" in tag or "facebook.com" in host or "fb.watch" in host:
return _FACEBOOK
if "reddit" in tag or "reddit.com" in host:
return _REDDIT
return _GENERIC
+10 -23
View File
@@ -11,40 +11,26 @@ Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
"""
import logging
from urllib.parse import urlparse
import yt_dlp
from sqlalchemy import select
from app.config import settings
from app.db import SessionLocal
from app.downloads import adapters
from app.downloads import edit as editmod
from app.downloads import formats, service
from app.models import MediaAsset
from app.worker import build_uploader_url
log = logging.getLogger("siftlode.backfill")
def _should_refetch(asset: MediaAsset) -> bool:
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
if asset.source_kind == "youtube":
return True
if asset.source_kind == "url":
host = urlparse(asset.source_ref).netloc.lower()
return "facebook.com" in host or "fb.watch" in host
return False
def _extract_meta(url: str) -> dict | None:
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
POT/player-client strategy even just to extract)."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
for clients in client_sets:
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
needs the POT/player-client strategy even just to extract)."""
for clients in adapter.client_sets():
opts = formats.build_ydl_opts(
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
)
opts["skip_download"] = True
try:
@@ -70,10 +56,11 @@ def run() -> dict:
.all()
)
for a in assets:
if a.uploader_url is None and _should_refetch(a):
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
if a.uploader_url is None and adapter.wants_metadata_backfill():
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
info = _extract_meta(url)
uu = build_uploader_url(info) if info else None
info = _extract_meta(url, adapter)
uu = adapter.uploader_url(info) if info else None
if uu:
a.uploader_url = uu
filled_url += 1
+7 -14
View File
@@ -96,13 +96,13 @@ def build_ydl_opts(
spec: dict,
outtmpl: str,
progress_hook,
pot_base_url: str | None = None,
player_clients: list[str] | None = None,
extra: dict | None = None,
) -> dict:
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
generic options below. Everything here is source-agnostic."""
s = normalize(spec)
h = s["max_height"]
opts: dict = {
@@ -127,16 +127,9 @@ def build_ydl_opts(
"socket_timeout": 30,
}
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
# are lists in the Python API.
ea: dict = {}
if player_clients:
ea["youtube"] = {"player_client": player_clients}
if pot_base_url:
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
if ea:
opts["extractor_args"] = ea
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
if extra:
opts.update(extra)
if s["mode"] == "a":
opts["format"] = "bestaudio/best"
+4 -4
View File
@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.config import settings
from app.downloads import adapters
from app.downloads import edit as editmod
from app.downloads import formats, quota, storage
from app.models import Channel, DownloadJob, MediaAsset, Video
@@ -27,10 +28,9 @@ class EditError(Exception):
def source_url(source_kind: str, source_ref: str) -> str:
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
if source_kind == "youtube":
return f"https://www.youtube.com/watch?v={source_ref}"
return source_ref
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
URL); ad-hoc sources store the URL verbatim."""
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
def reference_url(job, asset) -> str | None:
+4 -1
View File
@@ -59,6 +59,9 @@ class MediaMeta:
title: str
uploader: str
upload_date: date | None
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
# the SourceAdapter. Defaults to "youtube" only as a safety net for un-updated callers.
uniqueid_type: str = "youtube"
duration_s: int | None = None
description: str | None = None
thumbnail_url: str | None = None
@@ -170,7 +173,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
f" <premiered>{aired}</premiered>",
f" <year>{year}</year>",
f" <plot>{escape(meta.description or '')}</plot>",
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
]
if meta.duration_s:
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
+13 -38
View File
@@ -35,7 +35,7 @@ from app.config import settings
from app.db import SessionLocal
from app.downloads import edit as editmod
from app.downloads import errors as dlerrors
from app.downloads import formats, quota, service, storage
from app.downloads import adapters, formats, quota, service, storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
from app.titles import normalize_title
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
db.commit()
def build_uploader_url(info: dict) -> str | None:
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
no derivable channel page (e.g. reddit / a raw media URL)."""
url = info.get("channel_url") or info.get("uploader_url")
if url:
return url
uid = info.get("uploader_id")
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
if uid and "facebook" in tag:
return f"https://www.facebook.com/{uid}"
return None
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
# the final title with its own source_ref fallback.
asset.title = normalize_title(info.get("title")) or info.get("id")
asset.uploader = info.get("uploader") or info.get("channel")
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
asset.thumbnail_url = info.get("thumbnail")
asset.duration_s = int(info["duration"]) if info.get("duration") else None
asset.upload_date = _parse_upload_date(info.get("upload_date"))
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
return {**info, **chosen}
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE_MARKERS = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
client_sets = adapter.client_sets()
outtmpl = str(staging / "%(id)s.%(ext)s")
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
settings.download_pot_base_url, clients,
extra=adapter.ydl_opts_extra(clients),
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
return ydl.extract_info(url, download=True)
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
if i < len(client_sets) - 1 and retriable:
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
for p in list(staging.iterdir()): # drop partials before the retry
try:
@@ -815,8 +788,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
staging.mkdir(parents=True, exist_ok=True)
_set_job(job_id, phase="downloading", progress=0)
adapter = adapters.resolve_adapter(source_kind, source_ref)
url = service.source_url(source_kind, source_ref)
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
produced = _produced_file(info, staging)
produced = _ensure_browser_playable(job_id, produced, spec, info)
@@ -826,6 +800,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
title=normalize_title(info.get("title")) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=int(info["duration"]) if info.get("duration") else None,
description=info.get("description"),
thumbnail_url=info.get("thumbnail"),
@@ -859,7 +834,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapters.resolve_adapter(source_kind, source_ref, info).uploader_url(info)
asset.upload_date = meta.upload_date
asset.thumbnail_url = meta.thumbnail_url
# Record a self-hosted poster for every download: it's the reliable link-preview
+83
View File
@@ -0,0 +1,83 @@
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
from app.downloads.adapters import (
FacebookAdapter,
RedditAdapter,
SourceAdapter,
YouTubeAdapter,
resolve_adapter,
)
def test_resolve_by_source_kind_and_host_and_info():
assert resolve_adapter("youtube", "abc123").key == "youtube"
assert resolve_adapter("url", "https://www.youtube.com/watch?v=x").key == "youtube"
assert resolve_adapter("url", "https://youtu.be/x").key == "youtube"
assert resolve_adapter("url", "https://www.facebook.com/reel/123/").key == "facebook"
assert resolve_adapter("url", "https://fb.watch/abc/").key == "facebook"
assert resolve_adapter("url", "https://www.reddit.com/r/x/comments/y/z").key == "reddit"
assert resolve_adapter("url", "https://example.com/video.mp4").key == "generic"
# After extraction the info dict's extractor decides even for a bare/redirected url.
assert resolve_adapter("url", None, {"extractor_key": "facebook"}).key == "facebook"
assert resolve_adapter("url", None, {"extractor_key": "Reddit"}).key == "reddit"
def test_host_match_is_netloc_not_a_loose_substring():
# A path that merely CONTAINS "facebook.com" must not select the Facebook adapter.
assert resolve_adapter("url", "https://example.com/facebook.com-mirror").key == "generic"
def test_uploader_url_youtube_uses_channel_url():
a = YouTubeAdapter()
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
assert a.uploader_url({"uploader_url": "https://youtube.com/user/u"}) == "https://youtube.com/user/u"
assert a.uploader_url({}) is None
def test_uploader_url_facebook_falls_back_to_uploader_id():
a = FacebookAdapter()
assert a.uploader_url({"uploader_id": "12345"}) == "https://www.facebook.com/12345"
# A direct channel_url still wins over the id fallback.
assert a.uploader_url({"channel_url": "https://facebook.com/pageX", "uploader_id": "9"}) == (
"https://facebook.com/pageX"
)
assert a.uploader_url({}) is None
def test_uploader_url_reddit_and_generic_have_no_channel():
assert RedditAdapter().uploader_url({"uploader_id": "99"}) is None # not facebook → no fallback
assert SourceAdapter().uploader_url({}) is None
def test_generic_source_url_is_passthrough_youtube_reconstructs():
assert SourceAdapter().source_url("https://x/y") == "https://x/y"
assert YouTubeAdapter().source_url("VIDID12345_") == "https://www.youtube.com/watch?v=VIDID12345_"
def test_client_sets_and_retriable_are_youtube_only():
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
assert SourceAdapter().client_sets() == [None]
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
assert YouTubeAdapter().retriable("some unrelated network blip") is False
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
assert ea["youtube"]["player_client"] == ["android_vr"]
assert "youtubepot-bgutilhttp" in ea # the bgutil PO-token base_url is wired
# A client-less attempt still wires POT but no player_client key.
assert "youtube" not in YouTubeAdapter().ydl_opts_extra(None)["extractor_args"]
assert SourceAdapter().ydl_opts_extra(["android_vr"]) == {} # generic contributes nothing
def test_wants_metadata_backfill_matches_old_should_refetch():
assert YouTubeAdapter().wants_metadata_backfill() is True
assert FacebookAdapter().wants_metadata_backfill() is True
assert RedditAdapter().wants_metadata_backfill() is False
assert SourceAdapter().wants_metadata_backfill() is False
def test_nfo_uniqueid_type_is_the_adapter_key():
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
assert SourceAdapter().nfo_uniqueid_type() == "generic"
+9
View File
@@ -9,6 +9,15 @@ def _opts(spec):
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
def test_adapter_extra_opts_are_merged_in():
# The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`.
extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}}
opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra)
assert opts["extractor_args"]["youtube"]["player_client"] == ["web"]
# No extra → no extractor_args (a generic source contributes nothing).
assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None)
def test_mp4_default_selects_avc1_with_progressive_fallback():
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
+10
View File
@@ -137,3 +137,13 @@ class TestSafeAbsPath:
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
def test_nfo_uniqueid_type_reflects_the_source():
# The .nfo <uniqueid type=...> now follows the source (via SourceAdapter.nfo_uniqueid_type),
# instead of the old hardcoded "youtube" that mislabelled every url-source download.
from app.downloads.storage import _nfo_xml
base = dict(video_id="v1", title="t", uploader="c", upload_date=None)
assert '<uniqueid type="youtube"' in _nfo_xml(MediaMeta(**base)) # default preserved
assert '<uniqueid type="facebook"' in _nfo_xml(MediaMeta(**base, uniqueid_type="facebook"))