refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp
Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is policy only. Moved into adapters (behaviour-equivalent): - uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the worker/backfill duplication (build_uploader_url + _should_refetch gone). - the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts now takes source-agnostic opts plus an `extra` dict from the adapter). - source_url reconstruction (service.source_url delegates to the adapter). Two intentional (approved) behaviour changes: - the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction attempt instead of a pointless double attempt with youtube-only client args. - the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the hardcoded "youtube" that mislabelled every url-source download. Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path). Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the .nfo uniqueid type are correct per source.
This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||||
/ reddit / generic).
|
||||
|
||||
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||||
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||||
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||||
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||||
|
||||
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||||
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||||
current entry from `client_sets()`. Generic: nothing."""
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
return info.get("channel_url") or info.get("uploader_url")
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||||
return False
|
||||
|
||||
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||||
return source_ref
|
||||
|
||||
def nfo_uniqueid_type(self) -> str:
|
||||
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||||
return self.key
|
||||
|
||||
|
||||
class YouTubeAdapter(SourceAdapter):
|
||||
key = "youtube"
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
ea: dict = {}
|
||||
if clients:
|
||||
ea["youtube"] = {"player_client": clients}
|
||||
if settings.download_pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||||
return {"extractor_args": ea} if ea else {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return any(m in error_text for m in self._RETRIABLE)
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
# (the page id) that resolves at facebook.com/<id>.
|
||||
base = super().uploader_url(info)
|
||||
if base:
|
||||
return base
|
||||
uid = info.get("uploader_id")
|
||||
return f"https://www.facebook.com/{uid}" if uid else None
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
_YOUTUBE = YouTubeAdapter()
|
||||
_FACEBOOK = FacebookAdapter()
|
||||
_REDDIT = RedditAdapter()
|
||||
_GENERIC = SourceAdapter()
|
||||
|
||||
|
||||
def resolve_adapter(
|
||||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||||
) -> SourceAdapter:
|
||||
"""Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key`
|
||||
from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to
|
||||
the generic adapter — so behaviour is unchanged for any source without a dedicated one."""
|
||||
tag = ""
|
||||
if info:
|
||||
extractor = info.get("extractor_key") or info.get("extractor") or ""
|
||||
tag = f"{extractor} {info.get('webpage_url') or ''}".lower()
|
||||
host = urlparse(source_ref).netloc.lower() if source_ref else ""
|
||||
if source_kind == "youtube" or "youtube" in tag or "youtu.be" in host or "youtube.com" in host:
|
||||
return _YOUTUBE
|
||||
if "facebook" in tag or "facebook.com" in host or "fb.watch" in host:
|
||||
return _FACEBOOK
|
||||
if "reddit" in tag or "reddit.com" in host:
|
||||
return _REDDIT
|
||||
return _GENERIC
|
||||
@@ -11,40 +11,26 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import build_uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
@@ -70,10 +56,11 @@ def run() -> dict:
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = build_uploader_url(info) if info else None
|
||||
info = _extract_meta(url, adapter)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
|
||||
@@ -96,13 +96,13 @@ def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
progress_hook,
|
||||
pot_base_url: str | None = None,
|
||||
player_clients: list[str] | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||
|
||||
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
|
||||
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
|
||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
@@ -127,16 +127,9 @@ def build_ydl_opts(
|
||||
"socket_timeout": 30,
|
||||
}
|
||||
|
||||
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
|
||||
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
|
||||
# are lists in the Python API.
|
||||
ea: dict = {}
|
||||
if player_clients:
|
||||
ea["youtube"] = {"player_client": player_clients}
|
||||
if pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
|
||||
if ea:
|
||||
opts["extractor_args"] = ea
|
||||
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
@@ -27,10 +28,9 @@ class EditError(Exception):
|
||||
|
||||
|
||||
def source_url(source_kind: str, source_ref: str) -> str:
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
||||
if source_kind == "youtube":
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
return source_ref
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
|
||||
URL); ad-hoc sources store the URL verbatim."""
|
||||
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
|
||||
|
||||
|
||||
def reference_url(job, asset) -> str | None:
|
||||
|
||||
@@ -59,6 +59,9 @@ class MediaMeta:
|
||||
title: str
|
||||
uploader: str
|
||||
upload_date: date | None
|
||||
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
|
||||
# the SourceAdapter. Defaults to "youtube" only as a safety net for un-updated callers.
|
||||
uniqueid_type: str = "youtube"
|
||||
duration_s: int | None = None
|
||||
description: str | None = None
|
||||
thumbnail_url: str | None = None
|
||||
@@ -170,7 +173,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
f" <premiered>{aired}</premiered>",
|
||||
f" <year>{year}</year>",
|
||||
f" <plot>{escape(meta.description or '')}</plot>",
|
||||
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
]
|
||||
if meta.duration_s:
|
||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||
|
||||
Reference in New Issue
Block a user