refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp
Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is policy only. Moved into adapters (behaviour-equivalent): - uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the worker/backfill duplication (build_uploader_url + _should_refetch gone). - the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts now takes source-agnostic opts plus an `extra` dict from the adapter). - source_url reconstruction (service.source_url delegates to the adapter). Two intentional (approved) behaviour changes: - the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction attempt instead of a pointless double attempt with youtube-only client args. - the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the hardcoded "youtube" that mislabelled every url-source download. Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path). Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the .nfo uniqueid type are correct per source.
This commit is contained in:
@@ -11,40 +11,26 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import build_uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
@@ -70,10 +56,11 @@ def run() -> dict:
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = build_uploader_url(info) if info else None
|
||||
info = _extract_meta(url, adapter)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
|
||||
Reference in New Issue
Block a user