"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed. Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube / Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed URL may have expired — those just get a locally-generated poster instead). Run once per environment: docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())" """ from urllib.parse import urlsplit import yt_dlp from sqlalchemy import select from app import logs from app.config import settings from app.db import SessionLocal from app.downloads import adapters from app.downloads import edit as editmod from app.downloads import formats, service from app.models import MediaAsset log = logs.get_logger(__name__) def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None: """Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube needs the POT/player-client strategy even just to extract).""" with formats.hiding(url): return _extract_meta_scoped(url, adapter, asset_id) def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None: for clients in adapter.client_sets(): opts = formats.build_ydl_opts( {}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients) ) opts["skip_download"] = True try: with yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(url, download=False) if info: return info except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going # The asset id, not the URL. A source URL is a row of this user's download # history, and `redact_text` only takes credential params out of it — the rest # of it goes in whole. The id names the same asset to anyone holding the # database and nothing at all to anyone who is not; the host is kept because it # is what decides which client set was tried. yt-dlp's message goes through # `log_safe` for the same reason: it opens with the URL or the video id. log.info( "meta extract failed for asset %s on %s (%s): %s", asset_id, urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query clients, formats.log_safe(exc)[:100], ) return None def run() -> dict: filled_url = filled_poster = 0 with SessionLocal() as db: assets = ( db.execute( select(MediaAsset).where( MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None) ) ) .scalars() .all() ) for a in assets: adapter = adapters.resolve_adapter(a.source_kind, a.source_ref) if a.uploader_url is None and adapter.wants_metadata_backfill(): url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref) info = _extract_meta(url, adapter, a.id) uu = adapter.uploader_url(info) if info else None if uu: a.uploader_url = uu filled_url += 1 log.info( "asset %s uploader_url filled", a.id ) # the URL names what they downloaded if a.poster_path is None: # Record a self-hosted poster for every download (the reliable og:image source): # ensure_poster returns the existing .jpg thumbnail sidecar, or cuts a frame. rel = editmod.ensure_poster(a, settings.download_root) if rel: a.poster_path = rel filled_poster += 1 log.info("asset %s poster filled", a.id) # the path carries the title db.commit() result = { "assets": len(assets), "filled_uploader_url": filled_url, "filled_poster": filled_poster, } log.info("backfill done: %s", result) return result