peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp

Browse Source 
Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered
across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source
identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source
overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp
extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is
policy only.

Moved into adapters (behaviour-equivalent):
- uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the
  worker/backfill duplication (build_uploader_url + _should_refetch gone).
- the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts
  now takes source-agnostic opts plus an `extra` dict from the adapter).
- source_url reconstruction (service.source_url delegates to the adapter).

Two intentional (approved) behaviour changes:
- the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction
  attempt instead of a pointless double attempt with youtube-only client args.
- the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the
  hardcoded "youtube" that mislabelled every url-source download.

Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless
across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's
YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path).

Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the
.nfo uniqueid type are correct per source.
This commit is contained in:
peter
2026-08-13 00:34:42 +02:00
parent 44efdf328d
commit 8e4dee1bb6
9 changed files with 276 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/worker.py
+13 -38
View File
@@ -35,7 +35,7 @@ from app.config import settings
from app.db import SessionLocal
from app.downloads import edit as editmod
from app.downloads import errors as dlerrors
from app.downloads import formats, quota, service, storage
from app.downloads import adapters, formats, quota, service, storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
from app.titles import normalize_title
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
db.commit()
def build_uploader_url(info: dict) -> str | None:
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
no derivable channel page (e.g. reddit / a raw media URL)."""
url = info.get("channel_url") or info.get("uploader_url")
if url:
return url
uid = info.get("uploader_id")
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
if uid and "facebook" in tag:
return f"https://www.facebook.com/{uid}"
return None
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
# the final title with its own source_ref fallback.
asset.title = normalize_title(info.get("title")) or info.get("id")
asset.uploader = info.get("uploader") or info.get("channel")
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
asset.thumbnail_url = info.get("thumbnail")
asset.duration_s = int(info["duration"]) if info.get("duration") else None
asset.upload_date = _parse_upload_date(info.get("upload_date"))
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
return {**info, **chosen}
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE_MARKERS = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
client_sets = adapter.client_sets()
outtmpl = str(staging / "%(id)s.%(ext)s")
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
settings.download_pot_base_url, clients,
extra=adapter.ydl_opts_extra(clients),
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
return ydl.extract_info(url, download=True)
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
if i < len(client_sets) - 1 and retriable:
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
for p in list(staging.iterdir()): # drop partials before the retry
try:
@@ -815,8 +788,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
staging.mkdir(parents=True, exist_ok=True)
_set_job(job_id, phase="downloading", progress=0)
adapter = adapters.resolve_adapter(source_kind, source_ref)
url = service.source_url(source_kind, source_ref)
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
produced = _produced_file(info, staging)
produced = _ensure_browser_playable(job_id, produced, spec, info)
@@ -826,6 +800,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
title=normalize_title(info.get("title")) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=int(info["duration"]) if info.get("duration") else None,
description=info.get("description"),
thumbnail_url=info.get("thumbnail"),
@@ -859,7 +834,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapters.resolve_adapter(source_kind, source_ref, info).uploader_url(info)
asset.upload_date = meta.upload_date
asset.thumbnail_url = meta.thumbnail_url
# Record a self-hosted poster for every download: it's the reliable link-preview