refactor(downloads): introduce SourceAdapter — a thin per-source Strategy over yt-dlp
Consolidates source-specific policy (youtube / facebook / reddit / generic) that was scattered across worker.py, backfill.py, service.py, formats.py and storage.py with no shared source identity. app/downloads/adapters.py holds the base (= generic yt-dlp behaviour) + per-source overrides; resolve_adapter(source_kind, source_ref, info) picks one by source_kind, the yt-dlp extractor_key, or the URL host, falling back to generic. yt-dlp stays the engine — the adapter is policy only. Moved into adapters (behaviour-equivalent): - uploader_url derivation (YouTube channel_url / Facebook uploader_id / reddit None) — removes the worker/backfill duplication (build_uploader_url + _should_refetch gone). - the player-client fallback sets + retriable markers + YouTube extractor_args/POT (build_ydl_opts now takes source-agnostic opts plus an `extra` dict from the adapter). - source_url reconstruction (service.source_url delegates to the adapter). Two intentional (approved) behaviour changes: - the player-client fallback is now YouTube-only; non-YouTube sources make a single extraction attempt instead of a pointless double attempt with youtube-only client args. - the .nfo <uniqueid type=...> follows the source (adapter.nfo_uniqueid_type) instead of the hardcoded "youtube" that mislabelled every url-source download. Deliberately left generic (documented non-moves): title cleaning (normalize_title is harmless across sources), the classify marker table, thumbnail_to_jpg, the format selectors, and og.py's YouTube-hqdefault og:image preference (a self-contained thumbnail heuristic on the share path). Verified end-to-end on localdev across all four adapters: downloads succeed and uploader_url + the .nfo uniqueid type are correct per source.
This commit is contained in:
@@ -0,0 +1,136 @@
|
|||||||
|
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||||||
|
/ reddit / generic).
|
||||||
|
|
||||||
|
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||||||
|
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||||||
|
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||||||
|
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||||||
|
|
||||||
|
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||||||
|
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||||
|
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||||
|
"""
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
|
||||||
|
class SourceAdapter:
|
||||||
|
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||||||
|
|
||||||
|
key = "generic"
|
||||||
|
|
||||||
|
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||||
|
def client_sets(self) -> list:
|
||||||
|
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||||
|
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||||
|
return [None]
|
||||||
|
|
||||||
|
def ydl_opts_extra(self, clients) -> dict:
|
||||||
|
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||||||
|
current entry from `client_sets()`. Generic: nothing."""
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def retriable(self, error_text: str) -> bool:
|
||||||
|
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||||||
|
return False
|
||||||
|
|
||||||
|
# --- metadata -----------------------------------------------------------------------------
|
||||||
|
def uploader_url(self, info: dict) -> str | None:
|
||||||
|
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||||
|
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||||
|
return info.get("channel_url") or info.get("uploader_url")
|
||||||
|
|
||||||
|
def wants_metadata_backfill(self) -> bool:
|
||||||
|
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||||||
|
return False
|
||||||
|
|
||||||
|
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||||||
|
def source_url(self, source_ref: str) -> str:
|
||||||
|
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||||||
|
return source_ref
|
||||||
|
|
||||||
|
def nfo_uniqueid_type(self) -> str:
|
||||||
|
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||||||
|
return self.key
|
||||||
|
|
||||||
|
|
||||||
|
class YouTubeAdapter(SourceAdapter):
|
||||||
|
key = "youtube"
|
||||||
|
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||||
|
_RETRIABLE = (
|
||||||
|
"not a bot",
|
||||||
|
"DRM protected",
|
||||||
|
"Requested format is not available",
|
||||||
|
"Only images are available",
|
||||||
|
"Sign in to confirm",
|
||||||
|
)
|
||||||
|
|
||||||
|
def client_sets(self) -> list:
|
||||||
|
return [
|
||||||
|
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||||
|
] or [None]
|
||||||
|
|
||||||
|
def ydl_opts_extra(self, clients) -> dict:
|
||||||
|
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||||
|
ea: dict = {}
|
||||||
|
if clients:
|
||||||
|
ea["youtube"] = {"player_client": clients}
|
||||||
|
if settings.download_pot_base_url:
|
||||||
|
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||||||
|
return {"extractor_args": ea} if ea else {}
|
||||||
|
|
||||||
|
def retriable(self, error_text: str) -> bool:
|
||||||
|
return any(m in error_text for m in self._RETRIABLE)
|
||||||
|
|
||||||
|
def wants_metadata_backfill(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def source_url(self, source_ref: str) -> str:
|
||||||
|
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||||
|
|
||||||
|
|
||||||
|
class FacebookAdapter(SourceAdapter):
|
||||||
|
key = "facebook"
|
||||||
|
|
||||||
|
def uploader_url(self, info: dict) -> str | None:
|
||||||
|
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||||
|
# (the page id) that resolves at facebook.com/<id>.
|
||||||
|
base = super().uploader_url(info)
|
||||||
|
if base:
|
||||||
|
return base
|
||||||
|
uid = info.get("uploader_id")
|
||||||
|
return f"https://www.facebook.com/{uid}" if uid else None
|
||||||
|
|
||||||
|
def wants_metadata_backfill(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class RedditAdapter(SourceAdapter):
|
||||||
|
key = "reddit"
|
||||||
|
|
||||||
|
|
||||||
|
_YOUTUBE = YouTubeAdapter()
|
||||||
|
_FACEBOOK = FacebookAdapter()
|
||||||
|
_REDDIT = RedditAdapter()
|
||||||
|
_GENERIC = SourceAdapter()
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_adapter(
|
||||||
|
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||||||
|
) -> SourceAdapter:
|
||||||
|
"""Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key`
|
||||||
|
from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to
|
||||||
|
the generic adapter — so behaviour is unchanged for any source without a dedicated one."""
|
||||||
|
tag = ""
|
||||||
|
if info:
|
||||||
|
extractor = info.get("extractor_key") or info.get("extractor") or ""
|
||||||
|
tag = f"{extractor} {info.get('webpage_url') or ''}".lower()
|
||||||
|
host = urlparse(source_ref).netloc.lower() if source_ref else ""
|
||||||
|
if source_kind == "youtube" or "youtube" in tag or "youtu.be" in host or "youtube.com" in host:
|
||||||
|
return _YOUTUBE
|
||||||
|
if "facebook" in tag or "facebook.com" in host or "fb.watch" in host:
|
||||||
|
return _FACEBOOK
|
||||||
|
if "reddit" in tag or "reddit.com" in host:
|
||||||
|
return _REDDIT
|
||||||
|
return _GENERIC
|
||||||
@@ -11,40 +11,26 @@ Run once per environment:
|
|||||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
|
from app.downloads import adapters
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats, service
|
from app.downloads import formats, service
|
||||||
from app.models import MediaAsset
|
from app.models import MediaAsset
|
||||||
from app.worker import build_uploader_url
|
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.backfill")
|
log = logging.getLogger("siftlode.backfill")
|
||||||
|
|
||||||
|
|
||||||
def _should_refetch(asset: MediaAsset) -> bool:
|
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||||
if asset.source_kind == "youtube":
|
needs the POT/player-client strategy even just to extract)."""
|
||||||
return True
|
for clients in adapter.client_sets():
|
||||||
if asset.source_kind == "url":
|
|
||||||
host = urlparse(asset.source_ref).netloc.lower()
|
|
||||||
return "facebook.com" in host or "fb.watch" in host
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_meta(url: str) -> dict | None:
|
|
||||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
|
||||||
POT/player-client strategy even just to extract)."""
|
|
||||||
client_sets = [
|
|
||||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
|
||||||
] or [None]
|
|
||||||
for clients in client_sets:
|
|
||||||
opts = formats.build_ydl_opts(
|
opts = formats.build_ydl_opts(
|
||||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||||
)
|
)
|
||||||
opts["skip_download"] = True
|
opts["skip_download"] = True
|
||||||
try:
|
try:
|
||||||
@@ -70,10 +56,11 @@ def run() -> dict:
|
|||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
for a in assets:
|
for a in assets:
|
||||||
if a.uploader_url is None and _should_refetch(a):
|
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||||
|
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||||
info = _extract_meta(url)
|
info = _extract_meta(url, adapter)
|
||||||
uu = build_uploader_url(info) if info else None
|
uu = adapter.uploader_url(info) if info else None
|
||||||
if uu:
|
if uu:
|
||||||
a.uploader_url = uu
|
a.uploader_url = uu
|
||||||
filled_url += 1
|
filled_url += 1
|
||||||
|
|||||||
@@ -96,13 +96,13 @@ def build_ydl_opts(
|
|||||||
spec: dict,
|
spec: dict,
|
||||||
outtmpl: str,
|
outtmpl: str,
|
||||||
progress_hook,
|
progress_hook,
|
||||||
pot_base_url: str | None = None,
|
extra: dict | None = None,
|
||||||
player_clients: list[str] | None = None,
|
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||||
|
|
||||||
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
|
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||||
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
|
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||||
|
generic options below. Everything here is source-agnostic."""
|
||||||
s = normalize(spec)
|
s = normalize(spec)
|
||||||
h = s["max_height"]
|
h = s["max_height"]
|
||||||
opts: dict = {
|
opts: dict = {
|
||||||
@@ -127,16 +127,9 @@ def build_ydl_opts(
|
|||||||
"socket_timeout": 30,
|
"socket_timeout": 30,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
|
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
|
||||||
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
|
if extra:
|
||||||
# are lists in the Python API.
|
opts.update(extra)
|
||||||
ea: dict = {}
|
|
||||||
if player_clients:
|
|
||||||
ea["youtube"] = {"player_client": player_clients}
|
|
||||||
if pot_base_url:
|
|
||||||
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
|
|
||||||
if ea:
|
|
||||||
opts["extractor_args"] = ea
|
|
||||||
|
|
||||||
if s["mode"] == "a":
|
if s["mode"] == "a":
|
||||||
opts["format"] = "bestaudio/best"
|
opts["format"] = "bestaudio/best"
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.downloads import adapters
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats, quota, storage
|
from app.downloads import formats, quota, storage
|
||||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||||
@@ -27,10 +28,9 @@ class EditError(Exception):
|
|||||||
|
|
||||||
|
|
||||||
def source_url(source_kind: str, source_ref: str) -> str:
|
def source_url(source_kind: str, source_ref: str) -> str:
|
||||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
|
||||||
if source_kind == "youtube":
|
URL); ad-hoc sources store the URL verbatim."""
|
||||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
|
||||||
return source_ref
|
|
||||||
|
|
||||||
|
|
||||||
def reference_url(job, asset) -> str | None:
|
def reference_url(job, asset) -> str | None:
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ class MediaMeta:
|
|||||||
title: str
|
title: str
|
||||||
uploader: str
|
uploader: str
|
||||||
upload_date: date | None
|
upload_date: date | None
|
||||||
|
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
|
||||||
|
# the SourceAdapter. Defaults to "youtube" only as a safety net for un-updated callers.
|
||||||
|
uniqueid_type: str = "youtube"
|
||||||
duration_s: int | None = None
|
duration_s: int | None = None
|
||||||
description: str | None = None
|
description: str | None = None
|
||||||
thumbnail_url: str | None = None
|
thumbnail_url: str | None = None
|
||||||
@@ -170,7 +173,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
|||||||
f" <premiered>{aired}</premiered>",
|
f" <premiered>{aired}</premiered>",
|
||||||
f" <year>{year}</year>",
|
f" <year>{year}</year>",
|
||||||
f" <plot>{escape(meta.description or '')}</plot>",
|
f" <plot>{escape(meta.description or '')}</plot>",
|
||||||
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||||
]
|
]
|
||||||
if meta.duration_s:
|
if meta.duration_s:
|
||||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||||
|
|||||||
+13
-38
@@ -35,7 +35,7 @@ from app.config import settings
|
|||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads import formats, quota, service, storage
|
from app.downloads import adapters, formats, quota, service, storage
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.models import DownloadJob, MediaAsset
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.titles import normalize_title
|
from app.titles import normalize_title
|
||||||
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
|||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
def build_uploader_url(info: dict) -> str | None:
|
|
||||||
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
|
|
||||||
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
|
|
||||||
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
|
|
||||||
no derivable channel page (e.g. reddit / a raw media URL)."""
|
|
||||||
url = info.get("channel_url") or info.get("uploader_url")
|
|
||||||
if url:
|
|
||||||
return url
|
|
||||||
uid = info.get("uploader_id")
|
|
||||||
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
|
|
||||||
if uid and "facebook" in tag:
|
|
||||||
return f"https://www.facebook.com/{uid}"
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||||
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
||||||
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
||||||
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
|||||||
# the final title with its own source_ref fallback.
|
# the final title with its own source_ref fallback.
|
||||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||||
asset.uploader = info.get("uploader") or info.get("channel")
|
asset.uploader = info.get("uploader") or info.get("channel")
|
||||||
asset.uploader_url = build_uploader_url(info)
|
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||||
asset.thumbnail_url = info.get("thumbnail")
|
asset.thumbnail_url = info.get("thumbnail")
|
||||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||||
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
|
|||||||
return {**info, **chosen}
|
return {**info, **chosen}
|
||||||
|
|
||||||
|
|
||||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||||
_RETRIABLE_MARKERS = (
|
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||||
"not a bot",
|
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||||
"DRM protected",
|
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||||
"Requested format is not available",
|
client_sets = adapter.client_sets()
|
||||||
"Only images are available",
|
|
||||||
"Sign in to confirm",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
|
|
||||||
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
|
|
||||||
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
|
|
||||||
client_sets = [
|
|
||||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
|
||||||
] or [None]
|
|
||||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for i, clients in enumerate(client_sets):
|
for i, clients in enumerate(client_sets):
|
||||||
opts = formats.build_ydl_opts(
|
opts = formats.build_ydl_opts(
|
||||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
||||||
settings.download_pot_base_url, clients,
|
extra=adapter.ydl_opts_extra(clients),
|
||||||
)
|
)
|
||||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||||
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
|
|||||||
return ydl.extract_info(url, download=True)
|
return ydl.extract_info(url, download=True)
|
||||||
except yt_dlp.utils.DownloadError as exc:
|
except yt_dlp.utils.DownloadError as exc:
|
||||||
last_exc = exc
|
last_exc = exc
|
||||||
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
|
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||||
if i < len(client_sets) - 1 and retriable:
|
|
||||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
||||||
for p in list(staging.iterdir()): # drop partials before the retry
|
for p in list(staging.iterdir()): # drop partials before the retry
|
||||||
try:
|
try:
|
||||||
@@ -815,8 +788,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
staging.mkdir(parents=True, exist_ok=True)
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
_set_job(job_id, phase="downloading", progress=0)
|
_set_job(job_id, phase="downloading", progress=0)
|
||||||
|
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||||
url = service.source_url(source_kind, source_ref)
|
url = service.source_url(source_kind, source_ref)
|
||||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
|
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||||
|
|
||||||
produced = _produced_file(info, staging)
|
produced = _produced_file(info, staging)
|
||||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||||
@@ -826,6 +800,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
title=normalize_title(info.get("title")) or source_ref,
|
title=normalize_title(info.get("title")) or source_ref,
|
||||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||||
|
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||||
description=info.get("description"),
|
description=info.get("description"),
|
||||||
thumbnail_url=info.get("thumbnail"),
|
thumbnail_url=info.get("thumbnail"),
|
||||||
@@ -859,7 +834,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||||
asset.title = meta.title
|
asset.title = meta.title
|
||||||
asset.uploader = meta.uploader
|
asset.uploader = meta.uploader
|
||||||
asset.uploader_url = build_uploader_url(info)
|
asset.uploader_url = adapters.resolve_adapter(source_kind, source_ref, info).uploader_url(info)
|
||||||
asset.upload_date = meta.upload_date
|
asset.upload_date = meta.upload_date
|
||||||
asset.thumbnail_url = meta.thumbnail_url
|
asset.thumbnail_url = meta.thumbnail_url
|
||||||
# Record a self-hosted poster for every download: it's the reliable link-preview
|
# Record a self-hosted poster for every download: it's the reliable link-preview
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
|
||||||
|
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
|
||||||
|
from app.downloads.adapters import (
|
||||||
|
FacebookAdapter,
|
||||||
|
RedditAdapter,
|
||||||
|
SourceAdapter,
|
||||||
|
YouTubeAdapter,
|
||||||
|
resolve_adapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_by_source_kind_and_host_and_info():
|
||||||
|
assert resolve_adapter("youtube", "abc123").key == "youtube"
|
||||||
|
assert resolve_adapter("url", "https://www.youtube.com/watch?v=x").key == "youtube"
|
||||||
|
assert resolve_adapter("url", "https://youtu.be/x").key == "youtube"
|
||||||
|
assert resolve_adapter("url", "https://www.facebook.com/reel/123/").key == "facebook"
|
||||||
|
assert resolve_adapter("url", "https://fb.watch/abc/").key == "facebook"
|
||||||
|
assert resolve_adapter("url", "https://www.reddit.com/r/x/comments/y/z").key == "reddit"
|
||||||
|
assert resolve_adapter("url", "https://example.com/video.mp4").key == "generic"
|
||||||
|
# After extraction the info dict's extractor decides even for a bare/redirected url.
|
||||||
|
assert resolve_adapter("url", None, {"extractor_key": "facebook"}).key == "facebook"
|
||||||
|
assert resolve_adapter("url", None, {"extractor_key": "Reddit"}).key == "reddit"
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_match_is_netloc_not_a_loose_substring():
|
||||||
|
# A path that merely CONTAINS "facebook.com" must not select the Facebook adapter.
|
||||||
|
assert resolve_adapter("url", "https://example.com/facebook.com-mirror").key == "generic"
|
||||||
|
|
||||||
|
|
||||||
|
def test_uploader_url_youtube_uses_channel_url():
|
||||||
|
a = YouTubeAdapter()
|
||||||
|
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
|
||||||
|
assert a.uploader_url({"uploader_url": "https://youtube.com/user/u"}) == "https://youtube.com/user/u"
|
||||||
|
assert a.uploader_url({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_uploader_url_facebook_falls_back_to_uploader_id():
|
||||||
|
a = FacebookAdapter()
|
||||||
|
assert a.uploader_url({"uploader_id": "12345"}) == "https://www.facebook.com/12345"
|
||||||
|
# A direct channel_url still wins over the id fallback.
|
||||||
|
assert a.uploader_url({"channel_url": "https://facebook.com/pageX", "uploader_id": "9"}) == (
|
||||||
|
"https://facebook.com/pageX"
|
||||||
|
)
|
||||||
|
assert a.uploader_url({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_uploader_url_reddit_and_generic_have_no_channel():
|
||||||
|
assert RedditAdapter().uploader_url({"uploader_id": "99"}) is None # not facebook → no fallback
|
||||||
|
assert SourceAdapter().uploader_url({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_source_url_is_passthrough_youtube_reconstructs():
|
||||||
|
assert SourceAdapter().source_url("https://x/y") == "https://x/y"
|
||||||
|
assert YouTubeAdapter().source_url("VIDID12345_") == "https://www.youtube.com/watch?v=VIDID12345_"
|
||||||
|
|
||||||
|
|
||||||
|
def test_client_sets_and_retriable_are_youtube_only():
|
||||||
|
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
|
||||||
|
assert SourceAdapter().client_sets() == [None]
|
||||||
|
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
|
||||||
|
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
|
||||||
|
assert YouTubeAdapter().retriable("some unrelated network blip") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||||
|
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||||
|
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||||
|
assert "youtubepot-bgutilhttp" in ea # the bgutil PO-token base_url is wired
|
||||||
|
# A client-less attempt still wires POT but no player_client key.
|
||||||
|
assert "youtube" not in YouTubeAdapter().ydl_opts_extra(None)["extractor_args"]
|
||||||
|
assert SourceAdapter().ydl_opts_extra(["android_vr"]) == {} # generic contributes nothing
|
||||||
|
|
||||||
|
|
||||||
|
def test_wants_metadata_backfill_matches_old_should_refetch():
|
||||||
|
assert YouTubeAdapter().wants_metadata_backfill() is True
|
||||||
|
assert FacebookAdapter().wants_metadata_backfill() is True
|
||||||
|
assert RedditAdapter().wants_metadata_backfill() is False
|
||||||
|
assert SourceAdapter().wants_metadata_backfill() is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_nfo_uniqueid_type_is_the_adapter_key():
|
||||||
|
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
|
||||||
|
assert SourceAdapter().nfo_uniqueid_type() == "generic"
|
||||||
@@ -9,6 +9,15 @@ def _opts(spec):
|
|||||||
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
|
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapter_extra_opts_are_merged_in():
|
||||||
|
# The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`.
|
||||||
|
extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}}
|
||||||
|
opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra)
|
||||||
|
assert opts["extractor_args"]["youtube"]["player_client"] == ["web"]
|
||||||
|
# No extra → no extractor_args (a generic source contributes nothing).
|
||||||
|
assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None)
|
||||||
|
|
||||||
|
|
||||||
def test_mp4_default_selects_avc1_with_progressive_fallback():
|
def test_mp4_default_selects_avc1_with_progressive_fallback():
|
||||||
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
|
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
|
||||||
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
|
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
|
||||||
|
|||||||
@@ -137,3 +137,13 @@ class TestSafeAbsPath:
|
|||||||
|
|
||||||
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
|
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
|
||||||
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
|
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_nfo_uniqueid_type_reflects_the_source():
|
||||||
|
# The .nfo <uniqueid type=...> now follows the source (via SourceAdapter.nfo_uniqueid_type),
|
||||||
|
# instead of the old hardcoded "youtube" that mislabelled every url-source download.
|
||||||
|
from app.downloads.storage import _nfo_xml
|
||||||
|
|
||||||
|
base = dict(video_id="v1", title="t", uploader="c", upload_date=None)
|
||||||
|
assert '<uniqueid type="youtube"' in _nfo_xml(MediaMeta(**base)) # default preserved
|
||||||
|
assert '<uniqueid type="facebook"' in _nfo_xml(MediaMeta(**base, uniqueid_type="facebook"))
|
||||||
|
|||||||
Reference in New Issue
Block a user