release: 0.63.1 (improvement/source-adapters)
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||||
/ reddit / generic).
|
||||
|
||||
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||||
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||||
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||||
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||||
|
||||
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||||
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||||
current entry from `client_sets()`. Generic: nothing."""
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
return info.get("channel_url") or info.get("uploader_url")
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||||
return False
|
||||
|
||||
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||||
return source_ref
|
||||
|
||||
def nfo_uniqueid_type(self) -> str:
|
||||
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||||
return self.key
|
||||
|
||||
|
||||
class YouTubeAdapter(SourceAdapter):
|
||||
key = "youtube"
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
ea: dict = {}
|
||||
if clients:
|
||||
ea["youtube"] = {"player_client": clients}
|
||||
if settings.download_pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||||
return {"extractor_args": ea} if ea else {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return any(m in error_text for m in self._RETRIABLE)
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
# (the page id) that resolves at facebook.com/<id>.
|
||||
base = super().uploader_url(info)
|
||||
if base:
|
||||
return base
|
||||
uid = info.get("uploader_id")
|
||||
return f"https://www.facebook.com/{uid}" if uid else None
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
_YOUTUBE = YouTubeAdapter()
|
||||
_FACEBOOK = FacebookAdapter()
|
||||
_REDDIT = RedditAdapter()
|
||||
_GENERIC = SourceAdapter()
|
||||
|
||||
|
||||
def _host_is(host: str, *domains: str) -> bool:
|
||||
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
|
||||
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
|
||||
return any(host == d or host.endswith("." + d) for d in domains)
|
||||
|
||||
|
||||
def resolve_adapter(
|
||||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||||
) -> SourceAdapter:
|
||||
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
|
||||
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
|
||||
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
|
||||
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
|
||||
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
|
||||
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||||
hosts = []
|
||||
if source_ref:
|
||||
hosts.append(urlparse(source_ref).netloc.lower())
|
||||
if info and info.get("webpage_url"):
|
||||
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
|
||||
|
||||
def host_is(*domains: str) -> bool:
|
||||
return any(_host_is(h, *domains) for h in hosts)
|
||||
|
||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
||||
):
|
||||
return _YOUTUBE
|
||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||
return _FACEBOOK
|
||||
if "reddit" in extractor or host_is("reddit.com"):
|
||||
return _REDDIT
|
||||
return _GENERIC
|
||||
@@ -11,40 +11,26 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import build_uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
@@ -70,10 +56,11 @@ def run() -> dict:
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = build_uploader_url(info) if info else None
|
||||
info = _extract_meta(url, adapter)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
|
||||
@@ -96,13 +96,13 @@ def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
progress_hook,
|
||||
pot_base_url: str | None = None,
|
||||
player_clients: list[str] | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||
|
||||
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
|
||||
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
|
||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
@@ -127,16 +127,9 @@ def build_ydl_opts(
|
||||
"socket_timeout": 30,
|
||||
}
|
||||
|
||||
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
|
||||
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
|
||||
# are lists in the Python API.
|
||||
ea: dict = {}
|
||||
if player_clients:
|
||||
ea["youtube"] = {"player_client": player_clients}
|
||||
if pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
|
||||
if ea:
|
||||
opts["extractor_args"] = ea
|
||||
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
@@ -27,10 +28,9 @@ class EditError(Exception):
|
||||
|
||||
|
||||
def source_url(source_kind: str, source_ref: str) -> str:
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
||||
if source_kind == "youtube":
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
return source_ref
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
|
||||
URL); ad-hoc sources store the URL verbatim."""
|
||||
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
|
||||
|
||||
|
||||
def reference_url(job, asset) -> str | None:
|
||||
|
||||
@@ -59,6 +59,10 @@ class MediaMeta:
|
||||
title: str
|
||||
uploader: str
|
||||
upload_date: date | None
|
||||
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
|
||||
# the SourceAdapter. Defaults to the source-neutral "generic" so an un-updated caller is at worst
|
||||
# uninformative, never actively mislabelling a non-YouTube download as "youtube".
|
||||
uniqueid_type: str = "generic"
|
||||
duration_s: int | None = None
|
||||
description: str | None = None
|
||||
thumbnail_url: str | None = None
|
||||
@@ -170,7 +174,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
f" <premiered>{aired}</premiered>",
|
||||
f" <year>{year}</year>",
|
||||
f" <plot>{escape(meta.description or '')}</plot>",
|
||||
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
]
|
||||
if meta.duration_s:
|
||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||
@@ -178,16 +182,23 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _sidecar(media: Path, ext: str) -> Path:
|
||||
"""The sibling sidecar path (`.nfo` / `.jpg`) for a media file. Swaps the extension on the FULL
|
||||
media path — NOT via `media.with_suffix("")` first: stripping the real extension leaves a stem
|
||||
whose OWN last dot may be inside the title (e.g. "Sonka Házilag."), which `with_suffix` would
|
||||
then read as the extension and clobber, orphaning the sidecar under a truncated name."""
|
||||
return media.with_suffix(ext)
|
||||
|
||||
|
||||
def write_sidecars(
|
||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
||||
) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
try:
|
||||
base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg")))
|
||||
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
@@ -196,8 +207,7 @@ def write_sidecars(
|
||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")):
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
|
||||
+16
-38
@@ -35,7 +35,7 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, quota, service, storage
|
||||
from app.downloads import adapters, formats, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def build_uploader_url(info: dict) -> str | None:
|
||||
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
|
||||
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
|
||||
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
|
||||
no derivable channel page (e.g. reddit / a raw media URL)."""
|
||||
url = info.get("channel_url") or info.get("uploader_url")
|
||||
if url:
|
||||
return url
|
||||
uid = info.get("uploader_id")
|
||||
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
|
||||
if uid and "facebook" in tag:
|
||||
return f"https://www.facebook.com/{uid}"
|
||||
return None
|
||||
|
||||
|
||||
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
||||
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
||||
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = build_uploader_url(info)
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
|
||||
return {**info, **chosen}
|
||||
|
||||
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE_MARKERS = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
|
||||
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
|
||||
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
||||
settings.download_pot_base_url, clients,
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
|
||||
return ydl.extract_info(url, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
|
||||
if i < len(client_sets) - 1 and retriable:
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
@@ -815,8 +788,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
@@ -826,6 +803,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
@@ -859,7 +837,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = build_uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.upload_date = meta.upload_date
|
||||
asset.thumbnail_url = meta.thumbnail_url
|
||||
# Record a self-hosted poster for every download: it's the reliable link-preview
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
|
||||
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
|
||||
from app.downloads.adapters import (
|
||||
FacebookAdapter,
|
||||
RedditAdapter,
|
||||
SourceAdapter,
|
||||
YouTubeAdapter,
|
||||
resolve_adapter,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_by_source_kind_and_host_and_info():
|
||||
assert resolve_adapter("youtube", "abc123").key == "youtube"
|
||||
assert resolve_adapter("url", "https://www.youtube.com/watch?v=x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://youtu.be/x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://www.facebook.com/reel/123/").key == "facebook"
|
||||
assert resolve_adapter("url", "https://fb.watch/abc/").key == "facebook"
|
||||
assert resolve_adapter("url", "https://www.reddit.com/r/x/comments/y/z").key == "reddit"
|
||||
assert resolve_adapter("url", "https://example.com/video.mp4").key == "generic"
|
||||
# After extraction the info dict's extractor decides even for a bare/redirected url.
|
||||
assert resolve_adapter("url", None, {"extractor_key": "facebook"}).key == "facebook"
|
||||
assert resolve_adapter("url", None, {"extractor_key": "Reddit"}).key == "reddit"
|
||||
|
||||
|
||||
def test_host_match_is_netloc_not_a_loose_substring():
|
||||
# A path that merely CONTAINS "facebook.com" must not select the Facebook adapter.
|
||||
assert resolve_adapter("url", "https://example.com/facebook.com-mirror").key == "generic"
|
||||
|
||||
|
||||
def test_info_match_is_extractor_key_not_the_webpage_url():
|
||||
# The webpage_url path may carry a SIBLING source's name (a reddit slug about youtube); only the
|
||||
# yt-dlp extractor_key identifies the source, so this must resolve reddit, not youtube.
|
||||
info = {"extractor_key": "Reddit", "webpage_url": "https://reddit.com/r/x/comments/y/youtube_is_dead"}
|
||||
assert resolve_adapter("url", None, info).key == "reddit"
|
||||
|
||||
|
||||
def test_youtube_nocookie_host_still_gets_the_youtube_adapter():
|
||||
# An embed on youtube-nocookie.com is YouTube content and needs the POT/player-client strategy,
|
||||
# even though its host does not contain the substring "youtube.com".
|
||||
assert resolve_adapter("url", "https://www.youtube-nocookie.com/embed/abc123").key == "youtube"
|
||||
|
||||
|
||||
def test_lookalike_hosts_do_not_misresolve_but_subdomains_do():
|
||||
# Netloc-scoped match: a look-alike host that merely CONTAINS the domain string is NOT the source.
|
||||
assert resolve_adapter("url", "https://myyoutube.com/watch?v=x").key == "generic"
|
||||
assert resolve_adapter("url", "https://youtube.com.evil.example/x").key == "generic"
|
||||
assert resolve_adapter("url", "https://notreddit.com/r/x").key == "generic"
|
||||
# …but real subdomains still resolve.
|
||||
assert resolve_adapter("url", "https://m.youtube.com/watch?v=x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://music.youtube.com/watch?v=x").key == "youtube"
|
||||
|
||||
|
||||
def test_webpage_url_netloc_identifies_source_when_extractor_key_absent():
|
||||
# The early-fill path passes info with no source_ref; the webpage_url netloc still resolves the
|
||||
# source even if extractor_key is missing from the progress-hook info_dict.
|
||||
assert resolve_adapter(None, None, {"webpage_url": "https://www.facebook.com/reel/123"}).key == "facebook"
|
||||
# A sibling source's name in the webpage_url PATH must not misresolve (host, not substring).
|
||||
info = {"extractor_key": "Reddit", "webpage_url": "https://www.reddit.com/r/x/comments/y/youtube"}
|
||||
assert resolve_adapter(None, None, info).key == "reddit"
|
||||
|
||||
|
||||
def test_uploader_url_youtube_uses_channel_url():
|
||||
a = YouTubeAdapter()
|
||||
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
|
||||
assert a.uploader_url({"uploader_url": "https://youtube.com/user/u"}) == "https://youtube.com/user/u"
|
||||
assert a.uploader_url({}) is None
|
||||
|
||||
|
||||
def test_uploader_url_facebook_falls_back_to_uploader_id():
|
||||
a = FacebookAdapter()
|
||||
assert a.uploader_url({"uploader_id": "12345"}) == "https://www.facebook.com/12345"
|
||||
# A direct channel_url still wins over the id fallback.
|
||||
assert a.uploader_url({"channel_url": "https://facebook.com/pageX", "uploader_id": "9"}) == (
|
||||
"https://facebook.com/pageX"
|
||||
)
|
||||
assert a.uploader_url({}) is None
|
||||
|
||||
|
||||
def test_uploader_url_reddit_and_generic_have_no_channel():
|
||||
assert RedditAdapter().uploader_url({"uploader_id": "99"}) is None # not facebook → no fallback
|
||||
assert SourceAdapter().uploader_url({}) is None
|
||||
|
||||
|
||||
def test_generic_source_url_is_passthrough_youtube_reconstructs():
|
||||
assert SourceAdapter().source_url("https://x/y") == "https://x/y"
|
||||
assert YouTubeAdapter().source_url("VIDID12345_") == "https://www.youtube.com/watch?v=VIDID12345_"
|
||||
|
||||
|
||||
def test_client_sets_and_retriable_are_youtube_only():
|
||||
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
|
||||
assert SourceAdapter().client_sets() == [None]
|
||||
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
|
||||
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
|
||||
assert YouTubeAdapter().retriable("some unrelated network blip") is False
|
||||
|
||||
|
||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||
assert "youtubepot-bgutilhttp" in ea # the bgutil PO-token base_url is wired
|
||||
# A client-less attempt still wires POT but no player_client key.
|
||||
assert "youtube" not in YouTubeAdapter().ydl_opts_extra(None)["extractor_args"]
|
||||
assert SourceAdapter().ydl_opts_extra(["android_vr"]) == {} # generic contributes nothing
|
||||
|
||||
|
||||
def test_wants_metadata_backfill_matches_old_should_refetch():
|
||||
assert YouTubeAdapter().wants_metadata_backfill() is True
|
||||
assert FacebookAdapter().wants_metadata_backfill() is True
|
||||
assert RedditAdapter().wants_metadata_backfill() is False
|
||||
assert SourceAdapter().wants_metadata_backfill() is False
|
||||
|
||||
|
||||
def test_nfo_uniqueid_type_is_the_adapter_key():
|
||||
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
|
||||
assert SourceAdapter().nfo_uniqueid_type() == "generic"
|
||||
@@ -9,6 +9,15 @@ def _opts(spec):
|
||||
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
|
||||
|
||||
|
||||
def test_adapter_extra_opts_are_merged_in():
|
||||
# The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`.
|
||||
extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}}
|
||||
opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra)
|
||||
assert opts["extractor_args"]["youtube"]["player_client"] == ["web"]
|
||||
# No extra → no extractor_args (a generic source contributes nothing).
|
||||
assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None)
|
||||
|
||||
|
||||
def test_mp4_default_selects_avc1_with_progressive_fallback():
|
||||
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
|
||||
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
|
||||
|
||||
@@ -137,3 +137,38 @@ class TestSafeAbsPath:
|
||||
|
||||
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
|
||||
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
|
||||
|
||||
|
||||
def test_nfo_uniqueid_type_reflects_the_source():
|
||||
# The .nfo <uniqueid type=...> now follows the source (via SourceAdapter.nfo_uniqueid_type),
|
||||
# instead of the old hardcoded "youtube" that mislabelled every url-source download.
|
||||
from app.downloads.storage import _nfo_xml
|
||||
|
||||
base = dict(video_id="v1", title="t", uploader="c", upload_date=None)
|
||||
# An un-set caller gets the source-neutral "generic" default, never a false "youtube".
|
||||
assert '<uniqueid type="generic"' in _nfo_xml(MediaMeta(**base))
|
||||
assert '<uniqueid type="facebook"' in _nfo_xml(MediaMeta(**base, uniqueid_type="facebook"))
|
||||
assert '<uniqueid type="youtube"' in _nfo_xml(MediaMeta(**base, uniqueid_type="youtube"))
|
||||
|
||||
|
||||
def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path):
|
||||
# A title ending in a period ("Sonka Házilag.") puts a dot in the media slug; the .nfo/.jpg
|
||||
# sidecars must sit on the SAME (full) basename, not be truncated at that dot (which orphaned
|
||||
# them from the media so Plex/Kodi never matched them).
|
||||
from app.downloads.storage import write_sidecars
|
||||
|
||||
stem = "Chan_-_2026-08-12_-_Sonka_Hazilag._Villam_konyha_abc_a5"
|
||||
rel = f"Chan/Season_2026/{stem}.mp4"
|
||||
media = tmp_path / rel
|
||||
media.parent.mkdir(parents=True)
|
||||
media.write_text("x")
|
||||
thumb = tmp_path / "t.jpg"
|
||||
thumb.write_bytes(b"\xff\xd8\xff")
|
||||
meta = MediaMeta(video_id="abc", title="Sonka Házilag. | Villám konyha", uploader="Chan",
|
||||
upload_date=None)
|
||||
|
||||
assert write_sidecars(str(tmp_path), rel, meta, thumb) is True
|
||||
names = sorted(p.name for p in media.parent.iterdir())
|
||||
assert f"{stem}.nfo" in names, names
|
||||
assert f"{stem}.jpg" in names, names
|
||||
assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan
|
||||
|
||||
@@ -14,6 +14,17 @@ export interface ReleaseEntry {
|
||||
}
|
||||
|
||||
export const RELEASE_NOTES: ReleaseEntry[] = [
|
||||
{
|
||||
version: "0.63.1",
|
||||
date: "2026-08-13",
|
||||
fixes: [
|
||||
"Downloaded videos whose title ends in a period (some Facebook reels, for example) now save their Plex/Kodi info and poster files right next to the video so Plex picks them up — previously those files landed under a slightly different name and were missed.",
|
||||
"The info file for a non-YouTube download (Facebook, Reddit, a direct link) now records the correct source instead of always labelling it as YouTube.",
|
||||
],
|
||||
chores: [
|
||||
"Internal refactor: consolidated the per-source download behaviour (YouTube / Facebook / Reddit / generic) behind one small adapter layer. No change to how downloading looks or works.",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "0.63.0",
|
||||
date: "2026-08-12",
|
||||
|
||||
Reference in New Issue
Block a user