release: 0.63.1 (improvement/source-adapters)

This commit is contained in:
2026-08-13 01:21:50 +02:00
11 changed files with 375 additions and 86 deletions
+1 -1
View File
@@ -1 +1 @@
0.63.0
0.63.1
+151
View File
@@ -0,0 +1,151 @@
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
/ reddit / generic).
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
scattered across worker.py / backfill.py / service.py with no shared source identity.
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""
from urllib.parse import urlparse
from app.config import settings
class SourceAdapter:
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
key = "generic"
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
def client_sets(self) -> list:
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
return [None]
def ydl_opts_extra(self, clients) -> dict:
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
current entry from `client_sets()`. Generic: nothing."""
return {}
def retriable(self, error_text: str) -> bool:
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
return False
# --- metadata -----------------------------------------------------------------------------
def uploader_url(self, info: dict) -> str | None:
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
return info.get("channel_url") or info.get("uploader_url")
def wants_metadata_backfill(self) -> bool:
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
return False
# --- source ref ↔ URL ---------------------------------------------------------------------
def source_url(self, source_ref: str) -> str:
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
return source_ref
def nfo_uniqueid_type(self) -> str:
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
return self.key
class YouTubeAdapter(SourceAdapter):
key = "youtube"
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def client_sets(self) -> list:
return [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
def ydl_opts_extra(self, clients) -> dict:
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
ea: dict = {}
if clients:
ea["youtube"] = {"player_client": clients}
if settings.download_pot_base_url:
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
return {"extractor_args": ea} if ea else {}
def retriable(self, error_text: str) -> bool:
return any(m in error_text for m in self._RETRIABLE)
def wants_metadata_backfill(self) -> bool:
return True
def source_url(self, source_ref: str) -> str:
return f"https://www.youtube.com/watch?v={source_ref}"
class FacebookAdapter(SourceAdapter):
key = "facebook"
def uploader_url(self, info: dict) -> str | None:
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
# (the page id) that resolves at facebook.com/<id>.
base = super().uploader_url(info)
if base:
return base
uid = info.get("uploader_id")
return f"https://www.facebook.com/{uid}" if uid else None
def wants_metadata_backfill(self) -> bool:
return True
class RedditAdapter(SourceAdapter):
key = "reddit"
_YOUTUBE = YouTubeAdapter()
_FACEBOOK = FacebookAdapter()
_REDDIT = RedditAdapter()
_GENERIC = SourceAdapter()
def _host_is(host: str, *domains: str) -> bool:
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
return any(host == d or host.endswith("." + d) for d in domains)
def resolve_adapter(
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
) -> SourceAdapter:
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
hosts = []
if source_ref:
hosts.append(urlparse(source_ref).netloc.lower())
if info and info.get("webpage_url"):
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
def host_is(*domains: str) -> bool:
return any(_host_is(h, *domains) for h in hosts)
if source_kind == "youtube" or "youtube" in extractor or host_is(
"youtube.com", "youtu.be", "youtube-nocookie.com"
):
return _YOUTUBE
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
return _FACEBOOK
if "reddit" in extractor or host_is("reddit.com"):
return _REDDIT
return _GENERIC
+10 -23
View File
@@ -11,40 +11,26 @@ Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
"""
import logging
from urllib.parse import urlparse
import yt_dlp
from sqlalchemy import select
from app.config import settings
from app.db import SessionLocal
from app.downloads import adapters
from app.downloads import edit as editmod
from app.downloads import formats, service
from app.models import MediaAsset
from app.worker import build_uploader_url
log = logging.getLogger("siftlode.backfill")
def _should_refetch(asset: MediaAsset) -> bool:
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
if asset.source_kind == "youtube":
return True
if asset.source_kind == "url":
host = urlparse(asset.source_ref).netloc.lower()
return "facebook.com" in host or "fb.watch" in host
return False
def _extract_meta(url: str) -> dict | None:
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
POT/player-client strategy even just to extract)."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
for clients in client_sets:
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
needs the POT/player-client strategy even just to extract)."""
for clients in adapter.client_sets():
opts = formats.build_ydl_opts(
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
)
opts["skip_download"] = True
try:
@@ -70,10 +56,11 @@ def run() -> dict:
.all()
)
for a in assets:
if a.uploader_url is None and _should_refetch(a):
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
if a.uploader_url is None and adapter.wants_metadata_backfill():
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
info = _extract_meta(url)
uu = build_uploader_url(info) if info else None
info = _extract_meta(url, adapter)
uu = adapter.uploader_url(info) if info else None
if uu:
a.uploader_url = uu
filled_url += 1
+7 -14
View File
@@ -96,13 +96,13 @@ def build_ydl_opts(
spec: dict,
outtmpl: str,
progress_hook,
pot_base_url: str | None = None,
player_clients: list[str] | None = None,
extra: dict | None = None,
) -> dict:
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
generic options below. Everything here is source-agnostic."""
s = normalize(spec)
h = s["max_height"]
opts: dict = {
@@ -127,16 +127,9 @@ def build_ydl_opts(
"socket_timeout": 30,
}
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
# are lists in the Python API.
ea: dict = {}
if player_clients:
ea["youtube"] = {"player_client": player_clients}
if pot_base_url:
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
if ea:
opts["extractor_args"] = ea
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
if extra:
opts.update(extra)
if s["mode"] == "a":
opts["format"] = "bestaudio/best"
+4 -4
View File
@@ -13,6 +13,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.config import settings
from app.downloads import adapters
from app.downloads import edit as editmod
from app.downloads import formats, quota, storage
from app.models import Channel, DownloadJob, MediaAsset, Video
@@ -27,10 +28,9 @@ class EditError(Exception):
def source_url(source_kind: str, source_ref: str) -> str:
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
if source_kind == "youtube":
return f"https://www.youtube.com/watch?v={source_ref}"
return source_ref
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
URL); ad-hoc sources store the URL verbatim."""
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
def reference_url(job, asset) -> str | None:
+16 -6
View File
@@ -59,6 +59,10 @@ class MediaMeta:
title: str
uploader: str
upload_date: date | None
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
# the SourceAdapter. Defaults to the source-neutral "generic" so an un-updated caller is at worst
# uninformative, never actively mislabelling a non-YouTube download as "youtube".
uniqueid_type: str = "generic"
duration_s: int | None = None
description: str | None = None
thumbnail_url: str | None = None
@@ -170,7 +174,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
f" <premiered>{aired}</premiered>",
f" <year>{year}</year>",
f" <plot>{escape(meta.description or '')}</plot>",
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
]
if meta.duration_s:
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
@@ -178,16 +182,23 @@ def _nfo_xml(meta: MediaMeta) -> str:
return "\n".join(parts)
def _sidecar(media: Path, ext: str) -> Path:
"""The sibling sidecar path (`.nfo` / `.jpg`) for a media file. Swaps the extension on the FULL
media path — NOT via `media.with_suffix("")` first: stripping the real extension leaves a stem
whose OWN last dot may be inside the title (e.g. "Sonka Házilag."), which `with_suffix` would
then read as the extension and clobber, orphaning the sidecar under a truncated name."""
return media.with_suffix(ext)
def write_sidecars(
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
) -> bool:
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
media = abs_path(download_root, rel)
base = media.with_suffix("")
try:
base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
if thumb_src and thumb_src.exists():
shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg")))
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
return True
except OSError:
return False
@@ -196,8 +207,7 @@ def write_sidecars(
def delete_asset_files(download_root: str, rel: str) -> None:
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
media = abs_path(download_root, rel)
base = media.with_suffix("")
for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")):
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
try:
p.unlink()
except OSError:
+16 -38
View File
@@ -35,7 +35,7 @@ from app.config import settings
from app.db import SessionLocal
from app.downloads import edit as editmod
from app.downloads import errors as dlerrors
from app.downloads import formats, quota, service, storage
from app.downloads import adapters, formats, quota, service, storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
from app.titles import normalize_title
@@ -475,21 +475,6 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
db.commit()
def build_uploader_url(info: dict) -> str | None:
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
no derivable channel page (e.g. reddit / a raw media URL)."""
url = info.get("channel_url") or info.get("uploader_url")
if url:
return url
uid = info.get("uploader_id")
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
if uid and "facebook" in tag:
return f"https://www.facebook.com/{uid}"
return None
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
@@ -503,7 +488,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
# the final title with its own source_ref fallback.
asset.title = normalize_title(info.get("title")) or info.get("id")
asset.uploader = info.get("uploader") or info.get("channel")
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
asset.thumbnail_url = info.get("thumbnail")
asset.duration_s = int(info["duration"]) if info.get("duration") else None
asset.upload_date = _parse_upload_date(info.get("upload_date"))
@@ -713,28 +698,17 @@ def _media_info(info: dict) -> dict:
return {**info, **chosen}
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE_MARKERS = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
client_sets = adapter.client_sets()
outtmpl = str(staging / "%(id)s.%(ext)s")
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
settings.download_pot_base_url, clients,
extra=adapter.ydl_opts_extra(clients),
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
@@ -746,8 +720,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
return ydl.extract_info(url, download=True)
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
if i < len(client_sets) - 1 and retriable:
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
for p in list(staging.iterdir()): # drop partials before the retry
try:
@@ -815,8 +788,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
staging.mkdir(parents=True, exist_ok=True)
_set_job(job_id, phase="downloading", progress=0)
adapter = adapters.resolve_adapter(source_kind, source_ref)
url = service.source_url(source_kind, source_ref)
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
produced = _produced_file(info, staging)
produced = _ensure_browser_playable(job_id, produced, spec, info)
@@ -826,6 +803,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
title=normalize_title(info.get("title")) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=int(info["duration"]) if info.get("duration") else None,
description=info.get("description"),
thumbnail_url=info.get("thumbnail"),
@@ -859,7 +837,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = build_uploader_url(info)
asset.uploader_url = adapter.uploader_url(info)
asset.upload_date = meta.upload_date
asset.thumbnail_url = meta.thumbnail_url
# Record a self-hosted poster for every download: it's the reliable link-preview
+115
View File
@@ -0,0 +1,115 @@
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
from app.downloads.adapters import (
FacebookAdapter,
RedditAdapter,
SourceAdapter,
YouTubeAdapter,
resolve_adapter,
)
def test_resolve_by_source_kind_and_host_and_info():
assert resolve_adapter("youtube", "abc123").key == "youtube"
assert resolve_adapter("url", "https://www.youtube.com/watch?v=x").key == "youtube"
assert resolve_adapter("url", "https://youtu.be/x").key == "youtube"
assert resolve_adapter("url", "https://www.facebook.com/reel/123/").key == "facebook"
assert resolve_adapter("url", "https://fb.watch/abc/").key == "facebook"
assert resolve_adapter("url", "https://www.reddit.com/r/x/comments/y/z").key == "reddit"
assert resolve_adapter("url", "https://example.com/video.mp4").key == "generic"
# After extraction the info dict's extractor decides even for a bare/redirected url.
assert resolve_adapter("url", None, {"extractor_key": "facebook"}).key == "facebook"
assert resolve_adapter("url", None, {"extractor_key": "Reddit"}).key == "reddit"
def test_host_match_is_netloc_not_a_loose_substring():
# A path that merely CONTAINS "facebook.com" must not select the Facebook adapter.
assert resolve_adapter("url", "https://example.com/facebook.com-mirror").key == "generic"
def test_info_match_is_extractor_key_not_the_webpage_url():
# The webpage_url path may carry a SIBLING source's name (a reddit slug about youtube); only the
# yt-dlp extractor_key identifies the source, so this must resolve reddit, not youtube.
info = {"extractor_key": "Reddit", "webpage_url": "https://reddit.com/r/x/comments/y/youtube_is_dead"}
assert resolve_adapter("url", None, info).key == "reddit"
def test_youtube_nocookie_host_still_gets_the_youtube_adapter():
# An embed on youtube-nocookie.com is YouTube content and needs the POT/player-client strategy,
# even though its host does not contain the substring "youtube.com".
assert resolve_adapter("url", "https://www.youtube-nocookie.com/embed/abc123").key == "youtube"
def test_lookalike_hosts_do_not_misresolve_but_subdomains_do():
# Netloc-scoped match: a look-alike host that merely CONTAINS the domain string is NOT the source.
assert resolve_adapter("url", "https://myyoutube.com/watch?v=x").key == "generic"
assert resolve_adapter("url", "https://youtube.com.evil.example/x").key == "generic"
assert resolve_adapter("url", "https://notreddit.com/r/x").key == "generic"
# …but real subdomains still resolve.
assert resolve_adapter("url", "https://m.youtube.com/watch?v=x").key == "youtube"
assert resolve_adapter("url", "https://music.youtube.com/watch?v=x").key == "youtube"
def test_webpage_url_netloc_identifies_source_when_extractor_key_absent():
# The early-fill path passes info with no source_ref; the webpage_url netloc still resolves the
# source even if extractor_key is missing from the progress-hook info_dict.
assert resolve_adapter(None, None, {"webpage_url": "https://www.facebook.com/reel/123"}).key == "facebook"
# A sibling source's name in the webpage_url PATH must not misresolve (host, not substring).
info = {"extractor_key": "Reddit", "webpage_url": "https://www.reddit.com/r/x/comments/y/youtube"}
assert resolve_adapter(None, None, info).key == "reddit"
def test_uploader_url_youtube_uses_channel_url():
a = YouTubeAdapter()
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
assert a.uploader_url({"uploader_url": "https://youtube.com/user/u"}) == "https://youtube.com/user/u"
assert a.uploader_url({}) is None
def test_uploader_url_facebook_falls_back_to_uploader_id():
a = FacebookAdapter()
assert a.uploader_url({"uploader_id": "12345"}) == "https://www.facebook.com/12345"
# A direct channel_url still wins over the id fallback.
assert a.uploader_url({"channel_url": "https://facebook.com/pageX", "uploader_id": "9"}) == (
"https://facebook.com/pageX"
)
assert a.uploader_url({}) is None
def test_uploader_url_reddit_and_generic_have_no_channel():
assert RedditAdapter().uploader_url({"uploader_id": "99"}) is None # not facebook → no fallback
assert SourceAdapter().uploader_url({}) is None
def test_generic_source_url_is_passthrough_youtube_reconstructs():
assert SourceAdapter().source_url("https://x/y") == "https://x/y"
assert YouTubeAdapter().source_url("VIDID12345_") == "https://www.youtube.com/watch?v=VIDID12345_"
def test_client_sets_and_retriable_are_youtube_only():
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
assert SourceAdapter().client_sets() == [None]
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
assert YouTubeAdapter().retriable("some unrelated network blip") is False
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
assert ea["youtube"]["player_client"] == ["android_vr"]
assert "youtubepot-bgutilhttp" in ea # the bgutil PO-token base_url is wired
# A client-less attempt still wires POT but no player_client key.
assert "youtube" not in YouTubeAdapter().ydl_opts_extra(None)["extractor_args"]
assert SourceAdapter().ydl_opts_extra(["android_vr"]) == {} # generic contributes nothing
def test_wants_metadata_backfill_matches_old_should_refetch():
assert YouTubeAdapter().wants_metadata_backfill() is True
assert FacebookAdapter().wants_metadata_backfill() is True
assert RedditAdapter().wants_metadata_backfill() is False
assert SourceAdapter().wants_metadata_backfill() is False
def test_nfo_uniqueid_type_is_the_adapter_key():
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
assert SourceAdapter().nfo_uniqueid_type() == "generic"
+9
View File
@@ -9,6 +9,15 @@ def _opts(spec):
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
def test_adapter_extra_opts_are_merged_in():
# The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`.
extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}}
opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra)
assert opts["extractor_args"]["youtube"]["player_client"] == ["web"]
# No extra → no extractor_args (a generic source contributes nothing).
assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None)
def test_mp4_default_selects_avc1_with_progressive_fallback():
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
+35
View File
@@ -137,3 +137,38 @@ class TestSafeAbsPath:
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
def test_nfo_uniqueid_type_reflects_the_source():
# The .nfo <uniqueid type=...> now follows the source (via SourceAdapter.nfo_uniqueid_type),
# instead of the old hardcoded "youtube" that mislabelled every url-source download.
from app.downloads.storage import _nfo_xml
base = dict(video_id="v1", title="t", uploader="c", upload_date=None)
# An un-set caller gets the source-neutral "generic" default, never a false "youtube".
assert '<uniqueid type="generic"' in _nfo_xml(MediaMeta(**base))
assert '<uniqueid type="facebook"' in _nfo_xml(MediaMeta(**base, uniqueid_type="facebook"))
assert '<uniqueid type="youtube"' in _nfo_xml(MediaMeta(**base, uniqueid_type="youtube"))
def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path):
# A title ending in a period ("Sonka Házilag.") puts a dot in the media slug; the .nfo/.jpg
# sidecars must sit on the SAME (full) basename, not be truncated at that dot (which orphaned
# them from the media so Plex/Kodi never matched them).
from app.downloads.storage import write_sidecars
stem = "Chan_-_2026-08-12_-_Sonka_Hazilag._Villam_konyha_abc_a5"
rel = f"Chan/Season_2026/{stem}.mp4"
media = tmp_path / rel
media.parent.mkdir(parents=True)
media.write_text("x")
thumb = tmp_path / "t.jpg"
thumb.write_bytes(b"\xff\xd8\xff")
meta = MediaMeta(video_id="abc", title="Sonka Házilag. | Villám konyha", uploader="Chan",
upload_date=None)
assert write_sidecars(str(tmp_path), rel, meta, thumb) is True
names = sorted(p.name for p in media.parent.iterdir())
assert f"{stem}.nfo" in names, names
assert f"{stem}.jpg" in names, names
assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan
+11
View File
@@ -14,6 +14,17 @@ export interface ReleaseEntry {
}
export const RELEASE_NOTES: ReleaseEntry[] = [
{
version: "0.63.1",
date: "2026-08-13",
fixes: [
"Downloaded videos whose title ends in a period (some Facebook reels, for example) now save their Plex/Kodi info and poster files right next to the video so Plex picks them up — previously those files landed under a slightly different name and were missed.",
"The info file for a non-YouTube download (Facebook, Reddit, a direct link) now records the correct source instead of always labelling it as YouTube.",
],
chores: [
"Internal refactor: consolidated the per-source download behaviour (YouTube / Facebook / Reddit / generic) behind one small adapter layer. No change to how downloading looks or works.",
],
},
{
version: "0.63.0",
date: "2026-08-12",