diff --git a/backend/alembic/versions/0071_plex_tmdb_fields.py b/backend/alembic/versions/0071_plex_tmdb_fields.py new file mode 100644 index 00000000..5342f108 --- /dev/null +++ b/backend/alembic/versions/0071_plex_tmdb_fields.py @@ -0,0 +1,52 @@ +"""plex TMDB enrichment columns (E5 S2) + +Additive TMDB-sourced fields on plex_items + plex_shows — keywords, full crew, budget/revenue, TMDb +community score, production companies/countries, recommendations, and a backdrop-path pool — plus a +`tmdb_id` cache and a `tmdb_enriched_at` watermark. Filled by the new TMDB enrichment job (matched by +the stored canonical `guid`), SERVED FROM THE ROW (never a live TMDB call). Distinct from the existing +Plex-sourced `imdb_rating`/`rating`, which are left untouched. Plus `plex_people.tmdb_person_id` (the S2 +foundation for S6's person bios). ADD-only, all nullable → no table rewrite, no ACCESS EXCLUSIVE lock. +No watermark reset needed: `tmdb_enriched_at` starts NULL, so every eligible row is picked up once. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0071_plex_tmdb_fields" +down_revision: Union[str, None] = "0070_fold_maintenance_batch" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_TABLES = ("plex_items", "plex_shows") + +# (name, type) — same set on both movie/leaf rows and show rows. budget/revenue stay NULL for shows. +_COLS: tuple[tuple[str, sa.types.TypeEngine], ...] = ( + ("tmdb_id", sa.Integer()), + ("tmdb_enriched_at", sa.DateTime(timezone=True)), + ("tmdb_rating", sa.Float()), + ("tmdb_vote_count", sa.Integer()), + ("budget", sa.BigInteger()), + ("revenue", sa.BigInteger()), + ("keywords", postgresql.JSONB()), + ("crew", postgresql.JSONB()), + ("studios", postgresql.JSONB()), + ("production_countries", postgresql.JSONB()), + ("recommendations", postgresql.JSONB()), + ("backdrops", postgresql.JSONB()), +) + + +def upgrade() -> None: + for t in _TABLES: + for name, type_ in _COLS: + op.add_column(t, sa.Column(name, type_, nullable=True)) + op.add_column("plex_people", sa.Column("tmdb_person_id", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("plex_people", "tmdb_person_id") + for t in _TABLES: + for name, _ in reversed(_COLS): + op.drop_column(t, name) diff --git a/backend/app/config.py b/backend/app/config.py index ac399007..ec5d571e 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -245,6 +245,20 @@ class Settings(BaseSettings): # local disk, so the default is fine there. plex_hls_dir: str = "" + # --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) --- + # TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the + # admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget / + # revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the + # stored canonical guid, and SERVED FROM THE DB — never a live TMDB call at serve time. + tmdb_api_key: str = "" + tmdb_api_base: str = "https://api.themoviedb.org/3" + # Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default. + tmdb_language: str = "en-US" + # How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes. + tmdb_enrich_interval_min: int = 120 + # Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited). + tmdb_enrich_batch: int = 100 + @property def allowed_email_set(self) -> set[str]: return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()} diff --git a/backend/app/models.py b/backend/app/models.py index c1b44254..0a4cfe9b 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1067,6 +1067,20 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin): imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link tagline: Mapped[str | None] = mapped_column(Text) full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}] + # --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive; + # served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) --- + tmdb_id: Mapped[int | None] = mapped_column(Integer) + tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10) + tmdb_vote_count: Mapped[int | None] = mapped_column(Integer) + budget: Mapped[int | None] = mapped_column(BigInteger) + revenue: Mapped[int | None] = mapped_column(BigInteger) + keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str] + crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}] + studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str] + production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str] + recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) + backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched, # so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors. title_hu: Mapped[str | None] = mapped_column(String(512)) @@ -1184,6 +1198,24 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin): imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link tagline: Mapped[str | None] = mapped_column(Text) full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}] + # --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB + # backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating + # above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched; + # `tmdb_enriched_at` is the "tried" watermark (stamped on a definitive no-match too, so an + # unresolvable title isn't re-fetched every run; left NULL only on a transient fetch error). --- + tmdb_id: Mapped[int | None] = mapped_column(Integer) + tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10) + tmdb_vote_count: Mapped[int | None] = mapped_column(Integer) + budget: Mapped[int | None] = mapped_column(BigInteger) # USD (movies; TV has none) + revenue: Mapped[int | None] = mapped_column(BigInteger) + keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str] + crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}] + studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str] + production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str] + # [{tmdb_id, title, poster, media_type}] — resolved to library rows (by guid) at serve time. + recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) + backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB backdrop file_paths [str] (crossfade pool, S8) # Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched # (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors. title_hu: Mapped[str | None] = mapped_column(String(512)) @@ -1224,6 +1256,9 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(255), unique=True, index=True) photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None + # TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person + # biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person. + tmdb_person_id: Mapped[int | None] = mapped_column(Integer) class PlexCollection(Base, TimestampMixin, UpdatedAtMixin): diff --git a/backend/app/plex/tmdb.py b/backend/app/plex/tmdb.py new file mode 100644 index 00000000..5280900b --- /dev/null +++ b/backend/app/plex/tmdb.py @@ -0,0 +1,360 @@ +"""TMDB enrichment for the mirrored Plex catalog (E5 S2). + +A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items / +plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find). +Server-side ONLY (the api_key from sysconfig). Everything fetched is PERSISTED and served from the DB +row — there is never a live TMDB call at serve time (the user's explicit rule). Additive: the existing +Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the distinct TMDb score. + +One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes +the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module. +""" +import logging +import time +from datetime import datetime, timezone + +import httpx +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from app import sysconfig +from app.config import settings +from app.models import PlexItem, PlexPerson, PlexShow + +log = logging.getLogger("siftlode.tmdb") + +# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host. +_MIN_INTERVAL = 0.03 + +# The crew jobs worth surfacing on an info page (TMDB lists dozens per title). Order-preserving pick. +_CREW_JOBS = ( + "Director", + "Writer", + "Screenplay", + "Story", + "Producer", + "Executive Producer", + "Editor", + "Director of Photography", + "Original Music Composer", +) + +# append_to_response bundles the sub-resources into ONE request per title (keywords + credits + +# recommendations + images + external ids). TV uses aggregate_credits (per-person aggregated jobs). +_MOVIE_APPEND = "keywords,credits,recommendations,images,external_ids" +_TV_APPEND = "keywords,aggregate_credits,recommendations,images,external_ids" + + +class TmdbError(Exception): + """Any transient failure talking to TMDB (network, non-404 HTTP, bad payload). The caller leaves + the row's `tmdb_enriched_at` NULL so the next run retries.""" + + +class TmdbNotConfigured(TmdbError): + """No TMDB API key is set — the module can't do anything.""" + + +class TmdbClient: + """Bound to the configured api_key + language. Use as a context manager so the httpx client closes.""" + + def __init__(self, db: Session): + self.key = sysconfig.get_str(db, "tmdb_api_key") + if not self.key: + raise TmdbNotConfigured("TMDB API key is not configured") + self.base = settings.tmdb_api_base.rstrip("/") + self.language = sysconfig.get_str(db, "tmdb_language") or settings.tmdb_language + self._http = httpx.Client(timeout=30.0) + self._last = 0.0 + + def __enter__(self) -> "TmdbClient": + return self + + def __exit__(self, *exc) -> None: + self._http.close() + + def close(self) -> None: + self._http.close() + + def _throttle(self) -> None: + wait = self._last + _MIN_INTERVAL - time.monotonic() + if wait > 0: + time.sleep(wait) + self._last = time.monotonic() + + def _get(self, path: str, params: dict | None = None) -> dict | None: + """GET a TMDB path. Returns the JSON dict, None on a 404 (a definitive "no such id"), and raises + TmdbError on any transient failure (network / other HTTP status / non-JSON).""" + self._throttle() + p = {"api_key": self.key, "language": self.language, **(params or {})} + try: + r = self._http.get(f"{self.base}{path}", params=p) + except httpx.HTTPError as e: + raise TmdbError(str(e)) from e + if r.status_code == 404: + return None + try: + r.raise_for_status() + except httpx.HTTPStatusError as e: + raise TmdbError(str(e)) from e + try: + return r.json() + except ValueError as e: + raise TmdbError("TMDB returned a non-JSON response") from e + + def fetch(self, tmdb_id: int, media_type: str) -> dict | None: + """Full enrichment payload for a movie or show (one request via append_to_response).""" + if media_type == "tv": + return self._get(f"/tv/{tmdb_id}", {"append_to_response": _TV_APPEND}) + return self._get(f"/movie/{tmdb_id}", {"append_to_response": _MOVIE_APPEND}) + + def find(self, external_id: str, source: str) -> dict | None: + """Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find.""" + return self._get(f"/find/{external_id}", {"external_source": source}) + + +def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None: + """Map a stored canonical guid to a TMDB id. `tmdb:` is direct; `imdb:`/`tvdb:` go through /find + (picking the movie_results or tv_results list per `media_type`). Returns None for an unmatchable + guid — the caller then stamps the watermark so it isn't retried every run.""" + if not guid or ":" not in guid: + return None + provider, _, ext = guid.partition(":") + if provider == "tmdb": + try: + return int(ext) + except ValueError: + return None + source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider) + if not source: + return None + res = tmdb.find(ext, source) + if not res: + return None + items = res.get("tv_results" if media_type == "tv" else "movie_results") or [] + return items[0].get("id") if items else None + + +def _dedup_crew(crew: list[dict]) -> list[dict]: + seen: set[tuple] = set() + out: list[dict] = [] + for c in crew: + key = (c["name"], c["job"]) + if key in seen: + continue + seen.add(key) + out.append(c) + return out + + +def parse_tmdb(payload: dict, media_type: str) -> dict: + """Pure parser: pull the enrichment fields out of a TMDB movie/tv payload (append_to_response form). + Best-effort — any missing sub-resource is just omitted. Returns the row fields plus a `people` list + ([{name, tmdb_id}] over cast+crew) for the plex_people id-linking. TMDB image fields are stored as + bare paths ("/abc.jpg"); the client builds the image.tmdb.org URL.""" + is_tv = media_type == "tv" + votes = payload.get("vote_count") or 0 + + kw_container = payload.get("keywords") or {} + kw_list = kw_container.get("results") if is_tv else kw_container.get("keywords") + keywords = [k.get("name") for k in (kw_list or []) if k.get("name")][:20] + + credits = payload.get("aggregate_credits" if is_tv else "credits") or {} + crew: list[dict] = [] + for c in credits.get("crew") or []: + # Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person. + job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job") + if job in _CREW_JOBS and c.get("name"): + crew.append( + {"name": c["name"], "job": job, "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")} + ) + crew = _dedup_crew(crew)[:12] + + # name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids). + people: list[dict] = [] + for c in (credits.get("cast") or [])[:30]: + if c.get("name") and c.get("id"): + people.append({"name": c["name"], "tmdb_id": c["id"]}) + for c in crew: + if c.get("tmdb_id"): + people.append({"name": c["name"], "tmdb_id": c["tmdb_id"]}) + + recs: list[dict] = [] + for r in ((payload.get("recommendations") or {}).get("results") or [])[:20]: + recs.append( + { + "tmdb_id": r.get("id"), + "title": r.get("title") or r.get("name"), + "poster_path": r.get("poster_path"), + "media_type": r.get("media_type") or media_type, + } + ) + + backdrops = [ + b.get("file_path") + for b in ((payload.get("images") or {}).get("backdrops") or []) + if b.get("file_path") + ][:10] + + return { + "tmdb_id": payload.get("id"), + # vote_average is meaningless with no votes (TMDB returns 0.0) → treat as unknown. + "tmdb_rating": payload.get("vote_average") if votes else None, + "tmdb_vote_count": votes or None, + # 0 is TMDB's "unknown" for money fields; TV has neither. + "budget": (payload.get("budget") or None) if not is_tv else None, + "revenue": (payload.get("revenue") or None) if not is_tv else None, + "keywords": keywords or None, + "crew": crew or None, + "studios": [c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")][:8] + or None, + "production_countries": [ + c.get("name") for c in (payload.get("production_countries") or []) if c.get("name") + ] + or None, + "recommendations": recs or None, + "backdrops": backdrops or None, + "people": people, + } + + +def apply_tmdb_detail(row, parsed: dict) -> None: + """Persist the parsed TMDB fields onto a movie/show row. Does NOT touch full_cast / imdb_rating / + rating (Plex-owned). The caller stamps `tmdb_enriched_at` and links people separately.""" + row.tmdb_id = parsed["tmdb_id"] + row.tmdb_rating = parsed["tmdb_rating"] + row.tmdb_vote_count = parsed["tmdb_vote_count"] + row.budget = parsed["budget"] + row.revenue = parsed["revenue"] + row.keywords = parsed["keywords"] + row.crew = parsed["crew"] + row.studios = parsed["studios"] + row.production_countries = parsed["production_countries"] + row.recommendations = parsed["recommendations"] + row.backdrops = parsed["backdrops"] + + +def _link_people(db: Session, links: dict[str, int]) -> None: + """Upsert plex_people by name, stamping `tmdb_person_id` — keeping any id already set (first wins), + filling it where NULL, inserting a bare row where the person isn't cached yet. Deduped by name so a + single statement can't hit the conflict target twice (the same rule as people.enrich_row_people).""" + values = [{"name": nm, "tmdb_person_id": pid} for nm, pid in links.items() if nm and pid] + if not values: + return + stmt = pg_insert(PlexPerson).values(values) + db.execute( + stmt.on_conflict_do_update( + index_elements=[PlexPerson.name], + set_={"tmdb_person_id": func.coalesce(PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id)}, + ) + ) + + +def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int: + """Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder — + mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a + definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched + every run. One commit at the end. Returns the number processed (matched OR definitively unmatched).""" + movies = ( + db.execute( + select(PlexItem) + .where( + PlexItem.kind == "movie", + PlexItem.guid.is_not(None), + PlexItem.tmdb_enriched_at.is_(None), + ) + .order_by(PlexItem.id) + .limit(limit) + ) + .scalars() + .all() + ) + shows: list[PlexShow] = [] + if len(movies) < limit: + shows = ( + db.execute( + select(PlexShow) + .where(PlexShow.guid.is_not(None), PlexShow.tmdb_enriched_at.is_(None)) + .order_by(PlexShow.id) + .limit(limit - len(movies)) + ) + .scalars() + .all() + ) + + processed = 0 + links: dict[str, int] = {} + for row, media_type in [(m, "movie") for m in movies] + [(s, "tv") for s in shows]: + try: + tmdb_id = guid_to_tmdb_id(row.guid, media_type, tmdb) + payload = tmdb.fetch(tmdb_id, media_type) if tmdb_id else None + except TmdbError as e: + log.warning("tmdb enrich %s failed (will retry): %s", row.rating_key, e) + continue # transient — leave tmdb_enriched_at NULL for the next run + if payload: + parsed = parse_tmdb(payload, media_type) + apply_tmdb_detail(row, parsed) + for p in parsed["people"]: + links.setdefault(p["name"], p["tmdb_id"]) + # Stamp even on a definitive no-match so an unresolvable title isn't re-fetched every run. + row.tmdb_enriched_at = datetime.now(timezone.utc) + processed += 1 + + if links: + _link_people(db, links) + db.commit() + return processed + + +def run_enrich(db: Session) -> dict: + """Scheduler entry point. No-op (never an error) when the TMDB key isn't configured.""" + if not sysconfig.get_str(db, "tmdb_api_key"): + return {"enriched": 0, "skipped": "tmdb_api_key not set"} + with TmdbClient(db) as tmdb: + n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch) + return {"enriched": n} + + +def _resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]: + """Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:`) and, if so, + its rating_key — so the client can deep-link the ones we have and merely display the rest.""" + if not recs: + return [] + guids = [f"tmdb:{r['tmdb_id']}" for r in recs if r.get("tmdb_id")] + in_lib: dict[str, str] = {} + if guids: + for guid, rk in db.execute( + select(PlexItem.guid, PlexItem.rating_key).where( + PlexItem.kind == "movie", PlexItem.guid.in_(guids) + ) + ): + in_lib[guid] = rk + for guid, rk in db.execute( + select(PlexShow.guid, PlexShow.rating_key).where(PlexShow.guid.in_(guids)) + ): + in_lib.setdefault(guid, rk) + out: list[dict] = [] + for r in recs: + rk = in_lib.get(f"tmdb:{r.get('tmdb_id')}") + out.append({**r, "in_library": rk is not None, "rating_key": rk}) + return out + + +def serve_block(db: Session, row) -> dict | None: + """The TMDB extras for a movie/show detail response, served from the row (no live TMDB call). + None until the row has been TMDB-enriched (the client then shows nothing extra).""" + if row.tmdb_enriched_at is None: + return None + return { + "tmdb_id": row.tmdb_id, + "rating": row.tmdb_rating, + "vote_count": row.tmdb_vote_count, + "budget": row.budget, + "revenue": row.revenue, + "keywords": row.keywords or [], + "crew": row.crew or [], + "studios": row.studios or [], + "production_countries": row.production_countries or [], + "recommendations": _resolve_recs(db, row.recommendations), + "backdrops": row.backdrops or [], + } diff --git a/backend/app/routes/plex/detail.py b/backend/app/routes/plex/detail.py index a65ce916..36e3c6a0 100644 --- a/backend/app/routes/plex/detail.py +++ b/backend/app/routes/plex/detail.py @@ -23,6 +23,7 @@ from app.models import ( from app.plex import detail as plex_detail from app.plex import people as plex_people from app.plex import rollup, subtitles +from app.plex import tmdb as plex_tmdb from app.plex.cards import episode_card, movie_card, show_card from app.plex.client import PlexClient, PlexError, PlexNotConfigured from app.plex.lookups import by_rating_key @@ -172,6 +173,8 @@ def show_detail( "first": show_roll["first"], "episode_count": show_roll["episode_count"], "collection_keys": sh.collection_keys or [], + # TMDB enrichment (E5 S2), served from the row. + "tmdb": plex_tmdb.serve_block(db, sh), }, "seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons], "related": related, @@ -435,4 +438,6 @@ def item_detail( "prev_id": prev_id, "next_id": next_id, "collections": _collection_strips(db, it, user.id), + # TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched). + "tmdb": plex_tmdb.serve_block(db, it) if it.kind == "movie" else None, } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 1723cfc3..7a8b81b7 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -17,6 +17,7 @@ from app.demo import reset_demo_state from app.downloads.gc import run_download_gc from app.models import SchedulerSetting, User from app.plex.sync import sync as run_plex_sync +from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync from app.notifications import create_notification from app.state import is_sync_paused @@ -68,6 +69,7 @@ JOB_INTERVALS: dict[str, int] = { "plex_sync": settings.plex_sync_interval_min, "plex_watch_sync": settings.plex_watch_sync_interval_min, "plex_watch_reconcile": settings.plex_watch_reconcile_interval_min, + "tmdb_enrich": settings.tmdb_enrich_interval_min, "audit_gc": settings.audit_gc_minutes, } @@ -351,6 +353,13 @@ def _plex_watch_reconcile_job() -> None: _job("plex_watch_reconcile", run_plex_watch_reconcile) +def _tmdb_enrich_job() -> None: + # Enrich a batch of mirrored Plex titles from TMDB (keywords/crew/budget/revenue/TMDb score/ + # recommendations/backdrops), matched by guid. External metadata (no YouTube quota); a no-op when + # the TMDB key is unset or the Plex catalog is empty. + _job("tmdb_enrich", run_tmdb_enrich) + + def _audit_gc_job() -> None: # Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep # forever). Pure DB, no quota. @@ -378,6 +387,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = { "plex_sync": _plex_sync_job, "plex_watch_sync": _plex_watch_sync_job, "plex_watch_reconcile": _plex_watch_reconcile_job, + "tmdb_enrich": _tmdb_enrich_job, "audit_gc": _audit_gc_job, } diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 13c25d2f..2d94c74c 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -102,6 +102,11 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. + # --- TMDB enrichment (E5 S2; optional) — the API key is the only secret --- + ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True), + ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"), + # (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch + # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} diff --git a/backend/tests/test_plex_tmdb.py b/backend/tests/test_plex_tmdb.py new file mode 100644 index 00000000..07dac0dd --- /dev/null +++ b/backend/tests/test_plex_tmdb.py @@ -0,0 +1,210 @@ +"""TMDB enrichment engine (E5 S2): the pure parser + persister, guid→tmdb-id resolution, and the +DB-lane batch/serve integration (enrich stamps the watermark + links people; serve_block resolves +recommendations to library rows; the whole thing no-ops without an API key). + +The DB tests use the `db` fixture (real Postgres) — skipped when none is reachable.""" +from datetime import datetime, timezone + +from app.models import PlexItem, PlexLibrary, PlexPerson, PlexShow +from app.plex import tmdb + +# A TMDB /movie/{id}?append_to_response=keywords,credits,recommendations,images,external_ids payload. +MOVIE_PAYLOAD = { + "id": 238, + "vote_average": 8.7, + "vote_count": 20000, + "budget": 6000000, + "revenue": 245066411, + "keywords": {"keywords": [{"id": 1, "name": "mafia"}, {"id": 2, "name": "crime family"}]}, + "credits": { + "cast": [ + {"id": 3084, "name": "Marlon Brando", "character": "Vito", "profile_path": "/a.jpg", "order": 0}, + {"id": 1158, "name": "Al Pacino", "character": "Michael", "profile_path": "/b.jpg", "order": 1}, + ], + "crew": [ + {"id": 1776, "name": "Francis Ford Coppola", "job": "Director", "profile_path": "/c.jpg"}, + {"id": 1776, "name": "Francis Ford Coppola", "job": "Writer", "profile_path": "/c.jpg"}, + {"id": 9, "name": "Some Gaffer", "job": "Gaffer"}, # not a key job → dropped + ], + }, + "production_companies": [{"name": "Paramount"}, {"name": "Alfran"}], + "production_countries": [{"name": "United States of America"}], + "recommendations": { + "results": [ + {"id": 240, "title": "The Godfather Part II", "poster_path": "/p.jpg", "media_type": "movie"}, + {"id": 999, "title": "Not In Library", "poster_path": "/q.jpg", "media_type": "movie"}, + ] + }, + "images": {"backdrops": [{"file_path": "/bd1.jpg"}, {"file_path": "/bd2.jpg"}]}, + "external_ids": {"imdb_id": "tt0068646"}, +} + +# A TMDB /tv payload: keywords under `results`, aggregate_credits with per-person `jobs`, no budget. +TV_PAYLOAD = { + "id": 1396, + "vote_average": 9.0, + "vote_count": 12000, + "keywords": {"results": [{"name": "drugs"}, {"name": "new mexico"}]}, + "aggregate_credits": { + "cast": [{"id": 17419, "name": "Bryan Cranston"}], + "crew": [ + {"id": 66633, "name": "Vince Gilligan", "jobs": [{"job": "Executive Producer"}]}, + {"id": 5, "name": "Someone", "jobs": [{"job": "Sound"}]}, # dropped + ], + }, + "production_companies": [{"name": "Sony Pictures Television"}], + "production_countries": [{"name": "United States of America"}], + "recommendations": {"results": []}, + "images": {"backdrops": []}, +} + + +class FakeTmdb: + """Stand-in for TmdbClient in the parse/batch tests — no network.""" + + def __init__(self, payloads=None, find_map=None): + self.payloads = payloads or {} # (media_type, tmdb_id) -> payload + self.find_map = find_map or {} # external_id -> /find result + + def fetch(self, tmdb_id, media_type): + return self.payloads.get((media_type, tmdb_id)) + + def find(self, external_id, source): + return self.find_map.get(external_id) + + +# ---- pure parse/persist -------------------------------------------------------------------------- + +def test_parse_movie_pulls_all_fields(): + d = tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie") + assert d["tmdb_id"] == 238 + assert d["tmdb_rating"] == 8.7 + assert d["tmdb_vote_count"] == 20000 + assert d["budget"] == 6000000 + assert d["revenue"] == 245066411 + assert d["keywords"] == ["mafia", "crime family"] + assert d["crew"] == [ + {"name": "Francis Ford Coppola", "job": "Director", "tmdb_id": 1776, "profile_path": "/c.jpg"}, + {"name": "Francis Ford Coppola", "job": "Writer", "tmdb_id": 1776, "profile_path": "/c.jpg"}, + ] # the Gaffer is filtered; the two Coppola jobs survive (dedup is by name+job) + assert d["studios"] == ["Paramount", "Alfran"] + assert d["production_countries"] == ["United States of America"] + assert [r["tmdb_id"] for r in d["recommendations"]] == [240, 999] + assert d["backdrops"] == ["/bd1.jpg", "/bd2.jpg"] + # cast + crew names → tmdb person ids for the plex_people foundation + links = {(p["name"], p["tmdb_id"]) for p in d["people"]} + assert ("Marlon Brando", 3084) in links + assert ("Al Pacino", 1158) in links + assert ("Francis Ford Coppola", 1776) in links + + +def test_parse_tv_uses_aggregate_credits_and_no_money(): + d = tmdb.parse_tmdb(TV_PAYLOAD, "tv") + assert d["budget"] is None and d["revenue"] is None # TV has neither + assert d["keywords"] == ["drugs", "new mexico"] # from keywords.results + assert d["crew"] == [ + {"name": "Vince Gilligan", "job": "Executive Producer", "tmdb_id": 66633, "profile_path": None} + ] # the Sound person is filtered; job read from jobs[0] + assert d["tmdb_rating"] == 9.0 + assert ("Bryan Cranston", 17419) in {(p["name"], p["tmdb_id"]) for p in d["people"]} + + +def test_parse_zero_votes_and_money_are_unknown(): + d = tmdb.parse_tmdb({"id": 1, "vote_average": 0.0, "vote_count": 0, "budget": 0, "revenue": 0}, "movie") + assert d["tmdb_rating"] is None # 0.0 with no votes is meaningless + assert d["tmdb_vote_count"] is None + assert d["budget"] is None and d["revenue"] is None + + +def test_apply_writes_row_fields_without_touching_plex_owned(): + it = PlexItem(rating_key="m1", kind="movie", title="The Godfather", imdb_rating=8.7, full_cast=[{"x": 1}]) + tmdb.apply_tmdb_detail(it, tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie")) + assert it.tmdb_id == 238 + assert it.budget == 6000000 + assert it.keywords == ["mafia", "crime family"] + assert it.imdb_rating == 8.7 and it.full_cast == [{"x": 1}] # Plex-owned, untouched + + +def test_guid_to_tmdb_id_direct_find_and_bad(): + fake = FakeTmdb(find_map={"tt0068646": {"movie_results": [{"id": 238}], "tv_results": []}}) + assert tmdb.guid_to_tmdb_id("tmdb:238", "movie", fake) == 238 + assert tmdb.guid_to_tmdb_id("imdb:tt0068646", "movie", fake) == 238 # resolved via /find + assert tmdb.guid_to_tmdb_id("imdb:tt0000000", "movie", fake) is None # not in find_map + assert tmdb.guid_to_tmdb_id("tmdb:notanumber", "movie", fake) is None + assert tmdb.guid_to_tmdb_id(None, "movie", fake) is None + assert tmdb.guid_to_tmdb_id("local://abc", "movie", fake) is None + + +# ---- DB-lane: batch enrich + serve -------------------------------------------------------------- + +def _lib(db, key="1", kind="movie"): + lib = PlexLibrary(plex_key=key, title="L", kind=kind, enabled=True) + db.add(lib) + db.flush() + return lib + + +def test_enrich_batch_persists_stamps_and_links_people(db): + lib = _lib(db) + mv = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather", guid="tmdb:238") + unmatched = PlexItem(rating_key="m2", library_id=lib.id, kind="movie", title="Nope", guid="tmdb:777") + db.add_all([mv, unmatched]) + db.commit() + + fake = FakeTmdb(payloads={("movie", 238): MOVIE_PAYLOAD}) # 777 has no payload → definitive no-match + n = tmdb.enrich_batch(db, fake, 50) + assert n == 2 # both processed (one matched, one definitively unmatched) + + db.refresh(mv) + assert mv.tmdb_id == 238 + assert mv.keywords == ["mafia", "crime family"] + assert mv.revenue == 245066411 + assert mv.tmdb_enriched_at is not None + + db.refresh(unmatched) + assert unmatched.tmdb_enriched_at is not None # stamped so it isn't re-fetched every run + assert unmatched.tmdb_id is None # but nothing stored + + brando = db.execute( + PlexPerson.__table__.select().where(PlexPerson.name == "Marlon Brando") + ).mappings().first() + assert brando is not None and brando["tmdb_person_id"] == 3084 + + +def test_serve_block_none_until_enriched_and_resolves_recs(db): + lib = _lib(db) + # A recommendation target that IS in our library (guid tmdb:240) → deep-linkable. + sequel = PlexItem(rating_key="m240", library_id=lib.id, kind="movie", title="Part II", guid="tmdb:240") + fresh = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather", guid="tmdb:238") + db.add_all([sequel, fresh]) + db.commit() + + assert tmdb.serve_block(db, fresh) is None # not enriched yet + + tmdb.apply_tmdb_detail(fresh, tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie")) + fresh.tmdb_enriched_at = datetime.now(timezone.utc) + db.commit() + + block = tmdb.serve_block(db, fresh) + assert block is not None + assert block["budget"] == 6000000 + assert block["keywords"] == ["mafia", "crime family"] + recs = {r["tmdb_id"]: r for r in block["recommendations"]} + assert recs[240]["in_library"] is True and recs[240]["rating_key"] == "m240" + assert recs[999]["in_library"] is False and recs[999]["rating_key"] is None + + +def test_run_enrich_noops_without_key(db): + # No tmdb_api_key configured (default "") → clean no-op, never an error. + out = tmdb.run_enrich(db) + assert out["enriched"] == 0 and "skipped" in out + + +def test_enrich_batch_skips_shows_without_guid(db): + lib = _lib(db, key="2", kind="show") + sh = PlexShow(rating_key="s1", library_id=lib.id, title="No Guid Show") # guid None → not selected + db.add(sh) + db.commit() + assert tmdb.enrich_batch(db, FakeTmdb(), 50) == 0 + db.refresh(sh) + assert sh.tmdb_enriched_at is None