E5 S2: TMDB enrichment engine (Plex catalog)
Matches each mirrored Plex movie/show to TMDB by its stored canonical guid, fetches once (append_to_response), and PERSISTS the enrichment onto the row — served from the DB, never a live TMDB call at serve time. Additive: the Plex-sourced imdb_rating/rating are untouched; tmdb_rating is the distinct TMDb community score (ratings scope = TMDb + Plex's IMDb/audience, no OMDb/RT). - config.py: tmdb_api_key + api_base/language/interval/batch defaults. - sysconfig SPECS: tmdb_api_key (secret, Fernet) + tmdb_language, auto-rendered in the admin Config page; read like the Plex token. - migration 0071 (ADD-only nullable, reversible): plex_items + plex_shows gain tmdb_id / tmdb_enriched_at (watermark) / tmdb_rating / tmdb_vote_count / budget / revenue / keywords / crew / studios / production_countries / recommendations / backdrops; plex_people gains tmdb_person_id (the S2 foundation for S6 person bios). No watermark reset needed. - app/plex/tmdb.py: TmdbClient (mirrors PlexClient; throttled; 404->None, transient->TmdbError), parse_tmdb / apply_tmdb_detail (one-parser rule), guid_to_tmdb_id (tmdb: direct, imdb:/tvdb: via /find), enrich_batch (movies then shows, transient error leaves the watermark NULL to retry, definitive no-match still stamps so it isn't re-fetched), _link_people (upserts tmdb_person_id by name), serve_block (recommendations resolved to library rows by guid), run_enrich (no-op without a key). - scheduler: tmdb_enrich nightly job (default 120 min, no YouTube quota), runtime-tunable + "run now" from the Scheduler dashboard. - detail endpoints emit `tmdb` (movies + shows) so it's curl-verifiable now; the FE render lands in S5. Note: _link_people inserts plex_people rows for TMDB cast/crew; unreferenced crew (composers etc., not in cast_names/directors) get pruned by the orphan sweep — acceptable, the person page is cast-driven and cast rows survive. Gate: ruff + py_compile green; backend test lane FULL suite 387 passed (incl. migration drift downgrade->re-upgrade + test_plex_tmdb 9 tests). Frontend unchanged since S1. Not merged (epic branch, stacks on S1).
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
"""plex TMDB enrichment columns (E5 S2)
|
||||
|
||||
Additive TMDB-sourced fields on plex_items + plex_shows — keywords, full crew, budget/revenue, TMDb
|
||||
community score, production companies/countries, recommendations, and a backdrop-path pool — plus a
|
||||
`tmdb_id` cache and a `tmdb_enriched_at` watermark. Filled by the new TMDB enrichment job (matched by
|
||||
the stored canonical `guid`), SERVED FROM THE ROW (never a live TMDB call). Distinct from the existing
|
||||
Plex-sourced `imdb_rating`/`rating`, which are left untouched. Plus `plex_people.tmdb_person_id` (the S2
|
||||
foundation for S6's person bios). ADD-only, all nullable → no table rewrite, no ACCESS EXCLUSIVE lock.
|
||||
No watermark reset needed: `tmdb_enriched_at` starts NULL, so every eligible row is picked up once.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0071_plex_tmdb_fields"
|
||||
down_revision: Union[str, None] = "0070_fold_maintenance_batch"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_TABLES = ("plex_items", "plex_shows")
|
||||
|
||||
# (name, type) — same set on both movie/leaf rows and show rows. budget/revenue stay NULL for shows.
|
||||
_COLS: tuple[tuple[str, sa.types.TypeEngine], ...] = (
|
||||
("tmdb_id", sa.Integer()),
|
||||
("tmdb_enriched_at", sa.DateTime(timezone=True)),
|
||||
("tmdb_rating", sa.Float()),
|
||||
("tmdb_vote_count", sa.Integer()),
|
||||
("budget", sa.BigInteger()),
|
||||
("revenue", sa.BigInteger()),
|
||||
("keywords", postgresql.JSONB()),
|
||||
("crew", postgresql.JSONB()),
|
||||
("studios", postgresql.JSONB()),
|
||||
("production_countries", postgresql.JSONB()),
|
||||
("recommendations", postgresql.JSONB()),
|
||||
("backdrops", postgresql.JSONB()),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for t in _TABLES:
|
||||
for name, type_ in _COLS:
|
||||
op.add_column(t, sa.Column(name, type_, nullable=True))
|
||||
op.add_column("plex_people", sa.Column("tmdb_person_id", sa.Integer(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_people", "tmdb_person_id")
|
||||
for t in _TABLES:
|
||||
for name, _ in reversed(_COLS):
|
||||
op.drop_column(t, name)
|
||||
@@ -245,6 +245,20 @@ class Settings(BaseSettings):
|
||||
# local disk, so the default is fine there.
|
||||
plex_hls_dir: str = ""
|
||||
|
||||
# --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) ---
|
||||
# TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the
|
||||
# admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget /
|
||||
# revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the
|
||||
# stored canonical guid, and SERVED FROM THE DB — never a live TMDB call at serve time.
|
||||
tmdb_api_key: str = ""
|
||||
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
tmdb_enrich_batch: int = 100
|
||||
|
||||
@property
|
||||
def allowed_email_set(self) -> set[str]:
|
||||
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
||||
|
||||
@@ -1067,6 +1067,20 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||
tagline: Mapped[str | None] = mapped_column(Text)
|
||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive;
|
||||
# served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) ---
|
||||
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10)
|
||||
tmdb_vote_count: Mapped[int | None] = mapped_column(Integer)
|
||||
budget: Mapped[int | None] = mapped_column(BigInteger)
|
||||
revenue: Mapped[int | None] = mapped_column(BigInteger)
|
||||
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}]
|
||||
studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str]
|
||||
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
@@ -1184,6 +1198,24 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||
tagline: Mapped[str | None] = mapped_column(Text)
|
||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||
# backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating
|
||||
# above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched;
|
||||
# `tmdb_enriched_at` is the "tried" watermark (stamped on a definitive no-match too, so an
|
||||
# unresolvable title isn't re-fetched every run; left NULL only on a transient fetch error). ---
|
||||
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10)
|
||||
tmdb_vote_count: Mapped[int | None] = mapped_column(Integer)
|
||||
budget: Mapped[int | None] = mapped_column(BigInteger) # USD (movies; TV has none)
|
||||
revenue: Mapped[int | None] = mapped_column(BigInteger)
|
||||
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}]
|
||||
studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str]
|
||||
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
# [{tmdb_id, title, poster, media_type}] — resolved to library rows (by guid) at serve time.
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB backdrop file_paths [str] (crossfade pool, S8)
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
@@ -1224,6 +1256,9 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None
|
||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class PlexCollection(Base, TimestampMixin, UpdatedAtMixin):
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""TMDB enrichment for the mirrored Plex catalog (E5 S2).
|
||||
|
||||
A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items /
|
||||
plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find).
|
||||
Server-side ONLY (the api_key from sysconfig). Everything fetched is PERSISTED and served from the DB
|
||||
row — there is never a live TMDB call at serve time (the user's explicit rule). Additive: the existing
|
||||
Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the distinct TMDb score.
|
||||
|
||||
One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
|
||||
log = logging.getLogger("siftlode.tmdb")
|
||||
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
_MIN_INTERVAL = 0.03
|
||||
|
||||
# The crew jobs worth surfacing on an info page (TMDB lists dozens per title). Order-preserving pick.
|
||||
_CREW_JOBS = (
|
||||
"Director",
|
||||
"Writer",
|
||||
"Screenplay",
|
||||
"Story",
|
||||
"Producer",
|
||||
"Executive Producer",
|
||||
"Editor",
|
||||
"Director of Photography",
|
||||
"Original Music Composer",
|
||||
)
|
||||
|
||||
# append_to_response bundles the sub-resources into ONE request per title (keywords + credits +
|
||||
# recommendations + images + external ids). TV uses aggregate_credits (per-person aggregated jobs).
|
||||
_MOVIE_APPEND = "keywords,credits,recommendations,images,external_ids"
|
||||
_TV_APPEND = "keywords,aggregate_credits,recommendations,images,external_ids"
|
||||
|
||||
|
||||
class TmdbError(Exception):
|
||||
"""Any transient failure talking to TMDB (network, non-404 HTTP, bad payload). The caller leaves
|
||||
the row's `tmdb_enriched_at` NULL so the next run retries."""
|
||||
|
||||
|
||||
class TmdbNotConfigured(TmdbError):
|
||||
"""No TMDB API key is set — the module can't do anything."""
|
||||
|
||||
|
||||
class TmdbClient:
|
||||
"""Bound to the configured api_key + language. Use as a context manager so the httpx client closes."""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.key = sysconfig.get_str(db, "tmdb_api_key")
|
||||
if not self.key:
|
||||
raise TmdbNotConfigured("TMDB API key is not configured")
|
||||
self.base = settings.tmdb_api_base.rstrip("/")
|
||||
self.language = sysconfig.get_str(db, "tmdb_language") or settings.tmdb_language
|
||||
self._http = httpx.Client(timeout=30.0)
|
||||
self._last = 0.0
|
||||
|
||||
def __enter__(self) -> "TmdbClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self._http.close()
|
||||
|
||||
def close(self) -> None:
|
||||
self._http.close()
|
||||
|
||||
def _throttle(self) -> None:
|
||||
wait = self._last + _MIN_INTERVAL - time.monotonic()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self._last = time.monotonic()
|
||||
|
||||
def _get(self, path: str, params: dict | None = None) -> dict | None:
|
||||
"""GET a TMDB path. Returns the JSON dict, None on a 404 (a definitive "no such id"), and raises
|
||||
TmdbError on any transient failure (network / other HTTP status / non-JSON)."""
|
||||
self._throttle()
|
||||
p = {"api_key": self.key, "language": self.language, **(params or {})}
|
||||
try:
|
||||
r = self._http.get(f"{self.base}{path}", params=p)
|
||||
except httpx.HTTPError as e:
|
||||
raise TmdbError(str(e)) from e
|
||||
if r.status_code == 404:
|
||||
return None
|
||||
try:
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise TmdbError(str(e)) from e
|
||||
try:
|
||||
return r.json()
|
||||
except ValueError as e:
|
||||
raise TmdbError("TMDB returned a non-JSON response") from e
|
||||
|
||||
def fetch(self, tmdb_id: int, media_type: str) -> dict | None:
|
||||
"""Full enrichment payload for a movie or show (one request via append_to_response)."""
|
||||
if media_type == "tv":
|
||||
return self._get(f"/tv/{tmdb_id}", {"append_to_response": _TV_APPEND})
|
||||
return self._get(f"/movie/{tmdb_id}", {"append_to_response": _MOVIE_APPEND})
|
||||
|
||||
def find(self, external_id: str, source: str) -> dict | None:
|
||||
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
||||
return self._get(f"/find/{external_id}", {"external_source": source})
|
||||
|
||||
|
||||
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
||||
"""Map a stored canonical guid to a TMDB id. `tmdb:` is direct; `imdb:`/`tvdb:` go through /find
|
||||
(picking the movie_results or tv_results list per `media_type`). Returns None for an unmatchable
|
||||
guid — the caller then stamps the watermark so it isn't retried every run."""
|
||||
if not guid or ":" not in guid:
|
||||
return None
|
||||
provider, _, ext = guid.partition(":")
|
||||
if provider == "tmdb":
|
||||
try:
|
||||
return int(ext)
|
||||
except ValueError:
|
||||
return None
|
||||
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
||||
if not source:
|
||||
return None
|
||||
res = tmdb.find(ext, source)
|
||||
if not res:
|
||||
return None
|
||||
items = res.get("tv_results" if media_type == "tv" else "movie_results") or []
|
||||
return items[0].get("id") if items else None
|
||||
|
||||
|
||||
def _dedup_crew(crew: list[dict]) -> list[dict]:
|
||||
seen: set[tuple] = set()
|
||||
out: list[dict] = []
|
||||
for c in crew:
|
||||
key = (c["name"], c["job"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(c)
|
||||
return out
|
||||
|
||||
|
||||
def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
"""Pure parser: pull the enrichment fields out of a TMDB movie/tv payload (append_to_response form).
|
||||
Best-effort — any missing sub-resource is just omitted. Returns the row fields plus a `people` list
|
||||
([{name, tmdb_id}] over cast+crew) for the plex_people id-linking. TMDB image fields are stored as
|
||||
bare paths ("/abc.jpg"); the client builds the image.tmdb.org URL."""
|
||||
is_tv = media_type == "tv"
|
||||
votes = payload.get("vote_count") or 0
|
||||
|
||||
kw_container = payload.get("keywords") or {}
|
||||
kw_list = kw_container.get("results") if is_tv else kw_container.get("keywords")
|
||||
keywords = [k.get("name") for k in (kw_list or []) if k.get("name")][:20]
|
||||
|
||||
credits = payload.get("aggregate_credits" if is_tv else "credits") or {}
|
||||
crew: list[dict] = []
|
||||
for c in credits.get("crew") or []:
|
||||
# Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person.
|
||||
job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job")
|
||||
if job in _CREW_JOBS and c.get("name"):
|
||||
crew.append(
|
||||
{"name": c["name"], "job": job, "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")}
|
||||
)
|
||||
crew = _dedup_crew(crew)[:12]
|
||||
|
||||
# name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids).
|
||||
people: list[dict] = []
|
||||
for c in (credits.get("cast") or [])[:30]:
|
||||
if c.get("name") and c.get("id"):
|
||||
people.append({"name": c["name"], "tmdb_id": c["id"]})
|
||||
for c in crew:
|
||||
if c.get("tmdb_id"):
|
||||
people.append({"name": c["name"], "tmdb_id": c["tmdb_id"]})
|
||||
|
||||
recs: list[dict] = []
|
||||
for r in ((payload.get("recommendations") or {}).get("results") or [])[:20]:
|
||||
recs.append(
|
||||
{
|
||||
"tmdb_id": r.get("id"),
|
||||
"title": r.get("title") or r.get("name"),
|
||||
"poster_path": r.get("poster_path"),
|
||||
"media_type": r.get("media_type") or media_type,
|
||||
}
|
||||
)
|
||||
|
||||
backdrops = [
|
||||
b.get("file_path")
|
||||
for b in ((payload.get("images") or {}).get("backdrops") or [])
|
||||
if b.get("file_path")
|
||||
][:10]
|
||||
|
||||
return {
|
||||
"tmdb_id": payload.get("id"),
|
||||
# vote_average is meaningless with no votes (TMDB returns 0.0) → treat as unknown.
|
||||
"tmdb_rating": payload.get("vote_average") if votes else None,
|
||||
"tmdb_vote_count": votes or None,
|
||||
# 0 is TMDB's "unknown" for money fields; TV has neither.
|
||||
"budget": (payload.get("budget") or None) if not is_tv else None,
|
||||
"revenue": (payload.get("revenue") or None) if not is_tv else None,
|
||||
"keywords": keywords or None,
|
||||
"crew": crew or None,
|
||||
"studios": [c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")][:8]
|
||||
or None,
|
||||
"production_countries": [
|
||||
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
|
||||
]
|
||||
or None,
|
||||
"recommendations": recs or None,
|
||||
"backdrops": backdrops or None,
|
||||
"people": people,
|
||||
}
|
||||
|
||||
|
||||
def apply_tmdb_detail(row, parsed: dict) -> None:
|
||||
"""Persist the parsed TMDB fields onto a movie/show row. Does NOT touch full_cast / imdb_rating /
|
||||
rating (Plex-owned). The caller stamps `tmdb_enriched_at` and links people separately."""
|
||||
row.tmdb_id = parsed["tmdb_id"]
|
||||
row.tmdb_rating = parsed["tmdb_rating"]
|
||||
row.tmdb_vote_count = parsed["tmdb_vote_count"]
|
||||
row.budget = parsed["budget"]
|
||||
row.revenue = parsed["revenue"]
|
||||
row.keywords = parsed["keywords"]
|
||||
row.crew = parsed["crew"]
|
||||
row.studios = parsed["studios"]
|
||||
row.production_countries = parsed["production_countries"]
|
||||
row.recommendations = parsed["recommendations"]
|
||||
row.backdrops = parsed["backdrops"]
|
||||
|
||||
|
||||
def _link_people(db: Session, links: dict[str, int]) -> None:
|
||||
"""Upsert plex_people by name, stamping `tmdb_person_id` — keeping any id already set (first wins),
|
||||
filling it where NULL, inserting a bare row where the person isn't cached yet. Deduped by name so a
|
||||
single statement can't hit the conflict target twice (the same rule as people.enrich_row_people)."""
|
||||
values = [{"name": nm, "tmdb_person_id": pid} for nm, pid in links.items() if nm and pid]
|
||||
if not values:
|
||||
return
|
||||
stmt = pg_insert(PlexPerson).values(values)
|
||||
db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=[PlexPerson.name],
|
||||
set_={"tmdb_person_id": func.coalesce(PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id)},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||
movies = (
|
||||
db.execute(
|
||||
select(PlexItem)
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
PlexItem.tmdb_enriched_at.is_(None),
|
||||
)
|
||||
.order_by(PlexItem.id)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
shows: list[PlexShow] = []
|
||||
if len(movies) < limit:
|
||||
shows = (
|
||||
db.execute(
|
||||
select(PlexShow)
|
||||
.where(PlexShow.guid.is_not(None), PlexShow.tmdb_enriched_at.is_(None))
|
||||
.order_by(PlexShow.id)
|
||||
.limit(limit - len(movies))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
processed = 0
|
||||
links: dict[str, int] = {}
|
||||
for row, media_type in [(m, "movie") for m in movies] + [(s, "tv") for s in shows]:
|
||||
try:
|
||||
tmdb_id = guid_to_tmdb_id(row.guid, media_type, tmdb)
|
||||
payload = tmdb.fetch(tmdb_id, media_type) if tmdb_id else None
|
||||
except TmdbError as e:
|
||||
log.warning("tmdb enrich %s failed (will retry): %s", row.rating_key, e)
|
||||
continue # transient — leave tmdb_enriched_at NULL for the next run
|
||||
if payload:
|
||||
parsed = parse_tmdb(payload, media_type)
|
||||
apply_tmdb_detail(row, parsed)
|
||||
for p in parsed["people"]:
|
||||
links.setdefault(p["name"], p["tmdb_id"])
|
||||
# Stamp even on a definitive no-match so an unresolvable title isn't re-fetched every run.
|
||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
processed += 1
|
||||
|
||||
if links:
|
||||
_link_people(db, links)
|
||||
db.commit()
|
||||
return processed
|
||||
|
||||
|
||||
def run_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
||||
if not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
||||
with TmdbClient(db) as tmdb:
|
||||
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
||||
return {"enriched": n}
|
||||
|
||||
|
||||
def _resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`) and, if so,
|
||||
its rating_key — so the client can deep-link the ones we have and merely display the rest."""
|
||||
if not recs:
|
||||
return []
|
||||
guids = [f"tmdb:{r['tmdb_id']}" for r in recs if r.get("tmdb_id")]
|
||||
in_lib: dict[str, str] = {}
|
||||
if guids:
|
||||
for guid, rk in db.execute(
|
||||
select(PlexItem.guid, PlexItem.rating_key).where(
|
||||
PlexItem.kind == "movie", PlexItem.guid.in_(guids)
|
||||
)
|
||||
):
|
||||
in_lib[guid] = rk
|
||||
for guid, rk in db.execute(
|
||||
select(PlexShow.guid, PlexShow.rating_key).where(PlexShow.guid.in_(guids))
|
||||
):
|
||||
in_lib.setdefault(guid, rk)
|
||||
out: list[dict] = []
|
||||
for r in recs:
|
||||
rk = in_lib.get(f"tmdb:{r.get('tmdb_id')}")
|
||||
out.append({**r, "in_library": rk is not None, "rating_key": rk})
|
||||
return out
|
||||
|
||||
|
||||
def serve_block(db: Session, row) -> dict | None:
|
||||
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
||||
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
||||
if row.tmdb_enriched_at is None:
|
||||
return None
|
||||
return {
|
||||
"tmdb_id": row.tmdb_id,
|
||||
"rating": row.tmdb_rating,
|
||||
"vote_count": row.tmdb_vote_count,
|
||||
"budget": row.budget,
|
||||
"revenue": row.revenue,
|
||||
"keywords": row.keywords or [],
|
||||
"crew": row.crew or [],
|
||||
"studios": row.studios or [],
|
||||
"production_countries": row.production_countries or [],
|
||||
"recommendations": _resolve_recs(db, row.recommendations),
|
||||
"backdrops": row.backdrops or [],
|
||||
}
|
||||
@@ -23,6 +23,7 @@ from app.models import (
|
||||
from app.plex import detail as plex_detail
|
||||
from app.plex import people as plex_people
|
||||
from app.plex import rollup, subtitles
|
||||
from app.plex import tmdb as plex_tmdb
|
||||
from app.plex.cards import episode_card, movie_card, show_card
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
@@ -172,6 +173,8 @@ def show_detail(
|
||||
"first": show_roll["first"],
|
||||
"episode_count": show_roll["episode_count"],
|
||||
"collection_keys": sh.collection_keys or [],
|
||||
# TMDB enrichment (E5 S2), served from the row.
|
||||
"tmdb": plex_tmdb.serve_block(db, sh),
|
||||
},
|
||||
"seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons],
|
||||
"related": related,
|
||||
@@ -435,4 +438,6 @@ def item_detail(
|
||||
"prev_id": prev_id,
|
||||
"next_id": next_id,
|
||||
"collections": _collection_strips(db, it, user.id),
|
||||
# TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched).
|
||||
"tmdb": plex_tmdb.serve_block(db, it) if it.kind == "movie" else None,
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.demo import reset_demo_state
|
||||
from app.downloads.gc import run_download_gc
|
||||
from app.models import SchedulerSetting, User
|
||||
from app.plex.sync import sync as run_plex_sync
|
||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||
from app.notifications import create_notification
|
||||
from app.state import is_sync_paused
|
||||
@@ -68,6 +69,7 @@ JOB_INTERVALS: dict[str, int] = {
|
||||
"plex_sync": settings.plex_sync_interval_min,
|
||||
"plex_watch_sync": settings.plex_watch_sync_interval_min,
|
||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||
"audit_gc": settings.audit_gc_minutes,
|
||||
}
|
||||
|
||||
@@ -351,6 +353,13 @@ def _plex_watch_reconcile_job() -> None:
|
||||
_job("plex_watch_reconcile", run_plex_watch_reconcile)
|
||||
|
||||
|
||||
def _tmdb_enrich_job() -> None:
|
||||
# Enrich a batch of mirrored Plex titles from TMDB (keywords/crew/budget/revenue/TMDb score/
|
||||
# recommendations/backdrops), matched by guid. External metadata (no YouTube quota); a no-op when
|
||||
# the TMDB key is unset or the Plex catalog is empty.
|
||||
_job("tmdb_enrich", run_tmdb_enrich)
|
||||
|
||||
|
||||
def _audit_gc_job() -> None:
|
||||
# Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep
|
||||
# forever). Pure DB, no quota.
|
||||
@@ -378,6 +387,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
"plex_sync": _plex_sync_job,
|
||||
"plex_watch_sync": _plex_watch_sync_job,
|
||||
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
||||
"tmdb_enrich": _tmdb_enrich_job,
|
||||
"audit_gc": _audit_gc_job,
|
||||
}
|
||||
|
||||
|
||||
@@ -102,6 +102,11 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||
# NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job
|
||||
# (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS.
|
||||
# --- TMDB enrichment (E5 S2; optional) — the API key is the only secret ---
|
||||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True),
|
||||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"),
|
||||
# (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch
|
||||
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
||||
)
|
||||
|
||||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""TMDB enrichment engine (E5 S2): the pure parser + persister, guid→tmdb-id resolution, and the
|
||||
DB-lane batch/serve integration (enrich stamps the watermark + links people; serve_block resolves
|
||||
recommendations to library rows; the whole thing no-ops without an API key).
|
||||
|
||||
The DB tests use the `db` fixture (real Postgres) — skipped when none is reachable."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.models import PlexItem, PlexLibrary, PlexPerson, PlexShow
|
||||
from app.plex import tmdb
|
||||
|
||||
# A TMDB /movie/{id}?append_to_response=keywords,credits,recommendations,images,external_ids payload.
|
||||
MOVIE_PAYLOAD = {
|
||||
"id": 238,
|
||||
"vote_average": 8.7,
|
||||
"vote_count": 20000,
|
||||
"budget": 6000000,
|
||||
"revenue": 245066411,
|
||||
"keywords": {"keywords": [{"id": 1, "name": "mafia"}, {"id": 2, "name": "crime family"}]},
|
||||
"credits": {
|
||||
"cast": [
|
||||
{"id": 3084, "name": "Marlon Brando", "character": "Vito", "profile_path": "/a.jpg", "order": 0},
|
||||
{"id": 1158, "name": "Al Pacino", "character": "Michael", "profile_path": "/b.jpg", "order": 1},
|
||||
],
|
||||
"crew": [
|
||||
{"id": 1776, "name": "Francis Ford Coppola", "job": "Director", "profile_path": "/c.jpg"},
|
||||
{"id": 1776, "name": "Francis Ford Coppola", "job": "Writer", "profile_path": "/c.jpg"},
|
||||
{"id": 9, "name": "Some Gaffer", "job": "Gaffer"}, # not a key job → dropped
|
||||
],
|
||||
},
|
||||
"production_companies": [{"name": "Paramount"}, {"name": "Alfran"}],
|
||||
"production_countries": [{"name": "United States of America"}],
|
||||
"recommendations": {
|
||||
"results": [
|
||||
{"id": 240, "title": "The Godfather Part II", "poster_path": "/p.jpg", "media_type": "movie"},
|
||||
{"id": 999, "title": "Not In Library", "poster_path": "/q.jpg", "media_type": "movie"},
|
||||
]
|
||||
},
|
||||
"images": {"backdrops": [{"file_path": "/bd1.jpg"}, {"file_path": "/bd2.jpg"}]},
|
||||
"external_ids": {"imdb_id": "tt0068646"},
|
||||
}
|
||||
|
||||
# A TMDB /tv payload: keywords under `results`, aggregate_credits with per-person `jobs`, no budget.
|
||||
TV_PAYLOAD = {
|
||||
"id": 1396,
|
||||
"vote_average": 9.0,
|
||||
"vote_count": 12000,
|
||||
"keywords": {"results": [{"name": "drugs"}, {"name": "new mexico"}]},
|
||||
"aggregate_credits": {
|
||||
"cast": [{"id": 17419, "name": "Bryan Cranston"}],
|
||||
"crew": [
|
||||
{"id": 66633, "name": "Vince Gilligan", "jobs": [{"job": "Executive Producer"}]},
|
||||
{"id": 5, "name": "Someone", "jobs": [{"job": "Sound"}]}, # dropped
|
||||
],
|
||||
},
|
||||
"production_companies": [{"name": "Sony Pictures Television"}],
|
||||
"production_countries": [{"name": "United States of America"}],
|
||||
"recommendations": {"results": []},
|
||||
"images": {"backdrops": []},
|
||||
}
|
||||
|
||||
|
||||
class FakeTmdb:
|
||||
"""Stand-in for TmdbClient in the parse/batch tests — no network."""
|
||||
|
||||
def __init__(self, payloads=None, find_map=None):
|
||||
self.payloads = payloads or {} # (media_type, tmdb_id) -> payload
|
||||
self.find_map = find_map or {} # external_id -> /find result
|
||||
|
||||
def fetch(self, tmdb_id, media_type):
|
||||
return self.payloads.get((media_type, tmdb_id))
|
||||
|
||||
def find(self, external_id, source):
|
||||
return self.find_map.get(external_id)
|
||||
|
||||
|
||||
# ---- pure parse/persist --------------------------------------------------------------------------
|
||||
|
||||
def test_parse_movie_pulls_all_fields():
|
||||
d = tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie")
|
||||
assert d["tmdb_id"] == 238
|
||||
assert d["tmdb_rating"] == 8.7
|
||||
assert d["tmdb_vote_count"] == 20000
|
||||
assert d["budget"] == 6000000
|
||||
assert d["revenue"] == 245066411
|
||||
assert d["keywords"] == ["mafia", "crime family"]
|
||||
assert d["crew"] == [
|
||||
{"name": "Francis Ford Coppola", "job": "Director", "tmdb_id": 1776, "profile_path": "/c.jpg"},
|
||||
{"name": "Francis Ford Coppola", "job": "Writer", "tmdb_id": 1776, "profile_path": "/c.jpg"},
|
||||
] # the Gaffer is filtered; the two Coppola jobs survive (dedup is by name+job)
|
||||
assert d["studios"] == ["Paramount", "Alfran"]
|
||||
assert d["production_countries"] == ["United States of America"]
|
||||
assert [r["tmdb_id"] for r in d["recommendations"]] == [240, 999]
|
||||
assert d["backdrops"] == ["/bd1.jpg", "/bd2.jpg"]
|
||||
# cast + crew names → tmdb person ids for the plex_people foundation
|
||||
links = {(p["name"], p["tmdb_id"]) for p in d["people"]}
|
||||
assert ("Marlon Brando", 3084) in links
|
||||
assert ("Al Pacino", 1158) in links
|
||||
assert ("Francis Ford Coppola", 1776) in links
|
||||
|
||||
|
||||
def test_parse_tv_uses_aggregate_credits_and_no_money():
|
||||
d = tmdb.parse_tmdb(TV_PAYLOAD, "tv")
|
||||
assert d["budget"] is None and d["revenue"] is None # TV has neither
|
||||
assert d["keywords"] == ["drugs", "new mexico"] # from keywords.results
|
||||
assert d["crew"] == [
|
||||
{"name": "Vince Gilligan", "job": "Executive Producer", "tmdb_id": 66633, "profile_path": None}
|
||||
] # the Sound person is filtered; job read from jobs[0]
|
||||
assert d["tmdb_rating"] == 9.0
|
||||
assert ("Bryan Cranston", 17419) in {(p["name"], p["tmdb_id"]) for p in d["people"]}
|
||||
|
||||
|
||||
def test_parse_zero_votes_and_money_are_unknown():
|
||||
d = tmdb.parse_tmdb({"id": 1, "vote_average": 0.0, "vote_count": 0, "budget": 0, "revenue": 0}, "movie")
|
||||
assert d["tmdb_rating"] is None # 0.0 with no votes is meaningless
|
||||
assert d["tmdb_vote_count"] is None
|
||||
assert d["budget"] is None and d["revenue"] is None
|
||||
|
||||
|
||||
def test_apply_writes_row_fields_without_touching_plex_owned():
|
||||
it = PlexItem(rating_key="m1", kind="movie", title="The Godfather", imdb_rating=8.7, full_cast=[{"x": 1}])
|
||||
tmdb.apply_tmdb_detail(it, tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie"))
|
||||
assert it.tmdb_id == 238
|
||||
assert it.budget == 6000000
|
||||
assert it.keywords == ["mafia", "crime family"]
|
||||
assert it.imdb_rating == 8.7 and it.full_cast == [{"x": 1}] # Plex-owned, untouched
|
||||
|
||||
|
||||
def test_guid_to_tmdb_id_direct_find_and_bad():
|
||||
fake = FakeTmdb(find_map={"tt0068646": {"movie_results": [{"id": 238}], "tv_results": []}})
|
||||
assert tmdb.guid_to_tmdb_id("tmdb:238", "movie", fake) == 238
|
||||
assert tmdb.guid_to_tmdb_id("imdb:tt0068646", "movie", fake) == 238 # resolved via /find
|
||||
assert tmdb.guid_to_tmdb_id("imdb:tt0000000", "movie", fake) is None # not in find_map
|
||||
assert tmdb.guid_to_tmdb_id("tmdb:notanumber", "movie", fake) is None
|
||||
assert tmdb.guid_to_tmdb_id(None, "movie", fake) is None
|
||||
assert tmdb.guid_to_tmdb_id("local://abc", "movie", fake) is None
|
||||
|
||||
|
||||
# ---- DB-lane: batch enrich + serve --------------------------------------------------------------
|
||||
|
||||
def _lib(db, key="1", kind="movie"):
|
||||
lib = PlexLibrary(plex_key=key, title="L", kind=kind, enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
return lib
|
||||
|
||||
|
||||
def test_enrich_batch_persists_stamps_and_links_people(db):
|
||||
lib = _lib(db)
|
||||
mv = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather", guid="tmdb:238")
|
||||
unmatched = PlexItem(rating_key="m2", library_id=lib.id, kind="movie", title="Nope", guid="tmdb:777")
|
||||
db.add_all([mv, unmatched])
|
||||
db.commit()
|
||||
|
||||
fake = FakeTmdb(payloads={("movie", 238): MOVIE_PAYLOAD}) # 777 has no payload → definitive no-match
|
||||
n = tmdb.enrich_batch(db, fake, 50)
|
||||
assert n == 2 # both processed (one matched, one definitively unmatched)
|
||||
|
||||
db.refresh(mv)
|
||||
assert mv.tmdb_id == 238
|
||||
assert mv.keywords == ["mafia", "crime family"]
|
||||
assert mv.revenue == 245066411
|
||||
assert mv.tmdb_enriched_at is not None
|
||||
|
||||
db.refresh(unmatched)
|
||||
assert unmatched.tmdb_enriched_at is not None # stamped so it isn't re-fetched every run
|
||||
assert unmatched.tmdb_id is None # but nothing stored
|
||||
|
||||
brando = db.execute(
|
||||
PlexPerson.__table__.select().where(PlexPerson.name == "Marlon Brando")
|
||||
).mappings().first()
|
||||
assert brando is not None and brando["tmdb_person_id"] == 3084
|
||||
|
||||
|
||||
def test_serve_block_none_until_enriched_and_resolves_recs(db):
|
||||
lib = _lib(db)
|
||||
# A recommendation target that IS in our library (guid tmdb:240) → deep-linkable.
|
||||
sequel = PlexItem(rating_key="m240", library_id=lib.id, kind="movie", title="Part II", guid="tmdb:240")
|
||||
fresh = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather", guid="tmdb:238")
|
||||
db.add_all([sequel, fresh])
|
||||
db.commit()
|
||||
|
||||
assert tmdb.serve_block(db, fresh) is None # not enriched yet
|
||||
|
||||
tmdb.apply_tmdb_detail(fresh, tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie"))
|
||||
fresh.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
block = tmdb.serve_block(db, fresh)
|
||||
assert block is not None
|
||||
assert block["budget"] == 6000000
|
||||
assert block["keywords"] == ["mafia", "crime family"]
|
||||
recs = {r["tmdb_id"]: r for r in block["recommendations"]}
|
||||
assert recs[240]["in_library"] is True and recs[240]["rating_key"] == "m240"
|
||||
assert recs[999]["in_library"] is False and recs[999]["rating_key"] is None
|
||||
|
||||
|
||||
def test_run_enrich_noops_without_key(db):
|
||||
# No tmdb_api_key configured (default "") → clean no-op, never an error.
|
||||
out = tmdb.run_enrich(db)
|
||||
assert out["enriched"] == 0 and "skipped" in out
|
||||
|
||||
|
||||
def test_enrich_batch_skips_shows_without_guid(db):
|
||||
lib = _lib(db, key="2", kind="show")
|
||||
sh = PlexShow(rating_key="s1", library_id=lib.id, title="No Guid Show") # guid None → not selected
|
||||
db.add(sh)
|
||||
db.commit()
|
||||
assert tmdb.enrich_batch(db, FakeTmdb(), 50) == 0
|
||||
db.refresh(sh)
|
||||
assert sh.tmdb_enriched_at is None
|
||||
Reference in New Issue
Block a user