"""TMDB enrichment for the mirrored Plex catalog (E5 S2). A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items / plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find). Server-side ONLY (the api_key from sysconfig). Everything fetched is PERSISTED and served from the DB row — there is never a live TMDB call at serve time (the user's explicit rule). Additive: the existing Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the distinct TMDb score. One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module. """ import logging import time from datetime import datetime, timezone import httpx from sqlalchemy import func, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session from app import sysconfig from app.config import settings from app.models import PlexItem, PlexPerson, PlexShow log = logging.getLogger("siftlode.tmdb") # Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host. _MIN_INTERVAL = 0.03 # The crew jobs worth surfacing on an info page, in DISPLAY-priority order (the stored crew is sorted by # this — TMDB's own crew order is arbitrary and buries the lead roles). MOVIE and TV differ on purpose: # on a TV show `Director`/`Editor`/`Director of Photography` are PER-EPISODE credits (dozens of names # that would flood the list and bury the showrunner), so TV keys on `Creator` (from the payload's # top-level `created_by`, not the crew list) + the show-level producers/writers instead. _MOVIE_CREW_JOBS = ( "Director", "Writer", "Screenplay", "Story", "Producer", "Executive Producer", "Editor", "Director of Photography", "Original Music Composer", ) _TV_CREW_JOBS = ("Creator", "Executive Producer", "Writer", "Producer", "Original Music Composer") _MOVIE_PRIORITY = {job: i for i, job in enumerate(_MOVIE_CREW_JOBS)} _TV_PRIORITY = {job: i for i, job in enumerate(_TV_CREW_JOBS)} # append_to_response bundles the sub-resources into ONE request per title (keywords + credits + # recommendations + images + external ids). TV uses aggregate_credits (per-person aggregated jobs). _MOVIE_APPEND = "keywords,credits,recommendations,images,external_ids" _TV_APPEND = "keywords,aggregate_credits,recommendations,images,external_ids" class TmdbError(Exception): """Any transient failure talking to TMDB (network, non-404 HTTP, bad payload). The caller leaves the row's `tmdb_enriched_at` NULL so the next run retries.""" class TmdbNotConfigured(TmdbError): """No TMDB API key is set — the module can't do anything.""" class TmdbClient: """Bound to the configured api_key + language. Use as a context manager so the httpx client closes.""" def __init__(self, db: Session): self.key = sysconfig.get_str(db, "tmdb_api_key") if not self.key: raise TmdbNotConfigured("TMDB API key is not configured") self.base = settings.tmdb_api_base.rstrip("/") self.language = sysconfig.get_str(db, "tmdb_language") or settings.tmdb_language self._http = httpx.Client(timeout=30.0) self._last = 0.0 def __enter__(self) -> "TmdbClient": return self def __exit__(self, *exc) -> None: self._http.close() def close(self) -> None: self._http.close() def _throttle(self) -> None: wait = self._last + _MIN_INTERVAL - time.monotonic() if wait > 0: time.sleep(wait) self._last = time.monotonic() def _get(self, path: str, params: dict | None = None) -> dict | None: """GET a TMDB path. Returns the JSON dict, None on a 404 (a definitive "no such id"), and raises TmdbError on any transient failure (network / other HTTP status / non-JSON).""" self._throttle() p = {"api_key": self.key, "language": self.language, **(params or {})} try: r = self._http.get(f"{self.base}{path}", params=p) except httpx.HTTPError as e: raise TmdbError(str(e)) from e if r.status_code == 404: return None try: r.raise_for_status() except httpx.HTTPStatusError as e: raise TmdbError(str(e)) from e try: return r.json() except ValueError as e: raise TmdbError("TMDB returned a non-JSON response") from e def fetch(self, tmdb_id: int, media_type: str) -> dict | None: """Full enrichment payload for a movie or show (one request via append_to_response).""" if media_type == "tv": return self._get(f"/tv/{tmdb_id}", {"append_to_response": _TV_APPEND}) return self._get(f"/movie/{tmdb_id}", {"append_to_response": _MOVIE_APPEND}) def find(self, external_id: str, source: str) -> dict | None: """Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find.""" return self._get(f"/find/{external_id}", {"external_source": source}) def fetch_person(self, person_id: int) -> dict | None: """One person's bio + combined credits (E5 S6), in a single append_to_response request.""" return self._get(f"/person/{person_id}", {"append_to_response": "combined_credits"}) def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None: """Map a stored canonical guid to a TMDB id. `tmdb:` is direct; `imdb:`/`tvdb:` go through /find (picking the movie_results or tv_results list per `media_type`). Returns None for an unmatchable guid — the caller then stamps the watermark so it isn't retried every run.""" if not guid or ":" not in guid: return None provider, _, ext = guid.partition(":") if provider == "tmdb": try: return int(ext) except ValueError: return None source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider) if not source: return None res = tmdb.find(ext, source) if not res: return None items = res.get("tv_results" if media_type == "tv" else "movie_results") or [] return items[0].get("id") if items else None def _dedup_crew(crew: list[dict]) -> list[dict]: seen: set[tuple] = set() out: list[dict] = [] for c in crew: key = (c["name"], c["job"]) if key in seen: continue seen.add(key) out.append(c) return out def parse_tmdb(payload: dict, media_type: str) -> dict: """Pure parser: pull the enrichment fields out of a TMDB movie/tv payload (append_to_response form). Best-effort — any missing sub-resource is just omitted. Returns the row fields plus a `people` list ([{name, tmdb_id}] over cast+crew) for the plex_people id-linking. TMDB image fields are stored as bare paths ("/abc.jpg"); the client builds the image.tmdb.org URL.""" is_tv = media_type == "tv" votes = payload.get("vote_count") or 0 kw_container = payload.get("keywords") or {} kw_list = kw_container.get("results") if is_tv else kw_container.get("keywords") keywords = [k.get("name") for k in (kw_list or []) if k.get("name")][:20] priority = _TV_PRIORITY if is_tv else _MOVIE_PRIORITY credits = payload.get("aggregate_credits" if is_tv else "credits") or {} crew: list[dict] = [] # A TV show's creators live at the top level (created_by), not in the crew list — surface them. if is_tv: for c in payload.get("created_by") or []: if c.get("name"): crew.append( {"name": c["name"], "job": "Creator", "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")} ) for c in credits.get("crew") or []: # Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person. job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job") if job in priority and c.get("name"): crew.append( {"name": c["name"], "job": job, "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")} ) # Dedup by (name, job) — a person can hold two distinct key roles (Coppola: Director + Writer) — then # order by display priority so the lead roles come first, and cap. crew = _dedup_crew(crew) crew.sort(key=lambda c: priority.get(c["job"], 99)) crew = crew[:12] # name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids). people: list[dict] = [] for c in (credits.get("cast") or [])[:30]: if c.get("name") and c.get("id"): people.append({"name": c["name"], "tmdb_id": c["id"]}) for c in crew: if c.get("tmdb_id"): people.append({"name": c["name"], "tmdb_id": c["tmdb_id"]}) recs: list[dict] = [] for r in ((payload.get("recommendations") or {}).get("results") or [])[:20]: recs.append( { "tmdb_id": r.get("id"), "title": r.get("title") or r.get("name"), "poster_path": r.get("poster_path"), "media_type": r.get("media_type") or media_type, } ) backdrops = [ b.get("file_path") for b in ((payload.get("images") or {}).get("backdrops") or []) if b.get("file_path") ][:10] return { "tmdb_id": payload.get("id"), # vote_average is meaningless with no votes (TMDB returns 0.0) → treat as unknown. "tmdb_rating": payload.get("vote_average") if votes else None, "tmdb_vote_count": votes or None, # 0 is TMDB's "unknown" for money fields; TV has neither. "budget": (payload.get("budget") or None) if not is_tv else None, "revenue": (payload.get("revenue") or None) if not is_tv else None, "keywords": keywords or None, "crew": crew or None, "studios": [c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")][:8] or None, "production_countries": [ c.get("name") for c in (payload.get("production_countries") or []) if c.get("name") ] or None, "recommendations": recs or None, "backdrops": backdrops or None, "people": people, } def apply_tmdb_detail(row, parsed: dict) -> None: """Persist the parsed TMDB fields onto a movie/show row. Does NOT touch full_cast / imdb_rating / rating (Plex-owned). The caller stamps `tmdb_enriched_at` and links people separately.""" row.tmdb_id = parsed["tmdb_id"] row.tmdb_rating = parsed["tmdb_rating"] row.tmdb_vote_count = parsed["tmdb_vote_count"] row.budget = parsed["budget"] row.revenue = parsed["revenue"] row.keywords = parsed["keywords"] row.crew = parsed["crew"] row.studios = parsed["studios"] row.production_countries = parsed["production_countries"] row.recommendations = parsed["recommendations"] row.backdrops = parsed["backdrops"] def _link_people(db: Session, links: dict[str, int]) -> None: """Upsert plex_people by name, stamping `tmdb_person_id` — keeping any id already set (first wins), filling it where NULL, inserting a bare row where the person isn't cached yet. Deduped by name so a single statement can't hit the conflict target twice (the same rule as people.enrich_row_people).""" values = [{"name": nm, "tmdb_person_id": pid} for nm, pid in links.items() if nm and pid] if not values: return stmt = pg_insert(PlexPerson).values(values) db.execute( stmt.on_conflict_do_update( index_elements=[PlexPerson.name], set_={"tmdb_person_id": func.coalesce(PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id)}, ) ) def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int: """Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder — mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched every run. One commit at the end. Returns the number processed (matched OR definitively unmatched).""" movies = ( db.execute( select(PlexItem) .where( PlexItem.kind == "movie", PlexItem.guid.is_not(None), PlexItem.tmdb_enriched_at.is_(None), ) .order_by(PlexItem.id) .limit(limit) ) .scalars() .all() ) shows: list[PlexShow] = [] if len(movies) < limit: shows = ( db.execute( select(PlexShow) .where(PlexShow.guid.is_not(None), PlexShow.tmdb_enriched_at.is_(None)) .order_by(PlexShow.id) .limit(limit - len(movies)) ) .scalars() .all() ) processed = 0 links: dict[str, int] = {} for row, media_type in [(m, "movie") for m in movies] + [(s, "tv") for s in shows]: try: tmdb_id = guid_to_tmdb_id(row.guid, media_type, tmdb) payload = tmdb.fetch(tmdb_id, media_type) if tmdb_id else None except TmdbError as e: log.warning("tmdb enrich %s failed (will retry): %s", row.rating_key, e) continue # transient — leave tmdb_enriched_at NULL for the next run if payload: parsed = parse_tmdb(payload, media_type) apply_tmdb_detail(row, parsed) for p in parsed["people"]: links.setdefault(p["name"], p["tmdb_id"]) # Stamp even on a definitive no-match so an unresolvable title isn't re-fetched every run. row.tmdb_enriched_at = datetime.now(timezone.utc) processed += 1 if links: _link_people(db, links) db.commit() return processed def run_enrich(db: Session) -> dict: """Scheduler entry point. No-op (never an error) when the TMDB key isn't configured.""" if not sysconfig.get_str(db, "tmdb_api_key"): return {"enriched": 0, "skipped": "tmdb_api_key not set"} with TmdbClient(db) as tmdb: n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch) return {"enriched": n} def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]: """Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:`) and, if so, its rating_key — so the client can deep-link the ones we have and merely display the rest. TMDB movie ids and tv ids are independent sequences (id 603 can be BOTH a movie and a show), so the match is bucketed by each rec's own media_type: a movie rec only matches a library movie, a tv rec only a library show — otherwise a numeric-id collision would deep-link a rec to an unrelated title.""" if not recs: return [] movie_guids = [ f"tmdb:{r['tmdb_id']}" for r in recs if r.get("tmdb_id") and r.get("media_type") != "tv" ] tv_guids = [ f"tmdb:{r['tmdb_id']}" for r in recs if r.get("tmdb_id") and r.get("media_type") == "tv" ] movie_lib: dict[str, str] = {} show_lib: dict[str, str] = {} if movie_guids: for guid, rk in db.execute( select(PlexItem.guid, PlexItem.rating_key).where( PlexItem.kind == "movie", PlexItem.guid.in_(movie_guids) ) ): movie_lib.setdefault(guid, rk) if tv_guids: for guid, rk in db.execute( select(PlexShow.guid, PlexShow.rating_key).where(PlexShow.guid.in_(tv_guids)) ): show_lib.setdefault(guid, rk) out: list[dict] = [] for r in recs: g = f"tmdb:{r.get('tmdb_id')}" rk = show_lib.get(g) if r.get("media_type") == "tv" else movie_lib.get(g) out.append({**r, "in_library": rk is not None, "rating_key": rk}) return out def serve_block(db: Session, row) -> dict | None: """The TMDB extras for a movie/show detail response, served from the row (no live TMDB call). None until the row has been TMDB-enriched (the client then shows nothing extra).""" if row.tmdb_enriched_at is None: return None return { "tmdb_id": row.tmdb_id, "rating": row.tmdb_rating, "vote_count": row.tmdb_vote_count, "budget": row.budget, "revenue": row.revenue, "keywords": row.keywords or [], "crew": row.crew or [], "studios": row.studios or [], "production_countries": row.production_countries or [], "recommendations": resolve_recs(db, row.recommendations), "backdrops": row.backdrops or [], } # --- Person enrichment (E5 S6) ------------------------------------------------------------------- # A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open) # and served from the plex_people row. `known_for` stores the top combined-credits ids; the person # endpoint resolves them to library rows by guid at serve time (like recommendations). _KNOWN_FOR_MAX = 15 def parse_person(payload: dict) -> dict: """TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later.""" credits = (payload.get("combined_credits") or {}).get("cast") or [] seen: set[int] = set() known: list[dict] = [] for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True): tid, mt = c.get("id"), c.get("media_type") if not tid or mt not in ("movie", "tv") or tid in seen: continue seen.add(tid) known.append({"tmdb_id": tid, "media_type": mt, "title": c.get("title") or c.get("name")}) if len(known) >= _KNOWN_FOR_MAX: break return { "biography": (payload.get("biography") or "").strip() or None, "birthday": payload.get("birthday") or None, "deathday": payload.get("deathday") or None, "place_of_birth": payload.get("place_of_birth") or None, "known_for_department": payload.get("known_for_department") or None, "also_known_as": payload.get("also_known_as") or None, "known_for": known or None, } def apply_person(row: PlexPerson, parsed: dict) -> None: row.biography = parsed["biography"] row.birthday = parsed["birthday"] row.deathday = parsed["deathday"] row.place_of_birth = parsed["place_of_birth"] row.known_for_department = parsed["known_for_department"] row.also_known_as = parsed["also_known_as"] row.known_for = parsed["known_for"] row.tmdb_enriched_at = datetime.now(timezone.utc) def enrich_person(db: Session, person: PlexPerson) -> bool: """Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the watermark with empty fields so we don't refetch it every open.""" if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"): return False try: with TmdbClient(db) as tmdb: payload = tmdb.fetch_person(person.tmdb_person_id) except TmdbError: return False if payload is None: person.tmdb_enriched_at = datetime.now(timezone.utc) else: apply_person(person, parse_person(payload)) db.commit() return True