peter
4aaaaa7860
The code cited its plan instead of saying why: `E28 review J4`, `(R6 S3b)`, `S1 justified…`,
`deep-review round 5`. That plan lives in the private ops repo, so for every reader of this one the
citation was noise, and a sentence whose subject was the plan step ("round 2 showed…") hid its
lesson behind an id. Each one is gone and the "why" kept: a bare citation is deleted, a sentence
carrying a lesson is reworded around it ("a later attempt showed…").
Measured (`scripts/audit_metrics.py`, product code): 652 lines in 270 files carried an epic code by
the definition the next commit holds (507 / 241 by the old, two-digit-only one) -> 0. Across all
code, tests, scripts and configs: ~1100 epic-code lines plus ~320 sprint/round ids.
Behaviour unchanged, checked mechanically: every changed .py file has the same AST once docstrings
are blanked; css/sh/toml/yml/json files are identical once comments are stripped; the only
non-comment changes in TS/JS are test titles, one description string in a contract table, and five
ESLint messages, whose text is all the gates snapshot diff holds. Not touched: Markdown (docs and
the changelog keep their history), Alembic migrations, TV episode notation (`S2 · E5`), DESIGN.md
ids (`C1`, `B4`, `S3h`), tracker ids (`SEC-16`, `DX-17`).
One stale claim fixed on the way: test_hls_reaper's docstring said the player checkpoints "every
10 s whether or not it is playing", which plex/stream.py calls wrong; it now says what the code does.
677 lines
30 KiB
Python
677 lines
30 KiB
Python
"""TMDB enrichment for the mirrored Plex catalog.
|
|
|
|
A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items /
|
|
plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find).
|
|
Server-side ONLY (the api_key from sysconfig). Everything fetched is PERSISTED and served from the DB
|
|
row — there is never a live TMDB call at serve time (the user's explicit rule). Additive: the existing
|
|
Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the distinct TMDb score.
|
|
|
|
One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes
|
|
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
|
"""
|
|
|
|
import re
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from typing import cast
|
|
|
|
import httpx
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, sysconfig
|
|
from app.config import settings
|
|
from app.models import PlexItem, PlexPerson, PlexShow
|
|
from app.plex.cards import display_title
|
|
from app.plex.people import people_by_credits, warm_portraits
|
|
|
|
log = logs.get_logger(__name__)
|
|
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
|
_MIN_INTERVAL = 0.03
|
|
|
|
# The crew jobs worth surfacing on an info page, in DISPLAY-priority order (the stored crew is sorted by
|
|
# this — TMDB's own crew order is arbitrary and buries the lead roles). MOVIE and TV differ on purpose:
|
|
# on a TV show `Director`/`Editor`/`Director of Photography` are PER-EPISODE credits (dozens of names
|
|
# that would flood the list and bury the showrunner), so TV keys on `Creator` (from the payload's
|
|
# top-level `created_by`, not the crew list) + the show-level producers/writers instead.
|
|
_MOVIE_CREW_JOBS = (
|
|
"Director",
|
|
"Writer",
|
|
"Screenplay",
|
|
"Story",
|
|
"Producer",
|
|
"Executive Producer",
|
|
"Editor",
|
|
"Director of Photography",
|
|
"Original Music Composer",
|
|
)
|
|
_TV_CREW_JOBS = ("Creator", "Executive Producer", "Writer", "Producer", "Original Music Composer")
|
|
_MOVIE_PRIORITY = {job: i for i, job in enumerate(_MOVIE_CREW_JOBS)}
|
|
_TV_PRIORITY = {job: i for i, job in enumerate(_TV_CREW_JOBS)}
|
|
|
|
# append_to_response bundles the sub-resources into ONE request per title (keywords + credits +
|
|
# recommendations + images + external ids). TV uses aggregate_credits (per-person aggregated jobs).
|
|
_MOVIE_APPEND = "keywords,credits,recommendations,images,external_ids"
|
|
_TV_APPEND = "keywords,aggregate_credits,recommendations,images,external_ids"
|
|
|
|
|
|
class TmdbError(Exception):
|
|
"""Any transient failure talking to TMDB (network, non-404 HTTP, bad payload). The caller leaves
|
|
the row's `tmdb_enriched_at` NULL so the next run retries."""
|
|
|
|
|
|
class TmdbNotConfigured(TmdbError):
|
|
"""No TMDB API key is set — the module can't do anything."""
|
|
|
|
|
|
class TmdbClient:
|
|
"""Bound to the configured api_key + language. Use as a context manager so the httpx client closes."""
|
|
|
|
def __init__(self, db: Session, timeout: float = 30.0):
|
|
self.key = sysconfig.get_str(db, "tmdb_api_key")
|
|
if not self.key:
|
|
raise TmdbNotConfigured("TMDB API key is not configured")
|
|
self.base = settings.tmdb_api_base.rstrip("/")
|
|
self.language = sysconfig.get_str(db, "tmdb_language") or settings.tmdb_language
|
|
# `timeout` is caller-tunable so the request-path lazy person enrich can cap how long it may
|
|
# block a worker thread (the batch job keeps the generous default).
|
|
self._http = httpx.Client(timeout=timeout)
|
|
self._last = 0.0
|
|
|
|
def __enter__(self) -> "TmdbClient":
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self._http.close()
|
|
|
|
def close(self) -> None:
|
|
self._http.close()
|
|
|
|
def _throttle(self) -> None:
|
|
wait = self._last + _MIN_INTERVAL - time.monotonic()
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
self._last = time.monotonic()
|
|
|
|
def _get(self, path: str, params: dict | None = None) -> dict | None:
|
|
"""GET a TMDB path. Returns the JSON dict, None on a 404 (a definitive "no such id"), and raises
|
|
TmdbError on any transient failure (network / other HTTP status / non-JSON)."""
|
|
self._throttle()
|
|
p = {"api_key": self.key, "language": self.language, **(params or {})}
|
|
try:
|
|
r = self._http.get(f"{self.base}{path}", params=p)
|
|
except httpx.HTTPError as e:
|
|
# NEVER put str(e)/the URL in the message: the request URL carries `?api_key=<secret>` in
|
|
# its query string, and this message is logged (enrich_batch). Report the type + path only.
|
|
raise TmdbError(f"TMDB request error on {path}: {type(e).__name__}") from None
|
|
if r.status_code == 404:
|
|
return None
|
|
try:
|
|
r.raise_for_status()
|
|
except httpx.HTTPStatusError as e:
|
|
# Same reason: the HTTPStatusError string embeds the full URL incl. the api_key. Sever the
|
|
# cause (`from None`) so no traceback can resurface it, and log only status + path.
|
|
raise TmdbError(f"TMDB {path} returned {e.response.status_code}") from None
|
|
try:
|
|
return r.json()
|
|
except ValueError as e:
|
|
raise TmdbError("TMDB returned a non-JSON response") from e
|
|
|
|
def fetch(self, tmdb_id: int, media_type: str) -> dict | None:
|
|
"""Full enrichment payload for a movie or show (one request via append_to_response)."""
|
|
if media_type == "tv":
|
|
return self._get(f"/tv/{tmdb_id}", {"append_to_response": _TV_APPEND})
|
|
return self._get(f"/movie/{tmdb_id}", {"append_to_response": _MOVIE_APPEND})
|
|
|
|
def find(self, external_id: str, source: str) -> dict | None:
|
|
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
|
return self._get(f"/find/{external_id}", {"external_source": source})
|
|
|
|
def countries(self) -> list[dict] | None:
|
|
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
|
`_get`'s dict annotation does not describe)."""
|
|
return cast("list[dict] | None", self._get("/configuration/countries"))
|
|
|
|
def fetch_person(self, person_id: int) -> dict | None:
|
|
"""One person's bio + combined credits + the bio's translations, in a single
|
|
append_to_response request."""
|
|
return self._get(
|
|
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
|
)
|
|
|
|
|
|
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
|
"""Map a stored canonical guid to a TMDB id. `tmdb:` is direct; `imdb:`/`tvdb:` go through /find
|
|
(picking the movie_results or tv_results list per `media_type`). Returns None for an unmatchable
|
|
guid — the caller then stamps the watermark so it isn't retried every run."""
|
|
if not guid or ":" not in guid:
|
|
return None
|
|
provider, _, ext = guid.partition(":")
|
|
if provider == "tmdb":
|
|
try:
|
|
return int(ext)
|
|
except ValueError:
|
|
# The caller stamps the watermark on a None, so this title is never looked at again:
|
|
# one unreadable guid costs it its artwork and its cast permanently.
|
|
log.info(
|
|
"a Plex guid named TMDB but did not carry a numeric id",
|
|
extra=logs.event("plex.tmdb.guid_unparsable", provider=provider),
|
|
)
|
|
return None
|
|
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
|
if not source:
|
|
return None
|
|
res = tmdb.find(ext, source)
|
|
if not res:
|
|
return None
|
|
items = res.get("tv_results" if media_type == "tv" else "movie_results") or []
|
|
return items[0].get("id") if items else None
|
|
|
|
|
|
def _dedup_crew(crew: list[dict]) -> list[dict]:
|
|
seen: set[tuple] = set()
|
|
out: list[dict] = []
|
|
for c in crew:
|
|
key = (c["name"], c["job"])
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
out.append(c)
|
|
return out
|
|
|
|
|
|
def parse_tmdb(payload: dict, media_type: str) -> dict:
|
|
"""Pure parser: pull the enrichment fields out of a TMDB movie/tv payload (append_to_response form).
|
|
Best-effort — any missing sub-resource is just omitted. Returns the row fields plus a `people` list
|
|
([{name, tmdb_id}] over cast+crew) for the plex_people id-linking. TMDB image fields are stored as
|
|
bare paths ("/abc.jpg"); the client builds the image.tmdb.org URL."""
|
|
is_tv = media_type == "tv"
|
|
votes = payload.get("vote_count") or 0
|
|
|
|
kw_container = payload.get("keywords") or {}
|
|
kw_list = kw_container.get("results") if is_tv else kw_container.get("keywords")
|
|
keywords = [k.get("name") for k in (kw_list or []) if k.get("name")][:20]
|
|
|
|
priority = _TV_PRIORITY if is_tv else _MOVIE_PRIORITY
|
|
credits = payload.get("aggregate_credits" if is_tv else "credits") or {}
|
|
crew: list[dict] = []
|
|
# A TV show's creators live at the top level (created_by), not in the crew list — surface them.
|
|
if is_tv:
|
|
for c in payload.get("created_by") or []:
|
|
if c.get("name"):
|
|
crew.append(
|
|
{
|
|
"name": c["name"],
|
|
"job": "Creator",
|
|
"tmdb_id": c.get("id"),
|
|
"profile_path": c.get("profile_path"),
|
|
}
|
|
)
|
|
for c in credits.get("crew") or []:
|
|
# Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person.
|
|
job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job")
|
|
if job in priority and c.get("name"):
|
|
crew.append(
|
|
{
|
|
"name": c["name"],
|
|
"job": job,
|
|
"tmdb_id": c.get("id"),
|
|
"profile_path": c.get("profile_path"),
|
|
}
|
|
)
|
|
# Dedup by (name, job) — a person can hold two distinct key roles (Coppola: Director + Writer) — then
|
|
# order by display priority so the lead roles come first, and cap.
|
|
crew = _dedup_crew(crew)
|
|
crew.sort(key=lambda c: priority.get(c["job"], 99))
|
|
crew = crew[:12]
|
|
|
|
# name -> tmdb person id, for the plex_people foundation (bios are fetched off these ids).
|
|
people: list[dict] = []
|
|
for c in (credits.get("cast") or [])[:30]:
|
|
if c.get("name") and c.get("id"):
|
|
people.append({"name": c["name"], "tmdb_id": c["id"]})
|
|
for c in crew:
|
|
if c.get("tmdb_id"):
|
|
people.append({"name": c["name"], "tmdb_id": c["tmdb_id"]})
|
|
|
|
recs: list[dict] = []
|
|
for r in ((payload.get("recommendations") or {}).get("results") or [])[:20]:
|
|
recs.append(
|
|
{
|
|
"tmdb_id": r.get("id"),
|
|
"title": r.get("title") or r.get("name"),
|
|
"poster_path": r.get("poster_path"),
|
|
"media_type": r.get("media_type") or media_type,
|
|
}
|
|
)
|
|
|
|
backdrops = [
|
|
b.get("file_path")
|
|
for b in ((payload.get("images") or {}).get("backdrops") or [])
|
|
if b.get("file_path")
|
|
][:10]
|
|
|
|
return {
|
|
"tmdb_id": payload.get("id"),
|
|
# vote_average is meaningless with no votes (TMDB returns 0.0) → treat as unknown.
|
|
"tmdb_rating": payload.get("vote_average") if votes else None,
|
|
"tmdb_vote_count": votes or None,
|
|
# 0 is TMDB's "unknown" for money fields; TV has neither.
|
|
"budget": (payload.get("budget") or None) if not is_tv else None,
|
|
"revenue": (payload.get("revenue") or None) if not is_tv else None,
|
|
"keywords": keywords or None,
|
|
"crew": crew or None,
|
|
"studios": [
|
|
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
|
][:8]
|
|
or None,
|
|
# The ISO code beside TMDB's English name: the client names the country in the reader's
|
|
# language from the code; the name is the fallback for a code it cannot name.
|
|
"production_countries": [
|
|
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
|
for c in (payload.get("production_countries") or [])
|
|
if c.get("name")
|
|
]
|
|
or None,
|
|
"recommendations": recs or None,
|
|
"backdrops": backdrops or None,
|
|
"people": people,
|
|
}
|
|
|
|
|
|
def apply_tmdb_detail(row, parsed: dict) -> None:
|
|
"""Persist the parsed TMDB fields onto a movie/show row. Does NOT touch full_cast / imdb_rating /
|
|
rating (Plex-owned). The caller stamps `tmdb_enriched_at` and links people separately."""
|
|
row.tmdb_id = parsed["tmdb_id"]
|
|
row.tmdb_rating = parsed["tmdb_rating"]
|
|
row.tmdb_vote_count = parsed["tmdb_vote_count"]
|
|
row.budget = parsed["budget"]
|
|
row.revenue = parsed["revenue"]
|
|
row.keywords = parsed["keywords"]
|
|
row.crew = parsed["crew"]
|
|
row.studios = parsed["studios"]
|
|
row.production_countries = parsed["production_countries"]
|
|
row.recommendations = parsed["recommendations"]
|
|
row.backdrops = parsed["backdrops"]
|
|
|
|
|
|
def _link_people(db: Session, links: dict[str, int]) -> None:
|
|
"""Upsert plex_people by name, stamping `tmdb_person_id` — keeping any id already set (first wins),
|
|
filling it where NULL, inserting a bare row where the person isn't cached yet. Deduped by name so a
|
|
single statement can't hit the conflict target twice (the same rule as people.enrich_row_people)."""
|
|
values = [{"name": nm, "tmdb_person_id": pid} for nm, pid in links.items() if nm and pid]
|
|
if not values:
|
|
return
|
|
stmt = pg_insert(PlexPerson).values(values)
|
|
db.execute(
|
|
stmt.on_conflict_do_update(
|
|
index_elements=[PlexPerson.name],
|
|
set_={
|
|
"tmdb_person_id": func.coalesce(
|
|
PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id
|
|
)
|
|
},
|
|
)
|
|
)
|
|
|
|
|
|
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
|
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
|
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
|
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
|
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
|
movies = (
|
|
db.execute(
|
|
select(PlexItem)
|
|
.where(
|
|
PlexItem.kind == "movie",
|
|
PlexItem.guid.is_not(None),
|
|
PlexItem.tmdb_enriched_at.is_(None),
|
|
)
|
|
.order_by(PlexItem.id)
|
|
.limit(limit)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
shows: list[PlexShow] = []
|
|
if len(movies) < limit:
|
|
shows = (
|
|
db.execute(
|
|
select(PlexShow)
|
|
.where(PlexShow.guid.is_not(None), PlexShow.tmdb_enriched_at.is_(None))
|
|
.order_by(PlexShow.id)
|
|
.limit(limit - len(movies))
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
|
|
processed = 0
|
|
links: dict[str, int] = {}
|
|
for row, media_type in [(m, "movie") for m in movies] + [(s, "tv") for s in shows]:
|
|
try:
|
|
tmdb_id = guid_to_tmdb_id(row.guid, media_type, tmdb)
|
|
payload = tmdb.fetch(tmdb_id, media_type) if tmdb_id else None
|
|
except TmdbError as e:
|
|
log.warning("tmdb enrich %s failed (will retry): %s", row.rating_key, e)
|
|
continue # transient — leave tmdb_enriched_at NULL for the next run
|
|
if payload:
|
|
parsed = parse_tmdb(payload, media_type)
|
|
apply_tmdb_detail(row, parsed)
|
|
for p in parsed["people"]:
|
|
links.setdefault(p["name"], p["tmdb_id"])
|
|
# Stamp even on a definitive no-match so an unresolvable title isn't re-fetched every run.
|
|
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
processed += 1
|
|
|
|
if links:
|
|
_link_people(db, links)
|
|
db.commit()
|
|
return processed
|
|
|
|
|
|
def _country(c: dict | str) -> dict:
|
|
"""One stored production country as served: `{code, name}`. An older row stored the
|
|
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
|
shows the name)."""
|
|
return c if isinstance(c, dict) else {"code": None, "name": c}
|
|
|
|
|
|
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
|
"""Give the older countries (bare English names) their ISO codes, from TMDB's own
|
|
country list — the names came from that list, so they match it exactly. One request, and only
|
|
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
|
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
|
stale = [
|
|
row
|
|
for model in (PlexItem, PlexShow)
|
|
for row in db.execute(
|
|
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
|
).scalars()
|
|
]
|
|
if not stale:
|
|
return 0
|
|
codes = {
|
|
c["english_name"]: c["iso_3166_1"]
|
|
for c in tmdb.countries() or []
|
|
if c.get("english_name") and c.get("iso_3166_1")
|
|
}
|
|
for row in stale:
|
|
row.production_countries = [
|
|
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
|
for c in row.production_countries or []
|
|
]
|
|
db.commit()
|
|
return len(stale)
|
|
|
|
|
|
def run_enrich(db: Session) -> dict:
|
|
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
|
if not sysconfig.get_str(db, "tmdb_api_key"):
|
|
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
|
with TmdbClient(db) as tmdb:
|
|
coded = backfill_country_codes(db, tmdb)
|
|
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
|
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
|
|
|
|
|
def run_people_enrich(db: Session) -> dict:
|
|
"""Scheduler entry point of people_enrich: a batch of person bios (the most-credited
|
|
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
|
without Plex."""
|
|
batch = settings.people_enrich_batch
|
|
if sysconfig.get_str(db, "tmdb_api_key"):
|
|
with TmdbClient(db) as tmdb:
|
|
bios = enrich_people_batch(db, tmdb, batch)
|
|
else:
|
|
bios = {"skipped": "tmdb_api_key not set"}
|
|
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
|
|
|
|
|
def resolve_library_guids(
|
|
db: Session,
|
|
entries: list[dict] | None,
|
|
*,
|
|
movie_lib_ids: list[int] | None = None,
|
|
show_lib_ids: list[int] | None = None,
|
|
) -> tuple[dict[str, PlexItem], dict[str, PlexShow]]:
|
|
"""Match a list of TMDB credits/recs (each `{tmdb_id, media_type, …}`) to our library rows by the
|
|
canonical guid `tmdb:<id>`, returning `(movies_by_guid, shows_by_guid)` of the ORM rows.
|
|
|
|
⭐ TMDB movie ids and tv ids are INDEPENDENT sequences (id 603 can be BOTH a movie and a show), so
|
|
each entry matches ONLY its own media_type's table — a movie credit against a library movie, a tv
|
|
credit against a library show; a blind numeric match would resolve to an unrelated same-id title.
|
|
Pass `movie_lib_ids`/`show_lib_ids` to scope to the enabled libraries (the person page); leave them
|
|
None to match across all libraries (the detail-page recs). An explicit empty list means "that side
|
|
has no enabled library" → its bucket is skipped entirely."""
|
|
if not entries:
|
|
return {}, {}
|
|
movie_guids = [
|
|
f"tmdb:{e['tmdb_id']}" for e in entries if e.get("tmdb_id") and e.get("media_type") != "tv"
|
|
]
|
|
tv_guids = [
|
|
f"tmdb:{e['tmdb_id']}" for e in entries if e.get("tmdb_id") and e.get("media_type") == "tv"
|
|
]
|
|
movies: dict[str, PlexItem] = {}
|
|
if movie_guids and movie_lib_ids != []:
|
|
conds = [PlexItem.kind == "movie", PlexItem.guid.in_(movie_guids)]
|
|
if movie_lib_ids is not None:
|
|
conds.append(PlexItem.library_id.in_(movie_lib_ids))
|
|
for m in db.execute(select(PlexItem).where(*conds)).scalars():
|
|
movies.setdefault(m.guid, m)
|
|
shows: dict[str, PlexShow] = {}
|
|
if tv_guids and show_lib_ids != []:
|
|
conds = [PlexShow.guid.in_(tv_guids)]
|
|
if show_lib_ids is not None:
|
|
conds.append(PlexShow.library_id.in_(show_lib_ids))
|
|
for s in db.execute(select(PlexShow).where(*conds)).scalars():
|
|
shows.setdefault(s.guid, s)
|
|
return movies, shows
|
|
|
|
|
|
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
|
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
|
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
|
the ones we have and merely display the rest. One we have is named as the library names it, in
|
|
`lang`; one we lack keeps the title TMDB gave."""
|
|
if not recs:
|
|
return []
|
|
movies, shows = resolve_library_guids(db, recs)
|
|
out: list[dict] = []
|
|
for r in recs:
|
|
g = f"tmdb:{r.get('tmdb_id')}"
|
|
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
|
named = {"title": display_title(row, lang)} if row is not None else {}
|
|
out.append(
|
|
{
|
|
**r,
|
|
**named,
|
|
"in_library": row is not None,
|
|
"rating_key": row.rating_key if row else None,
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
|
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
|
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
|
if row.tmdb_enriched_at is None:
|
|
return None
|
|
return {
|
|
"tmdb_id": row.tmdb_id,
|
|
"rating": row.tmdb_rating,
|
|
"vote_count": row.tmdb_vote_count,
|
|
"budget": row.budget,
|
|
"revenue": row.revenue,
|
|
"keywords": row.keywords or [],
|
|
"crew": row.crew or [],
|
|
"studios": row.studios or [],
|
|
"production_countries": [_country(c) for c in row.production_countries or []],
|
|
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
|
"backdrops": row.backdrops or [],
|
|
}
|
|
|
|
|
|
# --- Person enrichment -------------------------------------------------------------------
|
|
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
|
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
|
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
|
|
|
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
|
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
|
# credits are often not the 15 a given library holds.
|
|
_KNOWN_FOR_MAX = 40
|
|
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
|
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows).
|
|
_GUEST_GENRES = {10763, 10764, 10767}
|
|
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
|
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
|
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
|
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
|
|
|
|
|
def _known_for_credit(c: dict, crew: bool) -> bool:
|
|
if crew:
|
|
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
|
if _SELF_CREDIT.match(c.get("character") or ""):
|
|
return False
|
|
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
|
|
|
|
|
def parse_person(payload: dict) -> dict:
|
|
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
|
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
|
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
|
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
|
combined = payload.get("combined_credits") or {}
|
|
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
|
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
|
]
|
|
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
|
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
|
# is two distinct credits and both must survive.
|
|
seen: set[tuple[int, str]] = set()
|
|
known: list[dict] = []
|
|
ranked = sorted(
|
|
credits,
|
|
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
|
reverse=True,
|
|
)
|
|
for c in ranked:
|
|
tid, mt = c.get("id"), c.get("media_type")
|
|
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
|
continue
|
|
seen.add((tid, mt))
|
|
known.append({"tmdb_id": tid, "media_type": mt, "title": c.get("title") or c.get("name")})
|
|
if len(known) >= _KNOWN_FOR_MAX:
|
|
break
|
|
return {
|
|
"biography": (payload.get("biography") or "").strip() or None,
|
|
"biography_hu": _translated_bio(payload, "hu"),
|
|
"birthday": payload.get("birthday") or None,
|
|
"deathday": payload.get("deathday") or None,
|
|
"place_of_birth": payload.get("place_of_birth") or None,
|
|
"known_for_department": payload.get("known_for_department") or None,
|
|
"also_known_as": payload.get("also_known_as") or None,
|
|
"known_for": known or None,
|
|
}
|
|
|
|
|
|
def _translated_bio(payload: dict, lang: str) -> str | None:
|
|
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
|
for tr in (payload.get("translations") or {}).get("translations") or []:
|
|
if tr.get("iso_639_1") == lang:
|
|
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
|
return None
|
|
|
|
|
|
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
|
row.biography = parsed["biography"]
|
|
row.biography_hu = parsed["biography_hu"]
|
|
row.birthday = parsed["birthday"]
|
|
row.deathday = parsed["deathday"]
|
|
row.place_of_birth = parsed["place_of_birth"]
|
|
row.known_for_department = parsed["known_for_department"]
|
|
row.also_known_as = parsed["also_known_as"]
|
|
row.known_for = parsed["known_for"]
|
|
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
|
|
|
|
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
|
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
|
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
|
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
|
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
|
if payload is None:
|
|
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
else:
|
|
apply_person(person, parse_person(payload))
|
|
|
|
|
|
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
|
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
|
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
|
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
|
open retries."""
|
|
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
|
return False
|
|
try:
|
|
# Called from a background task (off the request path — see people._enrich_person_bg), so it no
|
|
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
|
# doesn't tie up a threadpool worker for the default 30s.
|
|
with TmdbClient(db, timeout=8.0) as tmdb:
|
|
_fetch_person_into(tmdb, person)
|
|
except TmdbError as exc:
|
|
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
|
# front of the user right now shows an empty biography with nothing to say why.
|
|
log.warning(
|
|
"the TMDB person lookup failed",
|
|
extra=logs.event(
|
|
"plex.tmdb.person_enrich_failed",
|
|
person_id=person.id,
|
|
reason=type(exc).__name__,
|
|
),
|
|
)
|
|
return False
|
|
db.commit()
|
|
return True
|
|
|
|
|
|
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
|
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
|
_PEOPLE_FAILS_IN_A_ROW = 3
|
|
|
|
|
|
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
|
"""Fetch up to `limit` linked, not-yet-enriched people, the most-credited first, so a
|
|
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
|
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
|
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
|
ids = people_by_credits(db, "bio", limit)
|
|
done = failed = in_a_row = 0
|
|
for pid in ids:
|
|
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
|
break
|
|
person = db.get(PlexPerson, pid)
|
|
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
|
if person is None or person.tmdb_enriched_at is not None:
|
|
continue
|
|
try:
|
|
_fetch_person_into(tmdb, person)
|
|
except TmdbError as exc:
|
|
db.rollback()
|
|
failed, in_a_row = failed + 1, in_a_row + 1
|
|
log.warning(
|
|
"a TMDB person lookup in the batch failed",
|
|
extra=logs.event(
|
|
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
|
),
|
|
)
|
|
continue
|
|
db.commit()
|
|
done, in_a_row = done + 1, 0
|
|
return {"enriched": done, **({"failed": failed} if failed else {})}
|