peter
ab963c81d2
- people_enrich: a failed TMDB lookup or a dead headshot is skipped, not the end of the run (a dead file is marked and left for a week; three failures in a row end it); portraits wait for the Plex module switch; a no-Plex original keeps its own cache key, so the resize still replaces it; the job entry lives in the plex domain (run_people_enrich) - person-image takes only a headshot-shaped CDN url (no query variants of one file) - every synced row gets Hungarian genres, and a HU genre filter matches the word as picked too; migration 0094 fills the rows no overlay reached - a job run's audit row carries its result as data; audit_clear / demo_reset store their counts - one credit-column list for the prune and the enrichment order; _overlay_pass typed - tests: batch skip + stop, HU bio stored, dead headshots, module switch, the person-image route, the job body, genre filter + facet write + twin fallback, tagline/summary fallbacks, audit data, the reader's alphabet at every remaining collated() call site
677 lines
30 KiB
Python
677 lines
30 KiB
Python
"""TMDB enrichment for the mirrored Plex catalog (E5 S2).
|
|
|
|
A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items /
|
|
plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find).
|
|
Server-side ONLY (the api_key from sysconfig). Everything fetched is PERSISTED and served from the DB
|
|
row — there is never a live TMDB call at serve time (the user's explicit rule). Additive: the existing
|
|
Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the distinct TMDb score.
|
|
|
|
One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes
|
|
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
|
"""
|
|
|
|
import re
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from typing import cast
|
|
|
|
import httpx
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, sysconfig
|
|
from app.config import settings
|
|
from app.models import PlexItem, PlexPerson, PlexShow
|
|
from app.plex.cards import display_title
|
|
from app.plex.people import people_by_credits, warm_portraits
|
|
|
|
log = logs.get_logger(__name__)
|
|
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
|
_MIN_INTERVAL = 0.03
|
|
|
|
# The crew jobs worth surfacing on an info page, in DISPLAY-priority order (the stored crew is sorted by
|
|
# this — TMDB's own crew order is arbitrary and buries the lead roles). MOVIE and TV differ on purpose:
|
|
# on a TV show `Director`/`Editor`/`Director of Photography` are PER-EPISODE credits (dozens of names
|
|
# that would flood the list and bury the showrunner), so TV keys on `Creator` (from the payload's
|
|
# top-level `created_by`, not the crew list) + the show-level producers/writers instead.
|
|
_MOVIE_CREW_JOBS = (
|
|
"Director",
|
|
"Writer",
|
|
"Screenplay",
|
|
"Story",
|
|
"Producer",
|
|
"Executive Producer",
|
|
"Editor",
|
|
"Director of Photography",
|
|
"Original Music Composer",
|
|
)
|
|
_TV_CREW_JOBS = ("Creator", "Executive Producer", "Writer", "Producer", "Original Music Composer")
|
|
_MOVIE_PRIORITY = {job: i for i, job in enumerate(_MOVIE_CREW_JOBS)}
|
|
_TV_PRIORITY = {job: i for i, job in enumerate(_TV_CREW_JOBS)}
|
|
|
|
# append_to_response bundles the sub-resources into ONE request per title (keywords + credits +
|
|
# recommendations + images + external ids). TV uses aggregate_credits (per-person aggregated jobs).
|
|
_MOVIE_APPEND = "keywords,credits,recommendations,images,external_ids"
|
|
_TV_APPEND = "keywords,aggregate_credits,recommendations,images,external_ids"
|
|
|
|
|
|
class TmdbError(Exception):
|
|
"""Any transient failure talking to TMDB (network, non-404 HTTP, bad payload). The caller leaves
|
|
the row's `tmdb_enriched_at` NULL so the next run retries."""
|
|
|
|
|
|
class TmdbNotConfigured(TmdbError):
|
|
"""No TMDB API key is set — the module can't do anything."""
|
|
|
|
|
|
class TmdbClient:
|
|
"""Bound to the configured api_key + language. Use as a context manager so the httpx client closes."""
|
|
|
|
def __init__(self, db: Session, timeout: float = 30.0):
|
|
self.key = sysconfig.get_str(db, "tmdb_api_key")
|
|
if not self.key:
|
|
raise TmdbNotConfigured("TMDB API key is not configured")
|
|
self.base = settings.tmdb_api_base.rstrip("/")
|
|
self.language = sysconfig.get_str(db, "tmdb_language") or settings.tmdb_language
|
|
# `timeout` is caller-tunable so the request-path lazy person enrich can cap how long it may
|
|
# block a worker thread (the batch job keeps the generous default).
|
|
self._http = httpx.Client(timeout=timeout)
|
|
self._last = 0.0
|
|
|
|
def __enter__(self) -> "TmdbClient":
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self._http.close()
|
|
|
|
def close(self) -> None:
|
|
self._http.close()
|
|
|
|
def _throttle(self) -> None:
|
|
wait = self._last + _MIN_INTERVAL - time.monotonic()
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
self._last = time.monotonic()
|
|
|
|
def _get(self, path: str, params: dict | None = None) -> dict | None:
|
|
"""GET a TMDB path. Returns the JSON dict, None on a 404 (a definitive "no such id"), and raises
|
|
TmdbError on any transient failure (network / other HTTP status / non-JSON)."""
|
|
self._throttle()
|
|
p = {"api_key": self.key, "language": self.language, **(params or {})}
|
|
try:
|
|
r = self._http.get(f"{self.base}{path}", params=p)
|
|
except httpx.HTTPError as e:
|
|
# NEVER put str(e)/the URL in the message: the request URL carries `?api_key=<secret>` in
|
|
# its query string, and this message is logged (enrich_batch). Report the type + path only.
|
|
raise TmdbError(f"TMDB request error on {path}: {type(e).__name__}") from None
|
|
if r.status_code == 404:
|
|
return None
|
|
try:
|
|
r.raise_for_status()
|
|
except httpx.HTTPStatusError as e:
|
|
# Same reason: the HTTPStatusError string embeds the full URL incl. the api_key. Sever the
|
|
# cause (`from None`) so no traceback can resurface it, and log only status + path.
|
|
raise TmdbError(f"TMDB {path} returned {e.response.status_code}") from None
|
|
try:
|
|
return r.json()
|
|
except ValueError as e:
|
|
raise TmdbError("TMDB returned a non-JSON response") from e
|
|
|
|
def fetch(self, tmdb_id: int, media_type: str) -> dict | None:
|
|
"""Full enrichment payload for a movie or show (one request via append_to_response)."""
|
|
if media_type == "tv":
|
|
return self._get(f"/tv/{tmdb_id}", {"append_to_response": _TV_APPEND})
|
|
return self._get(f"/movie/{tmdb_id}", {"append_to_response": _MOVIE_APPEND})
|
|
|
|
def find(self, external_id: str, source: str) -> dict | None:
|
|
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
|
return self._get(f"/find/{external_id}", {"external_source": source})
|
|
|
|
def countries(self) -> list[dict] | None:
|
|
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
|
`_get`'s dict annotation does not describe)."""
|
|
return cast("list[dict] | None", self._get("/configuration/countries"))
|
|
|
|
def fetch_person(self, person_id: int) -> dict | None:
|
|
"""One person's bio + combined credits (E5 S6) + the bio's translations (E30 S4), in a single
|
|
append_to_response request."""
|
|
return self._get(
|
|
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
|
)
|
|
|
|
|
|
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
|
"""Map a stored canonical guid to a TMDB id. `tmdb:` is direct; `imdb:`/`tvdb:` go through /find
|
|
(picking the movie_results or tv_results list per `media_type`). Returns None for an unmatchable
|
|
guid — the caller then stamps the watermark so it isn't retried every run."""
|
|
if not guid or ":" not in guid:
|
|
return None
|
|
provider, _, ext = guid.partition(":")
|
|
if provider == "tmdb":
|
|
try:
|
|
return int(ext)
|
|
except ValueError:
|
|
# The caller stamps the watermark on a None, so this title is never looked at again:
|
|
# one unreadable guid costs it its artwork and its cast permanently.
|
|
log.info(
|
|
"a Plex guid named TMDB but did not carry a numeric id",
|
|
extra=logs.event("plex.tmdb.guid_unparsable", provider=provider),
|
|
)
|
|
return None
|
|
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
|
if not source:
|
|
return None
|
|
res = tmdb.find(ext, source)
|
|
if not res:
|
|
return None
|
|
items = res.get("tv_results" if media_type == "tv" else "movie_results") or []
|
|
return items[0].get("id") if items else None
|
|
|
|
|
|
def _dedup_crew(crew: list[dict]) -> list[dict]:
|
|
seen: set[tuple] = set()
|
|
out: list[dict] = []
|
|
for c in crew:
|
|
key = (c["name"], c["job"])
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
out.append(c)
|
|
return out
|
|
|
|
|
|
def parse_tmdb(payload: dict, media_type: str) -> dict:
|
|
"""Pure parser: pull the enrichment fields out of a TMDB movie/tv payload (append_to_response form).
|
|
Best-effort — any missing sub-resource is just omitted. Returns the row fields plus a `people` list
|
|
([{name, tmdb_id}] over cast+crew) for the plex_people id-linking. TMDB image fields are stored as
|
|
bare paths ("/abc.jpg"); the client builds the image.tmdb.org URL."""
|
|
is_tv = media_type == "tv"
|
|
votes = payload.get("vote_count") or 0
|
|
|
|
kw_container = payload.get("keywords") or {}
|
|
kw_list = kw_container.get("results") if is_tv else kw_container.get("keywords")
|
|
keywords = [k.get("name") for k in (kw_list or []) if k.get("name")][:20]
|
|
|
|
priority = _TV_PRIORITY if is_tv else _MOVIE_PRIORITY
|
|
credits = payload.get("aggregate_credits" if is_tv else "credits") or {}
|
|
crew: list[dict] = []
|
|
# A TV show's creators live at the top level (created_by), not in the crew list — surface them.
|
|
if is_tv:
|
|
for c in payload.get("created_by") or []:
|
|
if c.get("name"):
|
|
crew.append(
|
|
{
|
|
"name": c["name"],
|
|
"job": "Creator",
|
|
"tmdb_id": c.get("id"),
|
|
"profile_path": c.get("profile_path"),
|
|
}
|
|
)
|
|
for c in credits.get("crew") or []:
|
|
# Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person.
|
|
job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job")
|
|
if job in priority and c.get("name"):
|
|
crew.append(
|
|
{
|
|
"name": c["name"],
|
|
"job": job,
|
|
"tmdb_id": c.get("id"),
|
|
"profile_path": c.get("profile_path"),
|
|
}
|
|
)
|
|
# Dedup by (name, job) — a person can hold two distinct key roles (Coppola: Director + Writer) — then
|
|
# order by display priority so the lead roles come first, and cap.
|
|
crew = _dedup_crew(crew)
|
|
crew.sort(key=lambda c: priority.get(c["job"], 99))
|
|
crew = crew[:12]
|
|
|
|
# name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids).
|
|
people: list[dict] = []
|
|
for c in (credits.get("cast") or [])[:30]:
|
|
if c.get("name") and c.get("id"):
|
|
people.append({"name": c["name"], "tmdb_id": c["id"]})
|
|
for c in crew:
|
|
if c.get("tmdb_id"):
|
|
people.append({"name": c["name"], "tmdb_id": c["tmdb_id"]})
|
|
|
|
recs: list[dict] = []
|
|
for r in ((payload.get("recommendations") or {}).get("results") or [])[:20]:
|
|
recs.append(
|
|
{
|
|
"tmdb_id": r.get("id"),
|
|
"title": r.get("title") or r.get("name"),
|
|
"poster_path": r.get("poster_path"),
|
|
"media_type": r.get("media_type") or media_type,
|
|
}
|
|
)
|
|
|
|
backdrops = [
|
|
b.get("file_path")
|
|
for b in ((payload.get("images") or {}).get("backdrops") or [])
|
|
if b.get("file_path")
|
|
][:10]
|
|
|
|
return {
|
|
"tmdb_id": payload.get("id"),
|
|
# vote_average is meaningless with no votes (TMDB returns 0.0) → treat as unknown.
|
|
"tmdb_rating": payload.get("vote_average") if votes else None,
|
|
"tmdb_vote_count": votes or None,
|
|
# 0 is TMDB's "unknown" for money fields; TV has neither.
|
|
"budget": (payload.get("budget") or None) if not is_tv else None,
|
|
"revenue": (payload.get("revenue") or None) if not is_tv else None,
|
|
"keywords": keywords or None,
|
|
"crew": crew or None,
|
|
"studios": [
|
|
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
|
][:8]
|
|
or None,
|
|
# The ISO code beside TMDB's English name (E30): the client names the country in the reader's
|
|
# language from the code; the name is the fallback for a code it cannot name.
|
|
"production_countries": [
|
|
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
|
for c in (payload.get("production_countries") or [])
|
|
if c.get("name")
|
|
]
|
|
or None,
|
|
"recommendations": recs or None,
|
|
"backdrops": backdrops or None,
|
|
"people": people,
|
|
}
|
|
|
|
|
|
def apply_tmdb_detail(row, parsed: dict) -> None:
|
|
"""Persist the parsed TMDB fields onto a movie/show row. Does NOT touch full_cast / imdb_rating /
|
|
rating (Plex-owned). The caller stamps `tmdb_enriched_at` and links people separately."""
|
|
row.tmdb_id = parsed["tmdb_id"]
|
|
row.tmdb_rating = parsed["tmdb_rating"]
|
|
row.tmdb_vote_count = parsed["tmdb_vote_count"]
|
|
row.budget = parsed["budget"]
|
|
row.revenue = parsed["revenue"]
|
|
row.keywords = parsed["keywords"]
|
|
row.crew = parsed["crew"]
|
|
row.studios = parsed["studios"]
|
|
row.production_countries = parsed["production_countries"]
|
|
row.recommendations = parsed["recommendations"]
|
|
row.backdrops = parsed["backdrops"]
|
|
|
|
|
|
def _link_people(db: Session, links: dict[str, int]) -> None:
|
|
"""Upsert plex_people by name, stamping `tmdb_person_id` — keeping any id already set (first wins),
|
|
filling it where NULL, inserting a bare row where the person isn't cached yet. Deduped by name so a
|
|
single statement can't hit the conflict target twice (the same rule as people.enrich_row_people)."""
|
|
values = [{"name": nm, "tmdb_person_id": pid} for nm, pid in links.items() if nm and pid]
|
|
if not values:
|
|
return
|
|
stmt = pg_insert(PlexPerson).values(values)
|
|
db.execute(
|
|
stmt.on_conflict_do_update(
|
|
index_elements=[PlexPerson.name],
|
|
set_={
|
|
"tmdb_person_id": func.coalesce(
|
|
PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id
|
|
)
|
|
},
|
|
)
|
|
)
|
|
|
|
|
|
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
|
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
|
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
|
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
|
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
|
movies = (
|
|
db.execute(
|
|
select(PlexItem)
|
|
.where(
|
|
PlexItem.kind == "movie",
|
|
PlexItem.guid.is_not(None),
|
|
PlexItem.tmdb_enriched_at.is_(None),
|
|
)
|
|
.order_by(PlexItem.id)
|
|
.limit(limit)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
shows: list[PlexShow] = []
|
|
if len(movies) < limit:
|
|
shows = (
|
|
db.execute(
|
|
select(PlexShow)
|
|
.where(PlexShow.guid.is_not(None), PlexShow.tmdb_enriched_at.is_(None))
|
|
.order_by(PlexShow.id)
|
|
.limit(limit - len(movies))
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
|
|
processed = 0
|
|
links: dict[str, int] = {}
|
|
for row, media_type in [(m, "movie") for m in movies] + [(s, "tv") for s in shows]:
|
|
try:
|
|
tmdb_id = guid_to_tmdb_id(row.guid, media_type, tmdb)
|
|
payload = tmdb.fetch(tmdb_id, media_type) if tmdb_id else None
|
|
except TmdbError as e:
|
|
log.warning("tmdb enrich %s failed (will retry): %s", row.rating_key, e)
|
|
continue # transient — leave tmdb_enriched_at NULL for the next run
|
|
if payload:
|
|
parsed = parse_tmdb(payload, media_type)
|
|
apply_tmdb_detail(row, parsed)
|
|
for p in parsed["people"]:
|
|
links.setdefault(p["name"], p["tmdb_id"])
|
|
# Stamp even on a definitive no-match so an unresolvable title isn't re-fetched every run.
|
|
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
processed += 1
|
|
|
|
if links:
|
|
_link_people(db, links)
|
|
db.commit()
|
|
return processed
|
|
|
|
|
|
def _country(c: dict | str) -> dict:
|
|
"""One stored production country as served: `{code, name}`. A row enriched before E30 stored the
|
|
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
|
shows the name)."""
|
|
return c if isinstance(c, dict) else {"code": None, "name": c}
|
|
|
|
|
|
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
|
"""Give the countries stored before E30 (bare English names) their ISO codes, from TMDB's own
|
|
country list — the names came from that list, so they match it exactly. One request, and only
|
|
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
|
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
|
stale = [
|
|
row
|
|
for model in (PlexItem, PlexShow)
|
|
for row in db.execute(
|
|
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
|
).scalars()
|
|
]
|
|
if not stale:
|
|
return 0
|
|
codes = {
|
|
c["english_name"]: c["iso_3166_1"]
|
|
for c in tmdb.countries() or []
|
|
if c.get("english_name") and c.get("iso_3166_1")
|
|
}
|
|
for row in stale:
|
|
row.production_countries = [
|
|
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
|
for c in row.production_countries or []
|
|
]
|
|
db.commit()
|
|
return len(stale)
|
|
|
|
|
|
def run_enrich(db: Session) -> dict:
|
|
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
|
if not sysconfig.get_str(db, "tmdb_api_key"):
|
|
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
|
with TmdbClient(db) as tmdb:
|
|
coded = backfill_country_codes(db, tmdb)
|
|
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
|
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
|
|
|
|
|
def run_people_enrich(db: Session) -> dict:
|
|
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
|
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
|
without Plex."""
|
|
batch = settings.people_enrich_batch
|
|
if sysconfig.get_str(db, "tmdb_api_key"):
|
|
with TmdbClient(db) as tmdb:
|
|
bios = enrich_people_batch(db, tmdb, batch)
|
|
else:
|
|
bios = {"skipped": "tmdb_api_key not set"}
|
|
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
|
|
|
|
|
def resolve_library_guids(
|
|
db: Session,
|
|
entries: list[dict] | None,
|
|
*,
|
|
movie_lib_ids: list[int] | None = None,
|
|
show_lib_ids: list[int] | None = None,
|
|
) -> tuple[dict[str, PlexItem], dict[str, PlexShow]]:
|
|
"""Match a list of TMDB credits/recs (each `{tmdb_id, media_type, …}`) to our library rows by the
|
|
canonical guid `tmdb:<id>`, returning `(movies_by_guid, shows_by_guid)` of the ORM rows.
|
|
|
|
⭐ TMDB movie ids and tv ids are INDEPENDENT sequences (id 603 can be BOTH a movie and a show), so
|
|
each entry matches ONLY its own media_type's table — a movie credit against a library movie, a tv
|
|
credit against a library show; a blind numeric match would resolve to an unrelated same-id title.
|
|
Pass `movie_lib_ids`/`show_lib_ids` to scope to the enabled libraries (the person page); leave them
|
|
None to match across all libraries (the detail-page recs). An explicit empty list means "that side
|
|
has no enabled library" → its bucket is skipped entirely."""
|
|
if not entries:
|
|
return {}, {}
|
|
movie_guids = [
|
|
f"tmdb:{e['tmdb_id']}" for e in entries if e.get("tmdb_id") and e.get("media_type") != "tv"
|
|
]
|
|
tv_guids = [
|
|
f"tmdb:{e['tmdb_id']}" for e in entries if e.get("tmdb_id") and e.get("media_type") == "tv"
|
|
]
|
|
movies: dict[str, PlexItem] = {}
|
|
if movie_guids and movie_lib_ids != []:
|
|
conds = [PlexItem.kind == "movie", PlexItem.guid.in_(movie_guids)]
|
|
if movie_lib_ids is not None:
|
|
conds.append(PlexItem.library_id.in_(movie_lib_ids))
|
|
for m in db.execute(select(PlexItem).where(*conds)).scalars():
|
|
movies.setdefault(m.guid, m)
|
|
shows: dict[str, PlexShow] = {}
|
|
if tv_guids and show_lib_ids != []:
|
|
conds = [PlexShow.guid.in_(tv_guids)]
|
|
if show_lib_ids is not None:
|
|
conds.append(PlexShow.library_id.in_(show_lib_ids))
|
|
for s in db.execute(select(PlexShow).where(*conds)).scalars():
|
|
shows.setdefault(s.guid, s)
|
|
return movies, shows
|
|
|
|
|
|
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
|
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
|
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
|
the ones we have and merely display the rest. One we have is named as the library names it, in
|
|
`lang` (E30); one we lack keeps the title TMDB gave."""
|
|
if not recs:
|
|
return []
|
|
movies, shows = resolve_library_guids(db, recs)
|
|
out: list[dict] = []
|
|
for r in recs:
|
|
g = f"tmdb:{r.get('tmdb_id')}"
|
|
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
|
named = {"title": display_title(row, lang)} if row is not None else {}
|
|
out.append(
|
|
{
|
|
**r,
|
|
**named,
|
|
"in_library": row is not None,
|
|
"rating_key": row.rating_key if row else None,
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
|
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
|
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
|
if row.tmdb_enriched_at is None:
|
|
return None
|
|
return {
|
|
"tmdb_id": row.tmdb_id,
|
|
"rating": row.tmdb_rating,
|
|
"vote_count": row.tmdb_vote_count,
|
|
"budget": row.budget,
|
|
"revenue": row.revenue,
|
|
"keywords": row.keywords or [],
|
|
"crew": row.crew or [],
|
|
"studios": row.studios or [],
|
|
"production_countries": [_country(c) for c in row.production_countries or []],
|
|
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
|
"backdrops": row.backdrops or [],
|
|
}
|
|
|
|
|
|
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
|
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
|
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
|
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
|
|
|
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
|
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
|
# credits are often not the 15 a given library holds.
|
|
_KNOWN_FOR_MAX = 40
|
|
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
|
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
|
_GUEST_GENRES = {10763, 10764, 10767}
|
|
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
|
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
|
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
|
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
|
|
|
|
|
def _known_for_credit(c: dict, crew: bool) -> bool:
|
|
if crew:
|
|
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
|
if _SELF_CREDIT.match(c.get("character") or ""):
|
|
return False
|
|
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
|
|
|
|
|
def parse_person(payload: dict) -> dict:
|
|
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
|
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
|
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
|
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
|
combined = payload.get("combined_credits") or {}
|
|
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
|
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
|
]
|
|
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
|
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
|
# is two distinct credits and both must survive.
|
|
seen: set[tuple[int, str]] = set()
|
|
known: list[dict] = []
|
|
ranked = sorted(
|
|
credits,
|
|
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
|
reverse=True,
|
|
)
|
|
for c in ranked:
|
|
tid, mt = c.get("id"), c.get("media_type")
|
|
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
|
continue
|
|
seen.add((tid, mt))
|
|
known.append({"tmdb_id": tid, "media_type": mt, "title": c.get("title") or c.get("name")})
|
|
if len(known) >= _KNOWN_FOR_MAX:
|
|
break
|
|
return {
|
|
"biography": (payload.get("biography") or "").strip() or None,
|
|
"biography_hu": _translated_bio(payload, "hu"),
|
|
"birthday": payload.get("birthday") or None,
|
|
"deathday": payload.get("deathday") or None,
|
|
"place_of_birth": payload.get("place_of_birth") or None,
|
|
"known_for_department": payload.get("known_for_department") or None,
|
|
"also_known_as": payload.get("also_known_as") or None,
|
|
"known_for": known or None,
|
|
}
|
|
|
|
|
|
def _translated_bio(payload: dict, lang: str) -> str | None:
|
|
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
|
for tr in (payload.get("translations") or {}).get("translations") or []:
|
|
if tr.get("iso_639_1") == lang:
|
|
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
|
return None
|
|
|
|
|
|
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
|
row.biography = parsed["biography"]
|
|
row.biography_hu = parsed["biography_hu"]
|
|
row.birthday = parsed["birthday"]
|
|
row.deathday = parsed["deathday"]
|
|
row.place_of_birth = parsed["place_of_birth"]
|
|
row.known_for_department = parsed["known_for_department"]
|
|
row.also_known_as = parsed["also_known_as"]
|
|
row.known_for = parsed["known_for"]
|
|
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
|
|
|
|
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
|
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
|
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
|
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
|
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
|
if payload is None:
|
|
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
else:
|
|
apply_person(person, parse_person(payload))
|
|
|
|
|
|
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
|
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
|
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
|
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
|
open retries."""
|
|
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
|
return False
|
|
try:
|
|
# Called from a background task (off the request path — see people._enrich_person_bg), so it no
|
|
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
|
# doesn't tie up a threadpool worker for the default 30s.
|
|
with TmdbClient(db, timeout=8.0) as tmdb:
|
|
_fetch_person_into(tmdb, person)
|
|
except TmdbError as exc:
|
|
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
|
# front of the user right now shows an empty biography with nothing to say why.
|
|
log.warning(
|
|
"the TMDB person lookup failed",
|
|
extra=logs.event(
|
|
"plex.tmdb.person_enrich_failed",
|
|
person_id=person.id,
|
|
reason=type(exc).__name__,
|
|
),
|
|
)
|
|
return False
|
|
db.commit()
|
|
return True
|
|
|
|
|
|
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
|
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
|
_PEOPLE_FAILS_IN_A_ROW = 3
|
|
|
|
|
|
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
|
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
|
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
|
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
|
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
|
ids = people_by_credits(db, "bio", limit)
|
|
done = failed = in_a_row = 0
|
|
for pid in ids:
|
|
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
|
break
|
|
person = db.get(PlexPerson, pid)
|
|
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
|
if person is None or person.tmdb_enriched_at is not None:
|
|
continue
|
|
try:
|
|
_fetch_person_into(tmdb, person)
|
|
except TmdbError as exc:
|
|
db.rollback()
|
|
failed, in_a_row = failed + 1, in_a_row + 1
|
|
log.warning(
|
|
"a TMDB person lookup in the batch failed",
|
|
extra=logs.event(
|
|
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
|
),
|
|
)
|
|
continue
|
|
db.commit()
|
|
done, in_a_row = done + 1, 0
|
|
return {"enriched": done, **({"failed": failed} if failed else {})}
|