feat(plex): the people are local — bios and portraits fetched ahead, not on the click (E30 S6)
A person's TMDB bio was fetched only when their page was first opened, off the request path, so the first open showed no bio and the page never asked again (2 of 18,202 linked people were ever enriched). - people_enrich scheduler job (every 10 min, 300 per run): linked people's bios from TMDB and their portraits into the image cache, the most-credited people first. A failed TMDB lookup ends the batch. - Portraits are resized through Plex's photo transcoder to a 320 box (~15 KB instead of ~170 KB; the person-image proxy uses the same cache key, and a full-size original is removed as its small twin lands). - The person response says `enriching` while a bio is on its way (a person the job hasn't reached yet). - "Known for" is ranked by vote count, without guest visits as themselves (talk shows had crowded out the films: Tom Cruise had 14 of 15), with directed works, from a 40-deep pool; the page shows up to 15. Migration 0093 un-stamps enriched people so the job refetches them. - A phased job result (backfill, people_enrich) now counts as a change for the audit trail; before, an automatic backfill run that added videos was never audited. - HU: the watch reconcile job reads "egyeztetés" (a leftover from the S5 sweep).
This commit is contained in:
@@ -56,6 +56,28 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def has(key: str) -> bool:
|
||||
"""A cached, non-empty file exists for `key` (a prefetch's cheap check — no read)."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
if (CACHE_DIR / f"{key}.{ext}").stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: a miss, as in read().
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def drop(key: str) -> None:
|
||||
"""Remove `key`'s file, whatever its extension (a superseded variant). Best-effort."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
(CACHE_DIR / f"{key}.{ext}").unlink(missing_ok=True)
|
||||
# silent-ok: a leftover original only costs disk; the small twin is already written.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def write(key: str, data: bytes, ct: str) -> None:
|
||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||
try:
|
||||
|
||||
+125
-1
@@ -3,13 +3,21 @@ or show (cast_names + the people_text search blob + the plex_people headshot cac
|
||||
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
||||
re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
from urllib.parse import quote
|
||||
import hashlib
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexPerson
|
||||
from app.plex import imgcache
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
||||
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
||||
@@ -113,3 +121,119 @@ def prune_orphan_people(db: Session) -> int:
|
||||
)
|
||||
db.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||
# people_enrich job works in, so the faces met most often are local first.
|
||||
_CREDIT_COUNTS_SQL = """
|
||||
SELECT ref, count(*) AS n FROM (
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
) AS refs(ref) GROUP BY ref
|
||||
"""
|
||||
_NEEDS = {
|
||||
# linked to TMDB, bio never fetched
|
||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||
# has a Plex headshot (whether it is cached is the disk's answer, not the row's)
|
||||
"portrait": "p.photo_thumb IS NOT NULL",
|
||||
}
|
||||
|
||||
|
||||
def people_by_credits(
|
||||
db: Session, need: Literal["bio", "portrait"], limit: int | None
|
||||
) -> list[int]:
|
||||
"""plex_people ids that `need` something, the most-credited first (ties by id)."""
|
||||
rows = db.execute(
|
||||
text(
|
||||
f"SELECT p.id FROM plex_people p LEFT JOIN ({_CREDIT_COUNTS_SQL}) c ON c.ref = p.name "
|
||||
f"WHERE {_NEEDS[need]} ORDER BY coalesce(c.n, 0) DESC, p.id"
|
||||
+ (" LIMIT :limit" if limit is not None else "")
|
||||
),
|
||||
{"limit": limit} if limit is not None else {},
|
||||
)
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# --- Portraits (E30 S6) ---------------------------------------------------------------------------
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
# a new box re-caches instead of serving a stale size, and the full-size original under the old key is
|
||||
# removed as its small twin lands.
|
||||
PORTRAIT_BOX = 320
|
||||
|
||||
|
||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||
return f"{base}_{PORTRAIT_BOX}", base
|
||||
|
||||
|
||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||
return imgcache.read(_portrait_keys(url)[0])
|
||||
|
||||
|
||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||
"""Fetch one CDN headshot — resized through Plex when a Plex client is given, else the original
|
||||
straight from the public CDN — and cache it. Raises PlexError / httpx.HTTPError on a failed fetch."""
|
||||
key, legacy = _portrait_keys(url)
|
||||
if plex is not None:
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
else:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(url)
|
||||
r.raise_for_status()
|
||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(key, data, ct)
|
||||
if plex is not None:
|
||||
imgcache.drop(legacy)
|
||||
return data, ct
|
||||
|
||||
|
||||
def cdn_url(photo_thumb: str) -> str | None:
|
||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||
_, _, q = photo_thumb.partition("?u=")
|
||||
url = unquote(q)
|
||||
return url if url.startswith(PERSON_IMG_HOST) else None
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||
person's face comes from local disk. Needs Plex (the resize); without it, nothing is prefetched —
|
||||
full-size originals for every person would cost gigabytes. A failed fetch is skipped; one failing
|
||||
Plex ends the run."""
|
||||
try:
|
||||
plex = PlexClient(db)
|
||||
# silent-ok: Plex off is a setting, not a failure; the run's result says "disabled".
|
||||
except PlexNotConfigured:
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
photos: dict[int, str] = {
|
||||
pid: thumb
|
||||
for pid, thumb in db.execute(
|
||||
select(PlexPerson.id, PlexPerson.photo_thumb).where(PlexPerson.photo_thumb.is_not(None))
|
||||
).tuples()
|
||||
if thumb
|
||||
}
|
||||
done = failed = 0
|
||||
with plex:
|
||||
for pid in people_by_credits(db, "portrait", None):
|
||||
if done + failed >= limit:
|
||||
break
|
||||
url = cdn_url(photos.get(pid) or "")
|
||||
if url is None or imgcache.has(_portrait_keys(url)[0]):
|
||||
continue
|
||||
try:
|
||||
cache_portrait(plex, url)
|
||||
done += 1
|
||||
except (PlexError, httpx.HTTPError) as exc:
|
||||
failed += 1
|
||||
log.warning(
|
||||
"a portrait could not be cached",
|
||||
extra=logs.event(
|
||||
"plex.people.portrait_failed", person_id=pid, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
if failed >= 5:
|
||||
break
|
||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
+82
-17
@@ -10,6 +10,7 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import cast
|
||||
@@ -23,6 +24,7 @@ from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
from app.plex.cards import display_title
|
||||
from app.plex.people import people_by_credits
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
@@ -315,8 +317,8 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
|
||||
|
||||
|
||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
||||
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||
movies = (
|
||||
@@ -501,23 +503,51 @@ def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
|
||||
|
||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open)
|
||||
# and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||
|
||||
_KNOWN_FOR_MAX = 15
|
||||
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
||||
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
||||
# credits are often not the 15 a given library holds.
|
||||
_KNOWN_FOR_MAX = 40
|
||||
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
||||
_GUEST_GENRES = {10763, 10764, 10767}
|
||||
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
||||
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
||||
|
||||
|
||||
def _known_for_credit(c: dict, crew: bool) -> bool:
|
||||
if crew:
|
||||
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
||||
if _SELF_CREDIT.match(c.get("character") or ""):
|
||||
return False
|
||||
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
||||
|
||||
|
||||
def parse_person(payload: dict) -> dict:
|
||||
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
||||
most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later."""
|
||||
credits = (payload.get("combined_credits") or {}).get("cast") or []
|
||||
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
||||
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
||||
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
||||
combined = payload.get("combined_credits") or {}
|
||||
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
||||
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
||||
]
|
||||
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
||||
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
||||
# is two distinct credits and both must survive.
|
||||
seen: set[tuple[int, str]] = set()
|
||||
known: list[dict] = []
|
||||
for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True):
|
||||
ranked = sorted(
|
||||
credits,
|
||||
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
for c in ranked:
|
||||
tid, mt = c.get("id"), c.get("media_type")
|
||||
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
||||
continue
|
||||
@@ -557,11 +587,22 @@ def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
||||
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
||||
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
||||
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
||||
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
|
||||
|
||||
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
"""Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it
|
||||
survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient
|
||||
error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the
|
||||
watermark with empty fields so we don't refetch it every open."""
|
||||
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
||||
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
||||
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
||||
open retries."""
|
||||
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return False
|
||||
try:
|
||||
@@ -569,7 +610,7 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
@@ -582,9 +623,33 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||
lands: a page open during the run already sees it. A transient TmdbError ends the batch (TMDB is
|
||||
degraded; hammering it helps nobody) and the rest wait for the next run."""
|
||||
ids = people_by_credits(db, "bio", limit)
|
||||
done = 0
|
||||
for pid in ids:
|
||||
person = db.get(PlexPerson, pid)
|
||||
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||
if person is None or person.tmdb_enriched_at is not None:
|
||||
continue
|
||||
try:
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"the TMDB person batch stopped on a failed lookup",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
return {"enriched": done, "failed": 1}
|
||||
db.commit()
|
||||
done += 1
|
||||
return {"enriched": done}
|
||||
|
||||
Reference in New Issue
Block a user