From af6faf476f1f5cc3672b54f3a93c1254c4f2bd05 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 19 Jul 2026 04:50:45 +0200 Subject: [PATCH] feat(plex): full-library cast backfill via a batched rich re-sync (S1b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lazy enrichment only covers viewed titles, so an actor's count/results were partial. Add a background backfill: each plex_sync rich-fetches a bounded batch (60) of not-yet-enriched movies/shows and persists their FULL cast, stamping people_enriched_at so each is fetched once — the whole library is covered over successive runs (migration 0057 adds the marker). Extracted the enrichment into a shared app/plex/people.py (cast_from_meta / enrich_row_people / prune_orphan_people + the person-image host/proxy) so the info-page lazy path and the batch share ONE implementation — no duplication; the info page also stamps the marker so the batch skips it. Verified: an 8-item batch grows those movies' cast_names 3->24 and fills plex_people (32->224); the info-page cast is intact after the _rich_meta refactor. (Plex cast-explore S1b) --- .../versions/0057_plex_ppl_enriched.py | 25 +++++ backend/app/models.py | 4 + backend/app/plex/people.py | 97 +++++++++++++++++++ backend/app/plex/sync.py | 66 ++++++++----- backend/app/routes/plex.py | 79 +++------------ 5 files changed, 184 insertions(+), 87 deletions(-) create mode 100644 backend/alembic/versions/0057_plex_ppl_enriched.py create mode 100644 backend/app/plex/people.py diff --git a/backend/alembic/versions/0057_plex_ppl_enriched.py b/backend/alembic/versions/0057_plex_ppl_enriched.py new file mode 100644 index 0000000..528d855 --- /dev/null +++ b/backend/alembic/versions/0057_plex_ppl_enriched.py @@ -0,0 +1,25 @@ +"""plex_items/plex_shows.people_enriched_at: full-cast backfill marker + +Revision ID: 0057_plex_ppl_enriched +Revises: 0056_plex_people +Create Date: 2026-07-19 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0057_plex_ppl_enriched" +down_revision: Union[str, None] = "0056_plex_people" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + for tbl in ("plex_items", "plex_shows"): + op.add_column(tbl, sa.Column("people_enriched_at", sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + for tbl in ("plex_items", "plex_shows"): + op.drop_column(tbl, "people_enriched_at") diff --git a/backend/app/models.py b/backend/app/models.py index bbe598b..3e67cfd 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1027,6 +1027,8 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin): directors: Mapped[list | None] = mapped_column(JSONB) cast_names: Mapped[list | None] = mapped_column(JSONB) people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector + # Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3. + people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) search_vector: Mapped[object | None] = mapped_column( TSVECTOR, Computed( @@ -1118,6 +1120,8 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin): # Cast + director names, space-joined — folded into search_vector (weight B) so the search box # finds titles by a person's name (and ranks them above summary-only mentions). people_text: Mapped[str | None] = mapped_column(Text) + # Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3. + people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) search_vector: Mapped[object | None] = mapped_column( TSVECTOR, Computed( diff --git a/backend/app/plex/people.py b/backend/app/plex/people.py new file mode 100644 index 0000000..7bcb57c --- /dev/null +++ b/backend/app/plex/people.py @@ -0,0 +1,97 @@ +"""Shared Plex people helpers: parse cast/crew from rich metadata, persist the full cast onto a movie +or show (cast_names + the people_text search blob + the plex_people headshot cache), and prune +orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich +re-sync (plex.sync), so the two paths never diverge.""" + +from urllib.parse import quote + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.models import PlexPerson + +# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted) +# so the user's browser makes no third-party request (privacy + self-host CSP friendliness). +PERSON_IMG_HOST = "https://metadata-static.plex.tv/" + + +def person_thumb_url(thumb: str | None) -> str | None: + """Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN.""" + t = str(thumb or "") + return f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None + + +def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]: + """[{name, role, thumb}] for the cast (Role tags) in a full Plex metadata dict — thumb is the + proxied person-image URL or None. Shared by the info-page response, the lazy enrichment, and the + re-sync so the parsing lives in one place.""" + out: list[dict] = [] + for role in (meta.get("Role") or [])[:limit]: + name = role.get("tag") + if not name: + continue + out.append( + {"name": name, "role": role.get("role"), "thumb": person_thumb_url(role.get("thumb"))} + ) + return out + + +def enrich_row_people(db: Session, row, cast: list[dict]) -> bool: + """Persist the FULLER cast onto the row's `cast_names` + `people_text`, and cache each cast + member's headshot in plex_people. The cheap section listing that drives the main sync only carries + the top ~3 cast, so this — run whenever a rich fetch happens (info-page view + the rich re-sync) — + is what makes the full displayed cast filterable. Returns True if anything was written (the caller + commits). Best-effort by contract: never the reason a read 500s.""" + names = list(dict.fromkeys(c["name"] for c in cast if c.get("name")))[:24] + if not names: + return False + dirty = False + # Grow/refresh cast_names only when the rich set adds names the stored (listing top-3) set lacks — + # avoid churn and never downgrade a fuller set back to the thin listing. + if set(names) - set(row.cast_names or []): + row.cast_names = names + row.people_text = " ".join(dict.fromkeys(list(row.directors or []) + names)) or None + dirty = True + # Headshot cache: upsert name -> proxied thumb (a later fetch fills a prior NULL; names without a + # whitelisted photo still get a row so a card can show a placeholder). + existing = {p.name: p for p in db.query(PlexPerson).filter(PlexPerson.name.in_(names))} + for c in cast: + nm, th = c.get("name"), c.get("thumb") + if not nm: + continue + p = existing.get(nm) + if p is None: + p = PlexPerson(name=nm, photo_thumb=th) + db.add(p) + existing[nm] = p + dirty = True + elif th and not p.photo_thumb: + p.photo_thumb = th + dirty = True + return dirty + + +# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside +# this set are orphans left by a library removal. +_REFERENCED_PEOPLE_SQL = """ + SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array' + UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array' + UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array' + UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array' +""" + + +def prune_orphan_people(db: Session) -> int: + """Delete plex_people (headshot cache) rows whose name no longer appears in any movie/show cast or + directors — orphaned by a library removal. Returns the number pruned. Commits its own work.""" + # `WHERE ref IS NOT NULL` matters: a JSON-null array element surfaces as a SQL NULL in the + # referenced set, and `name NOT IN (…, NULL)` is NULL (never TRUE) for every row — the classic + # NOT IN trap that would silently prune nothing. Filtering NULLs out keeps the delete correct. + res = db.execute( + text( + "DELETE FROM plex_people WHERE name NOT IN " + f"(SELECT ref FROM ({_REFERENCED_PEOPLE_SQL}) AS refs(ref) WHERE ref IS NOT NULL)" + ) + ) + db.commit() + return res.rowcount or 0 diff --git a/backend/app/plex/sync.py b/backend/app/plex/sync.py index 0eca551..058349d 100644 --- a/backend/app/plex/sync.py +++ b/backend/app/plex/sync.py @@ -12,11 +12,11 @@ Cast + intro/credit markers are fetched lazily on the item-detail/player path (P import logging from datetime import datetime, timezone -from sqlalchemy import text from sqlalchemy.orm import Session from app import sysconfig from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow +from app.plex import people from app.plex.client import PlexClient, PlexError, PlexNotConfigured log = logging.getLogger("siftlode.plex") @@ -130,6 +130,12 @@ def sync(db: Session) -> dict: _sync_shows(db, plex, lib, stats) _sync_collections(db, plex, lib, stats) db.commit() + # Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library + # becomes filterable over successive runs (the cheap listing above only stored the top ~3). + try: + stats["people_enriched"] = _enrich_people_batch(db, plex) + except Exception as e: # best-effort — never fail the sync over the backfill + log.warning("Plex people enrich batch failed: %s", e) except PlexNotConfigured as e: return {"skipped": str(e)} except PlexError as e: @@ -139,37 +145,53 @@ def sync(db: Session) -> dict: # Prune the headshot cache: a reconcile may have removed titles, orphaning people who no longer # appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata. try: - stats["people_pruned"] = prune_orphan_people(db) + stats["people_pruned"] = people.prune_orphan_people(db) except Exception as e: # best-effort housekeeping — never fail the sync over it log.warning("Plex people prune failed: %s", e) log.info("Plex sync done: %s", stats) return stats -# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside -# this set are orphans left by a library removal. -_REFERENCED_PEOPLE_SQL = """ - SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array' - UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array' - UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array' - UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array' -""" +# How many titles to rich-fetch per sync run for the cast backfill — bounds the extra Plex load; the +# whole library is covered over successive runs (each fetched item is stamped people_enriched_at once). +_ENRICH_BATCH = 60 -def prune_orphan_people(db: Session) -> int: - """Delete plex_people (headshot cache) rows whose name no longer appears in any movie/show cast or - directors — orphaned by a library removal. Returns the number pruned. Commits its own work.""" - # `WHERE ref IS NOT NULL` matters: a JSON-null array element would surface as a SQL NULL in the - # referenced set, and `name NOT IN (…, NULL)` is NULL (never TRUE) for every row — the classic - # NOT IN trap that would silently prune nothing. Filtering NULLs out keeps the delete correct. - res = db.execute( - text( - "DELETE FROM plex_people WHERE name NOT IN " - f"(SELECT ref FROM ({_REFERENCED_PEOPLE_SQL}) AS refs(ref) WHERE ref IS NOT NULL)" - ) +def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BATCH) -> int: + """Rich-fetch the FULL cast of up to `limit` not-yet-enriched movies/shows and persist it (via the + shared enrich_row_people). Oldest-id first, movies before shows. Stamps people_enriched_at only on + a SUCCESSFUL fetch, so a transient per-item failure retries next run rather than being lost. + Returns how many items were enriched this run.""" + movies = ( + db.query(PlexItem) + .filter(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None)) + .order_by(PlexItem.id) + .limit(limit) + .all() ) + remaining = limit - len(movies) + shows = ( + db.query(PlexShow) + .filter(PlexShow.people_enriched_at.is_(None)) + .order_by(PlexShow.id) + .limit(remaining) + .all() + if remaining > 0 + else [] + ) + now = datetime.now(timezone.utc) + n = 0 + for row in [*movies, *shows]: + try: + meta = plex.metadata(row.rating_key) or {} + except Exception as e: + log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e) + continue # leave people_enriched_at NULL so it's retried on a later run + people.enrich_row_people(db, row, people.cast_from_meta(meta)) + row.people_enriched_at = now + n += 1 db.commit() - return res.rowcount or 0 + return n def _upsert_library(db: Session, key: str, title: str, kind: str) -> PlexLibrary: diff --git a/backend/app/routes/plex.py b/backend/app/routes/plex.py index 48c4112..95c42a5 100644 --- a/backend/app/routes/plex.py +++ b/backend/app/routes/plex.py @@ -17,7 +17,6 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path -from urllib.parse import quote import httpx from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request, Response @@ -43,6 +42,7 @@ from app.models import ( User, ) from app.plex import paths as plex_paths +from app.plex import people as plex_people from app.plex import stream as plex_stream from app.plex import sync as plex_sync from app.plex import watch_sync as plex_watch @@ -1227,8 +1227,11 @@ def show_detail( pass # Lazy enrichment: persist the show's full cast so its cast members become filterable (same as the - # movie info page — the sync only stored the listing's top ~3). Best-effort. - if _enrich_people_from_rich(db, sh, rich): + # movie info page — the sync only stored the listing's top ~3), and stamp people_enriched_at so the + # backfill skips it. Best-effort. + if rich.get("cast"): + plex_people.enrich_row_people(db, sh, rich["cast"]) + sh.people_enriched_at = datetime.now(timezone.utc) try: db.commit() except Exception: @@ -1582,11 +1585,6 @@ _PROGRESS_MIN_S = 5 _FINISH_MARGIN_S = 30 -# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted) -# so the user's browser makes no third-party request (privacy + self-host CSP friendliness). -_PERSON_IMG_HOST = "https://metadata-static.plex.tv/" - - def _rich_meta(meta: dict) -> dict: """Pull the "info page" extras out of a full Plex metadata dict: an IMDb score + link, content rating, genres, director(s), studio, tagline, and cast (name/role/photo). Best-effort — any @@ -1612,21 +1610,6 @@ def _rich_meta(meta: dict) -> dict: if gid.startswith("imdb://"): imdb_id = gid.split("imdb://", 1)[1] break - cast = [] - for role in (meta.get("Role") or [])[:24]: - name = role.get("tag") - if not name: - continue - thumb = str(role.get("thumb") or "") - cast.append( - { - "name": name, - "role": role.get("role"), - "thumb": f"/api/plex/person-image?u={quote(thumb, safe='')}" - if thumb.startswith(_PERSON_IMG_HOST) - else None, - } - ) return { "imdb_rating": imdb_rating, "imdb_id": imdb_id, @@ -1636,47 +1619,10 @@ def _rich_meta(meta: dict) -> dict: "directors": [d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")][:4], "studio": meta.get("studio"), "tagline": meta.get("tagline"), - "cast": cast, + "cast": plex_people.cast_from_meta(meta), } -def _enrich_people_from_rich(db: Session, row, rich: dict) -> bool: - """Persist the FULLER cast (from an item's rich metadata) onto the row's `cast_names` + the - `people_text` search blob, and cache each cast member's headshot in plex_people. The cheap section - listing that drives the main sync only carries the top ~3 cast, so this — run when a rich fetch - happens (info-page view + the rich re-sync) — is what makes the full displayed cast filterable. - Returns True if anything was written (the caller commits). Best-effort: never the reason a read - 500s (the caller wraps the commit).""" - cast = rich.get("cast") or [] - names = list(dict.fromkeys(c["name"] for c in cast if c.get("name")))[:24] - if not names: - return False - dirty = False - # Grow/refresh cast_names only when the rich set adds names the stored (listing top-3) set lacks — - # avoid churn and never downgrade a fuller set back to the thin listing. - if set(names) - set(row.cast_names or []): - row.cast_names = names - row.people_text = " ".join(dict.fromkeys(list(row.directors or []) + names)) or None - dirty = True - # Headshot cache: upsert name -> proxied thumb (a later fetch fills a prior NULL; names without a - # whitelisted photo still get a row so a card can show a placeholder). - existing = {p.name: p for p in db.query(PlexPerson).filter(PlexPerson.name.in_(names))} - for c in cast: - nm, th = c.get("name"), c.get("thumb") - if not nm: - continue - p = existing.get(nm) - if p is None: - p = PlexPerson(name=nm, photo_thumb=th) - db.add(p) - existing[nm] = p - dirty = True - elif th and not p.photo_thumb: - p.photo_thumb = th - dirty = True - return dirty - - @router.get("/people/cards") def people_cards( names: str, @@ -1727,7 +1673,7 @@ def person_image(u: str, request: Request) -> Response: """Proxy a cast/crew photo from Plex's public metadata CDN (host-whitelisted; no token needed). Signed-session auth (no DB) so it, like /image, never holds a pool connection during the fetch.""" _require_session(request) - if not u.startswith(_PERSON_IMG_HOST): + if not u.startswith(plex_people.PERSON_IMG_HOST): raise HTTPException(status_code=400, detail="Unsupported image host") cache_key = "person_" + hashlib.sha1(u.encode()).hexdigest() hit = _img_cache_read(cache_key) @@ -1875,9 +1821,12 @@ def item_detail( pass # metadata extras are best-effort; any live-Plex failure must degrade, not 500 — playback works without them # Lazy enrichment: the rich fetch above carries the FULL cast (the sync only stored the listing's - # top ~3), so persist it onto this movie so every cast member shown here becomes filterable. A - # movie only — episodes aren't in the grid; their show's cast is enriched from /show. Best-effort. - if it.kind == "movie" and _enrich_people_from_rich(db, it, rich): + # top ~3), so persist it onto this movie so every cast member shown here becomes filterable, and + # stamp people_enriched_at so the backfill skips it. Movie only — episodes aren't in the grid; a + # show's cast is enriched from /show. Best-effort. + if it.kind == "movie" and rich.get("cast"): + plex_people.enrich_row_people(db, it, rich["cast"]) + it.people_enriched_at = datetime.now(timezone.utc) try: db.commit() except Exception: