fix(plex,audit): E30 review round 1 — the people job keeps going, HU genres everywhere, results as data
- people_enrich: a failed TMDB lookup or a dead headshot is skipped, not the end of the run (a dead file is marked and left for a week; three failures in a row end it); portraits wait for the Plex module switch; a no-Plex original keeps its own cache key, so the resize still replaces it; the job entry lives in the plex domain (run_people_enrich) - person-image takes only a headshot-shaped CDN url (no query variants of one file) - every synced row gets Hungarian genres, and a HU genre filter matches the word as picked too; migration 0094 fills the rows no overlay reached - a job run's audit row carries its result as data; audit_clear / demo_reset store their counts - one credit-column list for the prune and the enrichment order; _overlay_pass typed - tests: batch skip + stop, HU bio stored, dead headshots, module switch, the person-image route, the job body, genre filter + facet write + twin fallback, tagline/summary fallbacks, audit data, the reader's alphabet at every remaining collated() call site
This commit is contained in:
+77
-41
@@ -4,6 +4,8 @@ orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and th
|
||||
re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
@@ -12,7 +14,7 @@ from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexPerson
|
||||
from app.plex import imgcache
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -97,14 +99,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
||||
return dirty
|
||||
|
||||
|
||||
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||
_CREDIT_COLUMNS = [
|
||||
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||
]
|
||||
|
||||
|
||||
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||
return f" {union} ".join(
|
||||
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||
for table, col in _CREDIT_COLUMNS
|
||||
)
|
||||
|
||||
|
||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||
# this set are orphans left by a library removal.
|
||||
_REFERENCED_PEOPLE_SQL = """
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
"""
|
||||
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||
|
||||
|
||||
def prune_orphan_people(db: Session) -> int:
|
||||
@@ -125,14 +136,9 @@ def prune_orphan_people(db: Session) -> int:
|
||||
|
||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||
# people_enrich job works in, so the faces met most often are local first.
|
||||
_CREDIT_COUNTS_SQL = """
|
||||
SELECT ref, count(*) AS n FROM (
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
) AS refs(ref) GROUP BY ref
|
||||
"""
|
||||
_CREDIT_COUNTS_SQL = (
|
||||
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||
)
|
||||
_NEEDS = {
|
||||
# linked to TMDB, bio never fetched
|
||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||
@@ -160,34 +166,49 @@ def people_by_credits(
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
# a new box re-caches instead of serving a stale size, and the full-size original under the old key is
|
||||
# removed as its small twin lands.
|
||||
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||
PORTRAIT_BOX = 320
|
||||
# A headshot Plex answered with an error for (a dead CDN file) is not asked for again this long, so a
|
||||
# few dead faces never hold up the rest of the prefetch.
|
||||
PORTRAIT_RETRY = timedelta(days=7)
|
||||
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): no query, no fragment. The
|
||||
# person-image proxy takes nothing else, so it cannot be made to cache endless variants of one file
|
||||
# (the image cache has no eviction).
|
||||
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + r"[A-Za-z0-9_./-]+")
|
||||
|
||||
|
||||
def is_portrait_url(url: str) -> bool:
|
||||
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||
|
||||
|
||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||
"""(the resized portrait's key, the full-size original's key)."""
|
||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||
return f"{base}_{PORTRAIT_BOX}", base
|
||||
|
||||
|
||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||
return imgcache.read(_portrait_keys(url)[0])
|
||||
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||
small, original = _portrait_keys(url)
|
||||
return imgcache.read(small) or imgcache.read(original)
|
||||
|
||||
|
||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||
"""Fetch one CDN headshot — resized through Plex when a Plex client is given, else the original
|
||||
straight from the public CDN — and cache it. Raises PlexError / httpx.HTTPError on a failed fetch."""
|
||||
key, legacy = _portrait_keys(url)
|
||||
if plex is not None:
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
else:
|
||||
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||
small, original = _portrait_keys(url)
|
||||
if plex is None:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(url)
|
||||
r.raise_for_status()
|
||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(key, data, ct)
|
||||
if plex is not None:
|
||||
imgcache.drop(legacy)
|
||||
imgcache.write(original, data, ct)
|
||||
return data, ct
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
imgcache.write(small, data, ct)
|
||||
imgcache.drop(original)
|
||||
return data, ct
|
||||
|
||||
|
||||
@@ -195,17 +216,23 @@ def cdn_url(photo_thumb: str) -> str | None:
|
||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||
_, _, q = photo_thumb.partition("?u=")
|
||||
url = unquote(q)
|
||||
return url if url.startswith(PERSON_IMG_HOST) else None
|
||||
return url if is_portrait_url(url) else None
|
||||
|
||||
|
||||
# Plex not answering this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||
_PLEX_DOWN_AFTER = 3
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||
person's face comes from local disk. Needs Plex (the resize); without it, nothing is prefetched —
|
||||
full-size originals for every person would cost gigabytes. A failed fetch is skipped; one failing
|
||||
Plex ends the run."""
|
||||
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||
answers with an error for is marked and left for PORTRAIT_RETRY; Plex not answering ends the run."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
try:
|
||||
plex = PlexClient(db)
|
||||
# silent-ok: Plex off is a setting, not a failure; the run's result says "disabled".
|
||||
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||
except PlexNotConfigured:
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
photos: dict[int, str] = {
|
||||
@@ -215,25 +242,34 @@ def warm_portraits(db: Session, limit: int) -> dict:
|
||||
).tuples()
|
||||
if thumb
|
||||
}
|
||||
done = failed = 0
|
||||
done = dead = down = 0
|
||||
with plex:
|
||||
for pid in people_by_credits(db, "portrait", None):
|
||||
if done + failed >= limit:
|
||||
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||
break
|
||||
url = cdn_url(photos.get(pid) or "")
|
||||
if url is None or imgcache.has(_portrait_keys(url)[0]):
|
||||
if url is None:
|
||||
continue
|
||||
key = _portrait_keys(url)[0]
|
||||
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||
continue
|
||||
try:
|
||||
cache_portrait(plex, url)
|
||||
done += 1
|
||||
except (PlexError, httpx.HTTPError) as exc:
|
||||
failed += 1
|
||||
done, down = done + 1, 0
|
||||
except PlexError as exc:
|
||||
# An error status is Plex's answer about this one file; no answer is Plex itself.
|
||||
if isinstance(exc.__cause__, httpx.HTTPStatusError):
|
||||
dead += 1
|
||||
imgcache.mark_miss(key)
|
||||
else:
|
||||
down += 1
|
||||
log.warning(
|
||||
"a portrait could not be cached",
|
||||
extra=logs.event(
|
||||
"plex.people.portrait_failed", person_id=pid, reason=type(exc).__name__
|
||||
"plex.people.portrait_failed",
|
||||
person_id=pid,
|
||||
reason=type(exc.__cause__).__name__,
|
||||
),
|
||||
)
|
||||
if failed >= 5:
|
||||
break
|
||||
failed = dead + down
|
||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
Reference in New Issue
Block a user