fix(plex,audit): E30 review round 1 — the people job keeps going, HU genres everywhere, results as data
- people_enrich: a failed TMDB lookup or a dead headshot is skipped, not the end of the run (a dead file is marked and left for a week; three failures in a row end it); portraits wait for the Plex module switch; a no-Plex original keeps its own cache key, so the resize still replaces it; the job entry lives in the plex domain (run_people_enrich) - person-image takes only a headshot-shaped CDN url (no query variants of one file) - every synced row gets Hungarian genres, and a HU genre filter matches the word as picked too; migration 0094 fills the rows no overlay reached - a job run's audit row carries its result as data; audit_clear / demo_reset store their counts - one credit-column list for the prune and the enrichment order; _overlay_pass typed - tests: batch skip + stop, HU bio stored, dead headshots, module switch, the person-image route, the job body, genre filter + facet write + twin fallback, tagline/summary fallbacks, audit data, the reader's alphabet at every remaining collated() call site
This commit is contained in:
@@ -0,0 +1,45 @@
|
|||||||
|
"""Hungarian genres on every Plex row, not only on overlaid ones (E30 review)
|
||||||
|
|
||||||
|
Revision ID: 0094_plex_genres_hu_everywhere
|
||||||
|
Revises: 0093_plex_person_known_for_votes
|
||||||
|
Create Date: 2026-09-30
|
||||||
|
|
||||||
|
The HU genre facets and filter read `genres_hu`, which only the overlay pass wrote — so a library with
|
||||||
|
no HU twin, or a guid-less row, offered English genres a Hungarian filter could never match. The sync
|
||||||
|
now writes `genres_hu` for every row (plex/sync.py `_apply_facet_fields`); this fills the rows it has
|
||||||
|
not reached yet, from their own English list through the same map.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from app.plex.genres import hu_genres
|
||||||
|
|
||||||
|
revision: str = "0094_plex_genres_hu_everywhere"
|
||||||
|
down_revision: str | None = "0093_plex_person_known_for_votes"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
for table in ("plex_items", "plex_shows"):
|
||||||
|
rows = conn.execute(
|
||||||
|
sa.text(
|
||||||
|
f"SELECT id, genres FROM {table} "
|
||||||
|
"WHERE genres_hu IS NULL AND jsonb_typeof(genres) = 'array'"
|
||||||
|
)
|
||||||
|
).fetchall()
|
||||||
|
updates = [
|
||||||
|
{"id": rid, "g": json.dumps(hu)} for rid, genres in rows if (hu := hu_genres(genres))
|
||||||
|
]
|
||||||
|
if updates:
|
||||||
|
conn.execute(
|
||||||
|
sa.text(f"UPDATE {table} SET genres_hu = CAST(:g AS jsonb) WHERE id = :id"), updates
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass # derived from `genres`; the sync writes the same values again
|
||||||
@@ -8,6 +8,9 @@ few hundred MB at most for a big library — acceptable, matching the download d
|
|||||||
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
from fastapi import Response
|
from fastapi import Response
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -78,6 +81,27 @@ def drop(key: str) -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def mark_miss(key: str) -> None:
|
||||||
|
"""Remember that `key`'s source could not be fetched (an empty `.miss` file; its mtime is when), so
|
||||||
|
a prefetch can leave a dead source alone for a while. Best-effort."""
|
||||||
|
try:
|
||||||
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
(CACHE_DIR / f"{key}.miss").touch()
|
||||||
|
# silent-ok: without the mark the dead source is only asked for again on the next run.
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def missed_within(key: str, age: timedelta) -> bool:
|
||||||
|
"""`key` was marked as a miss less than `age` ago."""
|
||||||
|
try:
|
||||||
|
mtime = (CACHE_DIR / f"{key}.miss").stat().st_mtime
|
||||||
|
# silent-ok: no mark (the usual case).
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return time.time() - mtime < age.total_seconds()
|
||||||
|
|
||||||
|
|
||||||
def write(key: str, data: bytes, ct: str) -> None:
|
def write(key: str, data: bytes, ct: str) -> None:
|
||||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||||
try:
|
try:
|
||||||
|
|||||||
+77
-41
@@ -4,6 +4,8 @@ orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and th
|
|||||||
re-sync (plex.sync), so the two paths never diverge."""
|
re-sync (plex.sync), so the two paths never diverge."""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import re
|
||||||
|
from datetime import timedelta
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
from urllib.parse import quote, unquote
|
from urllib.parse import quote, unquote
|
||||||
|
|
||||||
@@ -12,7 +14,7 @@ from sqlalchemy import select, text
|
|||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs
|
from app import logs, sysconfig
|
||||||
from app.models import PlexPerson
|
from app.models import PlexPerson
|
||||||
from app.plex import imgcache
|
from app.plex import imgcache
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
@@ -97,14 +99,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
|||||||
return dirty
|
return dirty
|
||||||
|
|
||||||
|
|
||||||
|
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||||
|
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||||
|
_CREDIT_COLUMNS = [
|
||||||
|
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||||
|
return f" {union} ".join(
|
||||||
|
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||||
|
for table, col in _CREDIT_COLUMNS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||||
# this set are orphans left by a library removal.
|
# this set are orphans left by a library removal.
|
||||||
_REFERENCED_PEOPLE_SQL = """
|
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def prune_orphan_people(db: Session) -> int:
|
def prune_orphan_people(db: Session) -> int:
|
||||||
@@ -125,14 +136,9 @@ def prune_orphan_people(db: Session) -> int:
|
|||||||
|
|
||||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||||
# people_enrich job works in, so the faces met most often are local first.
|
# people_enrich job works in, so the faces met most often are local first.
|
||||||
_CREDIT_COUNTS_SQL = """
|
_CREDIT_COUNTS_SQL = (
|
||||||
SELECT ref, count(*) AS n FROM (
|
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
)
|
||||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
UNION ALL SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
|
||||||
UNION ALL SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
) AS refs(ref) GROUP BY ref
|
|
||||||
"""
|
|
||||||
_NEEDS = {
|
_NEEDS = {
|
||||||
# linked to TMDB, bio never fetched
|
# linked to TMDB, bio never fetched
|
||||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||||
@@ -160,34 +166,49 @@ def people_by_credits(
|
|||||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||||
# a new box re-caches instead of serving a stale size, and the full-size original under the old key is
|
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||||
# removed as its small twin lands.
|
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||||
PORTRAIT_BOX = 320
|
PORTRAIT_BOX = 320
|
||||||
|
# A headshot Plex answered with an error for (a dead CDN file) is not asked for again this long, so a
|
||||||
|
# few dead faces never hold up the rest of the prefetch.
|
||||||
|
PORTRAIT_RETRY = timedelta(days=7)
|
||||||
|
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): no query, no fragment. The
|
||||||
|
# person-image proxy takes nothing else, so it cannot be made to cache endless variants of one file
|
||||||
|
# (the image cache has no eviction).
|
||||||
|
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + r"[A-Za-z0-9_./-]+")
|
||||||
|
|
||||||
|
|
||||||
|
def is_portrait_url(url: str) -> bool:
|
||||||
|
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||||
|
|
||||||
|
|
||||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||||
|
"""(the resized portrait's key, the full-size original's key)."""
|
||||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||||
return f"{base}_{PORTRAIT_BOX}", base
|
return f"{base}_{PORTRAIT_BOX}", base
|
||||||
|
|
||||||
|
|
||||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||||
return imgcache.read(_portrait_keys(url)[0])
|
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||||
|
small, original = _portrait_keys(url)
|
||||||
|
return imgcache.read(small) or imgcache.read(original)
|
||||||
|
|
||||||
|
|
||||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||||
"""Fetch one CDN headshot — resized through Plex when a Plex client is given, else the original
|
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||||
straight from the public CDN — and cache it. Raises PlexError / httpx.HTTPError on a failed fetch."""
|
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||||
key, legacy = _portrait_keys(url)
|
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||||
if plex is not None:
|
small, original = _portrait_keys(url)
|
||||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
if plex is None:
|
||||||
else:
|
|
||||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||||
r = c.get(url)
|
r = c.get(url)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||||
imgcache.write(key, data, ct)
|
imgcache.write(original, data, ct)
|
||||||
if plex is not None:
|
return data, ct
|
||||||
imgcache.drop(legacy)
|
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||||
|
imgcache.write(small, data, ct)
|
||||||
|
imgcache.drop(original)
|
||||||
return data, ct
|
return data, ct
|
||||||
|
|
||||||
|
|
||||||
@@ -195,17 +216,23 @@ def cdn_url(photo_thumb: str) -> str | None:
|
|||||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||||
_, _, q = photo_thumb.partition("?u=")
|
_, _, q = photo_thumb.partition("?u=")
|
||||||
url = unquote(q)
|
url = unquote(q)
|
||||||
return url if url.startswith(PERSON_IMG_HOST) else None
|
return url if is_portrait_url(url) else None
|
||||||
|
|
||||||
|
|
||||||
|
# Plex not answering this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||||
|
_PLEX_DOWN_AFTER = 3
|
||||||
|
|
||||||
|
|
||||||
def warm_portraits(db: Session, limit: int) -> dict:
|
def warm_portraits(db: Session, limit: int) -> dict:
|
||||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||||
person's face comes from local disk. Needs Plex (the resize); without it, nothing is prefetched —
|
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||||
full-size originals for every person would cost gigabytes. A failed fetch is skipped; one failing
|
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||||
Plex ends the run."""
|
answers with an error for is marked and left for PORTRAIT_RETRY; Plex not answering ends the run."""
|
||||||
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||||
|
return {"cached": 0, "skipped": "disabled"}
|
||||||
try:
|
try:
|
||||||
plex = PlexClient(db)
|
plex = PlexClient(db)
|
||||||
# silent-ok: Plex off is a setting, not a failure; the run's result says "disabled".
|
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||||
except PlexNotConfigured:
|
except PlexNotConfigured:
|
||||||
return {"cached": 0, "skipped": "disabled"}
|
return {"cached": 0, "skipped": "disabled"}
|
||||||
photos: dict[int, str] = {
|
photos: dict[int, str] = {
|
||||||
@@ -215,25 +242,34 @@ def warm_portraits(db: Session, limit: int) -> dict:
|
|||||||
).tuples()
|
).tuples()
|
||||||
if thumb
|
if thumb
|
||||||
}
|
}
|
||||||
done = failed = 0
|
done = dead = down = 0
|
||||||
with plex:
|
with plex:
|
||||||
for pid in people_by_credits(db, "portrait", None):
|
for pid in people_by_credits(db, "portrait", None):
|
||||||
if done + failed >= limit:
|
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||||
break
|
break
|
||||||
url = cdn_url(photos.get(pid) or "")
|
url = cdn_url(photos.get(pid) or "")
|
||||||
if url is None or imgcache.has(_portrait_keys(url)[0]):
|
if url is None:
|
||||||
|
continue
|
||||||
|
key = _portrait_keys(url)[0]
|
||||||
|
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
cache_portrait(plex, url)
|
cache_portrait(plex, url)
|
||||||
done += 1
|
done, down = done + 1, 0
|
||||||
except (PlexError, httpx.HTTPError) as exc:
|
except PlexError as exc:
|
||||||
failed += 1
|
# An error status is Plex's answer about this one file; no answer is Plex itself.
|
||||||
|
if isinstance(exc.__cause__, httpx.HTTPStatusError):
|
||||||
|
dead += 1
|
||||||
|
imgcache.mark_miss(key)
|
||||||
|
else:
|
||||||
|
down += 1
|
||||||
log.warning(
|
log.warning(
|
||||||
"a portrait could not be cached",
|
"a portrait could not be cached",
|
||||||
extra=logs.event(
|
extra=logs.event(
|
||||||
"plex.people.portrait_failed", person_id=pid, reason=type(exc).__name__
|
"plex.people.portrait_failed",
|
||||||
|
person_id=pid,
|
||||||
|
reason=type(exc.__cause__).__name__,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if failed >= 5:
|
failed = dead + down
|
||||||
break
|
|
||||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||||
|
|||||||
@@ -182,13 +182,15 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
|||||||
Genre chips are language-specific strings, so they filter the active-language genre column."""
|
Genre chips are language-specific strings, so they filter the active-language genre column."""
|
||||||
conds: list = []
|
conds: list = []
|
||||||
gsel = split_csv(p.get("genres"))
|
gsel = split_csv(p.get("genres"))
|
||||||
if gsel and lang == "hu":
|
|
||||||
# The HU column holds one Hungarian word per genre (plex/genres.py); a filter saved while it
|
|
||||||
# still offered "Action" is read as "Akció", so it keeps matching instead of emptying the grid.
|
|
||||||
gsel = list(dict.fromkeys(hu_genre(g) for g in gsel))
|
|
||||||
if gsel:
|
if gsel:
|
||||||
gcol = genres_col(model, lang)
|
gcol = genres_col(model, lang)
|
||||||
gc = [gcol.contains([g]) for g in gsel]
|
if lang == "hu":
|
||||||
|
# The HU column holds one Hungarian word per genre (plex/genres.py), so a filter saved while
|
||||||
|
# it offered "Action" is read as "Akció" too. The word as picked still matches as well: a row
|
||||||
|
# the sync has not reached since keeps its English (or Plex's mixed) list until it does.
|
||||||
|
gc = [or_(*(gcol.contains([w]) for w in dict.fromkeys((hu_genre(g), g)))) for g in gsel]
|
||||||
|
else:
|
||||||
|
gc = [gcol.contains([g]) for g in gsel]
|
||||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||||
crs = split_csv(p.get("content_ratings"))
|
crs = split_csv(p.get("content_ratings"))
|
||||||
if crs:
|
if crs:
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
|||||||
item-detail/player path (P2), not here.
|
item-detail/player path (P2), not here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
@@ -449,6 +450,10 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
|||||||
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
||||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||||
row.genres = _tags(meta, "Genre")
|
row.genres = _tags(meta, "Genre")
|
||||||
|
# Every row has Hungarian genres, not only one with an HU overlay twin (the overlay pass, which runs
|
||||||
|
# after this, puts the twin's own list over it): the HU genre facets and filter read this column,
|
||||||
|
# and an English word there could never match a filter read in Hungarian.
|
||||||
|
row.genres_hu = hu_genres(row.genres)
|
||||||
row.directors = _tags(meta, "Director")
|
row.directors = _tags(meta, "Director")
|
||||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||||
@@ -702,13 +707,19 @@ def _sync_overlay(
|
|||||||
|
|
||||||
|
|
||||||
def _overlay_pass(
|
def _overlay_pass(
|
||||||
db: Session, plex: PlexClient, lib: PlexLibrary, item_type: int, rows, stats: dict
|
db: Session,
|
||||||
):
|
plex: PlexClient,
|
||||||
|
lib: PlexLibrary,
|
||||||
|
item_type: int,
|
||||||
|
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||||
|
stats: dict,
|
||||||
|
) -> None:
|
||||||
"""One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
|
"""One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
|
||||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||||
by_guid: dict[str, list] = {}
|
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||||
for r in rows:
|
for r in rows:
|
||||||
by_guid.setdefault(r.guid, []).append(r)
|
if r.guid: # the passes select guid-bearing rows only
|
||||||
|
by_guid.setdefault(r.guid, []).append(r)
|
||||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
||||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
||||||
# The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
|
# The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from app import logs, sysconfig
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import PlexItem, PlexPerson, PlexShow
|
from app.models import PlexItem, PlexPerson, PlexShow
|
||||||
from app.plex.cards import display_title
|
from app.plex.cards import display_title
|
||||||
from app.plex.people import people_by_credits
|
from app.plex.people import people_by_credits, warm_portraits
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||||
@@ -417,6 +417,19 @@ def run_enrich(db: Session) -> dict:
|
|||||||
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
||||||
|
|
||||||
|
|
||||||
|
def run_people_enrich(db: Session) -> dict:
|
||||||
|
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
||||||
|
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||||
|
without Plex."""
|
||||||
|
batch = settings.people_enrich_batch
|
||||||
|
if sysconfig.get_str(db, "tmdb_api_key"):
|
||||||
|
with TmdbClient(db) as tmdb:
|
||||||
|
bios = enrich_people_batch(db, tmdb, batch)
|
||||||
|
else:
|
||||||
|
bios = {"skipped": "tmdb_api_key not set"}
|
||||||
|
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
||||||
|
|
||||||
|
|
||||||
def resolve_library_guids(
|
def resolve_library_guids(
|
||||||
db: Session,
|
db: Session,
|
||||||
entries: list[dict] | None,
|
entries: list[dict] | None,
|
||||||
@@ -627,14 +640,21 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
||||||
|
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
||||||
|
_PEOPLE_FAILS_IN_A_ROW = 3
|
||||||
|
|
||||||
|
|
||||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||||
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||||
lands: a page open during the run already sees it. A transient TmdbError ends the batch (TMDB is
|
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||||
degraded; hammering it helps nobody) and the rest wait for the next run."""
|
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||||
ids = people_by_credits(db, "bio", limit)
|
ids = people_by_credits(db, "bio", limit)
|
||||||
done = 0
|
done = failed = in_a_row = 0
|
||||||
for pid in ids:
|
for pid in ids:
|
||||||
|
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
||||||
|
break
|
||||||
person = db.get(PlexPerson, pid)
|
person = db.get(PlexPerson, pid)
|
||||||
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||||
if person is None or person.tmdb_enriched_at is not None:
|
if person is None or person.tmdb_enriched_at is not None:
|
||||||
@@ -643,13 +663,14 @@ def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
|||||||
_fetch_person_into(tmdb, person)
|
_fetch_person_into(tmdb, person)
|
||||||
except TmdbError as exc:
|
except TmdbError as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
|
failed, in_a_row = failed + 1, in_a_row + 1
|
||||||
log.warning(
|
log.warning(
|
||||||
"the TMDB person batch stopped on a failed lookup",
|
"a TMDB person lookup in the batch failed",
|
||||||
extra=logs.event(
|
extra=logs.event(
|
||||||
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return {"enriched": done, "failed": 1}
|
continue
|
||||||
db.commit()
|
db.commit()
|
||||||
done += 1
|
done, in_a_row = done + 1, 0
|
||||||
return {"enriched": done}
|
return {"enriched": done, **({"failed": failed} if failed else {})}
|
||||||
|
|||||||
@@ -834,7 +834,11 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
|||||||
demo = get_or_create_demo_user(db)
|
demo = get_or_create_demo_user(db)
|
||||||
seeded = reset_demo_state(db, demo)
|
seeded = reset_demo_state(db, demo)
|
||||||
audit.record(
|
audit.record(
|
||||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
db,
|
||||||
|
admin.id,
|
||||||
|
AuditAction.DEMO_RESET,
|
||||||
|
summary=f"{seeded} sample playlists re-seeded",
|
||||||
|
after={"playlists_seeded": seeded}, # the count as data: named in the reader's language
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||||
|
|||||||
@@ -66,6 +66,12 @@ def list_audit(
|
|||||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
audit.record(
|
||||||
|
db,
|
||||||
|
admin.id,
|
||||||
|
AuditAction.AUDIT_CLEAR,
|
||||||
|
summary=f"Cleared {n} audit entries",
|
||||||
|
after={"cleared": n}, # the count as data: the log names it in the reader's language
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "cleared": n}
|
return {"ok": True, "cleared": n}
|
||||||
|
|||||||
@@ -213,11 +213,12 @@ def _resume_push_superseded(user_id: int, item_id: int, intent: datetime) -> boo
|
|||||||
|
|
||||||
@router.get("/person-image")
|
@router.get("/person-image")
|
||||||
def person_image(u: str, request: Request) -> Response:
|
def person_image(u: str, request: Request) -> Response:
|
||||||
"""Proxy a cast/crew photo from Plex's public metadata CDN (host-whitelisted; no token needed),
|
"""Proxy a cast/crew photo from Plex's public metadata CDN (only a headshot-shaped URL on its host;
|
||||||
resized through Plex and cached (plex.people.cache_portrait; the people_enrich job fills the same
|
no token needed), resized through Plex and cached (plex.people.cache_portrait; the people_enrich job
|
||||||
cache ahead of time). Signed-session auth; like /image, the DB session is released before the fetch."""
|
fills the same cache ahead of time). Signed-session auth; like /image, the DB session is released
|
||||||
|
before the fetch."""
|
||||||
_require_session(request)
|
_require_session(request)
|
||||||
if not u.startswith(plex_people.PERSON_IMG_HOST):
|
if not plex_people.is_portrait_url(u):
|
||||||
raise refused(400, "image_host")
|
raise refused(400, "image_host")
|
||||||
hit = plex_people.portrait_cached(u)
|
hit = plex_people.portrait_cached(u)
|
||||||
if hit:
|
if hit:
|
||||||
|
|||||||
+10
-20
@@ -18,6 +18,7 @@ from app.downloads.gc import run_download_gc
|
|||||||
from app.models import SchedulerSetting, User
|
from app.models import SchedulerSetting, User
|
||||||
from app.plex.sync import sync as run_plex_sync
|
from app.plex.sync import sync as run_plex_sync
|
||||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||||
|
from app.plex.tmdb import run_people_enrich
|
||||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.state import is_sync_paused
|
from app.state import is_sync_paused
|
||||||
@@ -160,6 +161,9 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
|||||||
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
||||||
if actor_id is None and status == "ok" and not _run_changed(result):
|
if actor_id is None and status == "ok" and not _run_changed(result):
|
||||||
return
|
return
|
||||||
|
# The result as data too (E30): the audit log names it in the reader's language, as the inbox
|
||||||
|
# does; `summary` stays the English line of the row's own summary column.
|
||||||
|
data = _result_data(result)
|
||||||
try:
|
try:
|
||||||
audit.record(
|
audit.record(
|
||||||
db,
|
db,
|
||||||
@@ -168,7 +172,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
|||||||
target_type="job",
|
target_type="job",
|
||||||
target_id=job_id,
|
target_id=job_id,
|
||||||
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
||||||
after={"status": status, "summary": summary},
|
after={"status": status, "summary": summary, **({"result": data} if data else {})},
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -205,7 +209,7 @@ def _job(name: str, fn) -> None:
|
|||||||
)
|
)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||||
_audit_run(db, actor_id, name, "skipped", "sync paused", None)
|
_audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||||
return
|
return
|
||||||
with progress.bind(sink):
|
with progress.bind(sink):
|
||||||
result = fn(db)
|
result = fn(db)
|
||||||
@@ -252,7 +256,7 @@ _PAUSED = {"skipped": "sync paused"}
|
|||||||
def _result_data(result) -> dict | None:
|
def _result_data(result) -> dict | None:
|
||||||
"""A job's result as the client names it in the reader's language (E30): the dict itself (counts,
|
"""A job's result as the client names it in the reader's language (E30): the dict itself (counts,
|
||||||
a `skipped` / `reason` word, a nested dict per phase), `enrich`'s bare count as one. The one-line
|
a `skipped` / `reason` word, a nested dict per phase), `enrich`'s bare count as one. The one-line
|
||||||
`_summarize` is the English fallback — and what the audit log keeps."""
|
`_summarize` is the English fallback."""
|
||||||
if isinstance(result, dict):
|
if isinstance(result, dict):
|
||||||
return result or None
|
return result or None
|
||||||
if isinstance(result, int) and not isinstance(result, bool):
|
if isinstance(result, int) and not isinstance(result, bool):
|
||||||
@@ -410,23 +414,9 @@ def _tmdb_enrich_job() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _people_enrich_job() -> None:
|
def _people_enrich_job() -> None:
|
||||||
# Make the people local (E30 S6): a batch of person bios from TMDB (the most-credited first) and a
|
# Make the people local (E30 S6): a batch of person bios from TMDB and a batch of portraits resized
|
||||||
# batch of portraits resized through Plex into the image cache, so a person page and a cast strip
|
# through Plex, so a person page and a cast strip never wait on an outside service.
|
||||||
# never wait on an outside service. Each half is a no-op without its key / without Plex.
|
_job("people_enrich", run_people_enrich)
|
||||||
def work(db):
|
|
||||||
from app import sysconfig
|
|
||||||
from app.plex import people as plex_people
|
|
||||||
from app.plex import tmdb as plex_tmdb
|
|
||||||
|
|
||||||
batch = settings.people_enrich_batch
|
|
||||||
if sysconfig.get_str(db, "tmdb_api_key"):
|
|
||||||
with plex_tmdb.TmdbClient(db) as tmdb:
|
|
||||||
bios = plex_tmdb.enrich_people_batch(db, tmdb, batch)
|
|
||||||
else:
|
|
||||||
bios = {"skipped": "tmdb_api_key not set"}
|
|
||||||
return {"bios": bios, "portraits": plex_people.warm_portraits(db, batch)}
|
|
||||||
|
|
||||||
_job("people_enrich", work)
|
|
||||||
|
|
||||||
|
|
||||||
def _audit_gc_job() -> None:
|
def _audit_gc_job() -> None:
|
||||||
|
|||||||
@@ -207,3 +207,96 @@ def test_channel_discovery_keeps_that_order_after_enriching(db, monkeypatch, ove
|
|||||||
_seed_discovery(db, user, synced=False)
|
_seed_discovery(db, user, synced=False)
|
||||||
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
||||||
assert [c["title"] for c in r.json()] == HU
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
# --- The other lists the server puts in the reader's alphabet (E30 review): each call site, once. ---
|
||||||
|
|
||||||
|
|
||||||
|
def _others(db, n_offset: int = 0) -> list[User]:
|
||||||
|
users = [
|
||||||
|
User(email=f"{w.lower()}@example.com", display_name=w) for w in WORDS
|
||||||
|
] # emails and names in WORDS' order
|
||||||
|
db.add_all(users)
|
||||||
|
db.flush()
|
||||||
|
return users
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_share_and_message_recipients_are_in_the_readers_alphabet(
|
||||||
|
db, monkeypatch, override_dep
|
||||||
|
):
|
||||||
|
from app.models import MessageKey
|
||||||
|
|
||||||
|
user = _user(db, "hu")
|
||||||
|
others = _others(db)
|
||||||
|
db.add_all(
|
||||||
|
MessageKey(
|
||||||
|
user_id=u.id,
|
||||||
|
public_key="k",
|
||||||
|
wrapped_private_key="w",
|
||||||
|
salt="s",
|
||||||
|
wrap_iv="i",
|
||||||
|
key_check="c",
|
||||||
|
)
|
||||||
|
for u in others
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
client = _client(db, user, monkeypatch, override_dep)
|
||||||
|
emails = [r["email"] for r in client.get("/api/downloads/recipients").json()]
|
||||||
|
assert emails == [f"{w.lower()}@example.com" for w in HU]
|
||||||
|
names = [r["name"] for r in client.get("/api/messages/users").json()]
|
||||||
|
assert names == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_blocked_channels_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
from app.models import BlockedChannel
|
||||||
|
|
||||||
|
user = _user(db, "hu")
|
||||||
|
db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS))
|
||||||
|
db.flush()
|
||||||
|
db.add_all(BlockedChannel(user_id=user.id, channel_id=f"UC{i}") for i in range(len(WORDS)))
|
||||||
|
db.commit()
|
||||||
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/blocked/list")
|
||||||
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_plex_libraries_and_collections_are_in_the_readers_alphabet(
|
||||||
|
db, monkeypatch, override_dep
|
||||||
|
):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
libs = [
|
||||||
|
PlexLibrary(plex_key=str(i), title=w, kind="movie", enabled=True)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
]
|
||||||
|
db.add_all(libs)
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
PlexCollection(rating_key=f"c{i}", library_id=libs[0].id, title=w, child_count=1)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
client = _client(db, user, monkeypatch, override_dep)
|
||||||
|
assert [lib["title"] for lib in client.get("/api/plex/libraries").json()["libraries"]] == HU
|
||||||
|
cols = client.get("/api/plex/collections").json()["collections"]
|
||||||
|
assert [c["title"] for c in cols] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_persons_titles_of_one_year_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
PlexItem(
|
||||||
|
rating_key=f"m{i}",
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title=f"EN {i}",
|
||||||
|
title_hu=w,
|
||||||
|
year=2000,
|
||||||
|
cast_names=["Anna"],
|
||||||
|
)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
body = _client(db, user, monkeypatch, override_dep).get("/api/plex/person/Anna?lang=hu").json()
|
||||||
|
assert [m["title"] for m in body["movies"]] == HU
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from app.auth import current_user
|
|||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
|
from app.plex.cards import display_summary, display_title
|
||||||
from app.models import (
|
from app.models import (
|
||||||
PlexCollection,
|
PlexCollection,
|
||||||
PlexItem,
|
PlexItem,
|
||||||
@@ -57,7 +58,9 @@ def _seed(db):
|
|||||||
library_id=lib.id,
|
library_id=lib.id,
|
||||||
kind="movie",
|
kind="movie",
|
||||||
title="EN Only",
|
title="EN Only",
|
||||||
genres=["Comedy"], # no HU twin
|
summary="English only.",
|
||||||
|
tagline="English only tagline.",
|
||||||
|
genres=["Comedy"], # no HU twin, never reached by a sync since: no genres_hu
|
||||||
year=2002,
|
year=2002,
|
||||||
added_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
added_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||||
people_enriched_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
people_enriched_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||||
@@ -125,6 +128,33 @@ def test_a_genre_filter_saved_in_english_still_matches_in_hungarian(db, monkeypa
|
|||||||
assert _titles(resp) == {"Magyar Cím"}
|
assert _titles(resp) == {"Magyar Cím"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_genre_the_hungarian_list_offers_in_english_still_matches(db, monkeypatch, override_dep):
|
||||||
|
"""E30 review: a row the sync has not given Hungarian genres yet is listed under its English word
|
||||||
|
(the facets read coalesce(genres_hu, genres)); filtering by that word must find it."""
|
||||||
|
user = _seed(db)
|
||||||
|
facets = _get(db, user, monkeypatch, override_dep, "/api/plex/facets?scope=movie&lang=hu")
|
||||||
|
assert "Comedy" in {g["value"] for g in facets.json()["genres"]}
|
||||||
|
resp = _get(
|
||||||
|
db, user, monkeypatch, override_dep, "/api/plex/library?scope=movie&lang=hu&genres=Comedy"
|
||||||
|
)
|
||||||
|
assert _titles(resp) == {"EN Only"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_title_without_a_hungarian_twin_reads_in_english(db, monkeypatch, override_dep):
|
||||||
|
user = _seed(db)
|
||||||
|
body = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m2?lang=hu").json()
|
||||||
|
assert (body["title"], body["summary"], body["tagline"]) == (
|
||||||
|
"EN Only",
|
||||||
|
"English only.",
|
||||||
|
"English only tagline.",
|
||||||
|
)
|
||||||
|
row = PlexItem(title="T", summary="S")
|
||||||
|
assert (display_title(row, "hu"), display_summary(row, "hu")) == ("T", "S")
|
||||||
|
row.title_hu, row.summary_hu = "HT", "HS"
|
||||||
|
assert (display_title(row, "hu"), display_summary(row, "hu")) == ("HT", "HS")
|
||||||
|
assert (display_title(row, "en"), display_summary(row, "en")) == ("T", "S")
|
||||||
|
|
||||||
|
|
||||||
def test_item_detail_language_and_rating_split(db, monkeypatch, override_dep):
|
def test_item_detail_language_and_rating_split(db, monkeypatch, override_dep):
|
||||||
user = _seed(db)
|
user = _seed(db)
|
||||||
resp = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m1?lang=hu")
|
resp = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m1?lang=hu")
|
||||||
|
|||||||
@@ -334,6 +334,42 @@ def test_the_overlay_gives_every_row_one_hungarian_word_per_genre(db):
|
|||||||
assert lone.tagline_hu is None
|
assert lone.tagline_hu is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_twin_without_genres_keeps_the_rows_own_hungarian_list(db):
|
||||||
|
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||||
|
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||||
|
db.add_all([en_lib, hu_lib])
|
||||||
|
db.flush()
|
||||||
|
row = PlexItem(
|
||||||
|
rating_key="m1",
|
||||||
|
library_id=en_lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title="Heat",
|
||||||
|
guid="tmdb:1",
|
||||||
|
genres=["Crime"],
|
||||||
|
)
|
||||||
|
db.add(row)
|
||||||
|
db.commit()
|
||||||
|
_sync_overlay(
|
||||||
|
db,
|
||||||
|
_StubPlex([_hu_movie("h1", "Szemtől szemben", "HU.", [], "tmdb://1")]),
|
||||||
|
hu_lib,
|
||||||
|
en_lib.id,
|
||||||
|
{},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(row)
|
||||||
|
assert (row.title_hu, row.genres_hu) == ("Szemtől szemben", ["Bűnügyi"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_synced_row_gets_hungarian_genres():
|
||||||
|
"""E30 review: not only an overlaid row — the HU facets and filter read genres_hu for all."""
|
||||||
|
from app.plex.sync import _apply_facet_fields
|
||||||
|
|
||||||
|
row = PlexItem(title="Solo")
|
||||||
|
_apply_facet_fields(row, {"Genre": [{"tag": "Drama"}, {"tag": "Action"}]})
|
||||||
|
assert (row.genres, row.genres_hu) == (["Drama", "Action"], ["Dráma", "Akció"])
|
||||||
|
|
||||||
|
|
||||||
def test_the_genre_map_is_idempotent_and_keeps_an_unknown_word():
|
def test_the_genre_map_is_idempotent_and_keeps_an_unknown_word():
|
||||||
assert hu_genre("science fiction") == "Sci-Fi" # case-insensitive
|
assert hu_genre("science fiction") == "Sci-Fi" # case-insensitive
|
||||||
assert hu_genre("Dráma") == "Dráma"
|
assert hu_genre("Dráma") == "Dráma"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ DB lane (Postgres); the portrait cache runs in a temp directory.
|
|||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app import scheduler
|
from app import scheduler
|
||||||
@@ -32,13 +33,13 @@ def _library(db):
|
|||||||
class _Tmdb:
|
class _Tmdb:
|
||||||
"""A TmdbClient stand-in: records the ids asked for, fails on `fail_on`."""
|
"""A TmdbClient stand-in: records the ids asked for, fails on `fail_on`."""
|
||||||
|
|
||||||
def __init__(self, fail_on=None):
|
def __init__(self, fail_on=()):
|
||||||
self.asked: list[int] = []
|
self.asked: list[int] = []
|
||||||
self.fail_on = fail_on
|
self.fail_on = set(fail_on)
|
||||||
|
|
||||||
def fetch_person(self, pid):
|
def fetch_person(self, pid):
|
||||||
self.asked.append(pid)
|
self.asked.append(pid)
|
||||||
if pid == self.fail_on:
|
if pid in self.fail_on:
|
||||||
raise plex_tmdb.TmdbError("429")
|
raise plex_tmdb.TmdbError("429")
|
||||||
return {"biography": f"Bio {pid}."}
|
return {"biography": f"Bio {pid}."}
|
||||||
|
|
||||||
@@ -67,22 +68,52 @@ def test_the_batch_takes_the_most_credited_linked_people_first(db):
|
|||||||
assert db.query(PlexPerson).filter_by(name="Rare").one().tmdb_enriched_at is None # next run
|
assert db.query(PlexPerson).filter_by(name="Rare").one().tmdb_enriched_at is None # next run
|
||||||
|
|
||||||
|
|
||||||
def test_a_failed_lookup_ends_the_batch_and_keeps_what_landed(db):
|
def test_a_failed_lookup_is_skipped_and_the_rest_still_land(db):
|
||||||
db.add_all([PlexPerson(name="A", tmdb_person_id=1), PlexPerson(name="B", tmdb_person_id=2)])
|
db.add_all([PlexPerson(name="A", tmdb_person_id=1), PlexPerson(name="B", tmdb_person_id=2)])
|
||||||
db.add(PlexPerson(name="C", tmdb_person_id=3))
|
db.add(PlexPerson(name="C", tmdb_person_id=3))
|
||||||
db.commit()
|
db.commit()
|
||||||
tmdb = _Tmdb(fail_on=2)
|
tmdb = _Tmdb(fail_on={2})
|
||||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 1, "failed": 1} # type: ignore[arg-type]
|
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 2, "failed": 1} # type: ignore[arg-type]
|
||||||
assert tmdb.asked == [1, 2] # C is not asked: TMDB is struggling
|
assert tmdb.asked == [1, 2, 3] # B refused: skipped, not the end of the batch
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
stamped = {p.name: p.tmdb_enriched_at is not None for p in db.query(PlexPerson)}
|
stamped = {p.name: p.tmdb_enriched_at is not None for p in db.query(PlexPerson)}
|
||||||
assert stamped == {"A": True, "B": False, "C": False}
|
assert stamped == {"A": True, "B": False, "C": True} # B is tried again next run
|
||||||
|
|
||||||
|
|
||||||
|
def test_failures_in_a_row_end_the_batch(db):
|
||||||
|
db.add_all([PlexPerson(name=f"P{i}", tmdb_person_id=i) for i in range(1, 7)])
|
||||||
|
db.commit()
|
||||||
|
tmdb = _Tmdb(fail_on={1, 2, 3, 4, 5, 6}) # TMDB is down
|
||||||
|
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 0, "failed": 3} # type: ignore[arg-type]
|
||||||
|
assert tmdb.asked == [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_batch_stores_the_hungarian_biography(db):
|
||||||
|
db.add(PlexPerson(name="A", tmdb_person_id=1))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
class _WithHu(_Tmdb):
|
||||||
|
def fetch_person(self, pid):
|
||||||
|
return {
|
||||||
|
"biography": "English.",
|
||||||
|
"translations": {
|
||||||
|
"translations": [{"iso_639_1": "hu", "data": {"biography": "Magyar."}}]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
assert plex_tmdb.enrich_people_batch(db, _WithHu(), 10) == {"enriched": 1} # type: ignore[arg-type]
|
||||||
|
db.expire_all()
|
||||||
|
person = db.query(PlexPerson).one()
|
||||||
|
assert (person.biography, person.biography_hu) == ("English.", "Magyar.")
|
||||||
|
|
||||||
|
|
||||||
class _Plex:
|
class _Plex:
|
||||||
def __init__(self, fail=False):
|
"""A PlexClient stand-in. `down`: Plex does not answer; `dead`: urls Plex answers with an error."""
|
||||||
|
|
||||||
|
def __init__(self, down=False, dead=()):
|
||||||
self.asked: list[tuple[str, int | None]] = []
|
self.asked: list[tuple[str, int | None]] = []
|
||||||
self.fail = fail
|
self.down = down
|
||||||
|
self.dead = set(dead)
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
return self
|
return self
|
||||||
@@ -92,8 +123,14 @@ class _Plex:
|
|||||||
|
|
||||||
def image_bytes(self, url, width=None, height=None):
|
def image_bytes(self, url, width=None, height=None):
|
||||||
self.asked.append((url, width))
|
self.asked.append((url, width))
|
||||||
if self.fail:
|
if self.down:
|
||||||
raise plex_people.PlexError("down")
|
raise plex_people.PlexError("down") from httpx.ConnectError("refused")
|
||||||
|
if url in self.dead:
|
||||||
|
req = httpx.Request("GET", url)
|
||||||
|
status = httpx.HTTPStatusError(
|
||||||
|
"404", request=req, response=httpx.Response(404, request=req)
|
||||||
|
)
|
||||||
|
raise plex_people.PlexError("404") from status
|
||||||
return b"small", "image/jpeg"
|
return b"small", "image/jpeg"
|
||||||
|
|
||||||
|
|
||||||
@@ -103,7 +140,26 @@ def cache_dir(tmp_path, monkeypatch):
|
|||||||
return tmp_path
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache_dir, monkeypatch):
|
@pytest.fixture
|
||||||
|
def plex_on(monkeypatch):
|
||||||
|
"""The Plex module switched on (the job's gate), with `plex` as its client."""
|
||||||
|
state = {"enabled": True, "plex": _Plex()}
|
||||||
|
monkeypatch.setattr(plex_people.sysconfig, "get_bool", lambda d, k: state["enabled"])
|
||||||
|
monkeypatch.setattr(plex_people, "PlexClient", lambda d: state["plex"])
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def _faces(db, n):
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
PlexPerson(name=f"P{i}", photo_thumb=plex_people.person_thumb_url(CDN + f"{i}.jpg"))
|
||||||
|
for i in range(n)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache_dir, plex_on):
|
||||||
lib = _library(db)
|
lib = _library(db)
|
||||||
_movie(db, lib, "a", ["Often"])
|
_movie(db, lib, "a", ["Often"])
|
||||||
_movie(db, lib, "b", ["Often", "Cached"])
|
_movie(db, lib, "b", ["Often", "Cached"])
|
||||||
@@ -115,23 +171,21 @@ def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
small, legacy = plex_people._portrait_keys(CDN + "often.jpg")
|
small, original = plex_people._portrait_keys(CDN + "often.jpg")
|
||||||
imgcache.write(legacy, b"a big original", "image/jpeg") # from before E30 S6
|
imgcache.write(original, b"a big original", "image/jpeg") # cached without Plex
|
||||||
imgcache.write(plex_people._portrait_keys(CDN + "cached.jpg")[0], b"x", "image/jpeg")
|
imgcache.write(plex_people._portrait_keys(CDN + "cached.jpg")[0], b"x", "image/jpeg")
|
||||||
plex = _Plex()
|
plex = plex_on["plex"]
|
||||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: plex)
|
|
||||||
|
|
||||||
assert plex_people.warm_portraits(db, 10) == {"cached": 1}
|
assert plex_people.warm_portraits(db, 10) == {"cached": 1}
|
||||||
assert plex.asked == [
|
assert plex.asked == [
|
||||||
(CDN + "often.jpg", plex_people.PORTRAIT_BOX)
|
(CDN + "often.jpg", plex_people.PORTRAIT_BOX)
|
||||||
] # resized, the cached one skipped
|
] # resized, the cached one skipped
|
||||||
assert imgcache.read(small) == (b"small", "image/jpeg")
|
assert imgcache.read(small) == (b"small", "image/jpeg")
|
||||||
assert not imgcache.has(legacy)
|
assert not imgcache.has(original)
|
||||||
|
|
||||||
|
|
||||||
def test_portraits_wait_for_plex(db, cache_dir, monkeypatch):
|
def test_portraits_wait_for_plex(db, cache_dir, plex_on, monkeypatch):
|
||||||
db.add(PlexPerson(name="P", photo_thumb=plex_people.person_thumb_url(CDN + "p.jpg")))
|
_faces(db, 1)
|
||||||
db.commit()
|
|
||||||
|
|
||||||
def not_configured(d):
|
def not_configured(d):
|
||||||
raise plex_people.PlexNotConfigured()
|
raise plex_people.PlexNotConfigured()
|
||||||
@@ -141,18 +195,99 @@ def test_portraits_wait_for_plex(db, cache_dir, monkeypatch):
|
|||||||
assert list(cache_dir.iterdir()) == [] # no full-size originals behind Plex's back
|
assert list(cache_dir.iterdir()) == [] # no full-size originals behind Plex's back
|
||||||
|
|
||||||
|
|
||||||
def test_a_failing_plex_ends_the_portrait_run(db, cache_dir, monkeypatch):
|
def test_portraits_wait_for_the_plex_module_to_be_on(db, cache_dir, plex_on):
|
||||||
db.add_all(
|
_faces(db, 1)
|
||||||
[
|
plex_on["enabled"] = False # switched off, its server still set up
|
||||||
PlexPerson(name=f"P{i}", photo_thumb=plex_people.person_thumb_url(CDN + f"{i}.jpg"))
|
assert plex_people.warm_portraits(db, 10) == {"cached": 0, "skipped": "disabled"}
|
||||||
for i in range(8)
|
assert plex_on["plex"].asked == []
|
||||||
]
|
|
||||||
)
|
|
||||||
db.commit()
|
def test_plex_not_answering_ends_the_portrait_run(db, cache_dir, plex_on):
|
||||||
plex = _Plex(fail=True)
|
_faces(db, 8)
|
||||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: plex)
|
plex_on["plex"] = _Plex(down=True)
|
||||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 5}
|
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 3}
|
||||||
assert len(plex.asked) == 5
|
assert len(plex_on["plex"].asked) == 3
|
||||||
|
assert not list(cache_dir.glob("*.miss")) # Plex was down, the files are not to blame
|
||||||
|
|
||||||
|
|
||||||
|
def test_dead_headshots_are_left_alone_and_the_rest_still_land(db, cache_dir, plex_on):
|
||||||
|
_faces(db, 8)
|
||||||
|
dead = {CDN + f"{i}.jpg" for i in range(6)} # more dead files than any failure limit
|
||||||
|
plex_on["plex"] = _Plex(dead=dead)
|
||||||
|
assert plex_people.warm_portraits(db, 100) == {"cached": 2, "failed": 6}
|
||||||
|
|
||||||
|
plex_on["plex"] = again = _Plex(dead=dead)
|
||||||
|
assert plex_people.warm_portraits(db, 100) == {"cached": 0}
|
||||||
|
assert again.asked == [] # dead ones marked, live ones cached: nothing to ask for
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_person_image_route(db, cache_dir, monkeypatch):
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import app.state as state
|
||||||
|
from app.main import app
|
||||||
|
from app.routes.plex import playback
|
||||||
|
|
||||||
|
monkeypatch.setattr(state, "_configured_cache", True)
|
||||||
|
monkeypatch.setattr(playback, "_require_session", lambda request: 1)
|
||||||
|
client = TestClient(app)
|
||||||
|
url = CDN + "face.jpg"
|
||||||
|
small, original = plex_people._portrait_keys(url)
|
||||||
|
|
||||||
|
def get(u):
|
||||||
|
return client.get("/api/plex/person-image", params={"u": u})
|
||||||
|
|
||||||
|
# Only a headshot-shaped url on the CDN host: no query variants of one file, no other host.
|
||||||
|
for bad in (url + "?n=1", url + "#x", "https://example.com/people/a.jpg"):
|
||||||
|
assert get(bad).status_code == 400, bad
|
||||||
|
|
||||||
|
# No Plex: the original from the CDN, kept under its own key (the job resizes it later).
|
||||||
|
def not_configured(d):
|
||||||
|
raise plex_people.PlexNotConfigured()
|
||||||
|
|
||||||
|
class _Cdn:
|
||||||
|
def __init__(self, *a, **k):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get(self, u):
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=b"original",
|
||||||
|
headers={"content-type": "image/jpeg"},
|
||||||
|
request=httpx.Request("GET", u),
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(playback, "PlexClient", not_configured)
|
||||||
|
monkeypatch.setattr(plex_people.httpx, "Client", _Cdn)
|
||||||
|
resp = get(url)
|
||||||
|
assert (resp.status_code, resp.content) == (200, b"original")
|
||||||
|
assert imgcache.has(original) and not imgcache.has(small)
|
||||||
|
|
||||||
|
# With Plex: resized, and it replaces the original.
|
||||||
|
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex())
|
||||||
|
imgcache.drop(original)
|
||||||
|
resp = get(url)
|
||||||
|
assert (resp.status_code, resp.content) == (200, b"small")
|
||||||
|
assert imgcache.has(small)
|
||||||
|
assert get(url).content == b"small" # the cache answers now
|
||||||
|
|
||||||
|
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex(down=True))
|
||||||
|
assert get(CDN + "other.jpg").status_code == 502
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_people_job_runs_its_halves_without_a_tmdb_key(db, cache_dir, plex_on, monkeypatch):
|
||||||
|
_faces(db, 1)
|
||||||
|
monkeypatch.setattr(plex_tmdb.sysconfig, "get_str", lambda d, k: "")
|
||||||
|
assert plex_tmdb.run_people_enrich(db) == {
|
||||||
|
"bios": {"skipped": "tmdb_api_key not set"},
|
||||||
|
"portraits": {"cached": 1},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_a_phase_count_is_a_change_worth_an_audit_row():
|
def test_a_phase_count_is_a_change_worth_an_audit_row():
|
||||||
|
|||||||
@@ -67,3 +67,22 @@ def test_a_failed_run_drops_the_previous_result(db):
|
|||||||
rss = _snapshot_job("rss_poll")
|
rss = _snapshot_job("rss_poll")
|
||||||
assert rss["last_error"] == "boom"
|
assert rss["last_error"] == "boom"
|
||||||
assert rss["result"] is None
|
assert rss["result"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_job_runs_audit_row_carries_its_result_as_data(db):
|
||||||
|
"""E30 review: the audit log names a run's result in the reader's language, as the inbox does —
|
||||||
|
from data, not from the English one-liner (a nested phase there was a Python dict repr)."""
|
||||||
|
from app.models import AuditLog, User
|
||||||
|
|
||||||
|
admin = User(email="auditrun@example.com")
|
||||||
|
db.add(admin)
|
||||||
|
db.commit()
|
||||||
|
scheduler._audit_run(db, None, "people_enrich", "ok", "x", {"bios": {"enriched": 2}})
|
||||||
|
scheduler._audit_run(db, admin.id, "rss_poll", "skipped", "sync paused", scheduler._PAUSED)
|
||||||
|
scheduler._audit_run(db, admin.id, "rss_poll", "error", "boom", None)
|
||||||
|
rows = db.execute(select(AuditLog).order_by(AuditLog.id)).scalars().all()
|
||||||
|
assert [r.after for r in rows] == [
|
||||||
|
{"status": "ok", "summary": "x", "result": {"bios": {"enriched": 2}}},
|
||||||
|
{"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}},
|
||||||
|
{"status": "error", "summary": "boom"},
|
||||||
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user