fix(plex,audit): E30 review round 1 — the people job keeps going, HU genres everywhere, results as data
- people_enrich: a failed TMDB lookup or a dead headshot is skipped, not the end of the run (a dead file is marked and left for a week; three failures in a row end it); portraits wait for the Plex module switch; a no-Plex original keeps its own cache key, so the resize still replaces it; the job entry lives in the plex domain (run_people_enrich) - person-image takes only a headshot-shaped CDN url (no query variants of one file) - every synced row gets Hungarian genres, and a HU genre filter matches the word as picked too; migration 0094 fills the rows no overlay reached - a job run's audit row carries its result as data; audit_clear / demo_reset store their counts - one credit-column list for the prune and the enrichment order; _overlay_pass typed - tests: batch skip + stop, HU bio stored, dead headshots, module switch, the person-image route, the job body, genre filter + facet write + twin fallback, tagline/summary fallbacks, audit data, the reader's alphabet at every remaining collated() call site
This commit is contained in:
@@ -6,6 +6,7 @@ DB lane (Postgres); the portrait cache runs in a temp directory.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import scheduler
|
||||
@@ -32,13 +33,13 @@ def _library(db):
|
||||
class _Tmdb:
|
||||
"""A TmdbClient stand-in: records the ids asked for, fails on `fail_on`."""
|
||||
|
||||
def __init__(self, fail_on=None):
|
||||
def __init__(self, fail_on=()):
|
||||
self.asked: list[int] = []
|
||||
self.fail_on = fail_on
|
||||
self.fail_on = set(fail_on)
|
||||
|
||||
def fetch_person(self, pid):
|
||||
self.asked.append(pid)
|
||||
if pid == self.fail_on:
|
||||
if pid in self.fail_on:
|
||||
raise plex_tmdb.TmdbError("429")
|
||||
return {"biography": f"Bio {pid}."}
|
||||
|
||||
@@ -67,22 +68,52 @@ def test_the_batch_takes_the_most_credited_linked_people_first(db):
|
||||
assert db.query(PlexPerson).filter_by(name="Rare").one().tmdb_enriched_at is None # next run
|
||||
|
||||
|
||||
def test_a_failed_lookup_ends_the_batch_and_keeps_what_landed(db):
|
||||
def test_a_failed_lookup_is_skipped_and_the_rest_still_land(db):
|
||||
db.add_all([PlexPerson(name="A", tmdb_person_id=1), PlexPerson(name="B", tmdb_person_id=2)])
|
||||
db.add(PlexPerson(name="C", tmdb_person_id=3))
|
||||
db.commit()
|
||||
tmdb = _Tmdb(fail_on=2)
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 1, "failed": 1} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2] # C is not asked: TMDB is struggling
|
||||
tmdb = _Tmdb(fail_on={2})
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 2, "failed": 1} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2, 3] # B refused: skipped, not the end of the batch
|
||||
db.expire_all()
|
||||
stamped = {p.name: p.tmdb_enriched_at is not None for p in db.query(PlexPerson)}
|
||||
assert stamped == {"A": True, "B": False, "C": False}
|
||||
assert stamped == {"A": True, "B": False, "C": True} # B is tried again next run
|
||||
|
||||
|
||||
def test_failures_in_a_row_end_the_batch(db):
|
||||
db.add_all([PlexPerson(name=f"P{i}", tmdb_person_id=i) for i in range(1, 7)])
|
||||
db.commit()
|
||||
tmdb = _Tmdb(fail_on={1, 2, 3, 4, 5, 6}) # TMDB is down
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 0, "failed": 3} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2, 3]
|
||||
|
||||
|
||||
def test_the_batch_stores_the_hungarian_biography(db):
|
||||
db.add(PlexPerson(name="A", tmdb_person_id=1))
|
||||
db.commit()
|
||||
|
||||
class _WithHu(_Tmdb):
|
||||
def fetch_person(self, pid):
|
||||
return {
|
||||
"biography": "English.",
|
||||
"translations": {
|
||||
"translations": [{"iso_639_1": "hu", "data": {"biography": "Magyar."}}]
|
||||
},
|
||||
}
|
||||
|
||||
assert plex_tmdb.enrich_people_batch(db, _WithHu(), 10) == {"enriched": 1} # type: ignore[arg-type]
|
||||
db.expire_all()
|
||||
person = db.query(PlexPerson).one()
|
||||
assert (person.biography, person.biography_hu) == ("English.", "Magyar.")
|
||||
|
||||
|
||||
class _Plex:
|
||||
def __init__(self, fail=False):
|
||||
"""A PlexClient stand-in. `down`: Plex does not answer; `dead`: urls Plex answers with an error."""
|
||||
|
||||
def __init__(self, down=False, dead=()):
|
||||
self.asked: list[tuple[str, int | None]] = []
|
||||
self.fail = fail
|
||||
self.down = down
|
||||
self.dead = set(dead)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
@@ -92,8 +123,14 @@ class _Plex:
|
||||
|
||||
def image_bytes(self, url, width=None, height=None):
|
||||
self.asked.append((url, width))
|
||||
if self.fail:
|
||||
raise plex_people.PlexError("down")
|
||||
if self.down:
|
||||
raise plex_people.PlexError("down") from httpx.ConnectError("refused")
|
||||
if url in self.dead:
|
||||
req = httpx.Request("GET", url)
|
||||
status = httpx.HTTPStatusError(
|
||||
"404", request=req, response=httpx.Response(404, request=req)
|
||||
)
|
||||
raise plex_people.PlexError("404") from status
|
||||
return b"small", "image/jpeg"
|
||||
|
||||
|
||||
@@ -103,7 +140,26 @@ def cache_dir(tmp_path, monkeypatch):
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache_dir, monkeypatch):
|
||||
@pytest.fixture
|
||||
def plex_on(monkeypatch):
|
||||
"""The Plex module switched on (the job's gate), with `plex` as its client."""
|
||||
state = {"enabled": True, "plex": _Plex()}
|
||||
monkeypatch.setattr(plex_people.sysconfig, "get_bool", lambda d, k: state["enabled"])
|
||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: state["plex"])
|
||||
return state
|
||||
|
||||
|
||||
def _faces(db, n):
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name=f"P{i}", photo_thumb=plex_people.person_thumb_url(CDN + f"{i}.jpg"))
|
||||
for i in range(n)
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache_dir, plex_on):
|
||||
lib = _library(db)
|
||||
_movie(db, lib, "a", ["Often"])
|
||||
_movie(db, lib, "b", ["Often", "Cached"])
|
||||
@@ -115,23 +171,21 @@ def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
small, legacy = plex_people._portrait_keys(CDN + "often.jpg")
|
||||
imgcache.write(legacy, b"a big original", "image/jpeg") # from before E30 S6
|
||||
small, original = plex_people._portrait_keys(CDN + "often.jpg")
|
||||
imgcache.write(original, b"a big original", "image/jpeg") # cached without Plex
|
||||
imgcache.write(plex_people._portrait_keys(CDN + "cached.jpg")[0], b"x", "image/jpeg")
|
||||
plex = _Plex()
|
||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: plex)
|
||||
plex = plex_on["plex"]
|
||||
|
||||
assert plex_people.warm_portraits(db, 10) == {"cached": 1}
|
||||
assert plex.asked == [
|
||||
(CDN + "often.jpg", plex_people.PORTRAIT_BOX)
|
||||
] # resized, the cached one skipped
|
||||
assert imgcache.read(small) == (b"small", "image/jpeg")
|
||||
assert not imgcache.has(legacy)
|
||||
assert not imgcache.has(original)
|
||||
|
||||
|
||||
def test_portraits_wait_for_plex(db, cache_dir, monkeypatch):
|
||||
db.add(PlexPerson(name="P", photo_thumb=plex_people.person_thumb_url(CDN + "p.jpg")))
|
||||
db.commit()
|
||||
def test_portraits_wait_for_plex(db, cache_dir, plex_on, monkeypatch):
|
||||
_faces(db, 1)
|
||||
|
||||
def not_configured(d):
|
||||
raise plex_people.PlexNotConfigured()
|
||||
@@ -141,18 +195,99 @@ def test_portraits_wait_for_plex(db, cache_dir, monkeypatch):
|
||||
assert list(cache_dir.iterdir()) == [] # no full-size originals behind Plex's back
|
||||
|
||||
|
||||
def test_a_failing_plex_ends_the_portrait_run(db, cache_dir, monkeypatch):
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name=f"P{i}", photo_thumb=plex_people.person_thumb_url(CDN + f"{i}.jpg"))
|
||||
for i in range(8)
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
plex = _Plex(fail=True)
|
||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: plex)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 5}
|
||||
assert len(plex.asked) == 5
|
||||
def test_portraits_wait_for_the_plex_module_to_be_on(db, cache_dir, plex_on):
|
||||
_faces(db, 1)
|
||||
plex_on["enabled"] = False # switched off, its server still set up
|
||||
assert plex_people.warm_portraits(db, 10) == {"cached": 0, "skipped": "disabled"}
|
||||
assert plex_on["plex"].asked == []
|
||||
|
||||
|
||||
def test_plex_not_answering_ends_the_portrait_run(db, cache_dir, plex_on):
|
||||
_faces(db, 8)
|
||||
plex_on["plex"] = _Plex(down=True)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 3}
|
||||
assert len(plex_on["plex"].asked) == 3
|
||||
assert not list(cache_dir.glob("*.miss")) # Plex was down, the files are not to blame
|
||||
|
||||
|
||||
def test_dead_headshots_are_left_alone_and_the_rest_still_land(db, cache_dir, plex_on):
|
||||
_faces(db, 8)
|
||||
dead = {CDN + f"{i}.jpg" for i in range(6)} # more dead files than any failure limit
|
||||
plex_on["plex"] = _Plex(dead=dead)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 2, "failed": 6}
|
||||
|
||||
plex_on["plex"] = again = _Plex(dead=dead)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0}
|
||||
assert again.asked == [] # dead ones marked, live ones cached: nothing to ask for
|
||||
|
||||
|
||||
def test_the_person_image_route(db, cache_dir, monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.main import app
|
||||
from app.routes.plex import playback
|
||||
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
monkeypatch.setattr(playback, "_require_session", lambda request: 1)
|
||||
client = TestClient(app)
|
||||
url = CDN + "face.jpg"
|
||||
small, original = plex_people._portrait_keys(url)
|
||||
|
||||
def get(u):
|
||||
return client.get("/api/plex/person-image", params={"u": u})
|
||||
|
||||
# Only a headshot-shaped url on the CDN host: no query variants of one file, no other host.
|
||||
for bad in (url + "?n=1", url + "#x", "https://example.com/people/a.jpg"):
|
||||
assert get(bad).status_code == 400, bad
|
||||
|
||||
# No Plex: the original from the CDN, kept under its own key (the job resizes it later).
|
||||
def not_configured(d):
|
||||
raise plex_people.PlexNotConfigured()
|
||||
|
||||
class _Cdn:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def get(self, u):
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=b"original",
|
||||
headers={"content-type": "image/jpeg"},
|
||||
request=httpx.Request("GET", u),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(playback, "PlexClient", not_configured)
|
||||
monkeypatch.setattr(plex_people.httpx, "Client", _Cdn)
|
||||
resp = get(url)
|
||||
assert (resp.status_code, resp.content) == (200, b"original")
|
||||
assert imgcache.has(original) and not imgcache.has(small)
|
||||
|
||||
# With Plex: resized, and it replaces the original.
|
||||
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex())
|
||||
imgcache.drop(original)
|
||||
resp = get(url)
|
||||
assert (resp.status_code, resp.content) == (200, b"small")
|
||||
assert imgcache.has(small)
|
||||
assert get(url).content == b"small" # the cache answers now
|
||||
|
||||
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex(down=True))
|
||||
assert get(CDN + "other.jpg").status_code == 502
|
||||
|
||||
|
||||
def test_the_people_job_runs_its_halves_without_a_tmdb_key(db, cache_dir, plex_on, monkeypatch):
|
||||
_faces(db, 1)
|
||||
monkeypatch.setattr(plex_tmdb.sysconfig, "get_str", lambda d, k: "")
|
||||
assert plex_tmdb.run_people_enrich(db) == {
|
||||
"bios": {"skipped": "tmdb_api_key not set"},
|
||||
"portraits": {"cached": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_a_phase_count_is_a_change_worth_an_audit_row():
|
||||
|
||||
Reference in New Issue
Block a user