feat(plex): S5a HU-overlay backend — schema, guid, overlay sync

Adds the storage + sync for the Hungarian-metadata overlay (the browsable EN sections gain HU
twin fields written from a paired HU section, matched by guid; no browsable HU rows).

Schema (2 migrations, ADD-only; run before the first HU sync while *_hu is NULL so the generated
column build is trivial):
- 0063: plex_libraries.metadata_lang ('en' default) + self-FK overlay_of.
- 0064: plex_items/plex_shows.guid (indexed) + title_hu/summary_hu/genres_hu (+ GIN) + a STORED
  generated search_vector_hu mirroring the EN vector (title_hu A / people_text B / summary_hu C) + GIN.

Sync:
- includeGuids=1 on section_items so the canonical guid rides the cheap listing.
- _canonical_guid(meta) (tmdb > imdb > tvdb) captured onto every EN row in _apply_item/_sync_shows.
- Two-pass sync(): pass 1 mirrors browsable (non-overlay) sections; pass 2 reads plex_overlay_map
  ('huKey:enKey', new sysconfig), designates each HU section (metadata_lang='hu' + overlay_of), and
  _sync_overlay writes title_hu/summary_hu/genres_hu onto the guid-matched EN rows.

Browse: _content_libs now excludes overlay libraries (overlay_of is None); the unified_library and
facets inline library partitions fold into it, and list_libraries/collections exclude overlays too.

Tests: DB-lane coverage of the overlay write (matched vs unmatched rows, no browsable HU rows, the
generated search_vector_hu populates), the guid parse, and the browse exclusion. Full suite 272
passed; migrations verified reversible (upgrade→downgrade→upgrade). S5b (persist the live detail
fields + serve item/show detail from DB) is the follow-up.
This commit is contained in:
2026-08-16 22:16:54 +02:00
parent 072d8fc88f
commit c959859d29
9 changed files with 378 additions and 27 deletions
+4
View File
@@ -221,6 +221,10 @@ class Settings(BaseSettings):
plex_path_map: str = ""
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
plex_libraries: str = ""
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
plex_overlay_map: str = ""
# Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier).
plex_client_id: str = "siftlode-server"
# How often the catalog sync job mirrors Plex metadata, in minutes.
+43
View File
@@ -995,6 +995,13 @@ class PlexLibrary(Base, TimestampMixin, UpdatedAtMixin):
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# HU-overlay support (E5 S5). `metadata_lang` tags the section's metadata language. An OVERLAY
# library (overlay_of set) is the Hungarian-metadata twin of another section: it is never browsable
# on its own — its titles' HU fields are written onto the matched EN rows (paired by `guid`).
metadata_lang: Mapped[str] = mapped_column(String(8), default="en", server_default="en")
overlay_of: Mapped[int | None] = mapped_column(
ForeignKey("plex_libraries.id", ondelete="SET NULL"), index=True
)
class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
@@ -1007,6 +1014,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
Index("ix_plex_shows_genres_gin", "genres", postgresql_using="gin"),
Index("ix_plex_shows_directors_gin", "directors", postgresql_using="gin"),
Index("ix_plex_shows_cast_gin", "cast_names", postgresql_using="gin"),
Index("ix_plex_shows_genres_hu_gin", "genres_hu", postgresql_using="gin"),
# ix_plex_shows_collections_gin already created with collections (migration 0047).
)
@@ -1015,6 +1023,9 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
library_id: Mapped[int] = mapped_column(
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
)
# Canonical external id (e.g. "tmdb:12345") — the key that pairs this EN row with its HU-overlay
# twin during sync (E5 S5). Indexed for the guid->row overlay match.
guid: Mapped[str | None] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
year: Mapped[int | None] = mapped_column(Integer)
@@ -1037,6 +1048,11 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
title_hu: Mapped[str | None] = mapped_column(String(512))
summary_hu: Mapped[str | None] = mapped_column(Text)
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
search_vector: Mapped[object | None] = mapped_column(
TSVECTOR,
Computed(
@@ -1046,6 +1062,15 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
persisted=True,
),
)
search_vector_hu: Mapped[object | None] = mapped_column(
TSVECTOR,
Computed(
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
persisted=True,
),
)
class PlexSeason(Base, TimestampMixin, UpdatedAtMixin):
@@ -1077,6 +1102,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
Index("ix_plex_items_directors_gin", "directors", postgresql_using="gin"),
Index("ix_plex_items_cast_gin", "cast_names", postgresql_using="gin"),
Index("ix_plex_items_collections_gin", "collection_keys", postgresql_using="gin"),
Index("ix_plex_items_genres_hu_gin", "genres_hu", postgresql_using="gin"),
)
id: Mapped[int] = mapped_column(primary_key=True)
@@ -1085,6 +1111,9 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
)
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
# Canonical external id (e.g. "tmdb:12345") — pairs a MOVIE row with its HU-overlay twin during
# sync (E5 S5); indexed for the guid->row match. Episodes carry it too but it's unused for them.
guid: Mapped[str | None] = mapped_column(String(64), index=True)
show_id: Mapped[int | None] = mapped_column(
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
)
@@ -1130,6 +1159,11 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
people_text: Mapped[str | None] = mapped_column(Text)
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
title_hu: Mapped[str | None] = mapped_column(String(512))
summary_hu: Mapped[str | None] = mapped_column(Text)
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
search_vector: Mapped[object | None] = mapped_column(
TSVECTOR,
Computed(
@@ -1139,6 +1173,15 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
persisted=True,
),
)
search_vector_hu: Mapped[object | None] = mapped_column(
TSVECTOR,
Computed(
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
persisted=True,
),
)
class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
+4
View File
@@ -85,6 +85,10 @@ class PlexClient:
f"/library/sections/{key}/all",
params={
"type": item_type,
# includeGuids=1 carries each item's Guid array (tmdb/imdb/tvdb ids) in the cheap
# listing, so the sync can capture the canonical guid without a per-item rich fetch —
# the pairing key for the HU overlay (E5 S5).
"includeGuids": 1,
"X-Plex-Container-Start": start,
"X-Plex-Container-Size": size,
},
+112 -4
View File
@@ -82,6 +82,26 @@ def _rating(meta: dict) -> float | None:
return None
# Provider preference for the canonical guid (a stable id shared across a title's language twins).
_GUID_PREFERENCE = ("tmdb", "imdb", "tvdb")
def _canonical_guid(meta: dict) -> str | None:
"""The canonical external id used to pair an EN title with its HU-overlay twin (E5 S5). Reads the
`Guid` array (present only with includeGuids=1 in the listing), preferring tmdb > imdb > tvdb,
normalised to '<provider>:<id>' (e.g. 'tmdb:12345'). None when no known provider id is present."""
found: dict[str, str] = {}
for g in meta.get("Guid") or []:
raw = str(g.get("id") or "")
for prov in _GUID_PREFERENCE:
if raw.startswith(f"{prov}://"):
found.setdefault(prov, f"{prov}:{raw.split('://', 1)[1]}")
for prov in _GUID_PREFERENCE:
if prov in found:
return found[prov]
return None
def _media_facts(meta: dict) -> dict:
media = meta.get("Media") or []
if not media:
@@ -110,21 +130,48 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
return keys or None
def _overlay_map(db: Session) -> dict[str, str]:
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
Malformed pairs are skipped."""
raw = sysconfig.get_str(db, "plex_overlay_map")
out: dict[str, str] = {}
for pair in raw.split(","):
hu, sep, en = pair.partition(":")
hu, en = hu.strip(), en.strip()
if sep and hu and en:
out[hu] = en
return out
def sync(db: Session) -> dict:
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
if not sysconfig.get_bool(db, "plex_enabled"):
return {"skipped": "disabled"}
stats = {"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0, "collections": 0}
stats = {
"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0,
"collections": 0, "overlaid": 0,
}
wanted = _enabled_section_keys(db)
overlay_map = _overlay_map(db) # {huKey: enKey}
overlay_keys = set(overlay_map)
try:
with PlexClient(db) as plex:
for s in plex.sections():
if s.get("type") not in ("movie", "show"):
sections = {
str(s.get("key")): s
for s in plex.sections()
if s.get("type") in ("movie", "show")
}
libs_by_key: dict[str, PlexLibrary] = {}
# Pass 1 — browsable (non-overlay) sections. HU-overlay sections are handled in pass 2 and
# are NEVER mirrored as browsable libraries of their own.
for key, s in sections.items():
if key in overlay_keys:
continue
key = str(s.get("key"))
if wanted is not None and key not in wanted:
continue
lib = _upsert_library(db, key, s.get("title") or key, s.get("type"))
libs_by_key[key] = lib
stats["libraries"] += 1
if s.get("type") == "movie":
_sync_movies(db, plex, lib, stats)
@@ -135,6 +182,24 @@ def sync(db: Session) -> dict:
# libraries already mirrored (idempotent — the next run reconciles the rest), instead
# of the old single end-of-loop commit that a late failure rolled back entirely.
db.commit()
# Pass 2 — HU overlays. The EN twins are now synced (with guids), so write each overlay
# section's Hungarian metadata onto the matched EN rows. Requires the EN section to have been
# synced in pass 1 (in the allowlist / all) and to share the overlay's kind.
for hu_key, en_key in overlay_map.items():
s = sections.get(hu_key)
en_lib = libs_by_key.get(en_key)
if s is None or en_lib is None or s.get("type") != en_lib.kind:
log.warning(
"Plex overlay skipped: hu=%s en=%s (missing section/EN lib or kind mismatch)",
hu_key, en_key,
)
continue
lib = _upsert_library(db, hu_key, s.get("title") or hu_key, s.get("type"))
lib.metadata_lang = "hu"
lib.overlay_of = en_lib.id
db.flush()
_sync_overlay(db, plex, lib, en_lib.id, stats)
db.commit()
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
try:
@@ -246,6 +311,7 @@ def _apply_facet_fields(row, meta: dict) -> None:
def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
row.library_id = lib_id
row.kind = kind
row.guid = _canonical_guid(meta) # EN<->HU pairing key (movies; harmless/unused on episodes)
row.title = (meta.get("title") or "").strip() or "Untitled"
row.summary = meta.get("summary")
row.year = meta.get("year")
@@ -300,6 +366,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
db.add(sh)
shows[rk] = sh
sh.library_id = lib.id
sh.guid = _canonical_guid(meta) # EN<->HU pairing key
sh.title = (meta.get("title") or "").strip() or "Untitled"
sh.summary = meta.get("summary")
sh.year = meta.get("year")
@@ -359,6 +426,47 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
db.flush()
def _sync_overlay(
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
) -> None:
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
NO browsable rows — an overlay library is never listed on its own. EN rows with no HU match keep
their *_hu NULL (serving falls back to EN); HU items with no EN match are skipped. Only title/
summary/genres differ by language; the language-neutral people_text feeds both search vectors."""
if lib.kind == "movie":
by_guid = {
r.guid: r
for r in db.execute(
select(PlexItem).where(
PlexItem.library_id == en_lib_id,
PlexItem.kind == "movie",
PlexItem.guid.is_not(None),
)
).scalars()
}
item_type = _TYPE_MOVIE
else:
by_guid = {
r.guid: r
for r in db.execute(
select(PlexShow).where(
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
)
).scalars()
}
item_type = _TYPE_SHOW
for meta in _paginate(plex, lib.plex_key, item_type):
g = _canonical_guid(meta)
row = by_guid.get(g) if g else None
if row is None:
continue
row.title_hu = (meta.get("title") or "").strip() or None
row.summary_hu = meta.get("summary")
row.genres_hu = _tags(meta, "Genre")
stats["overlaid"] = stats.get("overlaid", 0) + 1
db.flush()
def _upsert_collection(
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
) -> PlexCollection:
+18 -23
View File
@@ -82,9 +82,12 @@ def _field_recheck(model, field: str, tsq):
def list_libraries(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""The mirrored libraries the user can browse (drives the Plex scope selector)."""
"""The mirrored libraries the user can browse (drives the Plex scope selector). HU overlay
libraries are excluded — they're not browsable, they only feed the EN twin's Hungarian metadata."""
libs = db.execute(
select(PlexLibrary).order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
select(PlexLibrary)
.where(PlexLibrary.overlay_of.is_(None))
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
).scalars().all()
out = []
for lib in libs:
@@ -291,11 +294,7 @@ def unified_library(
episodes come back in a separate `episodes` list (grouped result — the 'Episodes' section)."""
offset = max(0, offset)
p = f.as_dict()
libs = db.execute(
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
).scalars().all()
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
movie_lib_ids, show_lib_ids = _content_libs(db)
want_movies = scope in ("movie", "both") and bool(movie_lib_ids)
want_shows = scope in ("show", "both") and bool(show_lib_ids)
@@ -531,17 +530,9 @@ def facets(
"duration_min": None,
"duration_max": None,
}
libs = db.execute(
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
).scalars().all()
movie_ids = (
[lb.id for lb in libs if lb.kind == "movie"]
if scope in ("movie", "both")
else []
)
show_ids = (
[lb.id for lb in libs if lb.kind == "show"] if scope in ("show", "both") else []
)
all_movie_ids, all_show_ids = _content_libs(db)
movie_ids = all_movie_ids if scope in ("movie", "both") else []
show_ids = all_show_ids if scope in ("show", "both") else []
if not movie_ids and not show_ids:
return empty
p = f.as_dict()
@@ -758,7 +749,9 @@ def collections(
else:
enabled_ids = list(
db.execute(
select(PlexLibrary.id).where(PlexLibrary.enabled.is_(True))
select(PlexLibrary.id).where(
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
)
).scalars()
)
if not enabled_ids:
@@ -1545,11 +1538,13 @@ def _rich_meta(meta: dict) -> dict:
def _content_libs(db: Session) -> tuple[list[int], list[int]]:
"""The enabled content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
source of the browsable-library scope shared by people_cards and the person filmography endpoint.
(S5 will extend this to exclude HU overlay libraries and fold in the browse/facets partition sites.)"""
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5)."""
libs = db.execute(
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
select(PlexLibrary).where(
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
)
).scalars().all()
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
+1
View File
@@ -86,6 +86,7 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
# NOTE: plex_watch_sync_interval_min (config.py default) is surfaced here in Phase C, when the