feat(plex): S5a HU-overlay backend — schema, guid, overlay sync
Adds the storage + sync for the Hungarian-metadata overlay (the browsable EN sections gain HU
twin fields written from a paired HU section, matched by guid; no browsable HU rows).
Schema (2 migrations, ADD-only; run before the first HU sync while *_hu is NULL so the generated
column build is trivial):
- 0063: plex_libraries.metadata_lang ('en' default) + self-FK overlay_of.
- 0064: plex_items/plex_shows.guid (indexed) + title_hu/summary_hu/genres_hu (+ GIN) + a STORED
generated search_vector_hu mirroring the EN vector (title_hu A / people_text B / summary_hu C) + GIN.
Sync:
- includeGuids=1 on section_items so the canonical guid rides the cheap listing.
- _canonical_guid(meta) (tmdb > imdb > tvdb) captured onto every EN row in _apply_item/_sync_shows.
- Two-pass sync(): pass 1 mirrors browsable (non-overlay) sections; pass 2 reads plex_overlay_map
('huKey:enKey', new sysconfig), designates each HU section (metadata_lang='hu' + overlay_of), and
_sync_overlay writes title_hu/summary_hu/genres_hu onto the guid-matched EN rows.
Browse: _content_libs now excludes overlay libraries (overlay_of is None); the unified_library and
facets inline library partitions fold into it, and list_libraries/collections exclude overlays too.
Tests: DB-lane coverage of the overlay write (matched vs unmatched rows, no browsable HU rows, the
generated search_vector_hu populates), the guid parse, and the browse exclusion. Full suite 272
passed; migrations verified reversible (upgrade→downgrade→upgrade). S5b (persist the live detail
fields + serve item/show detail from DB) is the follow-up.
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
"""plex library HU-overlay columns (E5 S5a)
|
||||||
|
|
||||||
|
Adds `metadata_lang` (default 'en') and a self-referential `overlay_of` to plex_libraries. An
|
||||||
|
OVERLAY library is the Hungarian-metadata twin of another section: it is never browsed on its own —
|
||||||
|
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
||||||
|
metadata-only, no table rewrite.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0063_plex_library_overlay"
|
||||||
|
down_revision: Union[str, None] = "0062_drop_redundant_idx"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"plex_libraries",
|
||||||
|
sa.Column("metadata_lang", sa.String(8), nullable=False, server_default="en"),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"plex_libraries",
|
||||||
|
sa.Column(
|
||||||
|
"overlay_of",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_plex_libraries_overlay_of", "plex_libraries", ["overlay_of"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_plex_libraries_overlay_of", table_name="plex_libraries")
|
||||||
|
op.drop_column("plex_libraries", "overlay_of")
|
||||||
|
op.drop_column("plex_libraries", "metadata_lang")
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
"""plex HU-overlay item/show columns + guid (E5 S5a)
|
||||||
|
|
||||||
|
On plex_items and plex_shows: `guid` (canonical external id, the EN<->HU pairing key), the Hungarian
|
||||||
|
twin fields `title_hu`/`summary_hu`/`genres_hu`, and a STORED generated `search_vector_hu` mirroring
|
||||||
|
the EN vector (title_hu A / people_text B / left(summary_hu,1000) C) + a GIN index. people_text is
|
||||||
|
language-neutral, reused for both vectors.
|
||||||
|
|
||||||
|
Run BEFORE the first HU overlay sync: while the *_hu columns are all NULL the generated-column build
|
||||||
|
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
||||||
|
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0064_plex_hu_overlay_cols"
|
||||||
|
down_revision: Union[str, None] = "0063_plex_library_overlay"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
_TABLES = ("plex_items", "plex_shows")
|
||||||
|
|
||||||
|
# Mirror of the EN search_vector (models.py) but over the HU twin columns; people_text is shared.
|
||||||
|
_VECTOR_HU = (
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
for t in _TABLES:
|
||||||
|
op.add_column(t, sa.Column("guid", sa.String(64), nullable=True))
|
||||||
|
op.add_column(t, sa.Column("title_hu", sa.String(512), nullable=True))
|
||||||
|
op.add_column(t, sa.Column("summary_hu", sa.Text(), nullable=True))
|
||||||
|
op.add_column(t, sa.Column("genres_hu", postgresql.JSONB(), nullable=True))
|
||||||
|
op.create_index(f"ix_{t}_guid", t, ["guid"])
|
||||||
|
op.create_index(f"ix_{t}_genres_hu_gin", t, ["genres_hu"], postgresql_using="gin")
|
||||||
|
# Generated column can't be expressed via add_column — raw DDL, matching the EN idiom (0044).
|
||||||
|
op.execute(
|
||||||
|
f"ALTER TABLE {t} ADD COLUMN search_vector_hu tsvector "
|
||||||
|
f"GENERATED ALWAYS AS ({_VECTOR_HU}) STORED"
|
||||||
|
)
|
||||||
|
op.execute(f"CREATE INDEX ix_{t}_search_vector_hu ON {t} USING gin (search_vector_hu)")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
for t in _TABLES:
|
||||||
|
op.execute(f"DROP INDEX IF EXISTS ix_{t}_search_vector_hu")
|
||||||
|
op.execute(f"ALTER TABLE {t} DROP COLUMN IF EXISTS search_vector_hu")
|
||||||
|
op.drop_index(f"ix_{t}_genres_hu_gin", table_name=t)
|
||||||
|
op.drop_index(f"ix_{t}_guid", table_name=t)
|
||||||
|
op.drop_column(t, "genres_hu")
|
||||||
|
op.drop_column(t, "summary_hu")
|
||||||
|
op.drop_column(t, "title_hu")
|
||||||
|
op.drop_column(t, "guid")
|
||||||
@@ -221,6 +221,10 @@ class Settings(BaseSettings):
|
|||||||
plex_path_map: str = ""
|
plex_path_map: str = ""
|
||||||
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
||||||
plex_libraries: str = ""
|
plex_libraries: str = ""
|
||||||
|
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||||
|
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
|
||||||
|
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
|
||||||
|
plex_overlay_map: str = ""
|
||||||
# Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier).
|
# Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier).
|
||||||
plex_client_id: str = "siftlode-server"
|
plex_client_id: str = "siftlode-server"
|
||||||
# How often the catalog sync job mirrors Plex metadata, in minutes.
|
# How often the catalog sync job mirrors Plex metadata, in minutes.
|
||||||
|
|||||||
@@ -995,6 +995,13 @@ class PlexLibrary(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
|
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# HU-overlay support (E5 S5). `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||||
|
# library (overlay_of set) is the Hungarian-metadata twin of another section: it is never browsable
|
||||||
|
# on its own — its titles' HU fields are written onto the matched EN rows (paired by `guid`).
|
||||||
|
metadata_lang: Mapped[str] = mapped_column(String(8), default="en", server_default="en")
|
||||||
|
overlay_of: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("plex_libraries.id", ondelete="SET NULL"), index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
@@ -1007,6 +1014,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
Index("ix_plex_shows_genres_gin", "genres", postgresql_using="gin"),
|
Index("ix_plex_shows_genres_gin", "genres", postgresql_using="gin"),
|
||||||
Index("ix_plex_shows_directors_gin", "directors", postgresql_using="gin"),
|
Index("ix_plex_shows_directors_gin", "directors", postgresql_using="gin"),
|
||||||
Index("ix_plex_shows_cast_gin", "cast_names", postgresql_using="gin"),
|
Index("ix_plex_shows_cast_gin", "cast_names", postgresql_using="gin"),
|
||||||
|
Index("ix_plex_shows_genres_hu_gin", "genres_hu", postgresql_using="gin"),
|
||||||
# ix_plex_shows_collections_gin already created with collections (migration 0047).
|
# ix_plex_shows_collections_gin already created with collections (migration 0047).
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1015,6 +1023,9 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
library_id: Mapped[int] = mapped_column(
|
library_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
|
# Canonical external id (e.g. "tmdb:12345") — the key that pairs this EN row with its HU-overlay
|
||||||
|
# twin during sync (E5 S5). Indexed for the guid->row overlay match.
|
||||||
|
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||||
title: Mapped[str] = mapped_column(String(512))
|
title: Mapped[str] = mapped_column(String(512))
|
||||||
summary: Mapped[str | None] = mapped_column(Text)
|
summary: Mapped[str | None] = mapped_column(Text)
|
||||||
year: Mapped[int | None] = mapped_column(Integer)
|
year: Mapped[int | None] = mapped_column(Integer)
|
||||||
@@ -1037,6 +1048,11 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
|
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
|
||||||
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||||
|
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||||
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
|
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||||
|
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
search_vector: Mapped[object | None] = mapped_column(
|
search_vector: Mapped[object | None] = mapped_column(
|
||||||
TSVECTOR,
|
TSVECTOR,
|
||||||
Computed(
|
Computed(
|
||||||
@@ -1046,6 +1062,15 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
persisted=True,
|
persisted=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
search_vector_hu: Mapped[object | None] = mapped_column(
|
||||||
|
TSVECTOR,
|
||||||
|
Computed(
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
|
||||||
|
persisted=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PlexSeason(Base, TimestampMixin, UpdatedAtMixin):
|
class PlexSeason(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
@@ -1077,6 +1102,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
Index("ix_plex_items_directors_gin", "directors", postgresql_using="gin"),
|
Index("ix_plex_items_directors_gin", "directors", postgresql_using="gin"),
|
||||||
Index("ix_plex_items_cast_gin", "cast_names", postgresql_using="gin"),
|
Index("ix_plex_items_cast_gin", "cast_names", postgresql_using="gin"),
|
||||||
Index("ix_plex_items_collections_gin", "collection_keys", postgresql_using="gin"),
|
Index("ix_plex_items_collections_gin", "collection_keys", postgresql_using="gin"),
|
||||||
|
Index("ix_plex_items_genres_hu_gin", "genres_hu", postgresql_using="gin"),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
@@ -1085,6 +1111,9 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
|
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
|
||||||
|
# Canonical external id (e.g. "tmdb:12345") — pairs a MOVIE row with its HU-overlay twin during
|
||||||
|
# sync (E5 S5); indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||||
|
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||||
show_id: Mapped[int | None] = mapped_column(
|
show_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
|
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
@@ -1130,6 +1159,11 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
people_text: Mapped[str | None] = mapped_column(Text)
|
people_text: Mapped[str | None] = mapped_column(Text)
|
||||||
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||||
|
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||||
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
|
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||||
|
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
search_vector: Mapped[object | None] = mapped_column(
|
search_vector: Mapped[object | None] = mapped_column(
|
||||||
TSVECTOR,
|
TSVECTOR,
|
||||||
Computed(
|
Computed(
|
||||||
@@ -1139,6 +1173,15 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
persisted=True,
|
persisted=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
search_vector_hu: Mapped[object | None] = mapped_column(
|
||||||
|
TSVECTOR,
|
||||||
|
Computed(
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||||
|
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
|
||||||
|
persisted=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
|
|||||||
@@ -85,6 +85,10 @@ class PlexClient:
|
|||||||
f"/library/sections/{key}/all",
|
f"/library/sections/{key}/all",
|
||||||
params={
|
params={
|
||||||
"type": item_type,
|
"type": item_type,
|
||||||
|
# includeGuids=1 carries each item's Guid array (tmdb/imdb/tvdb ids) in the cheap
|
||||||
|
# listing, so the sync can capture the canonical guid without a per-item rich fetch —
|
||||||
|
# the pairing key for the HU overlay (E5 S5).
|
||||||
|
"includeGuids": 1,
|
||||||
"X-Plex-Container-Start": start,
|
"X-Plex-Container-Start": start,
|
||||||
"X-Plex-Container-Size": size,
|
"X-Plex-Container-Size": size,
|
||||||
},
|
},
|
||||||
|
|||||||
+112
-4
@@ -82,6 +82,26 @@ def _rating(meta: dict) -> float | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Provider preference for the canonical guid (a stable id shared across a title's language twins).
|
||||||
|
_GUID_PREFERENCE = ("tmdb", "imdb", "tvdb")
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_guid(meta: dict) -> str | None:
|
||||||
|
"""The canonical external id used to pair an EN title with its HU-overlay twin (E5 S5). Reads the
|
||||||
|
`Guid` array (present only with includeGuids=1 in the listing), preferring tmdb > imdb > tvdb,
|
||||||
|
normalised to '<provider>:<id>' (e.g. 'tmdb:12345'). None when no known provider id is present."""
|
||||||
|
found: dict[str, str] = {}
|
||||||
|
for g in meta.get("Guid") or []:
|
||||||
|
raw = str(g.get("id") or "")
|
||||||
|
for prov in _GUID_PREFERENCE:
|
||||||
|
if raw.startswith(f"{prov}://"):
|
||||||
|
found.setdefault(prov, f"{prov}:{raw.split('://', 1)[1]}")
|
||||||
|
for prov in _GUID_PREFERENCE:
|
||||||
|
if prov in found:
|
||||||
|
return found[prov]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _media_facts(meta: dict) -> dict:
|
def _media_facts(meta: dict) -> dict:
|
||||||
media = meta.get("Media") or []
|
media = meta.get("Media") or []
|
||||||
if not media:
|
if not media:
|
||||||
@@ -110,21 +130,48 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
|
|||||||
return keys or None
|
return keys or None
|
||||||
|
|
||||||
|
|
||||||
|
def _overlay_map(db: Session) -> dict[str, str]:
|
||||||
|
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||||
|
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||||
|
Malformed pairs are skipped."""
|
||||||
|
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for pair in raw.split(","):
|
||||||
|
hu, sep, en = pair.partition(":")
|
||||||
|
hu, en = hu.strip(), en.strip()
|
||||||
|
if sep and hu and en:
|
||||||
|
out[hu] = en
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def sync(db: Session) -> dict:
|
def sync(db: Session) -> dict:
|
||||||
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
||||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||||
return {"skipped": "disabled"}
|
return {"skipped": "disabled"}
|
||||||
stats = {"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0, "collections": 0}
|
stats = {
|
||||||
|
"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0,
|
||||||
|
"collections": 0, "overlaid": 0,
|
||||||
|
}
|
||||||
wanted = _enabled_section_keys(db)
|
wanted = _enabled_section_keys(db)
|
||||||
|
overlay_map = _overlay_map(db) # {huKey: enKey}
|
||||||
|
overlay_keys = set(overlay_map)
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
for s in plex.sections():
|
sections = {
|
||||||
if s.get("type") not in ("movie", "show"):
|
str(s.get("key")): s
|
||||||
|
for s in plex.sections()
|
||||||
|
if s.get("type") in ("movie", "show")
|
||||||
|
}
|
||||||
|
libs_by_key: dict[str, PlexLibrary] = {}
|
||||||
|
# Pass 1 — browsable (non-overlay) sections. HU-overlay sections are handled in pass 2 and
|
||||||
|
# are NEVER mirrored as browsable libraries of their own.
|
||||||
|
for key, s in sections.items():
|
||||||
|
if key in overlay_keys:
|
||||||
continue
|
continue
|
||||||
key = str(s.get("key"))
|
|
||||||
if wanted is not None and key not in wanted:
|
if wanted is not None and key not in wanted:
|
||||||
continue
|
continue
|
||||||
lib = _upsert_library(db, key, s.get("title") or key, s.get("type"))
|
lib = _upsert_library(db, key, s.get("title") or key, s.get("type"))
|
||||||
|
libs_by_key[key] = lib
|
||||||
stats["libraries"] += 1
|
stats["libraries"] += 1
|
||||||
if s.get("type") == "movie":
|
if s.get("type") == "movie":
|
||||||
_sync_movies(db, plex, lib, stats)
|
_sync_movies(db, plex, lib, stats)
|
||||||
@@ -135,6 +182,24 @@ def sync(db: Session) -> dict:
|
|||||||
# libraries already mirrored (idempotent — the next run reconciles the rest), instead
|
# libraries already mirrored (idempotent — the next run reconciles the rest), instead
|
||||||
# of the old single end-of-loop commit that a late failure rolled back entirely.
|
# of the old single end-of-loop commit that a late failure rolled back entirely.
|
||||||
db.commit()
|
db.commit()
|
||||||
|
# Pass 2 — HU overlays. The EN twins are now synced (with guids), so write each overlay
|
||||||
|
# section's Hungarian metadata onto the matched EN rows. Requires the EN section to have been
|
||||||
|
# synced in pass 1 (in the allowlist / all) and to share the overlay's kind.
|
||||||
|
for hu_key, en_key in overlay_map.items():
|
||||||
|
s = sections.get(hu_key)
|
||||||
|
en_lib = libs_by_key.get(en_key)
|
||||||
|
if s is None or en_lib is None or s.get("type") != en_lib.kind:
|
||||||
|
log.warning(
|
||||||
|
"Plex overlay skipped: hu=%s en=%s (missing section/EN lib or kind mismatch)",
|
||||||
|
hu_key, en_key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
lib = _upsert_library(db, hu_key, s.get("title") or hu_key, s.get("type"))
|
||||||
|
lib.metadata_lang = "hu"
|
||||||
|
lib.overlay_of = en_lib.id
|
||||||
|
db.flush()
|
||||||
|
_sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||||
|
db.commit()
|
||||||
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
||||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||||
try:
|
try:
|
||||||
@@ -246,6 +311,7 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
|||||||
def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
||||||
row.library_id = lib_id
|
row.library_id = lib_id
|
||||||
row.kind = kind
|
row.kind = kind
|
||||||
|
row.guid = _canonical_guid(meta) # EN<->HU pairing key (movies; harmless/unused on episodes)
|
||||||
row.title = (meta.get("title") or "").strip() or "Untitled"
|
row.title = (meta.get("title") or "").strip() or "Untitled"
|
||||||
row.summary = meta.get("summary")
|
row.summary = meta.get("summary")
|
||||||
row.year = meta.get("year")
|
row.year = meta.get("year")
|
||||||
@@ -300,6 +366,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
|||||||
db.add(sh)
|
db.add(sh)
|
||||||
shows[rk] = sh
|
shows[rk] = sh
|
||||||
sh.library_id = lib.id
|
sh.library_id = lib.id
|
||||||
|
sh.guid = _canonical_guid(meta) # EN<->HU pairing key
|
||||||
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
||||||
sh.summary = meta.get("summary")
|
sh.summary = meta.get("summary")
|
||||||
sh.year = meta.get("year")
|
sh.year = meta.get("year")
|
||||||
@@ -359,6 +426,47 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
|||||||
db.flush()
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_overlay(
|
||||||
|
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||||
|
) -> None:
|
||||||
|
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||||
|
NO browsable rows — an overlay library is never listed on its own. EN rows with no HU match keep
|
||||||
|
their *_hu NULL (serving falls back to EN); HU items with no EN match are skipped. Only title/
|
||||||
|
summary/genres differ by language; the language-neutral people_text feeds both search vectors."""
|
||||||
|
if lib.kind == "movie":
|
||||||
|
by_guid = {
|
||||||
|
r.guid: r
|
||||||
|
for r in db.execute(
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.library_id == en_lib_id,
|
||||||
|
PlexItem.kind == "movie",
|
||||||
|
PlexItem.guid.is_not(None),
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
item_type = _TYPE_MOVIE
|
||||||
|
else:
|
||||||
|
by_guid = {
|
||||||
|
r.guid: r
|
||||||
|
for r in db.execute(
|
||||||
|
select(PlexShow).where(
|
||||||
|
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
item_type = _TYPE_SHOW
|
||||||
|
for meta in _paginate(plex, lib.plex_key, item_type):
|
||||||
|
g = _canonical_guid(meta)
|
||||||
|
row = by_guid.get(g) if g else None
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
row.title_hu = (meta.get("title") or "").strip() or None
|
||||||
|
row.summary_hu = meta.get("summary")
|
||||||
|
row.genres_hu = _tags(meta, "Genre")
|
||||||
|
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
def _upsert_collection(
|
def _upsert_collection(
|
||||||
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
||||||
) -> PlexCollection:
|
) -> PlexCollection:
|
||||||
|
|||||||
@@ -82,9 +82,12 @@ def _field_recheck(model, field: str, tsq):
|
|||||||
def list_libraries(
|
def list_libraries(
|
||||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""The mirrored libraries the user can browse (drives the Plex scope selector)."""
|
"""The mirrored libraries the user can browse (drives the Plex scope selector). HU overlay
|
||||||
|
libraries are excluded — they're not browsable, they only feed the EN twin's Hungarian metadata."""
|
||||||
libs = db.execute(
|
libs = db.execute(
|
||||||
select(PlexLibrary).order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
|
select(PlexLibrary)
|
||||||
|
.where(PlexLibrary.overlay_of.is_(None))
|
||||||
|
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
|
||||||
).scalars().all()
|
).scalars().all()
|
||||||
out = []
|
out = []
|
||||||
for lib in libs:
|
for lib in libs:
|
||||||
@@ -291,11 +294,7 @@ def unified_library(
|
|||||||
episodes come back in a separate `episodes` list (grouped result — the 'Episodes' section)."""
|
episodes come back in a separate `episodes` list (grouped result — the 'Episodes' section)."""
|
||||||
offset = max(0, offset)
|
offset = max(0, offset)
|
||||||
p = f.as_dict()
|
p = f.as_dict()
|
||||||
libs = db.execute(
|
movie_lib_ids, show_lib_ids = _content_libs(db)
|
||||||
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
|
|
||||||
).scalars().all()
|
|
||||||
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
|
||||||
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
|
||||||
want_movies = scope in ("movie", "both") and bool(movie_lib_ids)
|
want_movies = scope in ("movie", "both") and bool(movie_lib_ids)
|
||||||
want_shows = scope in ("show", "both") and bool(show_lib_ids)
|
want_shows = scope in ("show", "both") and bool(show_lib_ids)
|
||||||
|
|
||||||
@@ -531,17 +530,9 @@ def facets(
|
|||||||
"duration_min": None,
|
"duration_min": None,
|
||||||
"duration_max": None,
|
"duration_max": None,
|
||||||
}
|
}
|
||||||
libs = db.execute(
|
all_movie_ids, all_show_ids = _content_libs(db)
|
||||||
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
|
movie_ids = all_movie_ids if scope in ("movie", "both") else []
|
||||||
).scalars().all()
|
show_ids = all_show_ids if scope in ("show", "both") else []
|
||||||
movie_ids = (
|
|
||||||
[lb.id for lb in libs if lb.kind == "movie"]
|
|
||||||
if scope in ("movie", "both")
|
|
||||||
else []
|
|
||||||
)
|
|
||||||
show_ids = (
|
|
||||||
[lb.id for lb in libs if lb.kind == "show"] if scope in ("show", "both") else []
|
|
||||||
)
|
|
||||||
if not movie_ids and not show_ids:
|
if not movie_ids and not show_ids:
|
||||||
return empty
|
return empty
|
||||||
p = f.as_dict()
|
p = f.as_dict()
|
||||||
@@ -758,7 +749,9 @@ def collections(
|
|||||||
else:
|
else:
|
||||||
enabled_ids = list(
|
enabled_ids = list(
|
||||||
db.execute(
|
db.execute(
|
||||||
select(PlexLibrary.id).where(PlexLibrary.enabled.is_(True))
|
select(PlexLibrary.id).where(
|
||||||
|
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
|
||||||
|
)
|
||||||
).scalars()
|
).scalars()
|
||||||
)
|
)
|
||||||
if not enabled_ids:
|
if not enabled_ids:
|
||||||
@@ -1545,11 +1538,13 @@ def _rich_meta(meta: dict) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def _content_libs(db: Session) -> tuple[list[int], list[int]]:
|
def _content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||||
"""The enabled content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||||
source of the browsable-library scope shared by people_cards and the person filmography endpoint.
|
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||||
(S5 will extend this to exclude HU overlay libraries and fold in the browse/facets partition sites.)"""
|
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5)."""
|
||||||
libs = db.execute(
|
libs = db.execute(
|
||||||
select(PlexLibrary).where(PlexLibrary.enabled.is_(True))
|
select(PlexLibrary).where(
|
||||||
|
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
|
||||||
|
)
|
||||||
).scalars().all()
|
).scalars().all()
|
||||||
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
||||||
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
|||||||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
|
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
|
||||||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
|
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
|
||||||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
|
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
|
||||||
|
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||||||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||||
# NOTE: plex_watch_sync_interval_min (config.py default) is surfaced here in Phase C, when the
|
# NOTE: plex_watch_sync_interval_min (config.py default) is surfaced here in Phase C, when the
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
"""DB-backed regression (E5 S5a): the HU-overlay sync writes Hungarian metadata onto the matched EN
|
||||||
|
rows (paired by guid), never creating browsable HU rows, and overlay libraries are excluded from the
|
||||||
|
browse scope. Also covers the canonical-guid parse and that the generated search_vector_hu populates.
|
||||||
|
|
||||||
|
Uses the DB lane (`db` fixture) — skipped when no Postgres is reachable. Runs against the real schema
|
||||||
|
(migrations 0063/0064) so the JSONB + STORED generated columns are exercised.
|
||||||
|
"""
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from app.models import PlexItem, PlexLibrary
|
||||||
|
from app.plex.sync import _canonical_guid, _sync_overlay
|
||||||
|
from app.routes.plex.browse import _content_libs
|
||||||
|
|
||||||
|
|
||||||
|
class _StubPlex:
|
||||||
|
"""Minimal PlexClient stand-in: _sync_overlay only calls section_items via _paginate."""
|
||||||
|
|
||||||
|
def __init__(self, items: list[dict]):
|
||||||
|
self._items = items
|
||||||
|
|
||||||
|
def section_items(self, key, item_type, start, size):
|
||||||
|
return self._items[start : start + size], len(self._items)
|
||||||
|
|
||||||
|
|
||||||
|
def _hu_movie(rk: str, title: str, summary: str, genres: list[str], guid: str | None) -> dict:
|
||||||
|
meta = {"ratingKey": rk, "title": title, "summary": summary, "Genre": [{"tag": g} for g in genres]}
|
||||||
|
if guid:
|
||||||
|
meta["Guid"] = [{"id": guid}]
|
||||||
|
return meta
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_guid_prefers_tmdb_then_imdb():
|
||||||
|
assert _canonical_guid({"Guid": [{"id": "imdb://tt1"}, {"id": "tmdb://55"}]}) == "tmdb:55"
|
||||||
|
assert _canonical_guid({"Guid": [{"id": "imdb://tt9"}, {"id": "tvdb://7"}]}) == "imdb:tt9"
|
||||||
|
assert _canonical_guid({"Guid": []}) is None
|
||||||
|
assert _canonical_guid({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_overlay_writes_hu_onto_matched_en_rows(db):
|
||||||
|
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||||
|
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||||
|
db.add_all([en_lib, hu_lib])
|
||||||
|
db.flush()
|
||||||
|
hu_lib.overlay_of = en_lib.id
|
||||||
|
matched = PlexItem(
|
||||||
|
rating_key="m1", library_id=en_lib.id, kind="movie", title="Heat", guid="tmdb:1"
|
||||||
|
)
|
||||||
|
unmatched = PlexItem(
|
||||||
|
rating_key="m2", library_id=en_lib.id, kind="movie", title="Solo", guid="tmdb:2"
|
||||||
|
)
|
||||||
|
db.add_all([matched, unmatched])
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
stub = _StubPlex(
|
||||||
|
[
|
||||||
|
_hu_movie("h1", "Öldöklés", "Egy rablás terve.", ["Akció", "Krimi"], "tmdb://1"),
|
||||||
|
_hu_movie("h2", "Ismeretlen", "Nincs EN párja.", ["Dráma"], "tmdb://999"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
stats = {"overlaid": 0}
|
||||||
|
_sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
db.refresh(matched)
|
||||||
|
db.refresh(unmatched)
|
||||||
|
assert matched.title_hu == "Öldöklés"
|
||||||
|
assert matched.summary_hu == "Egy rablás terve."
|
||||||
|
assert matched.genres_hu == ["Akció", "Krimi"]
|
||||||
|
assert matched.search_vector_hu is not None # the STORED generated column populated from title_hu
|
||||||
|
# The unmatched EN row keeps NULL HU fields (serving falls back to EN).
|
||||||
|
assert unmatched.title_hu is None
|
||||||
|
assert stats["overlaid"] == 1
|
||||||
|
# No browsable rows were created — the HU items never became plex_items of the overlay library.
|
||||||
|
hu_row_count = db.scalar(
|
||||||
|
select(func.count(PlexItem.id)).where(PlexItem.library_id == hu_lib.id)
|
||||||
|
)
|
||||||
|
assert hu_row_count == 0
|
||||||
|
# The EN library still holds exactly its two movies (no dup from the overlay pass).
|
||||||
|
en_row_count = db.scalar(
|
||||||
|
select(func.count(PlexItem.id)).where(PlexItem.library_id == en_lib.id)
|
||||||
|
)
|
||||||
|
assert en_row_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_libs_excludes_overlay_libraries(db):
|
||||||
|
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||||
|
en_shows = PlexLibrary(plex_key="1", title="TV Shows", kind="show")
|
||||||
|
db.add_all([en_lib, en_shows])
|
||||||
|
db.flush()
|
||||||
|
overlay = PlexLibrary(
|
||||||
|
plex_key="9", title="Filmek", kind="movie", metadata_lang="hu", overlay_of=en_lib.id
|
||||||
|
)
|
||||||
|
db.add(overlay)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
movie_ids, show_ids = _content_libs(db)
|
||||||
|
assert en_lib.id in movie_ids
|
||||||
|
assert overlay.id not in movie_ids # the HU overlay is never browsable
|
||||||
|
assert show_ids == [en_shows.id]
|
||||||
Reference in New Issue
Block a user