fix(plex): a listing cut short no longer clears watch state or Hungarian text

The daily watch reconcile reads "absent from Plex's scan" as "Plex un-watched it" and deletes the
row — but it walked the sections without asking _paginate whether the walk reached the server's
total, so a Plex hiccup mid-walk (an empty page early) un-watched every title it failed to reach
until the next complete run. The scan now reports completeness; an incomplete one still pulls in
what it saw, re-pushes dirty rows, and clears nothing (logged as plex.watch.reconcile_incomplete).

The HU overlay had the same shape: clear every row's HU text, then refill from the listing. It now
reads the listing first and clears only after a complete one. The overlay moved to plex/overlay.py
(plex/sync.py drops under the 800-line limit and off the size ratchet's list).

Tests: plex/paths (23% -> 100%: prefix boundary, longest prefix, dead mount), the reconcile's
clear/keep/re-push rules, the cut-short scan and listing (each seen red with the guard removed).

backend/app/plex/overlay.py

@@ -0,0 +1,150 @@
+"""The HU overlay: a Hungarian Plex section written onto its English twin's rows (E30), split out of
+`plex/sync.py` (E32 S2), which runs it once per paired section during the full sync.
+
+The pairing is admin-configured (`plex_overlay_map`, ``huKey:enKey,...``). An overlay section creates
+no browsable rows of its own; it only fills the `*_hu` columns of the EN rows it matches by guid.
+`plex/sync.py` and this module import each other as MODULES (attribute access at call time), which is
+what lets either be imported first.
+"""
+
+from collections.abc import Iterable
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app import logs, sysconfig
+from app.models import PlexItem, PlexLibrary, PlexShow
+from app.plex import sync as plex_sync
+from app.plex.client import PlexClient
+from app.plex.genres import hu_genres
+
+log = logs.get_logger(__name__)
+
+
+def overlay_map(db: Session) -> dict[str, str]:
+    """The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
+    ('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
+    Malformed pairs are skipped."""
+    raw = sysconfig.get_str(db, "plex_overlay_map")
+    out: dict[str, str] = {}
+    for pair in raw.split(","):
+        hu, sep, en = pair.partition(":")
+        hu, en = hu.strip(), en.strip()
+        if sep and hu and en:
+            out[hu] = en
+    return out
+
+
+def sync_overlay(
+    db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
+) -> None:
+    """Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
+    NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
+    differ by language; the language-neutral people_text feeds both search vectors. A show section's
+    episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
+    summary, and the episode rows live in the EN show library.
+
+    This is a full RECONCILE: after a COMPLETE HU listing every guid-bearing EN row's HU fields are
+    cleared, then the matches are repopulated — so an EN row whose HU twin vanished (or was re-guid'd)
+    falls back to EN instead of keeping stale Hungarian metadata. A listing cut short only adds. HU
+    items with no EN match are skipped."""
+    if lib.kind == "movie":
+        passes = [
+            (
+                plex_sync._TYPE_MOVIE,
+                select(PlexItem).where(
+                    PlexItem.library_id == en_lib_id,
+                    PlexItem.kind == "movie",
+                    PlexItem.guid.is_not(None),
+                ),
+            )
+        ]
+    else:
+        passes = [
+            (
+                plex_sync._TYPE_SHOW,
+                select(PlexShow).where(
+                    PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
+                ),
+            ),
+            (
+                plex_sync._TYPE_EPISODE,
+                select(PlexItem).where(
+                    PlexItem.library_id == en_lib_id,
+                    PlexItem.kind == "episode",
+                    PlexItem.guid.is_not(None),
+                ),
+            ),
+        ]
+    for item_type, rows_q in passes:
+        _overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
+    db.flush()
+
+
+def _overlay_pass(
+    db: Session,
+    plex: PlexClient,
+    lib: PlexLibrary,
+    item_type: int,
+    rows: Iterable[PlexItem] | Iterable[PlexShow],
+    stats: dict,
+) -> None:
+    """One kind of the overlay (movies, shows or episodes) — see `sync_overlay`."""
+    # guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
+    by_guid: dict[str, list[PlexItem | PlexShow]] = {}
+    for r in rows:
+        if r.guid:  # the passes select guid-bearing rows only
+            by_guid.setdefault(r.guid, []).append(r)
+    # The HU listing is read first and applied after, so the clear can wait for its verdict.
+    hu_by_guid, complete = _read_hu_listing(plex, lib, item_type, by_guid)
+    episodes = item_type == plex_sync._TYPE_EPISODE
+    if complete:
+        _clear_hu(by_guid, episodes)
+    else:
+        stats["overlay_incomplete"] = stats.get("overlay_incomplete", 0) + 1
+        log.warning(
+            "the HU listing ended before the server's total; existing HU text is kept",
+            extra=logs.event("plex.sync.overlay_incomplete", library=lib.plex_key),
+        )
+    for g, meta in hu_by_guid.items():
+        targets = by_guid[g]
+        title_hu = (meta.get("title") or "").strip() or None
+        summary_hu = meta.get("summary")
+        tagline_hu = (meta.get("tagline") or "").strip() or None
+        genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
+        for row in targets:
+            row.title_hu = title_hu
+            row.summary_hu = summary_hu
+            if not episodes:
+                row.tagline_hu = tagline_hu
+                row.genres_hu = genres_hu or row.genres_hu
+        stats["overlaid"] = stats.get("overlaid", 0) + 1
+
+
+def _read_hu_listing(
+    plex: PlexClient, lib: PlexLibrary, item_type: int, by_guid: dict
+) -> tuple[dict[str, dict], bool]:
+    """The HU section's entries for the guids the EN side carries, and whether the walk reached the
+    server's total. The first HU entry for a guid wins; a dup HU item can't overwrite or double-count."""
+    progress: dict = {}
+    hu_by_guid: dict[str, dict] = {}
+    for meta in plex_sync._paginate(plex, lib.plex_key, item_type, progress):
+        g = plex_sync._canonical_guid(meta)
+        if g and g in by_guid and g not in hu_by_guid:
+            hu_by_guid[g] = meta
+    return hu_by_guid, bool(progress.get("complete"))
+
+
+def _clear_hu(by_guid: dict, episodes: bool) -> None:
+    """Reconcile step: clear HU fields on all guid-bearing rows; the matches are re-set after. Only
+    rows that actually held HU data become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is
+    cheap. The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking
+    one, from its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
+    Only after a COMPLETE listing: one a Plex hiccup cut short would clear the HU text of every title
+    it failed to reach, until the next full sync (E32 S2, the watch reconcile's twin)."""
+    for group in by_guid.values():
+        for row in group:
+            row.title_hu = row.summary_hu = None
+            if not episodes:
+                row.tagline_hu = None
+                row.genres_hu = hu_genres(row.genres)

backend/app/plex/sync.py

@@ -15,7 +15,6 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
 item-detail/player path (P2), not here.
 """
 
-from collections.abc import Iterable
 from datetime import datetime, timezone
 
 from sqlalchemy import func, select
@@ -23,7 +22,7 @@ from sqlalchemy.orm import Session
 
 from app import logs, sysconfig
 from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
-from app.plex import detail, people
+from app.plex import detail, overlay, people
 from app.plex.client import PlexClient, PlexError, PlexNotConfigured
 from app.plex.genres import hu_genres
 from app.plex.lookups import by_rating_key
@@ -194,20 +193,6 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
     return keys or None
 
 
-def _overlay_map(db: Session) -> dict[str, str]:
-    """The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
-    ('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
-    Malformed pairs are skipped."""
-    raw = sysconfig.get_str(db, "plex_overlay_map")
-    out: dict[str, str] = {}
-    for pair in raw.split(","):
-        hu, sep, en = pair.partition(":")
-        hu, en = hu.strip(), en.strip()
-        if sep and hu and en:
-            out[hu] = en
-    return out
-
-
 def sync(db: Session) -> dict:
     """Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
     if not sysconfig.get_bool(db, "plex_enabled"):
@@ -223,7 +208,7 @@ def sync(db: Session) -> dict:
         "pruned": 0,
     }
     wanted = _enabled_section_keys(db)
-    overlay_map = _overlay_map(db)  # {huKey: enKey}
+    overlay_map = overlay.overlay_map(db)  # {huKey: enKey}
     overlay_keys = set(overlay_map)
     try:
         with PlexClient(db) as plex:
@@ -267,7 +252,7 @@ def sync(db: Session) -> dict:
                 lib.metadata_lang = "hu"
                 lib.overlay_of = en_lib.id
                 db.flush()
-                _sync_overlay(db, plex, lib, en_lib.id, stats)
+                overlay.sync_overlay(db, plex, lib, en_lib.id, stats)
                 db.commit()
             # Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
             # becomes filterable over successive runs (the cheap listing above only stored the top ~3).
@@ -661,98 +646,6 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
     _prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
 
 
-def _sync_overlay(
-    db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
-) -> None:
-    """Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
-    NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
-    differ by language; the language-neutral people_text feeds both search vectors. A show section's
-    episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
-    summary, and the episode rows live in the EN show library.
-
-    This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
-    section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
-    to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
-    if lib.kind == "movie":
-        passes = [
-            (
-                _TYPE_MOVIE,
-                select(PlexItem).where(
-                    PlexItem.library_id == en_lib_id,
-                    PlexItem.kind == "movie",
-                    PlexItem.guid.is_not(None),
-                ),
-            )
-        ]
-    else:
-        passes = [
-            (
-                _TYPE_SHOW,
-                select(PlexShow).where(
-                    PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
-                ),
-            ),
-            (
-                _TYPE_EPISODE,
-                select(PlexItem).where(
-                    PlexItem.library_id == en_lib_id,
-                    PlexItem.kind == "episode",
-                    PlexItem.guid.is_not(None),
-                ),
-            ),
-        ]
-    for item_type, rows_q in passes:
-        _overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
-    db.flush()
-
-
-def _overlay_pass(
-    db: Session,
-    plex: PlexClient,
-    lib: PlexLibrary,
-    item_type: int,
-    rows: Iterable[PlexItem] | Iterable[PlexShow],
-    stats: dict,
-) -> None:
-    """One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
-    # guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
-    by_guid: dict[str, list[PlexItem | PlexShow]] = {}
-    for r in rows:
-        if r.guid:  # the passes select guid-bearing rows only
-            by_guid.setdefault(r.guid, []).append(r)
-    # Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
-    # become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
-    # The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
-    # its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
-    episodes = item_type == _TYPE_EPISODE
-    for group in by_guid.values():
-        for row in group:
-            row.title_hu = row.summary_hu = None
-            if not episodes:
-                row.tagline_hu = None
-                row.genres_hu = hu_genres(row.genres)
-    seen: set[str] = (
-        set()
-    )  # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
-    for meta in _paginate(plex, lib.plex_key, item_type):
-        g = _canonical_guid(meta)
-        targets = by_guid.get(g) if g else None
-        if not targets or g in seen:
-            continue
-        seen.add(g)
-        title_hu = (meta.get("title") or "").strip() or None
-        summary_hu = meta.get("summary")
-        tagline_hu = (meta.get("tagline") or "").strip() or None
-        genres_hu = hu_genres(_tags(meta, "Genre"))
-        for row in targets:
-            row.title_hu = title_hu
-            row.summary_hu = summary_hu
-            if not episodes:
-                row.tagline_hu = tagline_hu
-                row.genres_hu = genres_hu or row.genres_hu
-        stats["overlaid"] = stats.get("overlaid", 0) + 1
-
-
 def _upsert_collection(
     db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
 ) -> PlexCollection:
@@ -815,6 +708,15 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
             select(PlexCollection).where(PlexCollection.library_id == lib.id)
         ).scalars()
     }
+    if not cols and existing:
+        # The catalog's own rule (see the module doc): an empty listing while we hold rows is a Plex
+        # glitch, not a library emptied of collections. Reconciling on it dropped every collection
+        # (its user-set `editable` flag with it) and every member's keys (E32 S2).
+        log.warning(
+            "Plex listed no collections for a library that has some; keeping them this run",
+            extra=logs.event("plex.sync.collections_empty", library=lib.plex_key),
+        )
+        return
     seen: set[str] = set()
     membership: dict[str, list[str]] = {}  # member rating_key -> [collection rating_key, ...]
     for meta in cols:

backend/app/plex/watch_sync.py

@@ -72,13 +72,17 @@ def _rk_to_id(db: Session) -> dict[str, int]:
 
 def _scan_plex_states(
     db: Session, plex: PlexClient
-) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int]:
+) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, bool]:
     """Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
-    prog_at)}`` for every mirrored leaf Plex holds a watch signal for, plus the scanned-leaf count.
-    Shared by the one-time import and the full reconcile (both need Plex's whole current picture)."""
+    prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, and
+    whether EVERY section's walk reached the server's total. Shared by the one-time import and the
+    full reconcile (both need Plex's whole current picture). The last one matters to the reconcile
+    alone: it reads "absent from the scan" as "Plex un-watched it", which a walk cut short by a
+    Plex hiccup does not mean (see `_paginate`)."""
     item_id_by_rk = _rk_to_id(db)
     wanted = _enabled_section_keys(db)
     scanned = 0
+    complete = True
     out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
     for s in plex.sections():
         stype = s.get("type")
@@ -88,7 +92,8 @@ def _scan_plex_states(
         if wanted is not None and key not in wanted:
             continue
         item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
-        for meta in _paginate(plex, key, item_type):
+        progress: dict = {}
+        for meta in _paginate(plex, key, item_type, progress):
             scanned += 1
             item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
             if item_id is None:
@@ -98,7 +103,8 @@ def _scan_plex_states(
             )
             if res is not None:
                 out[item_id] = res
-    return out, scanned
+        complete = complete and bool(progress.get("complete"))
+    return out, scanned, complete
 
 
 def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
@@ -111,7 +117,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
     stats = {"watched": 0, "in_progress": 0, "scanned": 0}
     try:
         with PlexClient(db) as plex:
-            scanned_states, stats["scanned"] = _scan_plex_states(db, plex)
+            # A partial scan is still safe here: the import only ever ADDS what Plex reports.
+            scanned_states, stats["scanned"], _complete = _scan_plex_states(db, plex)
             # One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
             # idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
             # while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
@@ -590,6 +597,20 @@ def run_plex_watch_sync(db: Session) -> dict:
     return total
 
 
+def _clear_unwatched(db: Session, user_id: int, plex_states: dict) -> int:
+    """Clear the user's rows Plex no longer has a record for — the mirrored ones (Plex un-watched
+    them); hidden and dirty rows stay (a dirty one is re-pushed by `_repush_dirty`). Called only
+    after a COMPLETE scan: a walk a Plex hiccup cut short leaves out rows it merely failed to reach,
+    and clearing those un-watched them here until the next complete run (E32 S2)."""
+    n = 0
+    for st in db.execute(select(PlexState).where(PlexState.user_id == user_id)).scalars().all():
+        if st.item_id in plex_states or st.status == "hidden" or not st.synced_to_plex:
+            continue
+        db.delete(st)
+        n += 1
+    return n
+
+
 def run_plex_watch_reconcile(db: Session) -> dict:
     """Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
     per item, uses `synced_to_plex` to settle what the incremental feed can't:
@@ -608,7 +629,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
         try:
             with PlexClient(db) as plex:
                 _resolve_account_id(db, plex, link)
-                plex_states, scanned = _scan_plex_states(db, plex)
+                plex_states, scanned, complete = _scan_plex_states(db, plex)
                 total["scanned"] += scanned
 
                 for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
@@ -616,18 +637,14 @@ def run_plex_watch_reconcile(db: Session) -> dict:
                     if ch:
                         total["applied"] += 1
 
-                # Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
-                # (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
-                for st in (
-                    db.execute(select(PlexState).where(PlexState.user_id == link.user_id))
-                    .scalars()
-                    .all()
-                ):
-                    if st.item_id in plex_states or st.status == "hidden":
-                        continue
-                    if st.synced_to_plex:
-                        db.delete(st)
-                        total["cleared"] += 1
+                if complete:
+                    total["cleared"] += _clear_unwatched(db, link.user_id, plex_states)
+                else:
+                    total["incomplete"] = total.get("incomplete", 0) + 1
+                    log.warning(
+                        "the Plex scan ended before the server's total; nothing is cleared this run",
+                        extra=logs.event("plex.watch.reconcile_incomplete", user_id=link.user_id),
+                    )
 
                 total["repushed"] += _repush_dirty(db, plex, link)
                 link.last_watch_sync_at = datetime.now(timezone.utc)