fix(sync): an empty answer the remote's own count contradicts is not believed, one level down

E32 review round 1 (B1, B2, B3, B8).

- Plex watch reconcile: a section the server SAYS is empty (totalSize 0, nothing listed) is a
  complete listing. It read as "cut short" forever, so one empty library stopped un-watches from
  ever syncing. A listing with no total at all stays incomplete (the client now returns None for
  it instead of 0), and a whole scan that listed nothing still clears nothing. Tests now cover
  completeness across two sections, both orders.
- Plex collections: no members from a collection whose listing counts some (childCount) keeps
  the membership we hold, in the full sync and in the targeted resync after a write.
- YouTube playlists: no items for a playlist whose listing counts some (itemCount) keeps the
  mirror in the bulk pull, and is refused as retryable in "reset to YouTube" and in the push
  plan/merge, where it read as "YouTube deleted them all". The rule sits beside vanished().
- A playlist deleted on YouTube answered "reconnect your account"; it now says it is gone
  (errors.youtube.gone, HU+EN).
- The pure 3-way merge moved to sync/playlist_merge.py; playlists.py is back under 800 lines and
  off the size list.
This commit is contained in:
2026-10-04 02:13:53 +02:00
parent 5d10a521a8
commit df4a79a1f1
15 changed files with 381 additions and 127 deletions
+5 -3
View File
@@ -78,9 +78,10 @@ class PlexClient:
# --- Catalog mirror (P1) ---
def section_items(
self, key: str, item_type: int, start: int = 0, size: int = 200
) -> tuple[list[dict], int]:
) -> tuple[list[dict], int | None]:
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
(items, total_size)."""
(items, total_size); the total is None when the answer carries none, so a container that
says "0" (a genuinely empty section) stays distinguishable from one that says nothing."""
mc = self._get(
f"/library/sections/{key}/all",
params={
@@ -93,7 +94,8 @@ class PlexClient:
"X-Plex-Container-Size": size,
},
)
return mc.get("Metadata", []) or [], int(mc.get("totalSize", mc.get("size", 0)) or 0)
total = mc.get("totalSize", mc.get("size"))
return mc.get("Metadata", []) or [], None if total is None else int(total)
def children(self, rating_key: str) -> list[dict]:
"""Direct children (show → seasons, season → episodes)."""
+43 -10
View File
@@ -418,7 +418,10 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
if progress is not None:
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
# watch reconcile); an empty mirror or overlay must not read it so (E32 S2).
progress["complete"] = bool(total) and start >= total
progress["total"] = total
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
@@ -671,9 +674,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
meta = plex.metadata(rk) or {}
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
Model = PlexItem if lib.kind == "movie" else PlexShow
new_members = {
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
}
children = _believed_children(plex, rk, meta)
if children is None:
db.flush()
return col
new_members = {str(c.get("ratingKey")) for c in children if c.get("ratingKey")}
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
for row in current: # drop rk from rows that are no longer members
if row.rating_key not in new_members:
@@ -686,6 +691,22 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
return col
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
"""A collection's members, or None when the answer is not believed: no members from a collection
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
empty-answer rule as the collection listing's (E32 review). Rebuilding on it showed the
collection empty until the next sync."""
children = plex.collection_children(rk)
count = str(meta.get("childCount") or "0")
if children or not count.isdigit() or int(count) == 0:
return children
log.warning(
"Plex listed no members for a non-empty collection; keeping its membership",
extra=logs.event("plex.sync.collection_children_empty", collection=rk),
)
return None
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
Model = PlexItem if lib.kind == "movie" else PlexShow
@@ -697,6 +718,19 @@ def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
db.flush()
def _rebuild_membership(
db: Session, lib: PlexLibrary, membership: dict[str, list[str]], kept: set[str]
) -> None:
"""Rebuild collection_keys on every member row of this library (clear + set), carrying over the
keys of a collection whose empty answer was not believed (`_believed_children`)."""
for Model in (PlexItem, PlexShow):
for row in db.execute(select(Model).where(Model.library_id == lib.id)).scalars():
held = [k for k in (row.collection_keys or []) if k in kept]
keys = [*membership.get(row.rating_key, []), *held]
row.collection_keys = list(dict.fromkeys(keys)) or None
db.flush()
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
@@ -718,6 +752,7 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
)
return
seen: set[str] = set()
kept: set[str] = set() # collections whose empty children answer is not believed
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
for meta in cols:
rk = str(meta.get("ratingKey") or "")
@@ -725,7 +760,10 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
continue
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
seen.add(rk)
for child in plex.collection_children(rk):
children = _believed_children(plex, rk, meta)
if children is None:
kept.add(rk)
for child in children or []:
crk = str(child.get("ratingKey") or "")
if crk:
membership.setdefault(crk, []).append(rk)
@@ -734,9 +772,4 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
for rk, col in existing.items():
if rk not in seen:
db.delete(col)
# Rebuild collection_keys on every member row of this library (clear + set).
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
row.collection_keys = membership.get(row.rating_key) or None
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
row.collection_keys = membership.get(row.rating_key) or None
db.flush()
_rebuild_membership(db, lib, membership, kept)
+12 -6
View File
@@ -75,9 +75,9 @@ def _scan_plex_states(
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, bool]:
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, and
whether EVERY section's walk reached the server's total. Shared by the one-time import and the
full reconcile (both need Plex's whole current picture). The last one matters to the reconcile
alone: it reads "absent from the scan" as "Plex un-watched it", which a walk cut short by a
whether EVERY section's walk reached the server's total (or the server said it holds none).
Shared by the one-time import and the full reconcile (both need Plex's whole current picture).
The last one matters to the reconcile alone: it reads "absent from the scan" as "Plex un-watched it", which a walk cut short by a
Plex hiccup does not mean (see `_paginate`)."""
item_id_by_rk = _rk_to_id(db)
wanted = _enabled_section_keys(db)
@@ -93,6 +93,7 @@ def _scan_plex_states(
continue
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
progress: dict = {}
before = scanned
for meta in _paginate(plex, key, item_type, progress):
scanned += 1
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
@@ -103,7 +104,10 @@ def _scan_plex_states(
)
if res is not None:
out[item_id] = res
complete = complete and bool(progress.get("complete"))
# A section the server SAYS is empty (total 0, nothing listed) is complete too: without this
# one genuinely empty library kept every run "incomplete" and un-watches never synced again.
said_empty = progress.get("total") == 0 and scanned == before
complete = complete and (bool(progress.get("complete")) or said_empty)
return out, scanned, complete
@@ -637,12 +641,14 @@ def run_plex_watch_reconcile(db: Session) -> dict:
if ch:
total["applied"] += 1
if complete:
# A whole scan that listed nothing at all is not "Plex un-watched everything" either:
# every section can say 0 only when the server lost its libraries, never as news.
if complete and scanned:
total["cleared"] += _clear_unwatched(db, link.user_id, plex_states)
else:
total["incomplete"] = total.get("incomplete", 0) + 1
log.warning(
"the Plex scan ended before the server's total; nothing is cleared this run",
"the Plex scan was cut short or listed nothing; nothing is cleared this run",
extra=logs.event("plex.watch.reconcile_incomplete", user_id=link.user_id),
)