fix(sync): an empty answer the remote's own count contradicts is not believed, one level down
E32 review round 1 (B1, B2, B3, B8). - Plex watch reconcile: a section the server SAYS is empty (totalSize 0, nothing listed) is a complete listing. It read as "cut short" forever, so one empty library stopped un-watches from ever syncing. A listing with no total at all stays incomplete (the client now returns None for it instead of 0), and a whole scan that listed nothing still clears nothing. Tests now cover completeness across two sections, both orders. - Plex collections: no members from a collection whose listing counts some (childCount) keeps the membership we hold, in the full sync and in the targeted resync after a write. - YouTube playlists: no items for a playlist whose listing counts some (itemCount) keeps the mirror in the bulk pull, and is refused as retryable in "reset to YouTube" and in the push plan/merge, where it read as "YouTube deleted them all". The rule sits beside vanished(). - A playlist deleted on YouTube answered "reconnect your account"; it now says it is gone (errors.youtube.gone, HU+EN). - The pure 3-way merge moved to sync/playlist_merge.py; playlists.py is back under 800 lines and off the size list.
This commit is contained in:
@@ -78,9 +78,10 @@ class PlexClient:
|
||||
# --- Catalog mirror (P1) ---
|
||||
def section_items(
|
||||
self, key: str, item_type: int, start: int = 0, size: int = 200
|
||||
) -> tuple[list[dict], int]:
|
||||
) -> tuple[list[dict], int | None]:
|
||||
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
||||
(items, total_size)."""
|
||||
(items, total_size); the total is None when the answer carries none, so a container that
|
||||
says "0" (a genuinely empty section) stays distinguishable from one that says nothing."""
|
||||
mc = self._get(
|
||||
f"/library/sections/{key}/all",
|
||||
params={
|
||||
@@ -93,7 +94,8 @@ class PlexClient:
|
||||
"X-Plex-Container-Size": size,
|
||||
},
|
||||
)
|
||||
return mc.get("Metadata", []) or [], int(mc.get("totalSize", mc.get("size", 0)) or 0)
|
||||
total = mc.get("totalSize", mc.get("size"))
|
||||
return mc.get("Metadata", []) or [], None if total is None else int(total)
|
||||
|
||||
def children(self, rating_key: str) -> list[dict]:
|
||||
"""Direct children (show → seasons, season → episodes)."""
|
||||
|
||||
+43
-10
@@ -418,7 +418,10 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
|
||||
if progress is not None:
|
||||
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
||||
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
||||
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
|
||||
# watch reconcile); an empty mirror or overlay must not read it so (E32 S2).
|
||||
progress["complete"] = bool(total) and start >= total
|
||||
progress["total"] = total
|
||||
|
||||
|
||||
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
||||
@@ -671,9 +674,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
meta = plex.metadata(rk) or {}
|
||||
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
new_members = {
|
||||
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
|
||||
}
|
||||
children = _believed_children(plex, rk, meta)
|
||||
if children is None:
|
||||
db.flush()
|
||||
return col
|
||||
new_members = {str(c.get("ratingKey")) for c in children if c.get("ratingKey")}
|
||||
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
||||
for row in current: # drop rk from rows that are no longer members
|
||||
if row.rating_key not in new_members:
|
||||
@@ -686,6 +691,22 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
return col
|
||||
|
||||
|
||||
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
|
||||
"""A collection's members, or None when the answer is not believed: no members from a collection
|
||||
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
|
||||
empty-answer rule as the collection listing's (E32 review). Rebuilding on it showed the
|
||||
collection empty until the next sync."""
|
||||
children = plex.collection_children(rk)
|
||||
count = str(meta.get("childCount") or "0")
|
||||
if children or not count.isdigit() or int(count) == 0:
|
||||
return children
|
||||
log.warning(
|
||||
"Plex listed no members for a non-empty collection; keeping its membership",
|
||||
extra=logs.event("plex.sync.collection_children_empty", collection=rk),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
@@ -697,6 +718,19 @@ def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||
db.flush()
|
||||
|
||||
|
||||
def _rebuild_membership(
|
||||
db: Session, lib: PlexLibrary, membership: dict[str, list[str]], kept: set[str]
|
||||
) -> None:
|
||||
"""Rebuild collection_keys on every member row of this library (clear + set), carrying over the
|
||||
keys of a collection whose empty answer was not believed (`_believed_children`)."""
|
||||
for Model in (PlexItem, PlexShow):
|
||||
for row in db.execute(select(Model).where(Model.library_id == lib.id)).scalars():
|
||||
held = [k for k in (row.collection_keys or []) if k in kept]
|
||||
keys = [*membership.get(row.rating_key, []), *held]
|
||||
row.collection_keys = list(dict.fromkeys(keys)) or None
|
||||
db.flush()
|
||||
|
||||
|
||||
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
||||
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
||||
@@ -718,6 +752,7 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
)
|
||||
return
|
||||
seen: set[str] = set()
|
||||
kept: set[str] = set() # collections whose empty children answer is not believed
|
||||
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
||||
for meta in cols:
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
@@ -725,7 +760,10 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
continue
|
||||
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
||||
seen.add(rk)
|
||||
for child in plex.collection_children(rk):
|
||||
children = _believed_children(plex, rk, meta)
|
||||
if children is None:
|
||||
kept.add(rk)
|
||||
for child in children or []:
|
||||
crk = str(child.get("ratingKey") or "")
|
||||
if crk:
|
||||
membership.setdefault(crk, []).append(rk)
|
||||
@@ -734,9 +772,4 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
for rk, col in existing.items():
|
||||
if rk not in seen:
|
||||
db.delete(col)
|
||||
# Rebuild collection_keys on every member row of this library (clear + set).
|
||||
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
db.flush()
|
||||
_rebuild_membership(db, lib, membership, kept)
|
||||
|
||||
@@ -75,9 +75,9 @@ def _scan_plex_states(
|
||||
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, bool]:
|
||||
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
||||
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, and
|
||||
whether EVERY section's walk reached the server's total. Shared by the one-time import and the
|
||||
full reconcile (both need Plex's whole current picture). The last one matters to the reconcile
|
||||
alone: it reads "absent from the scan" as "Plex un-watched it", which a walk cut short by a
|
||||
whether EVERY section's walk reached the server's total (or the server said it holds none).
|
||||
Shared by the one-time import and the full reconcile (both need Plex's whole current picture).
|
||||
The last one matters to the reconcile alone: it reads "absent from the scan" as "Plex un-watched it", which a walk cut short by a
|
||||
Plex hiccup does not mean (see `_paginate`)."""
|
||||
item_id_by_rk = _rk_to_id(db)
|
||||
wanted = _enabled_section_keys(db)
|
||||
@@ -93,6 +93,7 @@ def _scan_plex_states(
|
||||
continue
|
||||
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
||||
progress: dict = {}
|
||||
before = scanned
|
||||
for meta in _paginate(plex, key, item_type, progress):
|
||||
scanned += 1
|
||||
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
||||
@@ -103,7 +104,10 @@ def _scan_plex_states(
|
||||
)
|
||||
if res is not None:
|
||||
out[item_id] = res
|
||||
complete = complete and bool(progress.get("complete"))
|
||||
# A section the server SAYS is empty (total 0, nothing listed) is complete too: without this
|
||||
# one genuinely empty library kept every run "incomplete" and un-watches never synced again.
|
||||
said_empty = progress.get("total") == 0 and scanned == before
|
||||
complete = complete and (bool(progress.get("complete")) or said_empty)
|
||||
return out, scanned, complete
|
||||
|
||||
|
||||
@@ -637,12 +641,14 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
if ch:
|
||||
total["applied"] += 1
|
||||
|
||||
if complete:
|
||||
# A whole scan that listed nothing at all is not "Plex un-watched everything" either:
|
||||
# every section can say 0 only when the server lost its libraries, never as news.
|
||||
if complete and scanned:
|
||||
total["cleared"] += _clear_unwatched(db, link.user_id, plex_states)
|
||||
else:
|
||||
total["incomplete"] = total.get("incomplete", 0) + 1
|
||||
log.warning(
|
||||
"the Plex scan ended before the server's total; nothing is cleared this run",
|
||||
"the Plex scan was cut short or listed nothing; nothing is cleared this run",
|
||||
extra=logs.event("plex.watch.reconcile_incomplete", user_id=link.user_id),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user