fix(playlists): every sync path locks the list before its items, and holds it to the write

The mirror refresh re-ingested inside its row lock, and that ingest's commit let go of the lock
before the replace: a remove made then was undone and the list read as synced. Push, re-pull and a
delete touched items before the list row, a deadlock beside the playlist-first route edits. A push
now re-plans from the items under the lock and derives dirty after its YouTube writes.
lock_playlists moves to the sync package. Tests pin both ids of a move's lock and the claim's
own connection.

backend/app/routes/playlists.py

@@ -14,6 +14,7 @@ from app.db import get_db
 from app.feedquery import feed_columns, serialize
 from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
 from app.sync.playlists import (
+    lock_playlists,
     merge_playlist,
     plan_merge,
     recompute_dirty,
@@ -79,25 +80,11 @@ def _next_playlist_position(db: Session, user_id: int) -> int:
     )
 
 
-def _lock_playlists(db: Session, *playlist_ids: int) -> None:
-    """Row-lock playlists before touching their items, several in id order. Every item write takes
-    the PLAYLIST first: a writer that locked an item row first and the playlist second (a remove
-    beside a reorder) deadlocks against one that went the other way, and the loser is a 500. The
-    rows are re-read, so `dirty` and the sync baseline are the ones a finished sync committed."""
-    db.scalars(
-        select(Playlist)
-        .where(Playlist.id.in_(sorted(set(playlist_ids))))
-        .order_by(Playlist.id)
-        .with_for_update()
-        .execution_options(populate_existing=True)
-    ).all()  # consumed: rows fetched are what refreshes the loaded objects
-
-
 def _next_item_position(db: Session, playlist_id: int) -> int:
     """Append position for a new item at the end of a playlist. Locks the playlist row first: two
     adds (two tabs, the rail and the page) otherwise both read the same max and land on one
     position, and the list's order between them is then arbitrary."""
-    _lock_playlists(db, playlist_id)
+    lock_playlists(db, playlist_id)
     return (
         db.scalar(
             select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
@@ -113,7 +100,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
     was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
     Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
     insert hit uq_playlist_video as a 500."""
-    _lock_playlists(db, pl.id)
+    lock_playlists(db, pl.id)
     exists = db.scalar(
         select(PlaylistItem).where(
             PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
@@ -134,7 +121,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
 def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
     """Remove a video from a playlist if present (idempotent). Returns whether it was removed.
     `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
-    _lock_playlists(db, pl.id)
+    lock_playlists(db, pl.id)
     item = db.scalar(
         select(PlaylistItem).where(
             PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
@@ -541,6 +528,8 @@ def delete_playlist(
                     yt.delete_playlist(pl.yt_playlist_id)
         except YouTubeError as exc:
             raise youtube_refusal(exc)
+    # The playlist first, as every item write: the cascade deletes the items before the list row.
+    lock_playlists(db, pl.id)
     db.delete(pl)  # items cascade
     db.commit()
     return {"deleted": playlist_id}
@@ -625,7 +614,7 @@ def move_item(
             status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
         )
     # Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
-    _lock_playlists(db, src.id, dst.id)
+    lock_playlists(db, src.id, dst.id)
     item = db.scalar(
         select(PlaylistItem).where(
             PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id

backend/app/sync/playlists.py

@@ -78,6 +78,23 @@ def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
     return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
 
 
+def lock_playlists(db: Session, *playlist_ids: int) -> None:
+    """Row-lock playlists before touching their items, several in id order. Every item write takes
+    the PLAYLIST first — the route edits, a push, a re-pull, the scheduled mirror refresh and a
+    delete alike: a writer that locked an item row first and the playlist second (a push's item
+    replace beside a remove) deadlocks against one that went the other way, and the loser is a
+    500. The rows are re-read, so `dirty`, the name and the sync baseline are the ones a finished
+    writer committed. Held to the caller's next commit — so no commit may come between this and
+    the write it guards."""
+    db.scalars(
+        select(Playlist)
+        .where(Playlist.id.in_(sorted(set(playlist_ids))))
+        .order_by(Playlist.id)
+        .with_for_update()
+        .execution_options(populate_existing=True)
+    ).all()  # consumed: rows fetched are what refreshes the loaded objects
+
+
 def fingerprint(name: str, ids: list[str]) -> str:
     """A stable checksum of a playlist's name + ordered video ids. Stored as the synced
     baseline; the current fingerprint is compared to it to derive `dirty`."""
@@ -153,6 +170,18 @@ def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None =
     pl.dirty = False
 
 
+def _set_baseline_under_lock(
+    db: Session, pl: Playlist, ids: list[str], unavailable: list[str] | None = None
+) -> None:
+    """A push's closing baseline: `ids` is what YouTube now holds. The push spent seconds on
+    YouTube writes since it replaced the local items, so an edit may have landed meanwhile — lock
+    the list and derive `dirty` from what is there, or the edit would read as synced and never
+    reach YouTube."""
+    lock_playlists(db, pl.id)
+    _set_baseline(pl, ids, unavailable)
+    recompute_dirty(db, pl)
+
+
 def recompute_dirty(db: Session, pl: Playlist) -> None:
     """Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
     (and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
@@ -209,15 +238,13 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
     db.commit()
 
 
-def _replace_items_from_youtube(
-    db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
-) -> list[str]:
+def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
     """Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
-    replace): ensure the videos exist in the catalog, keep only those present in their YT
-    order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
-    `pl.name` must already be set to the desired name. The caller commits. Returns the final
-    ordered ids."""
-    _ensure_videos(db, yt, ids)
+    replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
+    items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
+    (`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
+    after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
+    video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
     present = (
         set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
     )
@@ -284,10 +311,12 @@ def pull_user_playlists(db: Session, user: User) -> dict:
             .scalars()
             .all()
         )
-        # Drop mirrors whose YouTube playlist no longer exists.
-        for ytid, pl in existing.items():
-            if ytid not in yt_ids:
-                db.delete(pl)
+        # Drop mirrors whose YouTube playlist no longer exists — locked first, as every item write.
+        gone = [pl for ytid, pl in existing.items() if ytid not in yt_ids]
+        if gone:
+            lock_playlists(db, *(pl.id for pl in gone))
+        for pl in gone:
+            db.delete(pl)
         # Committed now: a run where no list below commits (all dirty, or all claimed) would
         # otherwise drop the deletions with the session.
         db.commit()
@@ -325,18 +354,13 @@ def pull_user_playlists(db: Session, user: User) -> dict:
                 # loaded rows past a commit): a remove or reorder made meanwhile must not be
                 # replaced by YouTube's order. Item writes lock the playlist row, so this waits for
                 # one in flight and sees what it committed.
-                db.scalars(
-                    select(Playlist)
-                    .where(Playlist.id == pl.id)
-                    .with_for_update()
-                    .execution_options(populate_existing=True)
-                ).all()  # consumed: rows fetched are what refreshes the loaded objects
+                lock_playlists(db, pl.id)
                 if pl.dirty:
                     db.rollback()
                     continue
                 pl.name = p.get("title") or pl.name
                 # Mirror is authoritative: replace the items to match YouTube's order.
-                _replace_items_from_youtube(db, yt, pl, ids)
+                _replace_items_from_youtube(db, pl, ids)
                 db.commit()
             synced += 1
     return {"synced": synced}
@@ -588,7 +612,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
                 if not isinstance(added, _Failed):
                     inserted += 1
             if not failures:
-                _set_baseline(pl, local)
+                _set_baseline_under_lock(db, pl, local)
             db.commit()
             return {
                 "created": created,
@@ -629,6 +653,13 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
         # merged order authoritatively. Do this first so local reflects the merge even if a later
         # YouTube write trips the breaker.
         _ensure_videos(db, yt, mp.add_to_local)
+        # `local` was read before the YouTube calls above: an add or remove committed meanwhile
+        # (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
+        # behind us — and plan again from what is there now. Nothing new needs ingesting: a video
+        # the re-read adds is already a local row.
+        lock_playlists(db, pl.id)
+        local = _local_video_ids(db, pl)
+        mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
         # Only the ids that actually exist in the catalog can become local rows (a private/deleted
         # YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
         present = (
@@ -713,7 +744,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
                 (effective_unavailable | {v for v in mp.add_to_local if v not in present})
                 & yt_id_set
             )
-            _set_baseline(pl, final_local, unavailable)
+            _set_baseline_under_lock(db, pl, final_local, unavailable)
         db.commit()
     return {
         "created": created,
@@ -739,9 +770,11 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
         snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
         if snippet is None:
             raise YouTubeError("Playlist no longer exists on YouTube")
-        pl.name = snippet.get("title") or pl.name
         ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
-        ordered = _replace_items_from_youtube(db, yt, pl, ids)
+        _ensure_videos(db, yt, ids)
+        lock_playlists(db, pl.id)
+        pl.name = snippet.get("title") or pl.name
+        ordered = _replace_items_from_youtube(db, pl, ids)
         db.commit()
     return {"items": len(ordered)}