peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(playlists): every sync path locks the list before its items, and holds it to the write

Browse Source 
The mirror refresh re-ingested inside its row lock, and that ingest's commit let go of the lock
before the replace: a remove made then was undone and the list read as synced. Push, re-pull and a
delete touched items before the list row, a deadlock beside the playlist-first route edits. A push
now re-plans from the items under the lock and derives dirty after its YouTube writes.
lock_playlists moves to the sync package. Tests pin both ids of a move's lock and the claim's
own connection.
This commit is contained in:
peter
2026-09-27 21:56:23 +02:00
parent 2abd4ffcb6
commit 26697472cf
6 changed files with 329 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/playlists.py
+56 -23
View File
@@ -78,6 +78,23 @@ def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
def lock_playlists(db: Session, *playlist_ids: int) -> None:
"""Row-lock playlists before touching their items, several in id order. Every item write takes
the PLAYLIST first — the route edits, a push, a re-pull, the scheduled mirror refresh and a
delete alike: a writer that locked an item row first and the playlist second (a push's item
replace beside a remove) deadlocks against one that went the other way, and the loser is a
500. The rows are re-read, so `dirty`, the name and the sync baseline are the ones a finished
writer committed. Held to the caller's next commit — so no commit may come between this and
the write it guards."""
db.scalars(
select(Playlist)
.where(Playlist.id.in_(sorted(set(playlist_ids))))
.order_by(Playlist.id)
.with_for_update()
.execution_options(populate_existing=True)
).all() # consumed: rows fetched are what refreshes the loaded objects
def fingerprint(name: str, ids: list[str]) -> str:
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
baseline; the current fingerprint is compared to it to derive `dirty`."""
@@ -153,6 +170,18 @@ def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None =
pl.dirty = False
def _set_baseline_under_lock(
db: Session, pl: Playlist, ids: list[str], unavailable: list[str] | None = None
) -> None:
"""A push's closing baseline: `ids` is what YouTube now holds. The push spent seconds on
YouTube writes since it replaced the local items, so an edit may have landed meanwhile — lock
the list and derive `dirty` from what is there, or the edit would read as synced and never
reach YouTube."""
lock_playlists(db, pl.id)
_set_baseline(pl, ids, unavailable)
recompute_dirty(db, pl)
def recompute_dirty(db: Session, pl: Playlist) -> None:
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
@@ -209,15 +238,13 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
db.commit()
def _replace_items_from_youtube(
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
) -> list[str]:
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
replace): ensure the videos exist in the catalog, keep only those present in their YT
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
`pl.name` must already be set to the desired name. The caller commits. Returns the final
ordered ids."""
_ensure_videos(db, yt, ids)
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
present = (
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
)
@@ -284,10 +311,12 @@ def pull_user_playlists(db: Session, user: User) -> dict:
.scalars()
.all()
)
# Drop mirrors whose YouTube playlist no longer exists.
for ytid, pl in existing.items():
if ytid not in yt_ids:
db.delete(pl)
# Drop mirrors whose YouTube playlist no longer exists — locked first, as every item write.
gone = [pl for ytid, pl in existing.items() if ytid not in yt_ids]
if gone:
lock_playlists(db, *(pl.id for pl in gone))
for pl in gone:
db.delete(pl)
# Committed now: a run where no list below commits (all dirty, or all claimed) would
# otherwise drop the deletions with the session.
db.commit()
@@ -325,18 +354,13 @@ def pull_user_playlists(db: Session, user: User) -> dict:
# loaded rows past a commit): a remove or reorder made meanwhile must not be
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
# one in flight and sees what it committed.
db.scalars(
select(Playlist)
.where(Playlist.id == pl.id)
.with_for_update()
.execution_options(populate_existing=True)
).all() # consumed: rows fetched are what refreshes the loaded objects
lock_playlists(db, pl.id)
if pl.dirty:
db.rollback()
continue
pl.name = p.get("title") or pl.name
# Mirror is authoritative: replace the items to match YouTube's order.
_replace_items_from_youtube(db, yt, pl, ids)
_replace_items_from_youtube(db, pl, ids)
db.commit()
synced += 1
return {"synced": synced}
@@ -588,7 +612,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
if not isinstance(added, _Failed):
inserted += 1
if not failures:
_set_baseline(pl, local)
_set_baseline_under_lock(db, pl, local)
db.commit()
return {
"created": created,
@@ -629,6 +653,13 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
# merged order authoritatively. Do this first so local reflects the merge even if a later
# YouTube write trips the breaker.
_ensure_videos(db, yt, mp.add_to_local)
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
# the re-read adds is already a local row.
lock_playlists(db, pl.id)
local = _local_video_ids(db, pl)
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
present = (
@@ -713,7 +744,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
& yt_id_set
)
_set_baseline(pl, final_local, unavailable)
_set_baseline_under_lock(db, pl, final_local, unavailable)
db.commit()
return {
"created": created,
@@ -739,9 +770,11 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
if snippet is None:
raise YouTubeError("Playlist no longer exists on YouTube")
pl.name = snippet.get("title") or pl.name
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
ordered = _replace_items_from_youtube(db, yt, pl, ids)
_ensure_videos(db, yt, ids)
lock_playlists(db, pl.id)
pl.name = snippet.get("title") or pl.name
ordered = _replace_items_from_youtube(db, pl, ids)
db.commit()
return {"items": len(ordered)}