peter
13f65b4844
pg_locks is the whole server's: under the parallel lane a worker's own advisory claim on the same playlist id, or its own lock wait, was counted by another worker's probe (a leak reported where there was none, a commit made before the second tab blocked).
215 lines
8.4 KiB
Python
215 lines
8.4 KiB
Python
"""One YouTube sync of a playlist at a time, whoever starts it.
|
|
|
|
A push reads YouTube and then writes both sides for seconds, committing step by step, so the
|
|
client's in-flight flag (one page's menus) and a row lock (gone at the first commit) both leave two
|
|
tabs, or a push and the scheduled mirror refresh, free to interleave. The claim is an advisory lock
|
|
on a connection of its own; the test holds it from outside, the way a second request would.
|
|
"""
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.auth import require_write_scope
|
|
from app.db import SessionLocal
|
|
from app.reader.models import Playlist
|
|
from app.routes import playlists as plroutes
|
|
from app.sync import playlists as plsync
|
|
|
|
|
|
def _linked(db, user, *, source="local"):
|
|
pl = Playlist(user_id=user.id, name="L", source=source, yt_playlist_id="PL123")
|
|
db.add(pl)
|
|
db.commit()
|
|
return pl
|
|
|
|
|
|
class FakeYT:
|
|
"""YouTube as far as these routes read it: two playlists, none with items."""
|
|
|
|
def __init__(self, *_):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
return False
|
|
|
|
def iter_my_playlists(self):
|
|
return [{"id": "PL123", "title": "Renamed"}, {"id": "PL999", "title": "New"}]
|
|
|
|
def iter_my_playlist_video_ids(self, _ytid):
|
|
return []
|
|
|
|
def iter_playlist_items_with_ids(self, _ytid):
|
|
return []
|
|
|
|
def get_playlist_snippet(self, _ytid):
|
|
return {}
|
|
|
|
|
|
def test_push_and_revert_refuse_while_the_list_is_claimed(
|
|
db, as_user, make_user, override_dep, monkeypatch
|
|
):
|
|
user = make_user()
|
|
pl = _linked(db, user)
|
|
override_dep(require_write_scope, lambda: user)
|
|
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
|
ran: list[str] = []
|
|
monkeypatch.setattr(plroutes, "YouTubeClient", FakeYT)
|
|
monkeypatch.setattr(plroutes, "plan_merge", lambda d, u, p, live: {"units_estimate": 0})
|
|
monkeypatch.setattr(plroutes, "merge_playlist", lambda d, u, p, live: ran.append("push") or {})
|
|
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: ran.append("revert") or {})
|
|
c = as_user(user)
|
|
|
|
with plsync.sync_claim(db, pl.id) as free:
|
|
assert free
|
|
for path in ("push", "revert-youtube"):
|
|
r = c.post(f"/api/playlists/{pl.id}/{path}")
|
|
assert r.status_code == 409
|
|
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
|
# a delete too: a push mid-way would plan the deleted list as emptied
|
|
r = c.delete(f"/api/playlists/{pl.id}")
|
|
assert r.status_code == 409
|
|
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
|
assert ran == []
|
|
assert db.get(Playlist, pl.id) is not None
|
|
|
|
# Released on the way out: the same list syncs again, and a claim is per list.
|
|
assert c.post(f"/api/playlists/{pl.id}/push").status_code == 200
|
|
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
|
assert ran == ["push", "revert"]
|
|
|
|
|
|
def test_a_claim_is_released_when_its_block_raises(db, make_user):
|
|
pl = _linked(db, make_user())
|
|
try:
|
|
with plsync.sync_claim(db, pl.id) as free:
|
|
assert free
|
|
raise RuntimeError("boom")
|
|
except RuntimeError:
|
|
pass
|
|
# Asked of pg_locks, not by claiming again: a lock leaked on a pooled connection would be granted
|
|
# again to that same connection (advisory locks are re-entrant per session). pg_locks is the whole
|
|
# server's, so only this database's: a parallel worker's own claim on the same id counted here.
|
|
held = db.scalar(
|
|
text(
|
|
"SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = :id"
|
|
" AND database = (SELECT oid FROM pg_database WHERE datname = current_database())"
|
|
),
|
|
{"id": pl.id},
|
|
)
|
|
assert held == 0
|
|
|
|
|
|
def test_a_claim_draws_no_connection_from_the_app_pool(db, make_user):
|
|
"""A claim is held for a whole push beside the request's pooled session; drawn from the pool
|
|
too, a burst of pushes ran it dry and stalled every other request for pool_timeout."""
|
|
from app.db import engine
|
|
|
|
pl = _linked(db, make_user())
|
|
before = engine.pool.checkedout()
|
|
with plsync.sync_claim(db, pl.id) as free:
|
|
assert free
|
|
assert engine.pool.checkedout() == before
|
|
|
|
|
|
def test_the_scheduled_refresh_skips_a_mirror_being_synced(db, make_user, monkeypatch):
|
|
user = make_user()
|
|
busy = _linked(db, user, source="youtube")
|
|
replaced: list[int] = []
|
|
|
|
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
|
monkeypatch.setattr(
|
|
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
|
)
|
|
|
|
with plsync.sync_claim(db, busy.id):
|
|
out = plsync.sync_user_playlists(db, user)
|
|
|
|
assert out == {"synced": 1}
|
|
assert busy.id not in replaced and len(replaced) == 1
|
|
db.refresh(busy)
|
|
assert busy.name == "L"
|
|
|
|
|
|
def test_refused_reverts_do_not_spend_the_pull_cooldown(
|
|
db, as_user, make_user, override_dep, monkeypatch
|
|
):
|
|
"""The cooldown (three pulls per ten minutes) is checked after the claim, so clicking
|
|
Revert while a push runs is refused as busy without using up a pull."""
|
|
user = make_user()
|
|
pl = _linked(db, user)
|
|
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {})
|
|
c = as_user(user)
|
|
with plsync.sync_claim(db, pl.id):
|
|
for _ in range(4):
|
|
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 409
|
|
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
|
|
|
|
|
def test_a_bulk_pull_runs_once_per_user_at_a_time(db, as_user, make_user, monkeypatch):
|
|
"""Two bulk pulls at once both saw a YouTube list unmirrored and each created its mirror."""
|
|
user = make_user()
|
|
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
|
monkeypatch.setattr(plsync, "_replace_items_from_youtube", lambda d, p, ids: None)
|
|
c = as_user(user)
|
|
with plsync.user_sync_claim(db, user.id):
|
|
assert plsync.sync_user_playlists(db, user) == {"synced": 0, "reason": "busy"}
|
|
for _ in range(4):
|
|
r = c.post("/api/playlists/sync-youtube")
|
|
assert r.status_code == 409
|
|
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_all_busy"}
|
|
assert c.post("/api/playlists/sync-youtube").json() == {"ok": True, "synced": 2}
|
|
|
|
|
|
def test_a_gone_mirror_is_deleted_even_when_no_list_refreshes(db, make_user, monkeypatch):
|
|
"""Every remaining mirror has unpushed edits, so nothing below the deletion commits."""
|
|
user = make_user()
|
|
gone = Playlist(user_id=user.id, name="Gone", source="youtube", yt_playlist_id="PLgone")
|
|
kept = Playlist(user_id=user.id, name="K", source="youtube", yt_playlist_id="PL123", dirty=True)
|
|
other = Playlist(
|
|
user_id=user.id, name="O", source="youtube", yt_playlist_id="PL999", dirty=True
|
|
)
|
|
db.add_all([gone, kept, other])
|
|
db.commit()
|
|
gone_id = gone.id
|
|
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
|
|
|
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
|
db.rollback() # what the request's session close does to an uncommitted deletion
|
|
assert db.get(Playlist, gone_id) is None
|
|
|
|
|
|
def test_a_mirror_edited_during_the_pull_is_not_replaced(db, make_user, monkeypatch):
|
|
"""The loop's `dirty` is a snapshot from before the YouTube calls; a remove committed by another
|
|
request meanwhile must win over YouTube's order."""
|
|
user = make_user()
|
|
pl = _linked(db, user, source="youtube")
|
|
replaced: list[int] = []
|
|
|
|
class EditedMeanwhile(FakeYT):
|
|
def iter_my_playlists(self):
|
|
return [{"id": "PL123", "title": "Renamed"}]
|
|
|
|
def iter_my_playlist_video_ids(self, _ytid):
|
|
with SessionLocal() as other:
|
|
other.get(Playlist, pl.id).dirty = True
|
|
other.commit()
|
|
return []
|
|
|
|
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
|
monkeypatch.setattr(plsync, "YouTubeClient", EditedMeanwhile)
|
|
monkeypatch.setattr(
|
|
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
|
)
|
|
|
|
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
|
assert replaced == []
|
|
db.expire_all()
|
|
assert db.get(Playlist, pl.id).name == "L"
|