"""S7 — two-way (3-way) YouTube playlist merge. The load-bearing property: an item added (or deleted) on EITHER side survives the sync — the old push was one-directional (local wins, deletes everything not local from YouTube), so YouTube-side adds were lost. Pure-merge classification runs with no DB; the apply path uses the `db` lane with a fake YouTube client so both the local rows and the simulated YouTube state can be asserted. """ from datetime import datetime, timezone import pytest from app.models import Channel, Playlist, PlaylistItem, Video from app.sync import playlists as pl_mod from app.sync.playlists import ( merge_plan, merge_playlist, parse_baseline, parse_unavailable, plan_merge, ) from app.youtube.client import YouTubeError # --- pure merge classification (no DB) -------------------------------------------------------- class TestMergePlan: def test_no_baseline_unions_losslessly(self): # First-ever merge: can't tell an add from a delete, so keep everything, delete nothing. mp = merge_plan(None, local=["a", "b"], youtube=["b", "c"]) assert mp.add_to_yt == ["a"] # local-only pushed out assert mp.add_to_local == ["c"] # youtube-only pulled in assert mp.remove_from_yt == [] and mp.remove_from_local == [] assert mp.final == ["a", "b", "c"] # local order, youtube-only appended assert mp.baseline_known is False def test_adds_on_both_sides_both_survive(self): # baseline b,c ; local added 'a' ; youtube added 'd' — nothing deleted. mp = merge_plan(["b", "c"], local=["a", "b", "c"], youtube=["b", "c", "d"]) assert mp.add_to_yt == ["a"] assert mp.add_to_local == ["d"] assert mp.remove_from_yt == [] and mp.remove_from_local == [] assert mp.final == ["a", "b", "c", "d"] def test_genuine_local_delete_propagates_to_youtube(self): # baseline a,b,c ; local removed 'b' ; youtube unchanged -> remove b from youtube. mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "b", "c"]) assert mp.remove_from_yt == ["b"] assert mp.add_to_yt == [] and mp.add_to_local == [] assert mp.final == ["a", "c"] def test_genuine_youtube_delete_propagates_to_local(self): # baseline a,b,c ; youtube removed 'b' ; local unchanged -> remove b from local. mp = merge_plan(["a", "b", "c"], local=["a", "b", "c"], youtube=["a", "c"]) assert mp.remove_from_local == ["b"] assert mp.final == ["a", "c"] def test_delete_on_both_sides_is_gone_not_a_conflict(self): mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "c"]) assert mp.remove_from_yt == [] and mp.remove_from_local == [] assert mp.final == ["a", "c"] def test_a_local_delete_beats_a_stale_youtube_copy(self): # 'b' was synced, deleted locally, still present on youtube -> it must be removed there, # NOT resurrected into local as a youtube-add. mp = merge_plan(["a", "b"], local=["a"], youtube=["a", "b"]) assert mp.add_to_local == [] assert mp.remove_from_yt == ["b"] assert "b" not in mp.final # --- apply path (db lane + fake YouTube) ------------------------------------------------------ class FakeYouTube: """Enough of YouTubeClient for merge_playlist: an in-memory playlist of (item_id, video_id) and the write ops that mutate it. Every catalog video is pre-seeded so _ensure_videos never needs get_videos.""" def __init__( self, *, title: str, items: list[str], unavailable: set[str] | None = None, fail_add: set[str] | None = None, ): self.title = title self._seq = 0 self.items = [(self._new_id(), v) for v in items] # Ids the Data API won't return (private/deleted) — get_videos omits them, so _ensure_videos # can't ingest them. They still live in the playlist (`items`) like a real un-ingestable video. self.unavailable = set(unavailable or ()) # Ids whose add_playlist_item raises, to exercise the partial-failure baseline-hold path. self.fail_add = set(fail_add or ()) def _new_id(self) -> str: self._seq += 1 return f"it{self._seq}" def __enter__(self): return self def __exit__(self, *exc): return False def create_playlist(self, title, description="", privacy="private"): self.title = title return "PLnew" def get_playlist_snippet(self, pid): return {"title": self.title} def iter_playlist_items_with_ids(self, pid): return [{"item_id": iid, "video_id": v} for iid, v in self.items] def get_videos(self, ids): # Mirror the Data API: a minimal ingestable record for each id, EXCEPT the un-ingestable # ones (private/deleted), which are simply absent from the response. return [ { "id": v, "snippet": { "channelId": "UCtest", "channelTitle": "Test", "publishedAt": "2020-01-01T00:00:00Z", }, } for v in ids if v not in self.unavailable ] def update_playlist_title(self, pid, title): self.title = title def add_playlist_item(self, pid, vid, position=None): if vid in self.fail_add: raise YouTubeError("simulated add failure") iid = self._new_id() self.items.append((iid, vid)) return iid def delete_playlist_item(self, item_id): self.items = [(iid, v) for iid, v in self.items if iid != item_id] def move_playlist_item(self, item_id, pid, vid, position): row = next((r for r in self.items if r[0] == item_id), None) if row is not None: self.items.remove(row) self.items.insert(position, row) @property def video_ids(self) -> list[str]: return [v for _, v in self.items] def _seed_videos(db, ids): db.add(Channel(id="UCtest", title="Test")) now = datetime.now(timezone.utc) for v in ids: db.add(Video(id=v, channel_id="UCtest", published_at=now)) db.commit() def _linked_playlist(db, user, *, local_ids, baseline): pl = Playlist( user_id=user.id, name="Mix", source="local", yt_playlist_id="PLyt", synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline), ) db.add(pl) db.flush() for pos, v in enumerate(local_ids): db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos)) db.commit() return pl def _local_ids(db, pl): return pl_mod._local_video_ids(db, pl) @pytest.fixture def _fake_yt(monkeypatch): holder = {} def install(fake): holder["fake"] = fake monkeypatch.setattr(pl_mod, "YouTubeClient", lambda db, user: fake) return fake return install def test_adds_on_both_sides_survive_on_both_sides(db, make_user, _fake_yt): user = make_user() _seed_videos(db, ["a", "b", "c", "d"]) # baseline b,c ; local added 'a' ; youtube added 'd' pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c", "d"])) r = merge_playlist(db, user, pl) assert r["failures"] == [] assert set(_local_ids(db, pl)) == {"a", "b", "c", "d"}, "youtube-add 'd' pulled into Siftlode" assert set(fake.video_ids) == {"a", "b", "c", "d"}, "local-add 'a' pushed to YouTube" assert r["inserted"] == 1 and r["pulled"] == 1 # Baseline advanced to the merged set so a re-sync is a no-op. assert set(parse_baseline(pl)) == {"a", "b", "c", "d"} def test_a_genuine_delete_propagates_without_collateral_loss(db, make_user, _fake_yt): user = make_user() _seed_videos(db, ["a", "b", "c", "d"]) # baseline a,b,c ; local removed 'b' ; youtube added 'd' (a real add, must NOT be collateral) pl = _linked_playlist(db, user, local_ids=["a", "c"], baseline=["a", "b", "c"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c", "d"])) r = merge_playlist(db, user, pl) assert r["failures"] == [] assert set(fake.video_ids) == {"a", "c", "d"}, "'b' removed on YouTube, 'd' kept" assert set(_local_ids(db, pl)) == {"a", "c", "d"}, "'d' pulled in, 'b' stays deleted" assert r["deleted"] == 1 def test_first_merge_with_no_baseline_never_deletes(db, make_user, _fake_yt): user = make_user() _seed_videos(db, ["a", "b", "c"]) pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=None) fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"])) r = merge_playlist(db, user, pl) assert r["deleted"] == 0 and r["removed_local"] == 0 assert set(_local_ids(db, pl)) == {"a", "b", "c"} assert set(fake.video_ids) == {"a", "b", "c"} assert parse_baseline(pl) is not None, "a baseline is recorded after the first union" def test_create_path_appends_and_records_baseline(db, make_user, _fake_yt): user = make_user() _seed_videos(db, ["a", "b"]) pl = Playlist(user_id=user.id, name="Fresh", source="local") db.add(pl) db.flush() for pos, v in enumerate(["a", "b"]): db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos)) db.commit() _fake_yt(FakeYouTube(title="Fresh", items=[])) r = merge_playlist(db, user, pl) assert r["created"] == 1 and r["inserted"] == 2 assert pl.yt_playlist_id # created and linked assert parse_baseline(pl) == ["a", "b"] def test_reorder_reaches_the_merged_order_on_youtube(db, make_user, _fake_yt): # Same membership on both sides, but the user reordered locally to c,a,b. The apply-side # insertion-sort must drive YouTube to that exact order — asserted as an ordered list, not a # set, so an off-by-one or a dropped reorder pass can't pass unnoticed. user = make_user() _seed_videos(db, ["a", "b", "c"]) pl = _linked_playlist(db, user, local_ids=["c", "a", "b"], baseline=["a", "b", "c"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c"])) r = merge_playlist(db, user, pl) assert r["failures"] == [] assert fake.video_ids == ["c", "a", "b"], "YouTube order matches the merged (local) order" assert r["reordered"] >= 1 assert parse_baseline(pl) == ["c", "a", "b"] def test_partial_youtube_failure_holds_the_baseline_for_retry(db, make_user, _fake_yt): # A local-add whose YouTube insert fails must NOT advance the baseline — otherwise the retry # sync would read the un-pushed add as a genuine YouTube-side delete and silently drop it. user = make_user() _seed_videos(db, ["a", "b", "c"]) pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"], fail_add={"a"})) r = merge_playlist(db, user, pl) assert r["failures"] == ["a"], "the failed local-add is reported" assert "a" not in fake.video_ids, "the add never landed on YouTube" # Baseline is UNCHANGED so the next sync re-classifies 'a' as a local-add and re-pushes it, # rather than treating it as a YouTube delete and removing it locally. assert parse_baseline(pl) == ["b", "c"] assert "a" in set(_local_ids(db, pl)), "'a' stays local for the retry" def test_unavailable_youtube_video_converges(db, make_user, _fake_yt): # A YouTube-only video that can't be ingested (private/deleted) must not make the playlist # loop forever re-classifying it as a pending pull. It is recorded as un-ingestable and skipped # by the next merge; the playlist reports "up to date". user = make_user() _seed_videos(db, ["a", "b"]) # note: 'x' is NOT in the catalog and can't be ingested pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"})) r1 = merge_playlist(db, user, pl) assert r1["failures"] == [] assert r1["pulled"] == 0, "the un-ingestable video did not become a local row" assert set(_local_ids(db, pl)) == {"a", "b"} assert parse_unavailable(pl) == {"x"}, "'x' recorded as un-ingestable" assert "x" in fake.video_ids, "'x' left untouched on YouTube" # Second sync: the preview reports nothing pending (plan_merge filters 'x' via the stored # unavailable set). The sync itself re-attempts ingestion of 'x' (the heal path), but since it # still won't ingest, 'x' stays recorded as unavailable — no perpetual pending pull. plan = plan_merge(db, user, pl) assert plan["add_local"] == 0, "no perpetual pending pull — the playlist is up to date" r2 = merge_playlist(db, user, pl) assert r2["pulled"] == 0 and r2["failures"] == [] assert parse_unavailable(pl) == {"x"} assert "x" in fake.video_ids def test_unavailable_video_that_becomes_available_is_pulled_in(db, make_user, _fake_yt): # A video marked un-ingestable (e.g. private) that later becomes available (public) must # re-enter the merge on a normal sync — not stay skipped forever. user = make_user() _seed_videos(db, ["a", "b"]) pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"})) merge_playlist(db, user, pl) assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync" # 'x' becomes available; the next sync re-attempts ingestion and pulls it in. fake.unavailable.clear() r2 = merge_playlist(db, user, pl) assert r2["failures"] == [] assert "x" in set(_local_ids(db, pl)), "the now-available video is pulled into Siftlode" assert parse_unavailable(pl) == set(), "'x' is no longer marked un-ingestable" def test_unavailable_id_that_leaves_youtube_is_pruned(db, make_user, _fake_yt): # A recorded un-ingestable id that is later removed from the YouTube playlist must be pruned # from the persisted `unavailable` set (the `& yt_id_set` self-heal), not carried forever. user = make_user() _seed_videos(db, ["a", "b"]) pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"]) fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"})) merge_playlist(db, user, pl) assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync" # 'x' is removed from the YouTube playlist entirely; the next sync must drop it. fake.items = [row for row in fake.items if row[1] != "x"] r2 = merge_playlist(db, user, pl) assert r2["failures"] == [] assert parse_unavailable(pl) == set(), "'x' pruned once it left YouTube"