peter
df4a79a1f1
E32 review round 1 (B1, B2, B3, B8). - Plex watch reconcile: a section the server SAYS is empty (totalSize 0, nothing listed) is a complete listing. It read as "cut short" forever, so one empty library stopped un-watches from ever syncing. A listing with no total at all stays incomplete (the client now returns None for it instead of 0), and a whole scan that listed nothing still clears nothing. Tests now cover completeness across two sections, both orders. - Plex collections: no members from a collection whose listing counts some (childCount) keeps the membership we hold, in the full sync and in the targeted resync after a write. - YouTube playlists: no items for a playlist whose listing counts some (itemCount) keeps the mirror in the bulk pull, and is refused as retryable in "reset to YouTube" and in the push plan/merge, where it read as "YouTube deleted them all". The rule sits beside vanished(). - A playlist deleted on YouTube answered "reconnect your account"; it now says it is gone (errors.youtube.gone, HU+EN). - The pure 3-way merge moved to sync/playlist_merge.py; playlists.py is back under 800 lines and off the size list.
361 lines
14 KiB
Python
361 lines
14 KiB
Python
"""S7 — two-way (3-way) YouTube playlist merge.
|
|
|
|
The load-bearing property: an item added (or deleted) on EITHER side survives the sync — the old
|
|
push was one-directional (local wins, deletes everything not local from YouTube), so YouTube-side
|
|
adds were lost. Pure-merge classification runs with no DB; the apply path uses the `db` lane with a
|
|
fake YouTube client so both the local rows and the simulated YouTube state can be asserted.
|
|
"""
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
|
|
from app.models import Channel, Playlist, PlaylistItem, Video
|
|
from app.sync import playlists as pl_mod
|
|
from app.sync.playlist_merge import merge_plan
|
|
from app.sync.playlists import (
|
|
merge_playlist,
|
|
parse_baseline,
|
|
parse_unavailable,
|
|
plan_merge,
|
|
)
|
|
from app.youtube.client import YouTubeError
|
|
|
|
# --- pure merge classification (no DB) --------------------------------------------------------
|
|
|
|
|
|
class TestMergePlan:
|
|
def test_no_baseline_unions_losslessly(self):
|
|
# First-ever merge: can't tell an add from a delete, so keep everything, delete nothing.
|
|
mp = merge_plan(None, local=["a", "b"], youtube=["b", "c"])
|
|
assert mp.add_to_yt == ["a"] # local-only pushed out
|
|
assert mp.add_to_local == ["c"] # youtube-only pulled in
|
|
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
|
assert mp.final == ["a", "b", "c"] # local order, youtube-only appended
|
|
assert mp.baseline_known is False
|
|
|
|
def test_adds_on_both_sides_both_survive(self):
|
|
# baseline b,c ; local added 'a' ; youtube added 'd' — nothing deleted.
|
|
mp = merge_plan(["b", "c"], local=["a", "b", "c"], youtube=["b", "c", "d"])
|
|
assert mp.add_to_yt == ["a"]
|
|
assert mp.add_to_local == ["d"]
|
|
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
|
assert mp.final == ["a", "b", "c", "d"]
|
|
|
|
def test_genuine_local_delete_propagates_to_youtube(self):
|
|
# baseline a,b,c ; local removed 'b' ; youtube unchanged -> remove b from youtube.
|
|
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "b", "c"])
|
|
assert mp.remove_from_yt == ["b"]
|
|
assert mp.add_to_yt == [] and mp.add_to_local == []
|
|
assert mp.final == ["a", "c"]
|
|
|
|
def test_genuine_youtube_delete_propagates_to_local(self):
|
|
# baseline a,b,c ; youtube removed 'b' ; local unchanged -> remove b from local.
|
|
mp = merge_plan(["a", "b", "c"], local=["a", "b", "c"], youtube=["a", "c"])
|
|
assert mp.remove_from_local == ["b"]
|
|
assert mp.final == ["a", "c"]
|
|
|
|
def test_delete_on_both_sides_is_gone_not_a_conflict(self):
|
|
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "c"])
|
|
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
|
assert mp.final == ["a", "c"]
|
|
|
|
def test_a_local_delete_beats_a_stale_youtube_copy(self):
|
|
# 'b' was synced, deleted locally, still present on youtube -> it must be removed there,
|
|
# NOT resurrected into local as a youtube-add.
|
|
mp = merge_plan(["a", "b"], local=["a"], youtube=["a", "b"])
|
|
assert mp.add_to_local == []
|
|
assert mp.remove_from_yt == ["b"]
|
|
assert "b" not in mp.final
|
|
|
|
|
|
# --- apply path (db lane + fake YouTube) ------------------------------------------------------
|
|
|
|
|
|
class FakeYouTube:
|
|
"""Enough of YouTubeClient for merge_playlist: an in-memory playlist of (item_id, video_id)
|
|
and the write ops that mutate it. Every catalog video is pre-seeded so _ensure_videos never
|
|
needs get_videos."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
title: str,
|
|
items: list[str],
|
|
unavailable: set[str] | None = None,
|
|
fail_add: set[str] | None = None,
|
|
):
|
|
self.title = title
|
|
self._seq = 0
|
|
self.items = [(self._new_id(), v) for v in items]
|
|
# Ids the Data API won't return (private/deleted) — get_videos omits them, so _ensure_videos
|
|
# can't ingest them. They still live in the playlist (`items`) like a real un-ingestable video.
|
|
self.unavailable = set(unavailable or ())
|
|
# Ids whose add_playlist_item raises, to exercise the partial-failure baseline-hold path.
|
|
self.fail_add = set(fail_add or ())
|
|
|
|
def _new_id(self) -> str:
|
|
self._seq += 1
|
|
return f"it{self._seq}"
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def create_playlist(self, title, description="", privacy="private"):
|
|
self.title = title
|
|
return "PLnew"
|
|
|
|
def get_playlist_snippet(self, pid):
|
|
return {"title": self.title}
|
|
|
|
def iter_playlist_items_with_ids(self, pid):
|
|
return [{"item_id": iid, "video_id": v} for iid, v in self.items]
|
|
|
|
def get_videos(self, ids):
|
|
# Mirror the Data API: a minimal ingestable record for each id, EXCEPT the un-ingestable
|
|
# ones (private/deleted), which are simply absent from the response.
|
|
return [
|
|
{
|
|
"id": v,
|
|
"snippet": {
|
|
"channelId": "UCtest",
|
|
"channelTitle": "Test",
|
|
"publishedAt": "2020-01-01T00:00:00Z",
|
|
},
|
|
}
|
|
for v in ids
|
|
if v not in self.unavailable
|
|
]
|
|
|
|
def update_playlist_title(self, pid, title):
|
|
self.title = title
|
|
|
|
def add_playlist_item(self, pid, vid, position=None):
|
|
if vid in self.fail_add:
|
|
raise YouTubeError("simulated add failure")
|
|
iid = self._new_id()
|
|
self.items.append((iid, vid))
|
|
return iid
|
|
|
|
def delete_playlist_item(self, item_id):
|
|
self.items = [(iid, v) for iid, v in self.items if iid != item_id]
|
|
|
|
def move_playlist_item(self, item_id, pid, vid, position):
|
|
row = next((r for r in self.items if r[0] == item_id), None)
|
|
if row is not None:
|
|
self.items.remove(row)
|
|
self.items.insert(position, row)
|
|
|
|
@property
|
|
def video_ids(self) -> list[str]:
|
|
return [v for _, v in self.items]
|
|
|
|
|
|
def _seed_videos(db, ids):
|
|
db.add(Channel(id="UCtest", title="Test"))
|
|
now = datetime.now(timezone.utc)
|
|
for v in ids:
|
|
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
|
db.commit()
|
|
|
|
|
|
def _linked_playlist(db, user, *, local_ids, baseline):
|
|
pl = Playlist(
|
|
user_id=user.id,
|
|
name="Mix",
|
|
source="local",
|
|
yt_playlist_id="PLyt",
|
|
synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline),
|
|
)
|
|
db.add(pl)
|
|
db.flush()
|
|
for pos, v in enumerate(local_ids):
|
|
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
|
db.commit()
|
|
return pl
|
|
|
|
|
|
def _local_ids(db, pl):
|
|
return pl_mod._local_video_ids(db, pl)
|
|
|
|
|
|
@pytest.fixture
|
|
def _fake_yt(monkeypatch):
|
|
holder = {}
|
|
|
|
def install(fake):
|
|
holder["fake"] = fake
|
|
monkeypatch.setattr(pl_mod, "YouTubeClient", lambda db, user: fake)
|
|
return fake
|
|
|
|
return install
|
|
|
|
|
|
def test_adds_on_both_sides_survive_on_both_sides(db, make_user, _fake_yt):
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b", "c", "d"])
|
|
# baseline b,c ; local added 'a' ; youtube added 'd'
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c", "d"]))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["failures"] == []
|
|
assert set(_local_ids(db, pl)) == {"a", "b", "c", "d"}, "youtube-add 'd' pulled into Siftlode"
|
|
assert set(fake.video_ids) == {"a", "b", "c", "d"}, "local-add 'a' pushed to YouTube"
|
|
assert r["inserted"] == 1 and r["pulled"] == 1
|
|
# Baseline advanced to the merged set so a re-sync is a no-op.
|
|
assert set(parse_baseline(pl)) == {"a", "b", "c", "d"}
|
|
|
|
|
|
def test_a_genuine_delete_propagates_without_collateral_loss(db, make_user, _fake_yt):
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b", "c", "d"])
|
|
# baseline a,b,c ; local removed 'b' ; youtube added 'd' (a real add, must NOT be collateral)
|
|
pl = _linked_playlist(db, user, local_ids=["a", "c"], baseline=["a", "b", "c"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c", "d"]))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["failures"] == []
|
|
assert set(fake.video_ids) == {"a", "c", "d"}, "'b' removed on YouTube, 'd' kept"
|
|
assert set(_local_ids(db, pl)) == {"a", "c", "d"}, "'d' pulled in, 'b' stays deleted"
|
|
assert r["deleted"] == 1
|
|
|
|
|
|
def test_first_merge_with_no_baseline_never_deletes(db, make_user, _fake_yt):
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b", "c"])
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=None)
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"]))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["deleted"] == 0 and r["removed_local"] == 0
|
|
assert set(_local_ids(db, pl)) == {"a", "b", "c"}
|
|
assert set(fake.video_ids) == {"a", "b", "c"}
|
|
assert parse_baseline(pl) is not None, "a baseline is recorded after the first union"
|
|
|
|
|
|
def test_create_path_appends_and_records_baseline(db, make_user, _fake_yt):
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b"])
|
|
pl = Playlist(user_id=user.id, name="Fresh", source="local")
|
|
db.add(pl)
|
|
db.flush()
|
|
for pos, v in enumerate(["a", "b"]):
|
|
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
|
db.commit()
|
|
_fake_yt(FakeYouTube(title="Fresh", items=[]))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["created"] == 1 and r["inserted"] == 2
|
|
assert pl.yt_playlist_id # created and linked
|
|
assert parse_baseline(pl) == ["a", "b"]
|
|
|
|
|
|
def test_reorder_reaches_the_merged_order_on_youtube(db, make_user, _fake_yt):
|
|
# Same membership on both sides, but the user reordered locally to c,a,b. The apply-side
|
|
# insertion-sort must drive YouTube to that exact order — asserted as an ordered list, not a
|
|
# set, so an off-by-one or a dropped reorder pass can't pass unnoticed.
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b", "c"])
|
|
pl = _linked_playlist(db, user, local_ids=["c", "a", "b"], baseline=["a", "b", "c"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c"]))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["failures"] == []
|
|
assert fake.video_ids == ["c", "a", "b"], "YouTube order matches the merged (local) order"
|
|
assert r["reordered"] >= 1
|
|
assert parse_baseline(pl) == ["c", "a", "b"]
|
|
|
|
|
|
def test_partial_youtube_failure_holds_the_baseline_for_retry(db, make_user, _fake_yt):
|
|
# A local-add whose YouTube insert fails must NOT advance the baseline — otherwise the retry
|
|
# sync would read the un-pushed add as a genuine YouTube-side delete and silently drop it.
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b", "c"])
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"], fail_add={"a"}))
|
|
|
|
r = merge_playlist(db, user, pl)
|
|
|
|
assert r["failures"] == ["a"], "the failed local-add is reported"
|
|
assert "a" not in fake.video_ids, "the add never landed on YouTube"
|
|
# Baseline is UNCHANGED so the next sync re-classifies 'a' as a local-add and re-pushes it,
|
|
# rather than treating it as a YouTube delete and removing it locally.
|
|
assert parse_baseline(pl) == ["b", "c"]
|
|
assert "a" in set(_local_ids(db, pl)), "'a' stays local for the retry"
|
|
|
|
|
|
def test_unavailable_youtube_video_converges(db, make_user, _fake_yt):
|
|
# A YouTube-only video that can't be ingested (private/deleted) must not make the playlist
|
|
# loop forever re-classifying it as a pending pull. It is recorded as un-ingestable and skipped
|
|
# by the next merge; the playlist reports "up to date".
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b"]) # note: 'x' is NOT in the catalog and can't be ingested
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
|
|
|
r1 = merge_playlist(db, user, pl)
|
|
|
|
assert r1["failures"] == []
|
|
assert r1["pulled"] == 0, "the un-ingestable video did not become a local row"
|
|
assert set(_local_ids(db, pl)) == {"a", "b"}
|
|
assert parse_unavailable(pl) == {"x"}, "'x' recorded as un-ingestable"
|
|
assert "x" in fake.video_ids, "'x' left untouched on YouTube"
|
|
|
|
# Second sync: the preview reports nothing pending (plan_merge filters 'x' via the stored
|
|
# unavailable set). The sync itself re-attempts ingestion of 'x' (the heal path), but since it
|
|
# still won't ingest, 'x' stays recorded as unavailable — no perpetual pending pull.
|
|
plan = plan_merge(db, user, pl)
|
|
assert plan["add_local"] == 0, "no perpetual pending pull — the playlist is up to date"
|
|
r2 = merge_playlist(db, user, pl)
|
|
assert r2["pulled"] == 0 and r2["failures"] == []
|
|
assert parse_unavailable(pl) == {"x"}
|
|
assert "x" in fake.video_ids
|
|
|
|
|
|
def test_unavailable_video_that_becomes_available_is_pulled_in(db, make_user, _fake_yt):
|
|
# A video marked un-ingestable (e.g. private) that later becomes available (public) must
|
|
# re-enter the merge on a normal sync — not stay skipped forever.
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b"])
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
|
|
|
merge_playlist(db, user, pl)
|
|
assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync"
|
|
|
|
# 'x' becomes available; the next sync re-attempts ingestion and pulls it in.
|
|
fake.unavailable.clear()
|
|
r2 = merge_playlist(db, user, pl)
|
|
|
|
assert r2["failures"] == []
|
|
assert "x" in set(_local_ids(db, pl)), "the now-available video is pulled into Siftlode"
|
|
assert parse_unavailable(pl) == set(), "'x' is no longer marked un-ingestable"
|
|
|
|
|
|
def test_unavailable_id_that_leaves_youtube_is_pruned(db, make_user, _fake_yt):
|
|
# A recorded un-ingestable id that is later removed from the YouTube playlist must be pruned
|
|
# from the persisted `unavailable` set (the `& yt_id_set` self-heal), not carried forever.
|
|
user = make_user()
|
|
_seed_videos(db, ["a", "b"])
|
|
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
|
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
|
|
|
merge_playlist(db, user, pl)
|
|
assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync"
|
|
|
|
# 'x' is removed from the YouTube playlist entirely; the next sync must drop it.
|
|
fake.items = [row for row in fake.items if row[1] != "x"]
|
|
r2 = merge_playlist(db, user, pl)
|
|
|
|
assert r2["failures"] == []
|
|
assert parse_unavailable(pl) == set(), "'x' pruned once it left YouTube"
|