feat(playlists): two-way (3-way) merge YouTube sync + mandatory preview (S7)
"Sync to YouTube" was a one-directional destructive push (local wins, deletes from YouTube every item not in the local list), so items added on YouTube were lost and a dirty pull was skipped entirely. Replace it with a real 3-way merge: each item is classified against the last-synced baseline as a local-add, a YouTube-add, or a genuine delete, so an add or delete on either side survives. - Persist the ordered baseline id set (new playlists.synced_video_ids, migration 0075); the fingerprint hash alone couldn't drive a merge. NULL baseline (fresh export / pre-S7 link) unions losslessly on the first sync and records one. - Pure merge_plan(baseline, local, youtube) -> add/remove per side + merged order (local order, YouTube-only additions appended). merge_playlist applies the local side directly and the YouTube side through the failure breaker; the baseline only advances on full success so a partial failure re-converges. - plan_merge/push-plan now report what each side gains and loses; the confirm dialog shows the two-way plan before applying (pushPlanMerge, HU+EN). - Tests: 10 new (pure classification incl. delete-vs-stale-copy; DB apply proving a YouTube-add and a Siftlode-add both survive on both sides, a genuine delete propagates without collateral loss, no-baseline never deletes, create path).
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
"""playlist synced_video_ids (S7: 3-way merge baseline for two-way YouTube sync)
|
||||
|
||||
The one-directional "local wins" push is replaced by a 3-way merge that classifies each item
|
||||
against the last-synced state. That needs the raw ordered id set as a baseline, not just the
|
||||
`synced_fingerprint` hash. Add a nullable `synced_video_ids` (JSON list) column; NULL on every
|
||||
existing row means "no baseline yet" — the first merge then unions losslessly and records one.
|
||||
Forward-only in practice (a downgrade just drops the column).
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0075_playlist_sync_baseline"
|
||||
down_revision: Union[str, None] = "0074_download_credentials"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("playlists", sa.Column("synced_video_ids", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("playlists", "synced_video_ids")
|
||||
@@ -551,6 +551,10 @@ class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
||||
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
||||
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
||||
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
||||
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
||||
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
||||
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
items: Mapped[list["PlaylistItem"]] = relationship(
|
||||
|
||||
@@ -14,8 +14,8 @@ from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.sync.playlists import (
|
||||
plan_push,
|
||||
push_playlist,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
repull_playlist,
|
||||
sync_user_playlists,
|
||||
@@ -329,13 +329,13 @@ def push_plan(
|
||||
user: User = Depends(require_write_scope),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Dry run: what 'Sync to YouTube' would do + the quota estimate, so the UI can show a
|
||||
confirm. For a linked playlist this reads the live YouTube state (cheap, 1 unit/page).
|
||||
Write scope is required (require_write_scope) — syncing is a YouTube write."""
|
||||
"""Dry run: what a two-way 'Sync to YouTube' merge would do (what each side gains/loses) +
|
||||
the quota estimate, so the UI can show a confirm. For a linked playlist this reads the live
|
||||
YouTube state (cheap, 1 unit/page). Write scope is required — syncing is a YouTube write."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_push(db, user, pl)
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="Couldn't read the playlist on YouTube.")
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
@@ -349,14 +349,14 @@ def push(
|
||||
user: User = Depends(require_write_scope),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Push a local playlist to YouTube (create it on first push, else reconcile to match
|
||||
local — local wins). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big playlist can't strand half-pushed. Write scope required (require_write_scope)."""
|
||||
"""Two-way sync a local playlist with YouTube (create it on first push, else 3-way merge so
|
||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the push, halving read-quota and closing the second read→write TOCTOU
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
@@ -365,13 +365,13 @@ def push(
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_push(db, user, pl, live=live)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Not enough YouTube quota left today for this sync.",
|
||||
)
|
||||
return push_playlist(db, user, pl, live=live)
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
||||
|
||||
+182
-71
@@ -1,14 +1,16 @@
|
||||
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: push local
|
||||
edits out to YouTube (`plan_push` / `push_playlist`), behind the optional write scope.
|
||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
||||
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
||||
|
||||
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
||||
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
||||
ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
@@ -42,6 +44,30 @@ def current_fingerprint(db: Session, pl: Playlist) -> str:
|
||||
return fingerprint(pl.name, _local_video_ids(db, pl))
|
||||
|
||||
|
||||
def _dump_ids(ids: list[str]) -> str:
|
||||
"""Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`)."""
|
||||
return json.dumps(ids)
|
||||
|
||||
|
||||
def parse_baseline(pl: Playlist) -> list[str] | None:
|
||||
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
|
||||
export, or a link created before S7). None makes the first merge union losslessly."""
|
||||
if not pl.synced_video_ids:
|
||||
return None
|
||||
try:
|
||||
val = json.loads(pl.synced_video_ids)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return [str(v) for v in val] if isinstance(val, list) else None
|
||||
|
||||
|
||||
def _set_baseline(pl: Playlist, ids: list[str]) -> None:
|
||||
"""Record `ids` as both the hash (drives `dirty`) and the raw ordered set (drives merge)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
||||
pl.synced_video_ids = _dump_ids(ids)
|
||||
pl.dirty = False
|
||||
|
||||
|
||||
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
||||
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
||||
@@ -116,12 +142,18 @@ def _replace_items_from_youtube(
|
||||
if v in present and v not in seen:
|
||||
seen.add(v)
|
||||
ordered.append(v)
|
||||
_set_local_items(db, pl, ordered)
|
||||
# After a pull/reset local == YouTube, so this ordered set IS the new merge baseline.
|
||||
_set_baseline(pl, ordered)
|
||||
return ordered
|
||||
|
||||
|
||||
def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None:
|
||||
"""Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller
|
||||
is responsible for having ensured every id exists in the catalog."""
|
||||
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
|
||||
for pos, vid in enumerate(ordered):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ordered)
|
||||
pl.dirty = False
|
||||
return ordered
|
||||
|
||||
|
||||
def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
@@ -216,22 +248,81 @@ def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
||||
return moves
|
||||
|
||||
|
||||
def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Compute (without writing) what pushing this local playlist to YouTube would do:
|
||||
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
|
||||
YouTube has diverged (items present there that local will remove). Costs read units
|
||||
(1/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched
|
||||
(current_items, snippet) tuple so the caller can share one read with push_playlist."""
|
||||
desired = _local_video_ids(db, pl)
|
||||
@dataclass
|
||||
class MergePlan:
|
||||
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
||||
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
||||
|
||||
final: list[str] # merged order (local order, YouTube-only adds appended)
|
||||
add_to_yt: list[str] # local-adds to push out to YouTube
|
||||
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
||||
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
||||
remove_from_local: list[str] # items deleted on YouTube that are still local
|
||||
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
||||
baseline_known: bool
|
||||
|
||||
|
||||
def merge_plan(baseline: list[str] | None, local: list[str], youtube: list[str]) -> MergePlan:
|
||||
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
||||
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
||||
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
||||
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
||||
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
||||
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
||||
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
||||
additions appended in their YouTube order."""
|
||||
L = list(dict.fromkeys(local))
|
||||
Y = list(dict.fromkeys(youtube))
|
||||
Lset, Yset = set(L), set(Y)
|
||||
if baseline is None:
|
||||
add_to_yt = [v for v in L if v not in Yset]
|
||||
add_to_local = [v for v in Y if v not in Lset]
|
||||
remove_from_yt: list[str] = []
|
||||
remove_from_local: list[str] = []
|
||||
final = L + [v for v in Y if v not in Lset]
|
||||
else:
|
||||
B = set(baseline)
|
||||
deleted_local = B - Lset # was synced, removed on our side
|
||||
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
||||
gone = deleted_local | deleted_yt
|
||||
remove_from_yt = [v for v in Y if v in deleted_local]
|
||||
remove_from_local = [v for v in L if v in deleted_yt]
|
||||
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
||||
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
||||
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
||||
final_set = set(final)
|
||||
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
||||
# the reorder pass then moves items into `final`.
|
||||
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
||||
reorders = _reorder_moves(after_membership, final)
|
||||
return MergePlan(
|
||||
final=final,
|
||||
add_to_yt=add_to_yt,
|
||||
add_to_local=add_to_local,
|
||||
remove_from_yt=remove_from_yt,
|
||||
remove_from_local=remove_from_local,
|
||||
reorders=reorders,
|
||||
baseline_known=baseline is not None,
|
||||
)
|
||||
|
||||
|
||||
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
||||
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
||||
writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked
|
||||
playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple
|
||||
so the caller can share one read with merge_playlist."""
|
||||
local = _local_video_ids(db, pl)
|
||||
if not pl.yt_playlist_id:
|
||||
ops = 1 + len(desired) # create the playlist + one insert per video
|
||||
ops = 1 + len(local) # create the playlist + one insert per video
|
||||
return {
|
||||
"action": "create",
|
||||
"to_insert": len(desired),
|
||||
"to_delete": 0,
|
||||
"add_yt": len(local),
|
||||
"add_local": 0,
|
||||
"remove_yt": 0,
|
||||
"remove_local": 0,
|
||||
"to_reorder": 0,
|
||||
"rename": False,
|
||||
"yt_extra": 0,
|
||||
"units_estimate": ops * WRITE_UNIT_COST,
|
||||
}
|
||||
if live is not None:
|
||||
@@ -241,23 +332,17 @@ def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
||||
cur_vids = [it["video_id"] for it in current]
|
||||
cur_set = set(cur_vids)
|
||||
desired_set = set(desired)
|
||||
to_insert = [v for v in desired if v not in cur_set]
|
||||
to_delete = [v for v in cur_vids if v not in desired_set]
|
||||
# After membership reconciliation, the YT order is current-minus-deletes plus inserts
|
||||
# appended; the reorder pass then moves items into the desired order.
|
||||
after_membership = [v for v in cur_vids if v in desired_set] + to_insert
|
||||
reorders = _reorder_moves(after_membership, desired)
|
||||
ops = len(to_insert) + len(to_delete) + reorders + (1 if rename else 0)
|
||||
yt_ids = [it["video_id"] for it in current]
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids)
|
||||
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
||||
return {
|
||||
"action": "update",
|
||||
"to_insert": len(to_insert),
|
||||
"to_delete": len(to_delete),
|
||||
"to_reorder": reorders,
|
||||
"add_yt": len(mp.add_to_yt),
|
||||
"add_local": len(mp.add_to_local),
|
||||
"remove_yt": len(mp.remove_from_yt),
|
||||
"remove_local": len(mp.remove_from_local),
|
||||
"to_reorder": mp.reorders,
|
||||
"rename": rename,
|
||||
"yt_extra": len(to_delete),
|
||||
"units_estimate": ops * WRITE_UNIT_COST,
|
||||
}
|
||||
|
||||
@@ -327,13 +412,16 @@ class _Breaker:
|
||||
return out
|
||||
|
||||
|
||||
def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
|
||||
and order so YouTube matches local (local wins). Marks the playlist clean on success.
|
||||
def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE
|
||||
both sides so an item added (or deleted) on either side survives — no side wins wholesale.
|
||||
Local edits are applied directly; YouTube edits go through the failure breaker. On full
|
||||
success the merged set becomes the new baseline and the playlist is marked clean; on any
|
||||
YouTube failure the baseline is left untouched so the next sync retries and converges.
|
||||
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
|
||||
(current_items, snippet) tuple (from plan_push's read) to avoid a second live page-through."""
|
||||
desired = _local_video_ids(db, pl)
|
||||
inserted = deleted = reordered = created = 0
|
||||
(current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through."""
|
||||
local = _local_video_ids(db, pl)
|
||||
inserted = deleted = reordered = created = pulled = removed_local = 0
|
||||
failures: list[str] = []
|
||||
breaker = _Breaker()
|
||||
with YouTubeClient(db, user) as yt:
|
||||
@@ -341,8 +429,8 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
||||
db.commit()
|
||||
created = 1
|
||||
# Brand-new playlist: just append every video in order.
|
||||
for vid in desired:
|
||||
# Brand-new playlist: just append every local video in order.
|
||||
for vid in local:
|
||||
added = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
@@ -350,60 +438,82 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
if not isinstance(added, _Failed):
|
||||
inserted += 1
|
||||
if not failures:
|
||||
pl.synced_fingerprint = fingerprint(pl.name, desired)
|
||||
pl.dirty = False
|
||||
_set_baseline(pl, local)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
"inserted": inserted,
|
||||
"deleted": deleted,
|
||||
"reordered": reordered,
|
||||
"pulled": pulled,
|
||||
"removed_local": removed_local,
|
||||
"failures": failures,
|
||||
"aborted": breaker.tripped,
|
||||
"yt_playlist_id": pl.yt_playlist_id,
|
||||
}
|
||||
|
||||
# Existing link: diff against the live YouTube state (reuse plan_push's read if given).
|
||||
# Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read).
|
||||
if live is not None:
|
||||
current, snippet = live
|
||||
else:
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||
yt_ids = [it["video_id"] for it in current]
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids)
|
||||
|
||||
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
||||
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
||||
present = (
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all())
|
||||
if mp.final
|
||||
else set()
|
||||
)
|
||||
final_local = [v for v in mp.final if v in present]
|
||||
pulled = len([v for v in mp.add_to_local if v in present])
|
||||
removed_local = len(mp.remove_from_local)
|
||||
_set_local_items(db, pl, final_local)
|
||||
db.commit()
|
||||
|
||||
# --- YouTube side (metered, breaker-guarded).
|
||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
||||
try:
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
||||
except YouTubeError:
|
||||
failures.append("title")
|
||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||
cur_vids = [it["video_id"] for it in current]
|
||||
desired_set = set(desired)
|
||||
# Delete extras (local is authoritative).
|
||||
for it in current:
|
||||
if it["video_id"] not in desired_set:
|
||||
dropped = breaker.run(
|
||||
lambda i=it: yt.delete_playlist_item(i["item_id"]),
|
||||
lambda i=it: failures.append(i["video_id"]),
|
||||
)
|
||||
if not isinstance(dropped, _Failed):
|
||||
deleted += 1
|
||||
# Insert missing (append; the reorder pass below fixes positions).
|
||||
cur_set = set(cur_vids)
|
||||
for vid in desired:
|
||||
if vid not in cur_set:
|
||||
item_id = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(item_id, _Failed):
|
||||
item_id_by_vid[vid] = item_id
|
||||
inserted += 1
|
||||
# Reorder to match `desired` via insertion-sort moves over a local model.
|
||||
model = [v for v in cur_vids if v in desired_set] + [
|
||||
v for v in desired if v not in cur_set and v in item_id_by_vid
|
||||
# Propagate genuine local deletes.
|
||||
for vid in mp.remove_from_yt:
|
||||
item_id = item_id_by_vid.get(vid)
|
||||
if item_id is None:
|
||||
continue
|
||||
dropped = breaker.run(
|
||||
lambda i=item_id: yt.delete_playlist_item(i),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(dropped, _Failed):
|
||||
deleted += 1
|
||||
item_id_by_vid.pop(vid, None)
|
||||
# Insert local-adds (append; the reorder pass below fixes positions).
|
||||
for vid in mp.add_to_yt:
|
||||
item_id = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(item_id, _Failed):
|
||||
item_id_by_vid[vid] = item_id
|
||||
inserted += 1
|
||||
# Reorder YouTube to match `final` via insertion-sort moves over a local model.
|
||||
final_set = set(mp.final)
|
||||
model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [
|
||||
v for v in mp.add_to_yt if v in item_id_by_vid
|
||||
]
|
||||
for i, want in enumerate(desired):
|
||||
for i, want in enumerate(mp.final):
|
||||
if want not in item_id_by_vid:
|
||||
continue # failed insert; skip
|
||||
continue # deleted-from-YT or failed insert; skip
|
||||
if i < len(model) and model[i] == want:
|
||||
continue
|
||||
moved = breaker.run(
|
||||
@@ -419,14 +529,15 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
model.pop(j)
|
||||
model.insert(i, want)
|
||||
if not failures:
|
||||
pl.synced_fingerprint = fingerprint(pl.name, desired)
|
||||
pl.dirty = False
|
||||
_set_baseline(pl, final_local)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
"inserted": inserted,
|
||||
"deleted": deleted,
|
||||
"reordered": reordered,
|
||||
"pulled": pulled,
|
||||
"removed_local": removed_local,
|
||||
"failures": failures,
|
||||
"aborted": breaker.tripped,
|
||||
"yt_playlist_id": pl.yt_playlist_id,
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""S7 — two-way (3-way) YouTube playlist merge.
|
||||
|
||||
The load-bearing property: an item added (or deleted) on EITHER side survives the sync — the old
|
||||
push was one-directional (local wins, deletes everything not local from YouTube), so YouTube-side
|
||||
adds were lost. Pure-merge classification runs with no DB; the apply path uses the `db` lane with a
|
||||
fake YouTube client so both the local rows and the simulated YouTube state can be asserted.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
from app.sync import playlists as pl_mod
|
||||
from app.sync.playlists import merge_plan, merge_playlist, parse_baseline
|
||||
|
||||
# --- pure merge classification (no DB) --------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergePlan:
|
||||
def test_no_baseline_unions_losslessly(self):
|
||||
# First-ever merge: can't tell an add from a delete, so keep everything, delete nothing.
|
||||
mp = merge_plan(None, local=["a", "b"], youtube=["b", "c"])
|
||||
assert mp.add_to_yt == ["a"] # local-only pushed out
|
||||
assert mp.add_to_local == ["c"] # youtube-only pulled in
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "b", "c"] # local order, youtube-only appended
|
||||
assert mp.baseline_known is False
|
||||
|
||||
def test_adds_on_both_sides_both_survive(self):
|
||||
# baseline b,c ; local added 'a' ; youtube added 'd' — nothing deleted.
|
||||
mp = merge_plan(["b", "c"], local=["a", "b", "c"], youtube=["b", "c", "d"])
|
||||
assert mp.add_to_yt == ["a"]
|
||||
assert mp.add_to_local == ["d"]
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "b", "c", "d"]
|
||||
|
||||
def test_genuine_local_delete_propagates_to_youtube(self):
|
||||
# baseline a,b,c ; local removed 'b' ; youtube unchanged -> remove b from youtube.
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "b", "c"])
|
||||
assert mp.remove_from_yt == ["b"]
|
||||
assert mp.add_to_yt == [] and mp.add_to_local == []
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_genuine_youtube_delete_propagates_to_local(self):
|
||||
# baseline a,b,c ; youtube removed 'b' ; local unchanged -> remove b from local.
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "b", "c"], youtube=["a", "c"])
|
||||
assert mp.remove_from_local == ["b"]
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_delete_on_both_sides_is_gone_not_a_conflict(self):
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "c"])
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_a_local_delete_beats_a_stale_youtube_copy(self):
|
||||
# 'b' was synced, deleted locally, still present on youtube -> it must be removed there,
|
||||
# NOT resurrected into local as a youtube-add.
|
||||
mp = merge_plan(["a", "b"], local=["a"], youtube=["a", "b"])
|
||||
assert mp.add_to_local == []
|
||||
assert mp.remove_from_yt == ["b"]
|
||||
assert "b" not in mp.final
|
||||
|
||||
|
||||
# --- apply path (db lane + fake YouTube) ------------------------------------------------------
|
||||
|
||||
|
||||
class FakeYouTube:
|
||||
"""Enough of YouTubeClient for merge_playlist: an in-memory playlist of (item_id, video_id)
|
||||
and the write ops that mutate it. Every catalog video is pre-seeded so _ensure_videos never
|
||||
needs get_videos."""
|
||||
|
||||
def __init__(self, *, title: str, items: list[str]):
|
||||
self.title = title
|
||||
self._seq = 0
|
||||
self.items = [(self._new_id(), v) for v in items]
|
||||
|
||||
def _new_id(self) -> str:
|
||||
self._seq += 1
|
||||
return f"it{self._seq}"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def create_playlist(self, title, description="", privacy="private"):
|
||||
self.title = title
|
||||
return "PLnew"
|
||||
|
||||
def get_playlist_snippet(self, pid):
|
||||
return {"title": self.title}
|
||||
|
||||
def iter_playlist_items_with_ids(self, pid):
|
||||
return [{"item_id": iid, "video_id": v} for iid, v in self.items]
|
||||
|
||||
def update_playlist_title(self, pid, title):
|
||||
self.title = title
|
||||
|
||||
def add_playlist_item(self, pid, vid, position=None):
|
||||
iid = self._new_id()
|
||||
self.items.append((iid, vid))
|
||||
return iid
|
||||
|
||||
def delete_playlist_item(self, item_id):
|
||||
self.items = [(iid, v) for iid, v in self.items if iid != item_id]
|
||||
|
||||
def move_playlist_item(self, item_id, pid, vid, position):
|
||||
row = next((r for r in self.items if r[0] == item_id), None)
|
||||
if row is not None:
|
||||
self.items.remove(row)
|
||||
self.items.insert(position, row)
|
||||
|
||||
@property
|
||||
def video_ids(self) -> list[str]:
|
||||
return [v for _, v in self.items]
|
||||
|
||||
|
||||
def _seed_videos(db, ids):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ids:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _linked_playlist(db, user, *, local_ids, baseline):
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Mix",
|
||||
source="local",
|
||||
yt_playlist_id="PLyt",
|
||||
synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(local_ids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _local_ids(db, pl):
|
||||
return pl_mod._local_video_ids(db, pl)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fake_yt(monkeypatch):
|
||||
holder = {}
|
||||
|
||||
def install(fake):
|
||||
holder["fake"] = fake
|
||||
monkeypatch.setattr(pl_mod, "YouTubeClient", lambda db, user: fake)
|
||||
return fake
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def test_adds_on_both_sides_survive_on_both_sides(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c", "d"])
|
||||
# baseline b,c ; local added 'a' ; youtube added 'd'
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c", "d"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == []
|
||||
assert set(_local_ids(db, pl)) == {"a", "b", "c", "d"}, "youtube-add 'd' pulled into Siftlode"
|
||||
assert set(fake.video_ids) == {"a", "b", "c", "d"}, "local-add 'a' pushed to YouTube"
|
||||
assert r["inserted"] == 1 and r["pulled"] == 1
|
||||
# Baseline advanced to the merged set so a re-sync is a no-op.
|
||||
assert set(parse_baseline(pl)) == {"a", "b", "c", "d"}
|
||||
|
||||
|
||||
def test_a_genuine_delete_propagates_without_collateral_loss(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c", "d"])
|
||||
# baseline a,b,c ; local removed 'b' ; youtube added 'd' (a real add, must NOT be collateral)
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "c"], baseline=["a", "b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c", "d"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == []
|
||||
assert set(fake.video_ids) == {"a", "c", "d"}, "'b' removed on YouTube, 'd' kept"
|
||||
assert set(_local_ids(db, pl)) == {"a", "c", "d"}, "'d' pulled in, 'b' stays deleted"
|
||||
assert r["deleted"] == 1
|
||||
|
||||
|
||||
def test_first_merge_with_no_baseline_never_deletes(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c"])
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=None)
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["deleted"] == 0 and r["removed_local"] == 0
|
||||
assert set(_local_ids(db, pl)) == {"a", "b", "c"}
|
||||
assert set(fake.video_ids) == {"a", "b", "c"}
|
||||
assert parse_baseline(pl) is not None, "a baseline is recorded after the first union"
|
||||
|
||||
|
||||
def test_create_path_appends_and_records_baseline(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b"])
|
||||
pl = Playlist(user_id=user.id, name="Fresh", source="local")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(["a", "b"]):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
_fake_yt(FakeYouTube(title="Fresh", items=[]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["created"] == 1 and r["inserted"] == 2
|
||||
assert pl.yt_playlist_id # created and linked
|
||||
assert parse_baseline(pl) == ["a", "b"]
|
||||
@@ -362,7 +362,15 @@ export default function Playlists() {
|
||||
setPushing(true);
|
||||
try {
|
||||
const plan = await api.playlistPushPlan(selectedId);
|
||||
if (plan.action === "update" && !plan.to_insert && !plan.to_delete && !plan.to_reorder) {
|
||||
const nothingToDo =
|
||||
plan.action === "update" &&
|
||||
!plan.add_yt &&
|
||||
!plan.add_local &&
|
||||
!plan.remove_yt &&
|
||||
!plan.remove_local &&
|
||||
!plan.to_reorder &&
|
||||
!plan.rename;
|
||||
if (nothingToDo) {
|
||||
notify({ level: "info", message: t("playlists.pushUpToDate", { name: plName(detail) }) });
|
||||
return;
|
||||
}
|
||||
@@ -376,21 +384,21 @@ export default function Playlists() {
|
||||
});
|
||||
return;
|
||||
}
|
||||
let message =
|
||||
const message =
|
||||
plan.action === "create"
|
||||
? t("playlists.pushPlanCreate", {
|
||||
count: plan.to_insert,
|
||||
count: plan.add_yt,
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
})
|
||||
: t("playlists.pushPlanUpdate", {
|
||||
insert: plan.to_insert,
|
||||
del: plan.to_delete,
|
||||
reorder: plan.to_reorder,
|
||||
: t("playlists.pushPlanMerge", {
|
||||
addYt: plan.add_yt,
|
||||
addLocal: plan.add_local,
|
||||
removeYt: plan.remove_yt,
|
||||
removeLocal: plan.remove_local,
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
});
|
||||
if (plan.yt_extra > 0) message += " " + t("playlists.pushDiverged", { count: plan.yt_extra });
|
||||
const ok = await confirm({
|
||||
title: t("playlists.pushTitle"),
|
||||
message,
|
||||
|
||||
@@ -35,8 +35,7 @@
|
||||
"pushTitle": "Sync to YouTube",
|
||||
"pushConfirm": "Sync",
|
||||
"pushPlanCreate": "Create a new YouTube playlist with {{count}} videos? (~{{units}} quota units; {{left}} left today.)",
|
||||
"pushPlanUpdate": "Update on YouTube — add {{insert}}, remove {{del}}, reorder {{reorder}}? (~{{units}} quota units; {{left}} left today.)",
|
||||
"pushDiverged": "{{count}} video(s) currently on YouTube will be removed.",
|
||||
"pushPlanMerge": "Two-way sync — on YouTube: +{{addYt}} / −{{removeYt}}; here: +{{addLocal}} / −{{removeLocal}}. (~{{units}} quota units; {{left}} left today.)",
|
||||
"pushNoQuota": "Not enough YouTube quota left today ({{left}}) for this sync (~{{units}} needed). Try again tomorrow.",
|
||||
"pushUpToDate": "“{{name}}” is already in sync with YouTube.",
|
||||
"pushDone": "“{{name}}” synced to YouTube ✓",
|
||||
|
||||
@@ -35,8 +35,7 @@
|
||||
"pushTitle": "Szinkron YouTube-ra",
|
||||
"pushConfirm": "Szinkron",
|
||||
"pushPlanCreate": "Létrehozol egy új YouTube-listát {{count}} videóval? (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"pushPlanUpdate": "Frissítés a YouTube-on — {{insert}} hozzáadása, {{del}} eltávolítása, {{reorder}} átrendezése? (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"pushDiverged": "{{count}} videó, amely jelenleg a YouTube-on van, el lesz távolítva.",
|
||||
"pushPlanMerge": "Kétirányú szinkron — YouTube-on: +{{addYt}} / −{{removeYt}}; itt: +{{addLocal}} / −{{removeLocal}}. (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"pushNoQuota": "Nincs elég YouTube-kvóta mára ({{left}}) ehhez a szinkronhoz (~{{units}} kell). Próbáld újra holnap.",
|
||||
"pushUpToDate": "A(z) „{{name}}” már szinkronban van a YouTube-bal.",
|
||||
"pushDone": "„{{name}}” szinkronizálva a YouTube-ra ✓",
|
||||
|
||||
@@ -31,12 +31,16 @@ export interface PlaylistDetail {
|
||||
items: Video[];
|
||||
}
|
||||
|
||||
// Two-way sync plan: what each side gains/loses under the 3-way merge (neither side wins
|
||||
// wholesale). Local edits are free; only the YouTube writes cost quota.
|
||||
export interface PushPlan {
|
||||
action: "create" | "update";
|
||||
to_insert: number;
|
||||
to_delete: number;
|
||||
add_yt: number; // local-adds to push out to YouTube
|
||||
add_local: number; // YouTube-adds to pull into Siftlode
|
||||
remove_yt: number; // genuine local deletes propagated to YouTube
|
||||
remove_local: number; // genuine YouTube deletes propagated to Siftlode
|
||||
to_reorder: number;
|
||||
yt_extra: number; // items on YouTube that the push would remove (divergence)
|
||||
rename: boolean;
|
||||
units_estimate: number;
|
||||
remaining_today: number;
|
||||
affordable: boolean;
|
||||
@@ -44,9 +48,11 @@ export interface PushPlan {
|
||||
|
||||
export interface PushResult {
|
||||
created: number;
|
||||
inserted: number;
|
||||
deleted: number;
|
||||
inserted: number; // added on YouTube
|
||||
deleted: number; // removed on YouTube
|
||||
reordered: number;
|
||||
pulled: number; // added into Siftlode from YouTube
|
||||
removed_local: number; // removed from Siftlode (deleted on YouTube)
|
||||
failures: string[];
|
||||
yt_playlist_id: string | null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user