peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(playlists): two-way (3-way) merge YouTube sync + mandatory preview (S7)

Browse Source 
"Sync to YouTube" was a one-directional destructive push (local wins, deletes
from YouTube every item not in the local list), so items added on YouTube were
lost and a dirty pull was skipped entirely. Replace it with a real 3-way merge:
each item is classified against the last-synced baseline as a local-add, a
YouTube-add, or a genuine delete, so an add or delete on either side survives.

- Persist the ordered baseline id set (new playlists.synced_video_ids, migration
  0075); the fingerprint hash alone couldn't drive a merge. NULL baseline (fresh
  export / pre-S7 link) unions losslessly on the first sync and records one.
- Pure merge_plan(baseline, local, youtube) -> add/remove per side + merged order
  (local order, YouTube-only additions appended). merge_playlist applies the
  local side directly and the YouTube side through the failure breaker; the
  baseline only advances on full success so a partial failure re-converges.
- plan_merge/push-plan now report what each side gains and loses; the confirm
  dialog shows the two-way plan before applying (pushPlanMerge, HU+EN).
- Tests: 10 new (pure classification incl. delete-vs-stale-copy; DB apply proving
  a YouTube-add and a Siftlode-add both survive on both sides, a genuine delete
  propagates without collateral loss, no-baseline never deletes, create path).
This commit is contained in:
peter
2026-08-28 04:20:29 +02:00
parent bc33b2ed16
commit a0b725d779
9 changed files with 474 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/playlists.py
+182 -71
View File
@@ -1,14 +1,16 @@
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: push local
edits out to YouTube (`plan_push` / `push_playlist`), behind the optional write scope.
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
ingested (with a stub channel) so the mirror is faithful."""
import hashlib
import json
import logging
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import TypeVar
@@ -42,6 +44,30 @@ def current_fingerprint(db: Session, pl: Playlist) -> str:
return fingerprint(pl.name, _local_video_ids(db, pl))
def _dump_ids(ids: list[str]) -> str:
"""Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`)."""
return json.dumps(ids)
def parse_baseline(pl: Playlist) -> list[str] | None:
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
export, or a link created before S7). None makes the first merge union losslessly."""
if not pl.synced_video_ids:
return None
try:
val = json.loads(pl.synced_video_ids)
except (ValueError, TypeError):
return None
return [str(v) for v in val] if isinstance(val, list) else None
def _set_baseline(pl: Playlist, ids: list[str]) -> None:
"""Record `ids` as both the hash (drives `dirty`) and the raw ordered set (drives merge)."""
pl.synced_fingerprint = fingerprint(pl.name, ids)
pl.synced_video_ids = _dump_ids(ids)
pl.dirty = False
def recompute_dirty(db: Session, pl: Playlist) -> None:
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
@@ -116,12 +142,18 @@ def _replace_items_from_youtube(
if v in present and v not in seen:
seen.add(v)
ordered.append(v)
_set_local_items(db, pl, ordered)
# After a pull/reset local == YouTube, so this ordered set IS the new merge baseline.
_set_baseline(pl, ordered)
return ordered
def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None:
"""Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller
is responsible for having ensured every id exists in the catalog."""
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
for pos, vid in enumerate(ordered):
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
pl.synced_fingerprint = fingerprint(pl.name, ordered)
pl.dirty = False
return ordered
def sync_user_playlists(db: Session, user: User) -> dict:
@@ -216,22 +248,81 @@ def _reorder_moves(current: list[str], desired: list[str]) -> int:
return moves
def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Compute (without writing) what pushing this local playlist to YouTube would do:
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
YouTube has diverged (items present there that local will remove). Costs read units
(1/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched
(current_items, snippet) tuple so the caller can share one read with push_playlist."""
desired = _local_video_ids(db, pl)
@dataclass
class MergePlan:
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
loses an item, plus the merged order. `final` is the id set both sides end up with."""
final: list[str] # merged order (local order, YouTube-only adds appended)
add_to_yt: list[str] # local-adds to push out to YouTube
add_to_local: list[str] # YouTube-adds to pull into Siftlode
remove_from_yt: list[str] # items deleted locally that are still on YouTube
remove_from_local: list[str] # items deleted on YouTube that are still local
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
baseline_known: bool
def merge_plan(baseline: list[str] | None, local: list[str], youtube: list[str]) -> MergePlan:
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
- in local but not youtube, not deleted → add to YouTube (a local-add)
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
- in baseline but gone from local → a genuine local delete → propagate to YouTube
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
additions appended in their YouTube order."""
L = list(dict.fromkeys(local))
Y = list(dict.fromkeys(youtube))
Lset, Yset = set(L), set(Y)
if baseline is None:
add_to_yt = [v for v in L if v not in Yset]
add_to_local = [v for v in Y if v not in Lset]
remove_from_yt: list[str] = []
remove_from_local: list[str] = []
final = L + [v for v in Y if v not in Lset]
else:
B = set(baseline)
deleted_local = B - Lset # was synced, removed on our side
deleted_yt = B - Yset # was synced, removed on YouTube's side
gone = deleted_local | deleted_yt
remove_from_yt = [v for v in Y if v in deleted_local]
remove_from_local = [v for v in L if v in deleted_yt]
add_to_yt = [v for v in L if v not in Yset and v not in gone]
add_to_local = [v for v in Y if v not in Lset and v not in gone]
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
final_set = set(final)
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
# the reorder pass then moves items into `final`.
after_membership = [v for v in Y if v in final_set] + add_to_yt
reorders = _reorder_moves(after_membership, final)
return MergePlan(
final=final,
add_to_yt=add_to_yt,
add_to_local=add_to_local,
remove_from_yt=remove_from_yt,
remove_from_local=remove_from_local,
reorders=reorders,
baseline_known=baseline is not None,
)
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked
playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple
so the caller can share one read with merge_playlist."""
local = _local_video_ids(db, pl)
if not pl.yt_playlist_id:
ops = 1 + len(desired) # create the playlist + one insert per video
ops = 1 + len(local) # create the playlist + one insert per video
return {
"action": "create",
"to_insert": len(desired),
"to_delete": 0,
"add_yt": len(local),
"add_local": 0,
"remove_yt": 0,
"remove_local": 0,
"to_reorder": 0,
"rename": False,
"yt_extra": 0,
"units_estimate": ops * WRITE_UNIT_COST,
}
if live is not None:
@@ -241,23 +332,17 @@ def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
cur_vids = [it["video_id"] for it in current]
cur_set = set(cur_vids)
desired_set = set(desired)
to_insert = [v for v in desired if v not in cur_set]
to_delete = [v for v in cur_vids if v not in desired_set]
# After membership reconciliation, the YT order is current-minus-deletes plus inserts
# appended; the reorder pass then moves items into the desired order.
after_membership = [v for v in cur_vids if v in desired_set] + to_insert
reorders = _reorder_moves(after_membership, desired)
ops = len(to_insert) + len(to_delete) + reorders + (1 if rename else 0)
yt_ids = [it["video_id"] for it in current]
mp = merge_plan(parse_baseline(pl), local, yt_ids)
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
return {
"action": "update",
"to_insert": len(to_insert),
"to_delete": len(to_delete),
"to_reorder": reorders,
"add_yt": len(mp.add_to_yt),
"add_local": len(mp.add_to_local),
"remove_yt": len(mp.remove_from_yt),
"remove_local": len(mp.remove_from_local),
"to_reorder": mp.reorders,
"rename": rename,
"yt_extra": len(to_delete),
"units_estimate": ops * WRITE_UNIT_COST,
}
@@ -327,13 +412,16 @@ class _Breaker:
return out
def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
and order so YouTube matches local (local wins). Marks the playlist clean on success.
def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE
both sides so an item added (or deleted) on either side survives — no side wins wholesale.
Local edits are applied directly; YouTube edits go through the failure breaker. On full
success the merged set becomes the new baseline and the playlist is marked clean; on any
YouTube failure the baseline is left untouched so the next sync retries and converges.
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
(current_items, snippet) tuple (from plan_push's read) to avoid a second live page-through."""
desired = _local_video_ids(db, pl)
inserted = deleted = reordered = created = 0
(current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through."""
local = _local_video_ids(db, pl)
inserted = deleted = reordered = created = pulled = removed_local = 0
failures: list[str] = []
breaker = _Breaker()
with YouTubeClient(db, user) as yt:
@@ -341,8 +429,8 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
pl.yt_playlist_id = yt.create_playlist(pl.name)
db.commit()
created = 1
# Brand-new playlist: just append every video in order.
for vid in desired:
# Brand-new playlist: just append every local video in order.
for vid in local:
added = breaker.run(
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
lambda v=vid: failures.append(v),
@@ -350,60 +438,82 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
if not isinstance(added, _Failed):
inserted += 1
if not failures:
pl.synced_fingerprint = fingerprint(pl.name, desired)
pl.dirty = False
_set_baseline(pl, local)
db.commit()
return {
"created": created,
"inserted": inserted,
"deleted": deleted,
"reordered": reordered,
"pulled": pulled,
"removed_local": removed_local,
"failures": failures,
"aborted": breaker.tripped,
"yt_playlist_id": pl.yt_playlist_id,
}
# Existing link: diff against the live YouTube state (reuse plan_push's read if given).
# Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read).
if live is not None:
current, snippet = live
else:
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
yt_ids = [it["video_id"] for it in current]
mp = merge_plan(parse_baseline(pl), local, yt_ids)
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
# merged order authoritatively. Do this first so local reflects the merge even if a later
# YouTube write trips the breaker.
_ensure_videos(db, yt, mp.add_to_local)
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
present = (
set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all())
if mp.final
else set()
)
final_local = [v for v in mp.final if v in present]
pulled = len([v for v in mp.add_to_local if v in present])
removed_local = len(mp.remove_from_local)
_set_local_items(db, pl, final_local)
db.commit()
# --- YouTube side (metered, breaker-guarded).
if snippet is not None and (snippet.get("title") or "") != pl.name:
try:
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
except YouTubeError:
failures.append("title")
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
cur_vids = [it["video_id"] for it in current]
desired_set = set(desired)
# Delete extras (local is authoritative).
for it in current:
if it["video_id"] not in desired_set:
dropped = breaker.run(
lambda i=it: yt.delete_playlist_item(i["item_id"]),
lambda i=it: failures.append(i["video_id"]),
)
if not isinstance(dropped, _Failed):
deleted += 1
# Insert missing (append; the reorder pass below fixes positions).
cur_set = set(cur_vids)
for vid in desired:
if vid not in cur_set:
item_id = breaker.run(
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
lambda v=vid: failures.append(v),
)
if not isinstance(item_id, _Failed):
item_id_by_vid[vid] = item_id
inserted += 1
# Reorder to match `desired` via insertion-sort moves over a local model.
model = [v for v in cur_vids if v in desired_set] + [
v for v in desired if v not in cur_set and v in item_id_by_vid
# Propagate genuine local deletes.
for vid in mp.remove_from_yt:
item_id = item_id_by_vid.get(vid)
if item_id is None:
continue
dropped = breaker.run(
lambda i=item_id: yt.delete_playlist_item(i),
lambda v=vid: failures.append(v),
)
if not isinstance(dropped, _Failed):
deleted += 1
item_id_by_vid.pop(vid, None)
# Insert local-adds (append; the reorder pass below fixes positions).
for vid in mp.add_to_yt:
item_id = breaker.run(
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
lambda v=vid: failures.append(v),
)
if not isinstance(item_id, _Failed):
item_id_by_vid[vid] = item_id
inserted += 1
# Reorder YouTube to match `final` via insertion-sort moves over a local model.
final_set = set(mp.final)
model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [
v for v in mp.add_to_yt if v in item_id_by_vid
]
for i, want in enumerate(desired):
for i, want in enumerate(mp.final):
if want not in item_id_by_vid:
continue # failed insert; skip
continue # deleted-from-YT or failed insert; skip
if i < len(model) and model[i] == want:
continue
moved = breaker.run(
@@ -419,14 +529,15 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
model.pop(j)
model.insert(i, want)
if not failures:
pl.synced_fingerprint = fingerprint(pl.name, desired)
pl.dirty = False
_set_baseline(pl, final_local)
db.commit()
return {
"created": created,
"inserted": inserted,
"deleted": deleted,
"reordered": reordered,
"pulled": pulled,
"removed_local": removed_local,
"failures": failures,
"aborted": breaker.tripped,
"yt_playlist_id": pl.yt_playlist_id,