"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope. YouTube's special Watch Later / History playlists are not exposed by the Data API and are never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and ingested (with a stub channel) so the mirror is faithful.""" import hashlib import json from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timezone from typing import TypeVar from sqlalchemy import delete, select from sqlalchemy.orm import Session from app import logs, progress, quota from app.auth import has_read_scope from app.models import Channel, Playlist, PlaylistItem, User, Video from app.sync.runner import token_users from app.sync.videos import apply_video_details, parse_dt from app.youtube.client import YouTubeClient, YouTubeError log = logs.get_logger(__name__) # Cost of a single YouTube write op (playlists/playlistItems insert/update/delete). WRITE_UNIT_COST = 50 def fingerprint(name: str, ids: list[str]) -> str: """A stable checksum of a playlist's name + ordered video ids. Stored as the synced baseline; the current fingerprint is compared to it to derive `dirty`.""" h = hashlib.sha256() h.update((name or "").encode()) h.update(b"\n") h.update(",".join(ids).encode()) return h.hexdigest() def current_fingerprint(db: Session, pl: Playlist) -> str: return fingerprint(pl.name, _local_video_ids(db, pl)) def _dump_ids(ids: list[str]) -> str: """Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`).""" return json.dumps(ids) def _load_baseline(pl: Playlist) -> tuple[list[str] | None, set[str]]: """Parse `synced_video_ids` into (ordered id set, un-ingestable id set). Two on-disk forms: a plain JSON list (legacy / no un-ingestable ids) or `{"ids": [...], "unavailable": [...]}`. `unavailable` = YouTube-only videos that exist on YouTube but the Data API won't return (private/deleted/region-locked), so they never became local rows — tracked so the merge can skip them instead of forever re-classifying them as a YouTube-add (they'd never converge).""" if not pl.synced_video_ids: return None, set() try: val = json.loads(pl.synced_video_ids) except (ValueError, TypeError) as exc: # No baseline means the three-way merge has no "what we last agreed on" to subtract, so # it degrades to a union: nothing is lost, but a delete made on either side comes back. log.warning( "the sync baseline for a playlist could not be read; merging as a union", extra=logs.event( "sync.playlist.baseline_unreadable", playlist_id=pl.id, reason=type(exc).__name__, ), ) return None, set() if isinstance(val, list): return [str(v) for v in val], set() if isinstance(val, dict): raw_ids = val.get("ids") ids = [str(v) for v in raw_ids] if isinstance(raw_ids, list) else None raw_un = val.get("unavailable") un = {str(v) for v in raw_un} if isinstance(raw_un, list) else set() return ids, un return None, set() def parse_baseline(pl: Playlist) -> list[str] | None: """The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh export, or a link created before S7). None makes the first merge union losslessly.""" return _load_baseline(pl)[0] def parse_unavailable(pl: Playlist) -> set[str]: """The last-synced set of un-ingestable-but-live YouTube ids (see `_load_baseline`).""" return _load_baseline(pl)[1] def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None: """Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the next merge skips them; stored only when non-empty (keeps the common case a plain list).""" pl.synced_fingerprint = fingerprint(pl.name, ids) if unavailable: pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)}) else: pl.synced_video_ids = _dump_ids(ids) pl.dirty = False def recompute_dirty(db: Session, pl: Playlist) -> None: """Set `dirty` by comparing the current state to the synced baseline. Unlinked lists (and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state).""" if not pl.yt_playlist_id: pl.dirty = False return # The session has autoflush off, so the caller's pending item/order/name edits aren't # visible to current_fingerprint's query yet — flush first or dirty would lag one edit. db.flush() pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl) def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None: """Make sure every given video id exists in the catalog, fetching + ingesting the missing ones (and a stub channel for any unknown channel). Deleted/private videos that the API won't return are simply left out.""" if not video_ids: return have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all()) missing = [v for v in dict.fromkeys(video_ids) if v not in have] if not missing: return items = yt.get_videos(missing) chan_ids = { it.get("snippet", {}).get("channelId") for it in items if it.get("snippet", {}).get("channelId") } existing_ch = set( db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all() ) now = datetime.now(timezone.utc) for it in items: sn = it.get("snippet", {}) cid = sn.get("channelId") if cid and cid not in existing_ch: db.add(Channel(id=cid, title=sn.get("channelTitle"))) existing_ch.add(cid) db.flush() seen: set[str] = set() for it in items: vid = it.get("id") sn = it.get("snippet", {}) cid = sn.get("channelId") if not vid or not cid or vid in seen: continue seen.add(vid) # published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now() # fallback here too so the constructor never carries an explicit NULL. v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now) apply_video_details(v, it) v.enriched_at = now db.add(v) db.commit() def _replace_items_from_youtube( db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str] ) -> list[str]: """Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative replace): ensure the videos exist in the catalog, keep only those present in their YT order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag. `pl.name` must already be set to the desired name. The caller commits. Returns the final ordered ids.""" _ensure_videos(db, yt, ids) present = ( set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set() ) ordered: list[str] = [] seen: set[str] = set() for v in ids: if v in present and v not in seen: seen.add(v) ordered.append(v) _set_local_items(db, pl, ordered) # After a pull/reset local == YouTube, so this ordered set IS the new merge baseline. _set_baseline(pl, ordered) return ordered def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None: """Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller is responsible for having ensured every id exists in the catalog.""" db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id)) for pos, vid in enumerate(ordered): db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos)) def sync_user_playlists(db: Session, user: User) -> dict: """Mirror the user's YouTube playlists into local source='youtube' playlists. Leaves the user's local playlists and the built-in Watch later untouched.""" if not has_read_scope(user): return {"synced": 0, "reason": "no read scope"} synced = 0 with YouTubeClient(db, user) as yt: yt_playlists = list(yt.iter_my_playlists()) yt_ids = {p["id"] for p in yt_playlists if p.get("id")} existing = { pl.yt_playlist_id: pl for pl in db.execute( select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube") ).scalars() } # YouTube playlists that we already own as local (exported) playlists must not be # re-mirrored, or every exported list would also show up as a read-only duplicate. owned_yt_ids = set( db.execute( select(Playlist.yt_playlist_id).where( Playlist.user_id == user.id, Playlist.source == "local", Playlist.yt_playlist_id.is_not(None), ) ) .scalars() .all() ) # Drop mirrors whose YouTube playlist no longer exists. for ytid, pl in existing.items(): if ytid not in yt_ids: db.delete(pl) db.flush() for idx, p in enumerate(yt_playlists): ytid = p.get("id") if not ytid or ytid in owned_yt_ids: continue pl = existing.get(ytid) if pl is not None and pl.dirty: # Local edits not yet pushed: don't clobber them with YouTube's state. # A successful 'Sync to YouTube' clears dirty and lets the mirror refresh. continue if pl is None: pl = Playlist( user_id=user.id, name=p.get("title") or "Untitled", kind="user", source="youtube", yt_playlist_id=ytid, position=1000 + idx, # sort mirrored lists after local ones ) db.add(pl) db.flush() else: pl.name = p.get("title") or pl.name # Mirror is authoritative: replace the items to match YouTube's order. ids = list(yt.iter_my_playlist_video_ids(ytid)) _replace_items_from_youtube(db, yt, pl, ids) db.commit() synced += 1 return {"synced": synced} def _local_video_ids(db: Session, pl: Playlist) -> list[str]: return list( db.execute( select(PlaylistItem.video_id) .where(PlaylistItem.playlist_id == pl.id) .order_by(PlaylistItem.position) ) .scalars() .all() ) def _reorder_moves(current: list[str], desired: list[str]) -> int: """Count the playlistItems.update moves an insertion-sort would need to turn `current` order into `desired` order (both already the same set of video ids). Items already in place are skipped; this matches what apply_push actually executes.""" model = list(current) moves = 0 for i, want in enumerate(desired): if i < len(model) and model[i] == want: continue j = model.index(want, i) model.pop(j) model.insert(i, want) moves += 1 return moves @dataclass class MergePlan: """The 3-way merge of a linked playlist: what each side gains and loses so neither side loses an item, plus the merged order. `final` is the id set both sides end up with.""" final: list[str] # merged order (local order, YouTube-only adds appended) add_to_yt: list[str] # local-adds to push out to YouTube add_to_local: list[str] # YouTube-adds to pull into Siftlode remove_from_yt: list[str] # items deleted locally that are still on YouTube remove_from_local: list[str] # items deleted on YouTube that are still local reorders: int # estimated YouTube playlistItems.update moves to reach `final` baseline_known: bool def merge_plan( baseline: list[str] | None, local: list[str], youtube: list[str], unavailable: set[str] | None = None, ) -> MergePlan: """Pure 3-way merge. Classify each id against the last-synced `baseline`: - in local but not youtube, not deleted → add to YouTube (a local-add) - in youtube but not local, not deleted → add to Siftlode (a YouTube-add) - in baseline but gone from local → a genuine local delete → propagate to YouTube - in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both sides (never delete) — the lossless default. Merged order: local order first, YouTube-only additions appended in their YouTube order. `unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local delete (they were never local) — otherwise such a video would make the playlist never converge. They stay in place on YouTube, untouched by this merge.""" un = set(unavailable or ()) L = list(dict.fromkeys(local)) Y = [v for v in dict.fromkeys(youtube) if v not in un] Lset, Yset = set(L), set(Y) if baseline is None: add_to_yt = [v for v in L if v not in Yset] add_to_local = [v for v in Y if v not in Lset] remove_from_yt: list[str] = [] remove_from_local: list[str] = [] final = L + [v for v in Y if v not in Lset] else: B = set(baseline) deleted_local = B - Lset # was synced, removed on our side deleted_yt = B - Yset # was synced, removed on YouTube's side gone = deleted_local | deleted_yt remove_from_yt = [v for v in Y if v in deleted_local] remove_from_local = [v for v in L if v in deleted_yt] add_to_yt = [v for v in L if v not in Yset and v not in gone] add_to_local = [v for v in Y if v not in Lset and v not in gone] final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone] final_set = set(final) # On YouTube, after membership reconciliation the order is current-kept + inserts appended; # the reorder pass then moves items into `final`. after_membership = [v for v in Y if v in final_set] + add_to_yt reorders = _reorder_moves(after_membership, final) return MergePlan( final=final, add_to_yt=add_to_yt, add_to_local=add_to_local, remove_from_yt=remove_from_yt, remove_from_local=remove_from_local, reorders=reorders, baseline_known=baseline is not None, ) def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict: """Compute (without writing) what a two-way sync of this playlist would do: create vs update, how many items each side gains/loses, reorders, and the quota estimate (YouTube writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple so the caller can share one read with merge_playlist.""" local = _local_video_ids(db, pl) if not pl.yt_playlist_id: ops = 1 + len(local) # create the playlist + one insert per video return { "action": "create", "add_yt": len(local), "add_local": 0, "remove_yt": 0, "remove_local": 0, "to_reorder": 0, "rename": False, "units_estimate": ops * WRITE_UNIT_COST, } if live is not None: current, snippet = live else: with YouTubeClient(db, user) as yt: current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)) snippet = yt.get_playlist_snippet(pl.yt_playlist_id) rename = bool(snippet) and (snippet.get("title") or "") != pl.name yt_ids = [it["video_id"] for it in current] mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl)) ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0) return { "action": "update", "add_yt": len(mp.add_to_yt), "add_local": len(mp.add_to_local), "remove_yt": len(mp.remove_from_yt), "remove_local": len(mp.remove_from_local), "to_reorder": mp.reorders, "rename": rename, "units_estimate": ops * WRITE_UNIT_COST, } # After this many consecutive write failures we stop calling YouTube for the rest of the push. # The failure modes that matter here are all account-wide, not per-video: quota exhausted, the # write scope revoked, the token dead. Every remaining write then fails identically — but each # one still SPENDS 50 units, so a long playlist burns through the whole (already dead) budget, # and tomorrow's too, to produce a failure list we knew after the fifth item. _MAX_CONSECUTIVE_FAILURES = 5 class _Failed: """Type of the `FAILED` sentinel, so `run()` can stay generic in the op's return type: a call site gets back `str | _Failed` (not a bare `object`) and keeps its item id typed.""" #: Returned by `_Breaker.run` when the write did not happen (refused or failed). FAILED = _Failed() T = TypeVar("T") class _Breaker: """Consecutive-failure circuit breaker for a single push run. Any success re-arms it. Drive it with `run()` — it owns the whole open-check / call / record-outcome sequence, so a write site can't accidentally implement two thirds of the protocol (forgetting to record a failure would silently disable the breaker for that operation).""" def __init__( self, limit: int = _MAX_CONSECUTIVE_FAILURES, playlist_id: int | None = None ) -> None: self.limit = limit self.playlist_id = playlist_id # for the failure line only self.streak = 0 self.tripped = False def open(self) -> bool: """True while writes may still be attempted.""" return not self.tripped def hit(self, ok: bool) -> None: if ok: self.streak = 0 return self.streak += 1 if self.streak >= self.limit and not self.tripped: self.tripped = True log.warning( "playlist push: %d consecutive YouTube write failures — stopping this run " "(quota, scope or token); the remaining items are reported as failures", self.limit, ) def run( self, op: Callable[[], T], on_fail: Callable[[], None], action: str = "write" ) -> T | _Failed: """Attempt one YouTube write. Returns the op's result, or `FAILED` when the breaker is already tripped (op not called) or the op raised YouTubeError. `on_fail` records the item in the caller's failure list either way.""" if self.tripped: on_fail() return FAILED try: out = op() except YouTubeError as exc: # The client logged the call, but not which playlist it was for or what kind of write # — and the caller's failure list only ever reaches the user as a count. Not the video: # what is in someone's playlist is a row of their history, the same as a download. log.debug( "a playlist write to YouTube failed", extra=logs.event( "sync.playlist.item_push_failed", playlist_id=self.playlist_id, action=action, reason=exc.reason, ), ) on_fail() self.hit(False) return FAILED self.hit(True) return out def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict: """Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE both sides so an item added (or deleted) on either side survives — no side wins wholesale. Local edits are applied directly; YouTube edits go through the failure breaker. On full success the merged set becomes the new baseline and the playlist is marked clean; on any YouTube failure the baseline is left untouched so the next sync retries and converges. Caller must have checked the write scope and the quota budget. `live` = a pre-fetched (current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through.""" local = _local_video_ids(db, pl) inserted = deleted = reordered = created = pulled = removed_local = 0 failures: list[str] = [] breaker = _Breaker(playlist_id=pl.id) with YouTubeClient(db, user) as yt: if not pl.yt_playlist_id: pl.yt_playlist_id = yt.create_playlist(pl.name) db.commit() created = 1 # Brand-new playlist: just append every local video in order. for vid in local: added = breaker.run( lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v), lambda v=vid: failures.append(v), "add", ) if not isinstance(added, _Failed): inserted += 1 if not failures: _set_baseline(pl, local) db.commit() return { "created": created, "inserted": inserted, "deleted": deleted, "reordered": reordered, "pulled": pulled, "removed_local": removed_local, "failures": failures, "aborted": breaker.tripped, "yt_playlist_id": pl.yt_playlist_id, } # Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read). if live is not None: current, snippet = live else: snippet = yt.get_playlist_snippet(pl.yt_playlist_id) current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)) item_id_by_vid = {it["video_id"]: it["item_id"] for it in current} yt_ids = [it["video_id"] for it in current] # Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a # video that has since become available (e.g. private→public) should re-enter the merge, not # stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`). prev_unavailable = parse_unavailable(pl) still_listed = [v for v in yt_ids if v in prev_unavailable] if still_listed: _ensure_videos(db, yt, still_listed) healed = set( db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all() ) effective_unavailable = prev_unavailable - healed else: effective_unavailable = prev_unavailable mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable) # --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the # merged order authoritatively. Do this first so local reflects the merge even if a later # YouTube write trips the breaker. _ensure_videos(db, yt, mp.add_to_local) # Only the ids that actually exist in the catalog can become local rows (a private/deleted # YouTube video won't ingest); keep the merged order, dropping any that didn't materialize. present = ( set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all()) if mp.final else set() ) final_local = [v for v in mp.final if v in present] pulled = len([v for v in mp.add_to_local if v in present]) removed_local = len(mp.remove_from_local) _set_local_items(db, pl, final_local) db.commit() # --- YouTube side (metered, breaker-guarded). if snippet is not None and (snippet.get("title") or "") != pl.name: try: yt.update_playlist_title(pl.yt_playlist_id, pl.name) except YouTubeError as exc: # All the caller passes on is the word "title", which says the rename did not # happen but not that YouTube refused it, or why. log.debug( "a playlist rename did not reach YouTube", extra=logs.event( "sync.playlist.title_push_failed", playlist_id=pl.id, reason=exc.reason, ), ) failures.append("title") # Propagate genuine local deletes. for vid in mp.remove_from_yt: item_id = item_id_by_vid.get(vid) if item_id is None: continue dropped = breaker.run( lambda i=item_id: yt.delete_playlist_item(i), lambda v=vid: failures.append(v), "delete", ) if not isinstance(dropped, _Failed): deleted += 1 item_id_by_vid.pop(vid, None) # Insert local-adds (append; the reorder pass below fixes positions). for vid in mp.add_to_yt: item_id = breaker.run( lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v), lambda v=vid: failures.append(v), "add", ) if not isinstance(item_id, _Failed): item_id_by_vid[vid] = item_id inserted += 1 # Reorder YouTube to match `final` via insertion-sort moves over a local model. final_set = set(mp.final) model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [ v for v in mp.add_to_yt if v in item_id_by_vid ] for i, want in enumerate(mp.final): if want not in item_id_by_vid: continue # deleted-from-YT or failed insert; skip if i < len(model) and model[i] == want: continue moved = breaker.run( lambda w=want, pos=i: yt.move_playlist_item( item_id_by_vid[w], pl.yt_playlist_id, w, pos ), lambda w=want: failures.append(w), "move", ) if isinstance(moved, _Failed): continue reordered += 1 j = model.index(want) model.pop(j) model.insert(i, want) if not failures: # Carry forward un-ingestable-but-live YouTube ids so the next merge skips them # (see merge_plan): this run's failed pulls, plus prior ones still on YouTube, minus # any that have since left YouTube. Recomputed each success → self-healing. yt_id_set = set(yt_ids) unavailable = sorted( (effective_unavailable | {v for v in mp.add_to_local if v not in present}) & yt_id_set ) _set_baseline(pl, final_local, unavailable) db.commit() return { "created": created, "inserted": inserted, "deleted": deleted, "reordered": reordered, "pulled": pulled, "removed_local": removed_local, "failures": failures, "aborted": breaker.tripped, "yt_playlist_id": pl.yt_playlist_id, } def repull_playlist(db: Session, user: User, pl: Playlist) -> dict: """Force-refresh ONE YouTube-linked playlist from YouTube, discarding local edits and clearing dirty. This is the per-playlist 'reset to YouTube' — unlike the bulk read-sync it deliberately does NOT skip a dirty playlist (overwriting local changes is the point). Works for both mirrors (source='youtube') and exported local playlists (yt_playlist_id).""" if not pl.yt_playlist_id: raise YouTubeError("Playlist is not linked to YouTube") with YouTubeClient(db, user) as yt: snippet = yt.get_playlist_snippet(pl.yt_playlist_id) if snippet is None: raise YouTubeError("Playlist no longer exists on YouTube") pl.name = snippet.get("title") or pl.name ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id)) ordered = _replace_items_from_youtube(db, yt, pl, ids) db.commit() return {"items": len(ordered)} def sync_all_playlists(db: Session) -> dict: """Scheduler entry point: mirror playlists for every user with a read scope + token.""" users = token_users(db) total = 0 for i, user in enumerate(users, 1): if not has_read_scope(user): progress.report(i, len(users), "playlist_sync") continue try: with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL): total += sync_user_playlists(db, user).get("synced", 0) except YouTubeError: db.rollback() log.warning("Playlist sync failed for user %s", user.id) except Exception: db.rollback() log.exception("Playlist sync crashed for user %s", user.id) progress.report(i, len(users), "playlist_sync") return {"users": len(users), "playlists_synced": total}