peter
b7ce1d467c
- Every playlist item write (add, remove, move, reorder, rename; Plex remove and remove-bulk) row-locks the playlist before it touches an item, a move both lists in id order: remove beside reorder was a deadlock, and two adds of one video a unique-key 500. - The scheduled mirror refresh re-reads dirty under that lock, so an edit made during its YouTube calls is not replaced by YouTube's order. - A user's bulk pull holds a per-user advisory claim: two at once created duplicate mirrors. The manual sync answers 409 sync_all_busy instead of '0 synced', and like the re-pull spends its cooldown only on a pull that runs. - Deleted mirrors are committed at once, not only when a later list commits. - The claims open an unpooled connection of their own instead of a third pooled one per push.
776 lines
34 KiB
Python
776 lines
34 KiB
Python
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
|
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
|
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
|
|
|
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
|
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
|
ingested (with a stub channel) so the mirror is faithful."""
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Callable, Iterator
|
|
from contextlib import AbstractContextManager, contextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from typing import TypeVar
|
|
|
|
from sqlalchemy import Engine, create_engine, delete, func, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
from app import logs, progress, quota
|
|
from app.auth import has_read_scope
|
|
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
|
from app.sync.runner import token_users
|
|
from app.sync.videos import apply_video_details, parse_dt
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
|
|
log = logs.get_logger(__name__)
|
|
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
|
WRITE_UNIT_COST = 50
|
|
|
|
# The advisory-lock namespaces ("PL" per playlist, "PU" per user), so a key cannot meet another
|
|
# feature's lock on the same number (Postgres keeps the two-int keys apart from the one-bigint ones).
|
|
_SYNC_LOCK_NS = 0x504C
|
|
_USER_SYNC_LOCK_NS = 0x5055
|
|
|
|
# The claims' own connections, outside the app pool: a claim is held for a whole push (seconds of
|
|
# YouTube calls) beside the request's pooled session and quota's, so a burst of pushes drawing a
|
|
# third pooled connection each ran the pool dry and stalled every other request for pool_timeout.
|
|
# NullPool opens one per claim and closes it after, so a lock can never ride back into a pool either.
|
|
_claim_engines: dict[str, Engine] = {}
|
|
|
|
|
|
def _claim_engine(db: Session) -> Engine:
|
|
url = db.get_bind().engine.url
|
|
key = url.render_as_string(hide_password=False)
|
|
eng = _claim_engines.get(key)
|
|
if eng is None:
|
|
eng = _claim_engines[key] = create_engine(url, poolclass=NullPool, hide_parameters=True)
|
|
return eng
|
|
|
|
|
|
@contextmanager
|
|
def _advisory_claim(db: Session, ns: int, key: int) -> Iterator[bool]:
|
|
"""Try a session-level advisory lock for the block on a connection of its own; yields whether it
|
|
was free. It survives the block's commits, and a crashed worker's lock dies with its connection
|
|
— no claim column that a crash leaves stuck."""
|
|
conn = _claim_engine(db).connect()
|
|
held = False
|
|
try:
|
|
held = bool(conn.scalar(select(func.pg_try_advisory_lock(ns, key))))
|
|
conn.commit()
|
|
yield held
|
|
finally:
|
|
try:
|
|
if held:
|
|
conn.scalar(select(func.pg_advisory_unlock(ns, key)))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
|
|
"""Claim one playlist's YouTube sync (push, re-pull, the scheduled mirror refresh) for the block;
|
|
yields whether it was free. A push reads YouTube, then writes both sides for seconds and commits
|
|
step by step, so a row lock ends at its first commit and two tabs (or a push and the scheduler)
|
|
interleave into a doubled YouTube item or a baseline recorded over the other's writes."""
|
|
return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
|
|
|
|
|
|
def fingerprint(name: str, ids: list[str]) -> str:
|
|
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
|
baseline; the current fingerprint is compared to it to derive `dirty`."""
|
|
h = hashlib.sha256()
|
|
h.update((name or "").encode())
|
|
h.update(b"\n")
|
|
h.update(",".join(ids).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def current_fingerprint(db: Session, pl: Playlist) -> str:
|
|
return fingerprint(pl.name, _local_video_ids(db, pl))
|
|
|
|
|
|
def _dump_ids(ids: list[str]) -> str:
|
|
"""Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`)."""
|
|
return json.dumps(ids)
|
|
|
|
|
|
def _load_baseline(pl: Playlist) -> tuple[list[str] | None, set[str]]:
|
|
"""Parse `synced_video_ids` into (ordered id set, un-ingestable id set). Two on-disk forms:
|
|
a plain JSON list (legacy / no un-ingestable ids) or `{"ids": [...], "unavailable": [...]}`.
|
|
`unavailable` = YouTube-only videos that exist on YouTube but the Data API won't return
|
|
(private/deleted/region-locked), so they never became local rows — tracked so the merge can
|
|
skip them instead of forever re-classifying them as a YouTube-add (they'd never converge)."""
|
|
if not pl.synced_video_ids:
|
|
return None, set()
|
|
try:
|
|
val = json.loads(pl.synced_video_ids)
|
|
except (ValueError, TypeError) as exc:
|
|
# No baseline means the three-way merge has no "what we last agreed on" to subtract, so
|
|
# it degrades to a union: nothing is lost, but a delete made on either side comes back.
|
|
log.warning(
|
|
"the sync baseline for a playlist could not be read; merging as a union",
|
|
extra=logs.event(
|
|
"sync.playlist.baseline_unreadable",
|
|
playlist_id=pl.id,
|
|
reason=type(exc).__name__,
|
|
),
|
|
)
|
|
return None, set()
|
|
if isinstance(val, list):
|
|
return [str(v) for v in val], set()
|
|
if isinstance(val, dict):
|
|
raw_ids = val.get("ids")
|
|
ids = [str(v) for v in raw_ids] if isinstance(raw_ids, list) else None
|
|
raw_un = val.get("unavailable")
|
|
un = {str(v) for v in raw_un} if isinstance(raw_un, list) else set()
|
|
return ids, un
|
|
return None, set()
|
|
|
|
|
|
def parse_baseline(pl: Playlist) -> list[str] | None:
|
|
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
|
|
export, or a link created before S7). None makes the first merge union losslessly."""
|
|
return _load_baseline(pl)[0]
|
|
|
|
|
|
def parse_unavailable(pl: Playlist) -> set[str]:
|
|
"""The last-synced set of un-ingestable-but-live YouTube ids (see `_load_baseline`)."""
|
|
return _load_baseline(pl)[1]
|
|
|
|
|
|
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
|
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
|
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
|
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
|
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
|
if unavailable:
|
|
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
|
else:
|
|
pl.synced_video_ids = _dump_ids(ids)
|
|
pl.dirty = False
|
|
|
|
|
|
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
|
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
|
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
|
if not pl.yt_playlist_id:
|
|
pl.dirty = False
|
|
return
|
|
# The session has autoflush off, so the caller's pending item/order/name edits aren't
|
|
# visible to current_fingerprint's query yet — flush first or dirty would lag one edit.
|
|
db.flush()
|
|
pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
|
|
|
|
|
|
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
|
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
|
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
|
the API won't return are simply left out."""
|
|
if not video_ids:
|
|
return
|
|
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
|
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
|
if not missing:
|
|
return
|
|
items = yt.get_videos(missing)
|
|
chan_ids = {
|
|
it.get("snippet", {}).get("channelId")
|
|
for it in items
|
|
if it.get("snippet", {}).get("channelId")
|
|
}
|
|
existing_ch = set(
|
|
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
|
|
)
|
|
now = datetime.now(timezone.utc)
|
|
for it in items:
|
|
sn = it.get("snippet", {})
|
|
cid = sn.get("channelId")
|
|
if cid and cid not in existing_ch:
|
|
db.add(Channel(id=cid, title=sn.get("channelTitle")))
|
|
existing_ch.add(cid)
|
|
db.flush()
|
|
seen: set[str] = set()
|
|
for it in items:
|
|
vid = it.get("id")
|
|
sn = it.get("snippet", {})
|
|
cid = sn.get("channelId")
|
|
if not vid or not cid or vid in seen:
|
|
continue
|
|
seen.add(vid)
|
|
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
|
# fallback here too so the constructor never carries an explicit NULL.
|
|
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
|
apply_video_details(v, it)
|
|
v.enriched_at = now
|
|
db.add(v)
|
|
db.commit()
|
|
|
|
|
|
def _replace_items_from_youtube(
|
|
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
|
|
) -> list[str]:
|
|
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
|
replace): ensure the videos exist in the catalog, keep only those present in their YT
|
|
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
|
|
`pl.name` must already be set to the desired name. The caller commits. Returns the final
|
|
ordered ids."""
|
|
_ensure_videos(db, yt, ids)
|
|
present = (
|
|
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
|
)
|
|
ordered: list[str] = []
|
|
seen: set[str] = set()
|
|
for v in ids:
|
|
if v in present and v not in seen:
|
|
seen.add(v)
|
|
ordered.append(v)
|
|
_set_local_items(db, pl, ordered)
|
|
# After a pull/reset local == YouTube, so this ordered set IS the new merge baseline.
|
|
_set_baseline(pl, ordered)
|
|
return ordered
|
|
|
|
|
|
def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None:
|
|
"""Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller
|
|
is responsible for having ensured every id exists in the catalog."""
|
|
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
|
|
for pos, vid in enumerate(ordered):
|
|
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
|
|
|
|
|
|
def sync_user_playlists(db: Session, user: User) -> dict:
|
|
"""Mirror the user's YouTube playlists into local source='youtube' playlists. Leaves
|
|
the user's local playlists and the built-in Watch later untouched."""
|
|
if not has_read_scope(user):
|
|
return {"synced": 0, "reason": "no read scope"}
|
|
with user_sync_claim(db, user.id) as free:
|
|
if not free:
|
|
return {"synced": 0, "reason": "busy"}
|
|
return pull_user_playlists(db, user)
|
|
|
|
|
|
def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
|
"""Claim a user's bulk mirror pull for the block; yields whether it was free. Two runs at once
|
|
(the scheduler beside a manual "sync now", two tabs) both saw a YouTube list unmirrored and
|
|
each created its mirror — a duplicate row."""
|
|
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
|
|
|
|
|
def pull_user_playlists(db: Session, user: User) -> dict:
|
|
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
|
synced = 0
|
|
with YouTubeClient(db, user) as yt:
|
|
yt_playlists = list(yt.iter_my_playlists())
|
|
yt_ids = {p["id"] for p in yt_playlists if p.get("id")}
|
|
existing = {
|
|
pl.yt_playlist_id: pl
|
|
for pl in db.execute(
|
|
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
|
|
).scalars()
|
|
}
|
|
# YouTube playlists that we already own as local (exported) playlists must not be
|
|
# re-mirrored, or every exported list would also show up as a read-only duplicate.
|
|
owned_yt_ids = set(
|
|
db.execute(
|
|
select(Playlist.yt_playlist_id).where(
|
|
Playlist.user_id == user.id,
|
|
Playlist.source == "local",
|
|
Playlist.yt_playlist_id.is_not(None),
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
# Drop mirrors whose YouTube playlist no longer exists.
|
|
for ytid, pl in existing.items():
|
|
if ytid not in yt_ids:
|
|
db.delete(pl)
|
|
# Committed now: a run where no list below commits (all dirty, or all claimed) would
|
|
# otherwise drop the deletions with the session.
|
|
db.commit()
|
|
|
|
for idx, p in enumerate(yt_playlists):
|
|
ytid = p.get("id")
|
|
if not ytid or ytid in owned_yt_ids:
|
|
continue
|
|
pl = existing.get(ytid)
|
|
if pl is not None and pl.dirty:
|
|
# Local edits not yet pushed: don't clobber them with YouTube's state.
|
|
# A successful 'Sync to YouTube' clears dirty and lets the mirror refresh.
|
|
continue
|
|
if pl is None:
|
|
pl = Playlist(
|
|
user_id=user.id,
|
|
name=p.get("title") or "Untitled",
|
|
kind="user",
|
|
source="youtube",
|
|
yt_playlist_id=ytid,
|
|
position=1000 + idx, # sort mirrored lists after local ones
|
|
)
|
|
db.add(pl)
|
|
db.flush()
|
|
with sync_claim(db, pl.id) as free:
|
|
if not free:
|
|
# The user is pushing or re-pulling this very list: it writes the same rows.
|
|
# The next scheduled run refreshes it.
|
|
continue
|
|
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
|
# Ingest first — it commits — so the row lock below is held from the dirty check
|
|
# to the commit that replaces the items.
|
|
_ensure_videos(db, yt, ids)
|
|
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
|
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
|
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
|
# one in flight and sees what it committed.
|
|
db.scalars(
|
|
select(Playlist)
|
|
.where(Playlist.id == pl.id)
|
|
.with_for_update()
|
|
.execution_options(populate_existing=True)
|
|
).all() # consumed: rows fetched are what refreshes the loaded objects
|
|
if pl.dirty:
|
|
db.rollback()
|
|
continue
|
|
pl.name = p.get("title") or pl.name
|
|
# Mirror is authoritative: replace the items to match YouTube's order.
|
|
_replace_items_from_youtube(db, yt, pl, ids)
|
|
db.commit()
|
|
synced += 1
|
|
return {"synced": synced}
|
|
|
|
|
|
def _local_video_ids(db: Session, pl: Playlist) -> list[str]:
|
|
return list(
|
|
db.execute(
|
|
select(PlaylistItem.video_id)
|
|
.where(PlaylistItem.playlist_id == pl.id)
|
|
.order_by(PlaylistItem.position)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
|
|
|
|
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
|
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
|
`current` order into `desired` order (both already the same set of video ids).
|
|
Items already in place are skipped; this matches what apply_push actually executes."""
|
|
model = list(current)
|
|
moves = 0
|
|
for i, want in enumerate(desired):
|
|
if i < len(model) and model[i] == want:
|
|
continue
|
|
j = model.index(want, i)
|
|
model.pop(j)
|
|
model.insert(i, want)
|
|
moves += 1
|
|
return moves
|
|
|
|
|
|
@dataclass
|
|
class MergePlan:
|
|
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
|
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
|
|
|
final: list[str] # merged order (local order, YouTube-only adds appended)
|
|
add_to_yt: list[str] # local-adds to push out to YouTube
|
|
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
|
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
|
remove_from_local: list[str] # items deleted on YouTube that are still local
|
|
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
|
baseline_known: bool
|
|
|
|
|
|
def merge_plan(
|
|
baseline: list[str] | None,
|
|
local: list[str],
|
|
youtube: list[str],
|
|
unavailable: set[str] | None = None,
|
|
) -> MergePlan:
|
|
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
|
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
|
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
|
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
|
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
|
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
|
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
|
additions appended in their YouTube order.
|
|
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
|
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
|
delete (they were never local) — otherwise such a video would make the playlist never
|
|
converge. They stay in place on YouTube, untouched by this merge."""
|
|
un = set(unavailable or ())
|
|
L = list(dict.fromkeys(local))
|
|
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
|
Lset, Yset = set(L), set(Y)
|
|
if baseline is None:
|
|
add_to_yt = [v for v in L if v not in Yset]
|
|
add_to_local = [v for v in Y if v not in Lset]
|
|
remove_from_yt: list[str] = []
|
|
remove_from_local: list[str] = []
|
|
final = L + [v for v in Y if v not in Lset]
|
|
else:
|
|
B = set(baseline)
|
|
deleted_local = B - Lset # was synced, removed on our side
|
|
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
|
gone = deleted_local | deleted_yt
|
|
remove_from_yt = [v for v in Y if v in deleted_local]
|
|
remove_from_local = [v for v in L if v in deleted_yt]
|
|
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
|
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
|
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
|
final_set = set(final)
|
|
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
|
# the reorder pass then moves items into `final`.
|
|
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
|
reorders = _reorder_moves(after_membership, final)
|
|
return MergePlan(
|
|
final=final,
|
|
add_to_yt=add_to_yt,
|
|
add_to_local=add_to_local,
|
|
remove_from_yt=remove_from_yt,
|
|
remove_from_local=remove_from_local,
|
|
reorders=reorders,
|
|
baseline_known=baseline is not None,
|
|
)
|
|
|
|
|
|
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
|
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
|
writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked
|
|
playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple
|
|
so the caller can share one read with merge_playlist."""
|
|
local = _local_video_ids(db, pl)
|
|
if not pl.yt_playlist_id:
|
|
ops = 1 + len(local) # create the playlist + one insert per video
|
|
return {
|
|
"action": "create",
|
|
"add_yt": len(local),
|
|
"add_local": 0,
|
|
"remove_yt": 0,
|
|
"remove_local": 0,
|
|
"to_reorder": 0,
|
|
"rename": False,
|
|
"units_estimate": ops * WRITE_UNIT_COST,
|
|
}
|
|
if live is not None:
|
|
current, snippet = live
|
|
else:
|
|
with YouTubeClient(db, user) as yt:
|
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
|
yt_ids = [it["video_id"] for it in current]
|
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
|
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
|
return {
|
|
"action": "update",
|
|
"add_yt": len(mp.add_to_yt),
|
|
"add_local": len(mp.add_to_local),
|
|
"remove_yt": len(mp.remove_from_yt),
|
|
"remove_local": len(mp.remove_from_local),
|
|
"to_reorder": mp.reorders,
|
|
"rename": rename,
|
|
"units_estimate": ops * WRITE_UNIT_COST,
|
|
}
|
|
|
|
|
|
# After this many consecutive write failures we stop calling YouTube for the rest of the push.
|
|
# The failure modes that matter here are all account-wide, not per-video: quota exhausted, the
|
|
# write scope revoked, the token dead. Every remaining write then fails identically — but each
|
|
# one still SPENDS 50 units, so a long playlist burns through the whole (already dead) budget,
|
|
# and tomorrow's too, to produce a failure list we knew after the fifth item.
|
|
_MAX_CONSECUTIVE_FAILURES = 5
|
|
|
|
|
|
class _Failed:
|
|
"""Type of the `FAILED` sentinel, so `run()` can stay generic in the op's return type: a call
|
|
site gets back `str | _Failed` (not a bare `object`) and keeps its item id typed."""
|
|
|
|
|
|
#: Returned by `_Breaker.run` when the write did not happen (refused or failed).
|
|
FAILED = _Failed()
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
class _Breaker:
|
|
"""Consecutive-failure circuit breaker for a single push run. Any success re-arms it.
|
|
|
|
Drive it with `run()` — it owns the whole open-check / call / record-outcome sequence, so a
|
|
write site can't accidentally implement two thirds of the protocol (forgetting to record a
|
|
failure would silently disable the breaker for that operation)."""
|
|
|
|
def __init__(
|
|
self, limit: int = _MAX_CONSECUTIVE_FAILURES, playlist_id: int | None = None
|
|
) -> None:
|
|
self.limit = limit
|
|
self.playlist_id = playlist_id # for the failure line only
|
|
self.streak = 0
|
|
self.tripped = False
|
|
|
|
def open(self) -> bool:
|
|
"""True while writes may still be attempted."""
|
|
return not self.tripped
|
|
|
|
def hit(self, ok: bool) -> None:
|
|
if ok:
|
|
self.streak = 0
|
|
return
|
|
self.streak += 1
|
|
if self.streak >= self.limit and not self.tripped:
|
|
self.tripped = True
|
|
log.warning(
|
|
"playlist push: %d consecutive YouTube write failures — stopping this run "
|
|
"(quota, scope or token); the remaining items are reported as failures",
|
|
self.limit,
|
|
)
|
|
|
|
def run(
|
|
self, op: Callable[[], T], on_fail: Callable[[], None], action: str = "write"
|
|
) -> T | _Failed:
|
|
"""Attempt one YouTube write. Returns the op's result, or `FAILED` when the breaker is
|
|
already tripped (op not called) or the op raised YouTubeError. `on_fail` records the item
|
|
in the caller's failure list either way."""
|
|
if self.tripped:
|
|
on_fail()
|
|
return FAILED
|
|
try:
|
|
out = op()
|
|
except YouTubeError as exc:
|
|
# The client logged the call, but not which playlist it was for or what kind of write
|
|
# — and the caller's failure list only ever reaches the user as a count. Not the video:
|
|
# what is in someone's playlist is a row of their history, the same as a download.
|
|
log.debug(
|
|
"a playlist write to YouTube failed",
|
|
extra=logs.event(
|
|
"sync.playlist.item_push_failed",
|
|
playlist_id=self.playlist_id,
|
|
action=action,
|
|
reason=exc.reason,
|
|
),
|
|
)
|
|
on_fail()
|
|
self.hit(False)
|
|
return FAILED
|
|
self.hit(True)
|
|
return out
|
|
|
|
|
|
def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|
"""Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE
|
|
both sides so an item added (or deleted) on either side survives — no side wins wholesale.
|
|
Local edits are applied directly; YouTube edits go through the failure breaker. On full
|
|
success the merged set becomes the new baseline and the playlist is marked clean; on any
|
|
YouTube failure the baseline is left untouched so the next sync retries and converges.
|
|
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
|
|
(current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through."""
|
|
local = _local_video_ids(db, pl)
|
|
inserted = deleted = reordered = created = pulled = removed_local = 0
|
|
failures: list[str] = []
|
|
breaker = _Breaker(playlist_id=pl.id)
|
|
with YouTubeClient(db, user) as yt:
|
|
if not pl.yt_playlist_id:
|
|
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
|
db.commit()
|
|
created = 1
|
|
# Brand-new playlist: just append every local video in order.
|
|
for vid in local:
|
|
added = breaker.run(
|
|
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
|
lambda v=vid: failures.append(v),
|
|
"add",
|
|
)
|
|
if not isinstance(added, _Failed):
|
|
inserted += 1
|
|
if not failures:
|
|
_set_baseline(pl, local)
|
|
db.commit()
|
|
return {
|
|
"created": created,
|
|
"inserted": inserted,
|
|
"deleted": deleted,
|
|
"reordered": reordered,
|
|
"pulled": pulled,
|
|
"removed_local": removed_local,
|
|
"failures": failures,
|
|
"aborted": breaker.tripped,
|
|
"yt_playlist_id": pl.yt_playlist_id,
|
|
}
|
|
|
|
# Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read).
|
|
if live is not None:
|
|
current, snippet = live
|
|
else:
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
|
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
|
yt_ids = [it["video_id"] for it in current]
|
|
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
|
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
|
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
|
prev_unavailable = parse_unavailable(pl)
|
|
still_listed = [v for v in yt_ids if v in prev_unavailable]
|
|
if still_listed:
|
|
_ensure_videos(db, yt, still_listed)
|
|
healed = set(
|
|
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
|
|
)
|
|
effective_unavailable = prev_unavailable - healed
|
|
else:
|
|
effective_unavailable = prev_unavailable
|
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
|
|
|
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
|
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
|
# YouTube write trips the breaker.
|
|
_ensure_videos(db, yt, mp.add_to_local)
|
|
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
|
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
|
present = (
|
|
set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all())
|
|
if mp.final
|
|
else set()
|
|
)
|
|
final_local = [v for v in mp.final if v in present]
|
|
pulled = len([v for v in mp.add_to_local if v in present])
|
|
removed_local = len(mp.remove_from_local)
|
|
_set_local_items(db, pl, final_local)
|
|
db.commit()
|
|
|
|
# --- YouTube side (metered, breaker-guarded).
|
|
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
|
try:
|
|
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
|
except YouTubeError as exc:
|
|
# All the caller passes on is the word "title", which says the rename did not
|
|
# happen but not that YouTube refused it, or why.
|
|
log.debug(
|
|
"a playlist rename did not reach YouTube",
|
|
extra=logs.event(
|
|
"sync.playlist.title_push_failed",
|
|
playlist_id=pl.id,
|
|
reason=exc.reason,
|
|
),
|
|
)
|
|
failures.append("title")
|
|
# Propagate genuine local deletes.
|
|
for vid in mp.remove_from_yt:
|
|
item_id = item_id_by_vid.get(vid)
|
|
if item_id is None:
|
|
continue
|
|
dropped = breaker.run(
|
|
lambda i=item_id: yt.delete_playlist_item(i),
|
|
lambda v=vid: failures.append(v),
|
|
"delete",
|
|
)
|
|
if not isinstance(dropped, _Failed):
|
|
deleted += 1
|
|
item_id_by_vid.pop(vid, None)
|
|
# Insert local-adds (append; the reorder pass below fixes positions).
|
|
for vid in mp.add_to_yt:
|
|
item_id = breaker.run(
|
|
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
|
lambda v=vid: failures.append(v),
|
|
"add",
|
|
)
|
|
if not isinstance(item_id, _Failed):
|
|
item_id_by_vid[vid] = item_id
|
|
inserted += 1
|
|
# Reorder YouTube to match `final` via insertion-sort moves over a local model.
|
|
final_set = set(mp.final)
|
|
model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [
|
|
v for v in mp.add_to_yt if v in item_id_by_vid
|
|
]
|
|
for i, want in enumerate(mp.final):
|
|
if want not in item_id_by_vid:
|
|
continue # deleted-from-YT or failed insert; skip
|
|
if i < len(model) and model[i] == want:
|
|
continue
|
|
moved = breaker.run(
|
|
lambda w=want, pos=i: yt.move_playlist_item(
|
|
item_id_by_vid[w], pl.yt_playlist_id, w, pos
|
|
),
|
|
lambda w=want: failures.append(w),
|
|
"move",
|
|
)
|
|
if isinstance(moved, _Failed):
|
|
continue
|
|
reordered += 1
|
|
j = model.index(want)
|
|
model.pop(j)
|
|
model.insert(i, want)
|
|
if not failures:
|
|
# Carry forward un-ingestable-but-live YouTube ids so the next merge skips them
|
|
# (see merge_plan): this run's failed pulls, plus prior ones still on YouTube, minus
|
|
# any that have since left YouTube. Recomputed each success → self-healing.
|
|
yt_id_set = set(yt_ids)
|
|
unavailable = sorted(
|
|
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
|
& yt_id_set
|
|
)
|
|
_set_baseline(pl, final_local, unavailable)
|
|
db.commit()
|
|
return {
|
|
"created": created,
|
|
"inserted": inserted,
|
|
"deleted": deleted,
|
|
"reordered": reordered,
|
|
"pulled": pulled,
|
|
"removed_local": removed_local,
|
|
"failures": failures,
|
|
"aborted": breaker.tripped,
|
|
"yt_playlist_id": pl.yt_playlist_id,
|
|
}
|
|
|
|
|
|
def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
|
"""Force-refresh ONE YouTube-linked playlist from YouTube, discarding local edits and
|
|
clearing dirty. This is the per-playlist 'reset to YouTube' — unlike the bulk read-sync
|
|
it deliberately does NOT skip a dirty playlist (overwriting local changes is the point).
|
|
Works for both mirrors (source='youtube') and exported local playlists (yt_playlist_id)."""
|
|
if not pl.yt_playlist_id:
|
|
raise YouTubeError("Playlist is not linked to YouTube")
|
|
with YouTubeClient(db, user) as yt:
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
if snippet is None:
|
|
raise YouTubeError("Playlist no longer exists on YouTube")
|
|
pl.name = snippet.get("title") or pl.name
|
|
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
|
ordered = _replace_items_from_youtube(db, yt, pl, ids)
|
|
db.commit()
|
|
return {"items": len(ordered)}
|
|
|
|
|
|
def sync_all_playlists(db: Session) -> dict:
|
|
"""Scheduler entry point: mirror playlists for every user with a read scope + token."""
|
|
users = token_users(db)
|
|
total = 0
|
|
for i, user in enumerate(users, 1):
|
|
if not has_read_scope(user):
|
|
progress.report(i, len(users), "playlist_sync")
|
|
continue
|
|
try:
|
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
|
total += sync_user_playlists(db, user).get("synced", 0)
|
|
except YouTubeError as exc:
|
|
db.rollback()
|
|
log.warning(
|
|
"Playlist sync failed for user %s",
|
|
user.id,
|
|
extra=logs.event("sync.playlists.failed", user_id=user.id, reason=exc.reason),
|
|
)
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception(
|
|
"Playlist sync crashed for user %s",
|
|
user.id,
|
|
extra=logs.event("sync.playlists.crashed", user_id=user.id),
|
|
)
|
|
progress.report(i, len(users), "playlist_sync")
|
|
return {"users": len(users), "playlists_synced": total}
|