peter
54d19cc386
- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
`formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
file paths, and the job-failed, refused, edit-failed and backfill lines use
it. Done lines log the asset id, not the title-built path. The PII contract
now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
a process's last batch closes the repeat window, so a short-lived download
child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
`download.job.failed` with a cause; they left no line at all before. The
crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
carry reason `transport`. Subscribe recognises a duplicate by reason code
only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
681 lines
29 KiB
Python
681 lines
29 KiB
Python
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
|
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
|
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
|
|
|
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
|
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
|
ingested (with a stub channel) so the mirror is faithful."""
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from typing import TypeVar
|
|
|
|
from sqlalchemy import delete, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, progress, quota
|
|
from app.auth import has_read_scope
|
|
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
|
from app.sync.runner import token_users
|
|
from app.sync.videos import apply_video_details, parse_dt
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
|
|
log = logs.get_logger(__name__)
|
|
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
|
WRITE_UNIT_COST = 50
|
|
|
|
|
|
def fingerprint(name: str, ids: list[str]) -> str:
|
|
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
|
baseline; the current fingerprint is compared to it to derive `dirty`."""
|
|
h = hashlib.sha256()
|
|
h.update((name or "").encode())
|
|
h.update(b"\n")
|
|
h.update(",".join(ids).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def current_fingerprint(db: Session, pl: Playlist) -> str:
|
|
return fingerprint(pl.name, _local_video_ids(db, pl))
|
|
|
|
|
|
def _dump_ids(ids: list[str]) -> str:
|
|
"""Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`)."""
|
|
return json.dumps(ids)
|
|
|
|
|
|
def _load_baseline(pl: Playlist) -> tuple[list[str] | None, set[str]]:
|
|
"""Parse `synced_video_ids` into (ordered id set, un-ingestable id set). Two on-disk forms:
|
|
a plain JSON list (legacy / no un-ingestable ids) or `{"ids": [...], "unavailable": [...]}`.
|
|
`unavailable` = YouTube-only videos that exist on YouTube but the Data API won't return
|
|
(private/deleted/region-locked), so they never became local rows — tracked so the merge can
|
|
skip them instead of forever re-classifying them as a YouTube-add (they'd never converge)."""
|
|
if not pl.synced_video_ids:
|
|
return None, set()
|
|
try:
|
|
val = json.loads(pl.synced_video_ids)
|
|
except (ValueError, TypeError) as exc:
|
|
# No baseline means the three-way merge has no "what we last agreed on" to subtract, so
|
|
# it degrades to a union: nothing is lost, but a delete made on either side comes back.
|
|
log.warning(
|
|
"the sync baseline for a playlist could not be read; merging as a union",
|
|
extra=logs.event(
|
|
"sync.playlist.baseline_unreadable",
|
|
playlist_id=pl.id,
|
|
reason=type(exc).__name__,
|
|
),
|
|
)
|
|
return None, set()
|
|
if isinstance(val, list):
|
|
return [str(v) for v in val], set()
|
|
if isinstance(val, dict):
|
|
raw_ids = val.get("ids")
|
|
ids = [str(v) for v in raw_ids] if isinstance(raw_ids, list) else None
|
|
raw_un = val.get("unavailable")
|
|
un = {str(v) for v in raw_un} if isinstance(raw_un, list) else set()
|
|
return ids, un
|
|
return None, set()
|
|
|
|
|
|
def parse_baseline(pl: Playlist) -> list[str] | None:
|
|
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
|
|
export, or a link created before S7). None makes the first merge union losslessly."""
|
|
return _load_baseline(pl)[0]
|
|
|
|
|
|
def parse_unavailable(pl: Playlist) -> set[str]:
|
|
"""The last-synced set of un-ingestable-but-live YouTube ids (see `_load_baseline`)."""
|
|
return _load_baseline(pl)[1]
|
|
|
|
|
|
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
|
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
|
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
|
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
|
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
|
if unavailable:
|
|
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
|
else:
|
|
pl.synced_video_ids = _dump_ids(ids)
|
|
pl.dirty = False
|
|
|
|
|
|
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
|
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
|
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
|
if not pl.yt_playlist_id:
|
|
pl.dirty = False
|
|
return
|
|
# The session has autoflush off, so the caller's pending item/order/name edits aren't
|
|
# visible to current_fingerprint's query yet — flush first or dirty would lag one edit.
|
|
db.flush()
|
|
pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
|
|
|
|
|
|
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
|
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
|
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
|
the API won't return are simply left out."""
|
|
if not video_ids:
|
|
return
|
|
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
|
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
|
if not missing:
|
|
return
|
|
items = yt.get_videos(missing)
|
|
chan_ids = {
|
|
it.get("snippet", {}).get("channelId")
|
|
for it in items
|
|
if it.get("snippet", {}).get("channelId")
|
|
}
|
|
existing_ch = set(
|
|
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
|
|
)
|
|
now = datetime.now(timezone.utc)
|
|
for it in items:
|
|
sn = it.get("snippet", {})
|
|
cid = sn.get("channelId")
|
|
if cid and cid not in existing_ch:
|
|
db.add(Channel(id=cid, title=sn.get("channelTitle")))
|
|
existing_ch.add(cid)
|
|
db.flush()
|
|
seen: set[str] = set()
|
|
for it in items:
|
|
vid = it.get("id")
|
|
sn = it.get("snippet", {})
|
|
cid = sn.get("channelId")
|
|
if not vid or not cid or vid in seen:
|
|
continue
|
|
seen.add(vid)
|
|
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
|
# fallback here too so the constructor never carries an explicit NULL.
|
|
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
|
apply_video_details(v, it)
|
|
v.enriched_at = now
|
|
db.add(v)
|
|
db.commit()
|
|
|
|
|
|
def _replace_items_from_youtube(
|
|
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
|
|
) -> list[str]:
|
|
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
|
replace): ensure the videos exist in the catalog, keep only those present in their YT
|
|
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
|
|
`pl.name` must already be set to the desired name. The caller commits. Returns the final
|
|
ordered ids."""
|
|
_ensure_videos(db, yt, ids)
|
|
present = (
|
|
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
|
)
|
|
ordered: list[str] = []
|
|
seen: set[str] = set()
|
|
for v in ids:
|
|
if v in present and v not in seen:
|
|
seen.add(v)
|
|
ordered.append(v)
|
|
_set_local_items(db, pl, ordered)
|
|
# After a pull/reset local == YouTube, so this ordered set IS the new merge baseline.
|
|
_set_baseline(pl, ordered)
|
|
return ordered
|
|
|
|
|
|
def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None:
|
|
"""Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller
|
|
is responsible for having ensured every id exists in the catalog."""
|
|
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
|
|
for pos, vid in enumerate(ordered):
|
|
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
|
|
|
|
|
|
def sync_user_playlists(db: Session, user: User) -> dict:
|
|
"""Mirror the user's YouTube playlists into local source='youtube' playlists. Leaves
|
|
the user's local playlists and the built-in Watch later untouched."""
|
|
if not has_read_scope(user):
|
|
return {"synced": 0, "reason": "no read scope"}
|
|
synced = 0
|
|
with YouTubeClient(db, user) as yt:
|
|
yt_playlists = list(yt.iter_my_playlists())
|
|
yt_ids = {p["id"] for p in yt_playlists if p.get("id")}
|
|
existing = {
|
|
pl.yt_playlist_id: pl
|
|
for pl in db.execute(
|
|
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
|
|
).scalars()
|
|
}
|
|
# YouTube playlists that we already own as local (exported) playlists must not be
|
|
# re-mirrored, or every exported list would also show up as a read-only duplicate.
|
|
owned_yt_ids = set(
|
|
db.execute(
|
|
select(Playlist.yt_playlist_id).where(
|
|
Playlist.user_id == user.id,
|
|
Playlist.source == "local",
|
|
Playlist.yt_playlist_id.is_not(None),
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
# Drop mirrors whose YouTube playlist no longer exists.
|
|
for ytid, pl in existing.items():
|
|
if ytid not in yt_ids:
|
|
db.delete(pl)
|
|
db.flush()
|
|
|
|
for idx, p in enumerate(yt_playlists):
|
|
ytid = p.get("id")
|
|
if not ytid or ytid in owned_yt_ids:
|
|
continue
|
|
pl = existing.get(ytid)
|
|
if pl is not None and pl.dirty:
|
|
# Local edits not yet pushed: don't clobber them with YouTube's state.
|
|
# A successful 'Sync to YouTube' clears dirty and lets the mirror refresh.
|
|
continue
|
|
if pl is None:
|
|
pl = Playlist(
|
|
user_id=user.id,
|
|
name=p.get("title") or "Untitled",
|
|
kind="user",
|
|
source="youtube",
|
|
yt_playlist_id=ytid,
|
|
position=1000 + idx, # sort mirrored lists after local ones
|
|
)
|
|
db.add(pl)
|
|
db.flush()
|
|
else:
|
|
pl.name = p.get("title") or pl.name
|
|
# Mirror is authoritative: replace the items to match YouTube's order.
|
|
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
|
_replace_items_from_youtube(db, yt, pl, ids)
|
|
db.commit()
|
|
synced += 1
|
|
return {"synced": synced}
|
|
|
|
|
|
def _local_video_ids(db: Session, pl: Playlist) -> list[str]:
|
|
return list(
|
|
db.execute(
|
|
select(PlaylistItem.video_id)
|
|
.where(PlaylistItem.playlist_id == pl.id)
|
|
.order_by(PlaylistItem.position)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
|
|
|
|
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
|
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
|
`current` order into `desired` order (both already the same set of video ids).
|
|
Items already in place are skipped; this matches what apply_push actually executes."""
|
|
model = list(current)
|
|
moves = 0
|
|
for i, want in enumerate(desired):
|
|
if i < len(model) and model[i] == want:
|
|
continue
|
|
j = model.index(want, i)
|
|
model.pop(j)
|
|
model.insert(i, want)
|
|
moves += 1
|
|
return moves
|
|
|
|
|
|
@dataclass
|
|
class MergePlan:
|
|
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
|
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
|
|
|
final: list[str] # merged order (local order, YouTube-only adds appended)
|
|
add_to_yt: list[str] # local-adds to push out to YouTube
|
|
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
|
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
|
remove_from_local: list[str] # items deleted on YouTube that are still local
|
|
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
|
baseline_known: bool
|
|
|
|
|
|
def merge_plan(
|
|
baseline: list[str] | None,
|
|
local: list[str],
|
|
youtube: list[str],
|
|
unavailable: set[str] | None = None,
|
|
) -> MergePlan:
|
|
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
|
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
|
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
|
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
|
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
|
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
|
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
|
additions appended in their YouTube order.
|
|
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
|
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
|
delete (they were never local) — otherwise such a video would make the playlist never
|
|
converge. They stay in place on YouTube, untouched by this merge."""
|
|
un = set(unavailable or ())
|
|
L = list(dict.fromkeys(local))
|
|
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
|
Lset, Yset = set(L), set(Y)
|
|
if baseline is None:
|
|
add_to_yt = [v for v in L if v not in Yset]
|
|
add_to_local = [v for v in Y if v not in Lset]
|
|
remove_from_yt: list[str] = []
|
|
remove_from_local: list[str] = []
|
|
final = L + [v for v in Y if v not in Lset]
|
|
else:
|
|
B = set(baseline)
|
|
deleted_local = B - Lset # was synced, removed on our side
|
|
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
|
gone = deleted_local | deleted_yt
|
|
remove_from_yt = [v for v in Y if v in deleted_local]
|
|
remove_from_local = [v for v in L if v in deleted_yt]
|
|
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
|
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
|
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
|
final_set = set(final)
|
|
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
|
# the reorder pass then moves items into `final`.
|
|
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
|
reorders = _reorder_moves(after_membership, final)
|
|
return MergePlan(
|
|
final=final,
|
|
add_to_yt=add_to_yt,
|
|
add_to_local=add_to_local,
|
|
remove_from_yt=remove_from_yt,
|
|
remove_from_local=remove_from_local,
|
|
reorders=reorders,
|
|
baseline_known=baseline is not None,
|
|
)
|
|
|
|
|
|
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
|
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
|
writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked
|
|
playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple
|
|
so the caller can share one read with merge_playlist."""
|
|
local = _local_video_ids(db, pl)
|
|
if not pl.yt_playlist_id:
|
|
ops = 1 + len(local) # create the playlist + one insert per video
|
|
return {
|
|
"action": "create",
|
|
"add_yt": len(local),
|
|
"add_local": 0,
|
|
"remove_yt": 0,
|
|
"remove_local": 0,
|
|
"to_reorder": 0,
|
|
"rename": False,
|
|
"units_estimate": ops * WRITE_UNIT_COST,
|
|
}
|
|
if live is not None:
|
|
current, snippet = live
|
|
else:
|
|
with YouTubeClient(db, user) as yt:
|
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
|
yt_ids = [it["video_id"] for it in current]
|
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
|
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
|
return {
|
|
"action": "update",
|
|
"add_yt": len(mp.add_to_yt),
|
|
"add_local": len(mp.add_to_local),
|
|
"remove_yt": len(mp.remove_from_yt),
|
|
"remove_local": len(mp.remove_from_local),
|
|
"to_reorder": mp.reorders,
|
|
"rename": rename,
|
|
"units_estimate": ops * WRITE_UNIT_COST,
|
|
}
|
|
|
|
|
|
# After this many consecutive write failures we stop calling YouTube for the rest of the push.
|
|
# The failure modes that matter here are all account-wide, not per-video: quota exhausted, the
|
|
# write scope revoked, the token dead. Every remaining write then fails identically — but each
|
|
# one still SPENDS 50 units, so a long playlist burns through the whole (already dead) budget,
|
|
# and tomorrow's too, to produce a failure list we knew after the fifth item.
|
|
_MAX_CONSECUTIVE_FAILURES = 5
|
|
|
|
|
|
class _Failed:
|
|
"""Type of the `FAILED` sentinel, so `run()` can stay generic in the op's return type: a call
|
|
site gets back `str | _Failed` (not a bare `object`) and keeps its item id typed."""
|
|
|
|
|
|
#: Returned by `_Breaker.run` when the write did not happen (refused or failed).
|
|
FAILED = _Failed()
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
class _Breaker:
|
|
"""Consecutive-failure circuit breaker for a single push run. Any success re-arms it.
|
|
|
|
Drive it with `run()` — it owns the whole open-check / call / record-outcome sequence, so a
|
|
write site can't accidentally implement two thirds of the protocol (forgetting to record a
|
|
failure would silently disable the breaker for that operation)."""
|
|
|
|
def __init__(
|
|
self, limit: int = _MAX_CONSECUTIVE_FAILURES, playlist_id: int | None = None
|
|
) -> None:
|
|
self.limit = limit
|
|
self.playlist_id = playlist_id # for the failure line only
|
|
self.streak = 0
|
|
self.tripped = False
|
|
|
|
def open(self) -> bool:
|
|
"""True while writes may still be attempted."""
|
|
return not self.tripped
|
|
|
|
def hit(self, ok: bool) -> None:
|
|
if ok:
|
|
self.streak = 0
|
|
return
|
|
self.streak += 1
|
|
if self.streak >= self.limit and not self.tripped:
|
|
self.tripped = True
|
|
log.warning(
|
|
"playlist push: %d consecutive YouTube write failures — stopping this run "
|
|
"(quota, scope or token); the remaining items are reported as failures",
|
|
self.limit,
|
|
)
|
|
|
|
def run(
|
|
self, op: Callable[[], T], on_fail: Callable[[], None], action: str = "write"
|
|
) -> T | _Failed:
|
|
"""Attempt one YouTube write. Returns the op's result, or `FAILED` when the breaker is
|
|
already tripped (op not called) or the op raised YouTubeError. `on_fail` records the item
|
|
in the caller's failure list either way."""
|
|
if self.tripped:
|
|
on_fail()
|
|
return FAILED
|
|
try:
|
|
out = op()
|
|
except YouTubeError as exc:
|
|
# The client logged the call, but not which playlist it was for or what kind of write
|
|
# — and the caller's failure list only ever reaches the user as a count. Not the video:
|
|
# what is in someone's playlist is a row of their history, the same as a download.
|
|
log.debug(
|
|
"a playlist write to YouTube failed",
|
|
extra=logs.event(
|
|
"sync.playlist.item_push_failed",
|
|
playlist_id=self.playlist_id,
|
|
action=action,
|
|
reason=exc.reason,
|
|
),
|
|
)
|
|
on_fail()
|
|
self.hit(False)
|
|
return FAILED
|
|
self.hit(True)
|
|
return out
|
|
|
|
|
|
def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|
"""Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE
|
|
both sides so an item added (or deleted) on either side survives — no side wins wholesale.
|
|
Local edits are applied directly; YouTube edits go through the failure breaker. On full
|
|
success the merged set becomes the new baseline and the playlist is marked clean; on any
|
|
YouTube failure the baseline is left untouched so the next sync retries and converges.
|
|
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
|
|
(current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through."""
|
|
local = _local_video_ids(db, pl)
|
|
inserted = deleted = reordered = created = pulled = removed_local = 0
|
|
failures: list[str] = []
|
|
breaker = _Breaker(playlist_id=pl.id)
|
|
with YouTubeClient(db, user) as yt:
|
|
if not pl.yt_playlist_id:
|
|
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
|
db.commit()
|
|
created = 1
|
|
# Brand-new playlist: just append every local video in order.
|
|
for vid in local:
|
|
added = breaker.run(
|
|
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
|
lambda v=vid: failures.append(v),
|
|
"add",
|
|
)
|
|
if not isinstance(added, _Failed):
|
|
inserted += 1
|
|
if not failures:
|
|
_set_baseline(pl, local)
|
|
db.commit()
|
|
return {
|
|
"created": created,
|
|
"inserted": inserted,
|
|
"deleted": deleted,
|
|
"reordered": reordered,
|
|
"pulled": pulled,
|
|
"removed_local": removed_local,
|
|
"failures": failures,
|
|
"aborted": breaker.tripped,
|
|
"yt_playlist_id": pl.yt_playlist_id,
|
|
}
|
|
|
|
# Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read).
|
|
if live is not None:
|
|
current, snippet = live
|
|
else:
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
|
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
|
yt_ids = [it["video_id"] for it in current]
|
|
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
|
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
|
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
|
prev_unavailable = parse_unavailable(pl)
|
|
still_listed = [v for v in yt_ids if v in prev_unavailable]
|
|
if still_listed:
|
|
_ensure_videos(db, yt, still_listed)
|
|
healed = set(
|
|
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
|
|
)
|
|
effective_unavailable = prev_unavailable - healed
|
|
else:
|
|
effective_unavailable = prev_unavailable
|
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
|
|
|
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
|
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
|
# YouTube write trips the breaker.
|
|
_ensure_videos(db, yt, mp.add_to_local)
|
|
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
|
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
|
present = (
|
|
set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all())
|
|
if mp.final
|
|
else set()
|
|
)
|
|
final_local = [v for v in mp.final if v in present]
|
|
pulled = len([v for v in mp.add_to_local if v in present])
|
|
removed_local = len(mp.remove_from_local)
|
|
_set_local_items(db, pl, final_local)
|
|
db.commit()
|
|
|
|
# --- YouTube side (metered, breaker-guarded).
|
|
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
|
try:
|
|
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
|
except YouTubeError as exc:
|
|
# All the caller passes on is the word "title", which says the rename did not
|
|
# happen but not that YouTube refused it, or why.
|
|
log.debug(
|
|
"a playlist rename did not reach YouTube",
|
|
extra=logs.event(
|
|
"sync.playlist.title_push_failed",
|
|
playlist_id=pl.id,
|
|
reason=exc.reason,
|
|
),
|
|
)
|
|
failures.append("title")
|
|
# Propagate genuine local deletes.
|
|
for vid in mp.remove_from_yt:
|
|
item_id = item_id_by_vid.get(vid)
|
|
if item_id is None:
|
|
continue
|
|
dropped = breaker.run(
|
|
lambda i=item_id: yt.delete_playlist_item(i),
|
|
lambda v=vid: failures.append(v),
|
|
"delete",
|
|
)
|
|
if not isinstance(dropped, _Failed):
|
|
deleted += 1
|
|
item_id_by_vid.pop(vid, None)
|
|
# Insert local-adds (append; the reorder pass below fixes positions).
|
|
for vid in mp.add_to_yt:
|
|
item_id = breaker.run(
|
|
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
|
lambda v=vid: failures.append(v),
|
|
"add",
|
|
)
|
|
if not isinstance(item_id, _Failed):
|
|
item_id_by_vid[vid] = item_id
|
|
inserted += 1
|
|
# Reorder YouTube to match `final` via insertion-sort moves over a local model.
|
|
final_set = set(mp.final)
|
|
model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [
|
|
v for v in mp.add_to_yt if v in item_id_by_vid
|
|
]
|
|
for i, want in enumerate(mp.final):
|
|
if want not in item_id_by_vid:
|
|
continue # deleted-from-YT or failed insert; skip
|
|
if i < len(model) and model[i] == want:
|
|
continue
|
|
moved = breaker.run(
|
|
lambda w=want, pos=i: yt.move_playlist_item(
|
|
item_id_by_vid[w], pl.yt_playlist_id, w, pos
|
|
),
|
|
lambda w=want: failures.append(w),
|
|
"move",
|
|
)
|
|
if isinstance(moved, _Failed):
|
|
continue
|
|
reordered += 1
|
|
j = model.index(want)
|
|
model.pop(j)
|
|
model.insert(i, want)
|
|
if not failures:
|
|
# Carry forward un-ingestable-but-live YouTube ids so the next merge skips them
|
|
# (see merge_plan): this run's failed pulls, plus prior ones still on YouTube, minus
|
|
# any that have since left YouTube. Recomputed each success → self-healing.
|
|
yt_id_set = set(yt_ids)
|
|
unavailable = sorted(
|
|
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
|
& yt_id_set
|
|
)
|
|
_set_baseline(pl, final_local, unavailable)
|
|
db.commit()
|
|
return {
|
|
"created": created,
|
|
"inserted": inserted,
|
|
"deleted": deleted,
|
|
"reordered": reordered,
|
|
"pulled": pulled,
|
|
"removed_local": removed_local,
|
|
"failures": failures,
|
|
"aborted": breaker.tripped,
|
|
"yt_playlist_id": pl.yt_playlist_id,
|
|
}
|
|
|
|
|
|
def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
|
"""Force-refresh ONE YouTube-linked playlist from YouTube, discarding local edits and
|
|
clearing dirty. This is the per-playlist 'reset to YouTube' — unlike the bulk read-sync
|
|
it deliberately does NOT skip a dirty playlist (overwriting local changes is the point).
|
|
Works for both mirrors (source='youtube') and exported local playlists (yt_playlist_id)."""
|
|
if not pl.yt_playlist_id:
|
|
raise YouTubeError("Playlist is not linked to YouTube")
|
|
with YouTubeClient(db, user) as yt:
|
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
|
if snippet is None:
|
|
raise YouTubeError("Playlist no longer exists on YouTube")
|
|
pl.name = snippet.get("title") or pl.name
|
|
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
|
ordered = _replace_items_from_youtube(db, yt, pl, ids)
|
|
db.commit()
|
|
return {"items": len(ordered)}
|
|
|
|
|
|
def sync_all_playlists(db: Session) -> dict:
|
|
"""Scheduler entry point: mirror playlists for every user with a read scope + token."""
|
|
users = token_users(db)
|
|
total = 0
|
|
for i, user in enumerate(users, 1):
|
|
if not has_read_scope(user):
|
|
progress.report(i, len(users), "playlist_sync")
|
|
continue
|
|
try:
|
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
|
total += sync_user_playlists(db, user).get("synced", 0)
|
|
except YouTubeError:
|
|
db.rollback()
|
|
log.warning("Playlist sync failed for user %s", user.id)
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("Playlist sync crashed for user %s", user.id)
|
|
progress.report(i, len(users), "playlist_sync")
|
|
return {"users": len(users), "playlists_synced": total}
|