refactor(reader): the HTTP layer reads through queries and writes through the service
Feed, playlists, saved views, tags and the Home pins do no database work in their routes any more: reads go through reader/queries (and feedquery for a feed page, its counts and facets), writes through reader/service and discover/pins, which flush; the handlers commit. Reader is fenced below its HTTP layer too: its owner filters go through userscope, and the catalog's jobs ask Reader instead of joining its tables (the auto-tagger's system tags, the cleanup jobs' keep rule, the maintenance notice). A playlist sync's catalog ingest moves to sync/videos.
This commit is contained in:
@@ -12,7 +12,6 @@ import json
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import Engine, create_engine, delete, func, select
|
||||
@@ -21,13 +20,14 @@ from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
from app.models import Channel, User, Video
|
||||
from app.models import User, Video
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
from app.sync.playlist_merge import merge_plan
|
||||
from app.sync.runner import token_users
|
||||
from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.sync.videos import ensure_videos
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||
from app.userscope import owned_by
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||
@@ -228,55 +228,11 @@ def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
|
||||
|
||||
|
||||
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
||||
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
||||
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
||||
the API won't return are simply left out."""
|
||||
if not video_ids:
|
||||
return
|
||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
||||
if not missing:
|
||||
return
|
||||
items = yt.get_videos(missing)
|
||||
chan_ids = {
|
||||
it.get("snippet", {}).get("channelId")
|
||||
for it in items
|
||||
if it.get("snippet", {}).get("channelId")
|
||||
}
|
||||
existing_ch = set(
|
||||
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
for it in items:
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if cid and cid not in existing_ch:
|
||||
db.add(Channel(id=cid, title=sn.get("channelTitle")))
|
||||
existing_ch.add(cid)
|
||||
db.flush()
|
||||
seen: set[str] = set()
|
||||
for it in items:
|
||||
vid = it.get("id")
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if not vid or not cid or vid in seen:
|
||||
continue
|
||||
seen.add(vid)
|
||||
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
||||
# fallback here too so the constructor never carries an explicit NULL.
|
||||
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
||||
apply_video_details(v, it)
|
||||
v.enriched_at = now
|
||||
db.add(v)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
|
||||
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
||||
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
|
||||
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
|
||||
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
(`sync.videos.ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
|
||||
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
|
||||
present = (
|
||||
@@ -347,7 +303,9 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
existing = {
|
||||
pl.yt_playlist_id: pl
|
||||
for pl in db.execute(
|
||||
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
|
||||
select(Playlist).where(
|
||||
owned_by(Playlist.user_id, user.id), Playlist.source == "youtube"
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
# YouTube playlists that we already own as local (exported) playlists must not be
|
||||
@@ -355,7 +313,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
owned_yt_ids = set(
|
||||
db.execute(
|
||||
select(Playlist.yt_playlist_id).where(
|
||||
Playlist.user_id == user.id,
|
||||
owned_by(Playlist.user_id, user.id),
|
||||
Playlist.source == "local",
|
||||
Playlist.yt_playlist_id.is_not(None),
|
||||
)
|
||||
@@ -402,7 +360,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
continue
|
||||
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||
# to the commit that replaces the items.
|
||||
_ensure_videos(db, yt, ids)
|
||||
ensure_videos(db, yt, ids)
|
||||
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
||||
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
||||
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
||||
@@ -613,7 +571,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
prev_unavailable = parse_unavailable(pl)
|
||||
still_listed = [v for v in yt_ids if v in prev_unavailable]
|
||||
if still_listed:
|
||||
_ensure_videos(db, yt, still_listed)
|
||||
ensure_videos(db, yt, still_listed)
|
||||
healed = set(
|
||||
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
|
||||
)
|
||||
@@ -625,7 +583,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
ensure_videos(db, yt, mp.add_to_local)
|
||||
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
|
||||
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
|
||||
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
|
||||
@@ -750,7 +708,7 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
||||
)
|
||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||
refuse_blank_listing(ids, snippet.get("item_count"))
|
||||
_ensure_videos(db, yt, ids)
|
||||
ensure_videos(db, yt, ids)
|
||||
_lock_or_gone(db, pl)
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ordered = _replace_items_from_youtube(db, pl, ids)
|
||||
|
||||
Reference in New Issue
Block a user