refactor(reader): the HTTP layer reads through queries and writes through the service

Feed, playlists, saved views, tags and the Home pins do no database work in their routes any more:
reads go through reader/queries (and feedquery for a feed page, its counts and facets), writes
through reader/service and discover/pins, which flush; the handlers commit. Reader is fenced below
its HTTP layer too: its owner filters go through userscope, and the catalog's jobs ask Reader
instead of joining its tables (the auto-tagger's system tags, the cleanup jobs' keep rule, the
maintenance notice). A playlist sync's catalog ingest moves to sync/videos.

backend/app/sync/autotag.py

@@ -8,12 +8,13 @@ regenerated freely; user tags are never touched here.
 
 import re
 
-from sqlalchemy import and_, exists, func, select
+from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from app import logs, progress, sysconfig
 from app.models import Channel, Video
-from app.reader.models import ChannelTag, SavedView, Tag
+from app.reader import queries as reader_queries
+from app.reader import service as reader_service
 
 log = logs.get_logger(__name__)
 
@@ -210,71 +211,30 @@ def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
     return topics
 
 
-def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
-    tag = db.execute(
-        select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
-    ).scalar_one_or_none()
-    if tag is None:
-        tag = Tag(user_id=None, name=name, category=category, color=color)
-        db.add(tag)
-        db.flush()
-    return tag
-
-
 def apply_channel_autotags(db: Session, channel: Channel) -> None:
     desired: dict[int, float | None] = {}
 
     lang, confidence = detect_channel_language(db, channel)
     if lang:
-        tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR)
+        tag = reader_service.ensure_system_tag(
+            db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR
+        )
         desired[tag.id] = confidence
 
     for topic in compute_channel_topics(db, channel):
-        tag = ensure_system_tag(db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR))
+        tag = reader_service.ensure_system_tag(
+            db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
+        )
         desired.setdefault(tag.id, None)
 
-    existing = {
-        ct.tag_id: ct
-        for ct in db.execute(
-            select(ChannelTag).where(
-                ChannelTag.channel_id == channel.id,
-                ChannelTag.user_id.is_(None),
-                ChannelTag.source == "auto",
-            )
-        ).scalars()
-    }
-    for tag_id, ct in existing.items():
-        if tag_id not in desired:
-            db.delete(ct)
-    for tag_id, conf in desired.items():
-        ct = existing.get(tag_id)
-        if ct is None:
-            db.add(
-                ChannelTag(
-                    channel_id=channel.id,
-                    tag_id=tag_id,
-                    user_id=None,
-                    source="auto",
-                    confidence=conf,
-                )
-            )
-        else:
-            ct.confidence = conf
+    reader_service.set_auto_tags(db, channel.id, desired)
     db.commit()
 
 
 def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
     query = select(Channel)
     if only_missing:
-        query = query.where(
-            ~exists().where(
-                and_(
-                    ChannelTag.channel_id == Channel.id,
-                    ChannelTag.source == "auto",
-                    ChannelTag.user_id.is_(None),
-                )
-            )
-        )
+        query = query.where(~reader_queries.has_auto_tags(Channel.id))
     channels = db.execute(query).scalars().all()
     tagged = 0
     for i, channel in enumerate(channels, 1):
@@ -285,40 +245,6 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
             db.rollback()
             log.exception("Auto-tagging failed for channel %s", channel.id)
         progress.report(i, len(channels), "autotag")
-    removed = _cleanup_orphan_system_tags(db)
-    return {"channels_tagged": tagged, "orphan_tags_removed": removed}
-
-
-def _view_tag_ids(filters: object) -> list[int]:
-    """The tag ids a stored view's filters name; anything else in that slot names none."""
-    tags = filters.get("tags") if isinstance(filters, dict) else None
-    if not isinstance(tags, list):
-        return []
-    return [t for t in tags if type(t) is int]  # `type`, not isinstance: True is not a tag id
-
-
-def _cleanup_orphan_system_tags(db: Session) -> int:
-    """Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
-    one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
-    when a channel earns it again, so the view's filter would point at nothing; the feed drops an
-    unknown id, and the view showed the whole feed under its old name.
-
-    The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
-    stored before its route checked the shape (or written around it) must not raise for everyone's
-    job, nor push a `NOT IN` past the driver's bind-parameter limit."""
-    in_views = {
-        t
-        for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
-        for t in _view_tag_ids(filters)
-    }
-    orphans = [
-        tag
-        for tag in db.execute(
-            select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
-        ).scalars()
-        if tag.id not in in_views
-    ]
-    for tag in orphans:
-        db.delete(tag)
+    removed = reader_service.prune_orphan_system_tags(db)
     db.commit()
-    return len(orphans)
+    return {"channels_tagged": tagged, "orphan_tags_removed": removed}

backend/app/sync/explore.py

@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
 
 from app import logs, sysconfig
 from app.models import Channel, ExploredChannel, SearchFind, Subscription, Video
-from app.reader.models import PlaylistItem, VideoState
+from app.reader import queries as reader_queries
 from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
 from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
 
@@ -122,15 +122,12 @@ def purge_unkept_explored(db: Session) -> dict:
     if not orphan_ids:
         return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0}
 
-    state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
-    pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
     find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
     videos_deleted = db.execute(
         delete(Video).where(
             Video.channel_id.in_(orphan_ids),
             Video.via_explore.is_(True),
-            ~state_exists,
-            ~pl_exists,
+            ~reader_queries.kept_by_a_reader(Video.id),
             ~find_exists,
         )
     ).rowcount
@@ -159,15 +156,12 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
     0 quota."""
     grace = sysconfig.get_int(db, "search_grace_days") if grace_days is None else grace_days
     cutoff = datetime.now(timezone.utc) - timedelta(days=grace)
-    state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
-    pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
     sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
     deleted = db.execute(
         delete(Video).where(
             Video.via_search.is_(True),
             Video.created_at < cutoff,
-            ~state_exists,
-            ~pl_exists,
+            ~reader_queries.kept_by_a_reader(Video.id),
             ~sub_exists,
         )
     ).rowcount

backend/app/sync/maintenance.py

@@ -28,7 +28,7 @@ from sqlalchemy.orm import Session
 from app import logs, progress, quota, sysconfig
 from app.config import settings
 from app.models import Video
-from app.reader.models import Playlist, PlaylistItem
+from app.reader import queries as reader_queries
 from app.notifications import create_notification
 from app.sync.runner import get_service_user
 from app.sync.videos import apply_video_details, parse_dt
@@ -89,11 +89,7 @@ def _collect_user_impact(db: Session, videos: list[Video]) -> dict[int, list[dic
     # Anyone who had it in a playlist — which now also covers "saved": Watch later became a
     # built-in playlist, so "saved" is no longer a VideoState status (the old status=="saved"
     # scan matched nothing and under-counted impacted users).
-    for user_id, video_id in db.execute(
-        select(Playlist.user_id, PlaylistItem.video_id)
-        .join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id)
-        .where(PlaylistItem.video_id.in_(ids))
-    ):
+    for user_id, video_id in reader_queries.playlisted_by(db, ids):
         impact.setdefault(user_id, set()).add(video_id)
     return {
         user_id: [

backend/app/sync/playlists.py

@@ -12,7 +12,6 @@ import json
 from collections.abc import Callable, Iterator
 from contextlib import AbstractContextManager, contextmanager
 
-from datetime import datetime, timezone
 from typing import TypeVar
 
 from sqlalchemy import Engine, create_engine, delete, func, select
@@ -21,13 +20,14 @@ from sqlalchemy.pool import NullPool
 
 from app import logs, progress, quota
 from app.auth import has_read_scope
-from app.models import Channel, User, Video
+from app.models import User, Video
 from app.reader.models import Playlist, PlaylistItem
 from app.sync.playlist_merge import merge_plan
 from app.sync.runner import token_users
 from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
-from app.sync.videos import apply_video_details, parse_dt
+from app.sync.videos import ensure_videos
 from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
+from app.userscope import owned_by
 
 log = logs.get_logger(__name__)
 # Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
@@ -228,55 +228,11 @@ def recompute_dirty(db: Session, pl: Playlist) -> None:
     pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
 
 
-def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
-    """Make sure every given video id exists in the catalog, fetching + ingesting the
-    missing ones (and a stub channel for any unknown channel). Deleted/private videos that
-    the API won't return are simply left out."""
-    if not video_ids:
-        return
-    have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
-    missing = [v for v in dict.fromkeys(video_ids) if v not in have]
-    if not missing:
-        return
-    items = yt.get_videos(missing)
-    chan_ids = {
-        it.get("snippet", {}).get("channelId")
-        for it in items
-        if it.get("snippet", {}).get("channelId")
-    }
-    existing_ch = set(
-        db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
-    )
-    now = datetime.now(timezone.utc)
-    for it in items:
-        sn = it.get("snippet", {})
-        cid = sn.get("channelId")
-        if cid and cid not in existing_ch:
-            db.add(Channel(id=cid, title=sn.get("channelTitle")))
-            existing_ch.add(cid)
-    db.flush()
-    seen: set[str] = set()
-    for it in items:
-        vid = it.get("id")
-        sn = it.get("snippet", {})
-        cid = sn.get("channelId")
-        if not vid or not cid or vid in seen:
-            continue
-        seen.add(vid)
-        # published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
-        # fallback here too so the constructor never carries an explicit NULL.
-        v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
-        apply_video_details(v, it)
-        v.enriched_at = now
-        db.add(v)
-    db.commit()
-
-
 def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
     """Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
     replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
     items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
-    (`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
+    (`sync.videos.ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
     after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
     video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
     present = (
@@ -347,7 +303,9 @@ def pull_user_playlists(db: Session, user: User) -> dict:
         existing = {
             pl.yt_playlist_id: pl
             for pl in db.execute(
-                select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
+                select(Playlist).where(
+                    owned_by(Playlist.user_id, user.id), Playlist.source == "youtube"
+                )
             ).scalars()
         }
         # YouTube playlists that we already own as local (exported) playlists must not be
@@ -355,7 +313,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
         owned_yt_ids = set(
             db.execute(
                 select(Playlist.yt_playlist_id).where(
-                    Playlist.user_id == user.id,
+                    owned_by(Playlist.user_id, user.id),
                     Playlist.source == "local",
                     Playlist.yt_playlist_id.is_not(None),
                 )
@@ -402,7 +360,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
                     continue
                 # Ingest first — it commits — so the row lock below is held from the dirty check
                 # to the commit that replaces the items.
-                _ensure_videos(db, yt, ids)
+                ensure_videos(db, yt, ids)
                 # `dirty` read above is a snapshot from before the YouTube calls (sessions keep
                 # loaded rows past a commit): a remove or reorder made meanwhile must not be
                 # replaced by YouTube's order. Item writes lock the playlist row, so this waits for
@@ -613,7 +571,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
         prev_unavailable = parse_unavailable(pl)
         still_listed = [v for v in yt_ids if v in prev_unavailable]
         if still_listed:
-            _ensure_videos(db, yt, still_listed)
+            ensure_videos(db, yt, still_listed)
             healed = set(
                 db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
             )
@@ -625,7 +583,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
         # --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
         # merged order authoritatively. Do this first so local reflects the merge even if a later
         # YouTube write trips the breaker.
-        _ensure_videos(db, yt, mp.add_to_local)
+        ensure_videos(db, yt, mp.add_to_local)
         # `local` was read before the YouTube calls above: an add or remove committed meanwhile
         # (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
         # behind us — and plan again from what is there now. Nothing new needs ingesting: a video
@@ -750,7 +708,7 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
             )
         ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
         refuse_blank_listing(ids, snippet.get("item_count"))
-        _ensure_videos(db, yt, ids)
+        ensure_videos(db, yt, ids)
         _lock_or_gone(db, pl)
         pl.name = snippet.get("title") or pl.name
         ordered = _replace_items_from_youtube(db, pl, ids)

backend/app/sync/tagsuggest.py

@@ -25,6 +25,7 @@ from sqlalchemy.orm import Session
 
 from app.models import Subscription, User
 from app.reader.models import ChannelTag, Tag
+from app.userscope import owned_by
 
 # A user tag needs at least this many manually-tagged example channels before we trust its
 # profile — fewer is noise (one gaming-cooking channel shouldn't make "cooking" mean "gaming").
@@ -58,7 +59,9 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
     # Channel Manager shows ghost chips only on non-hidden rows, so suggesting for hidden channels would
     # inflate the header count past what's shown and let bulk-accept tag channels the user can't see).
     subs = db.execute(
-        select(Subscription.channel_id, Subscription.hidden).where(Subscription.user_id == user.id)
+        select(Subscription.channel_id, Subscription.hidden).where(
+            owned_by(Subscription.user_id, user.id)
+        )
     ).all()
     subscribed: set[str] = {cid for cid, _ in subs}
     hidden: set[str] = {cid for cid, h in subs if h}
@@ -75,7 +78,7 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
     user_tagged: set[str] = set()
     for tag_id, cid in db.execute(
         select(ChannelTag.tag_id, ChannelTag.channel_id).where(
-            ChannelTag.user_id == user.id, ChannelTag.source == "user"
+            owned_by(ChannelTag.user_id, user.id), ChannelTag.source == "user"
         )
     ).all():
         if cid not in subscribed:

backend/app/sync/videos.py

@@ -418,3 +418,48 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
                 progress.report(probed, len(candidates), "probe")
     db.commit()
     return {"probed": probed, "shorts": shorts}
+
+
+def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
+    """Make sure every given video id exists in the catalog, fetching + ingesting the
+    missing ones (and a stub channel for any unknown channel). Deleted/private videos that
+    the API won't return are simply left out. The catalog's write for a playlist sync that names
+    videos it has never seen; commits."""
+    if not video_ids:
+        return
+    have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
+    missing = [v for v in dict.fromkeys(video_ids) if v not in have]
+    if not missing:
+        return
+    items = yt.get_videos(missing)
+    chan_ids = {
+        it.get("snippet", {}).get("channelId")
+        for it in items
+        if it.get("snippet", {}).get("channelId")
+    }
+    existing_ch = set(
+        db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
+    )
+    now = datetime.now(timezone.utc)
+    for it in items:
+        sn = it.get("snippet", {})
+        cid = sn.get("channelId")
+        if cid and cid not in existing_ch:
+            db.add(Channel(id=cid, title=sn.get("channelTitle")))
+            existing_ch.add(cid)
+    db.flush()
+    seen: set[str] = set()
+    for it in items:
+        vid = it.get("id")
+        sn = it.get("snippet", {})
+        cid = sn.get("channelId")
+        if not vid or not cid or vid in seen:
+            continue
+        seen.add(vid)
+        # published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
+        # fallback here too so the constructor never carries an explicit NULL.
+        v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
+        apply_video_details(v, it)
+        v.enriched_at = now
+        db.add(v)
+    db.commit()