peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(catalog): the HTTP layer reads through queries and writes through the service

Browse Source 
Channels, search and sync do no database work in their routes any more: reads go through
catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search.
What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed
projection of results) it asks reader/queries and reader/service. The scheduler and the runs page
read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP
layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest
jobs commit only where they are named.
This commit is contained in:
peter
2026-10-19 02:32:20 +02:00
parent 238e6f8a61
commit 5229fd96be
21 changed files with 1223 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routes/channels.py
+69 -257
View File
@@ -4,28 +4,22 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w
from datetime import datetime, timezone
from functools import partial
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import Select, func, select, update
from sqlalchemy.orm import Session
from app.collation import collated, sorted_words
from app.collation import sorted_words
from app.mailtext import lang_for_user
from app import logs, quota, scheduler
from app.auth import admin_user, current_user, require_write_scope
from app.db import get_db
from app.models import User
from app.catalog.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
ExploredChannel,
Subscription,
Video,
)
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
from app.catalog import queries, service
from app.reader import queries as reader_queries
from app.reader import service as reader_service
from app.routes._common import not_found, youtube_refusal
from app.sync.explore import explore_ingest_page
from app.sync.runner import backfill_channel_now, run_recent_backfill
from app.sync.subscriptions import apply_channel_details, details_stale
@@ -33,35 +27,17 @@ from app.utils import iso
from app.youtube.client import YouTubeClient, YouTubeError
from app.refusal import refused
if TYPE_CHECKING:
from app.catalog.models import Channel, Subscription
router = APIRouter(prefix="/api/channels", tags=["channels"])
log = logs.get_logger(__name__)
def channel_aggregates(channel_ids: list[str]) -> Select:
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
alone; a test holds that (test_channel_list_index_only)."""
return (
select(
Video.channel_id,
func.count(Video.id),
func.max(Video.published_at),
func.coalesce(func.sum(Video.duration_seconds), 0),
)
.where(Video.channel_id.in_(channel_ids))
.group_by(Video.channel_id)
)
@router.get("")
def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = db.execute(
select(Subscription, Channel)
.join(Channel, Channel.id == Subscription.channel_id)
.where(Subscription.user_id == user.id)
.order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user)))
).all()
rows = queries.subscriptions(db, user, lang_for_user(user))
# Per-channel aggregates over the shared catalog (channel_aggregates), computed on read rather than
# denormalised: an Index Only Scan over a ~20 MB index, where it used to read every video's heap page
@@ -70,36 +46,21 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
# computed here — the manager table dropped its N/S/L column.)
channel_ids = [c.id for _, c in rows]
agg: dict[str, dict] = {}
if channel_ids:
for cid, n, last_up, total_dur in db.execute(channel_aggregates(channel_ids)).all():
agg[cid] = {
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
}
for cid, n, last_up, total_dur in queries.aggregates(db, channel_ids):
agg[cid] = {
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
}
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
# user) has requested full history and it isn't done yet. Their whole back-catalogue is
# coming for everyone (video data is shared), so the UI shouldn't offer "get full
# history" to a user who simply hasn't personally opted in.
deep_queued: set[str] = set()
if channel_ids:
for (cid,) in db.execute(
select(Subscription.channel_id)
.where(
Subscription.channel_id.in_(channel_ids),
Subscription.deep_requested.is_(True),
)
.distinct()
).all():
deep_queued.add(cid)
deep_queued = queries.deep_queued(db, channel_ids)
# The user's personal tag links, grouped by channel.
tags_by_channel: dict[str, list[int]] = {}
for cid, tag_id in db.execute(
select(ChannelTag.channel_id, ChannelTag.tag_id).where(ChannelTag.user_id == user.id)
).all():
tags_by_channel.setdefault(cid, []).append(tag_id)
tags_by_channel = reader_queries.user_tag_links(db, user)
return [
{
@@ -128,27 +89,7 @@ DISCOVERY_ENRICH_CAP = 200
def _discovery_rows(db: Session, user: User) -> list:
"""The user's playlist channels they don't subscribe to (and that aren't their own),
each with how many playlist videos come from it and across how many playlists."""
subscribed = select(Subscription.channel_id).where(Subscription.user_id == user.id)
q = (
select(
Channel,
func.count(func.distinct(PlaylistItem.video_id)),
func.count(func.distinct(Playlist.id)),
)
.select_from(PlaylistItem)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.join(Video, Video.id == PlaylistItem.video_id)
.join(Channel, Channel.id == Video.channel_id)
.where(Playlist.user_id == user.id, Channel.id.not_in(subscribed))
)
if user.yt_channel_id:
q = q.where(Channel.id != user.yt_channel_id)
return db.execute(
q.group_by(Channel.id).order_by(
func.count(func.distinct(PlaylistItem.video_id)).desc(),
collated(Channel.title, lang_for_user(user)),
)
).all()
return reader_queries.playlist_channels(db, user, user.yt_channel_id, lang_for_user(user))
@router.get("/discovery")
@@ -170,7 +111,7 @@ def discover_channels(
):
own = yt.get_my_channel_id()
if own:
user.yt_channel_id = own
service.remember_own_channel(user, own)
db.commit()
except YouTubeError as exc:
# No traceback anywhere in this file's YouTube handlers: for an upstream answer
@@ -226,7 +167,7 @@ def discover_channels(
]
def _channel_summary(ch: Channel) -> dict:
def _channel_summary(ch: "Channel") -> dict:
"""The channel fields common to the list and discovery projections."""
return {
"id": ch.id,
@@ -240,7 +181,7 @@ def _channel_summary(ch: Channel) -> dict:
def _channel_detail_dict(
channel: Channel,
channel: "Channel",
*,
subscribed: bool,
explored: bool,
@@ -309,7 +250,7 @@ def channel_detail(
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
cached after)."""
channel = get_or_404(db, Channel, channel_id, what="channel")
channel = _channel_or_404(db, channel_id)
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
stale = details_stale(channel, datetime.now(timezone.utc))
@@ -321,7 +262,7 @@ def channel_detail(
):
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
db.commit()
channel = db.get(Channel, channel_id)
channel = _channel_or_404(db, channel_id)
except YouTubeError as exc:
log.warning(
"channel detail enrich failed: %s",
@@ -331,41 +272,15 @@ def channel_detail(
),
)
db.rollback()
sub = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
sub = queries.subscription(db, user, channel_id)
subscribed = sub is not None
explored = (
db.execute(
select(ExploredChannel.id).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).first()
is not None
)
blocked = _is_blocked(db, user, channel_id)
explored = queries.exploration(db, user, channel_id) is not None
blocked = queries.is_blocked(db, user, channel_id)
# Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager
# list), so the channel page can show the per-type breakdown that moved off the table.
n_all, n_short, n_live = db.execute(
select(
func.count(Video.id),
func.count(Video.id).filter(Video.is_short.is_(True)),
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
).where(Video.channel_id == channel_id)
).one()
stored = int(n_all or 0)
count_short = int(n_short or 0)
count_live = int(n_live or 0)
stored, count_short, count_live = queries.video_counts(db, channel_id)
count_normal = max(0, stored - count_short - count_live)
tag_ids = list(
db.scalars(
select(ChannelTag.tag_id).where(
ChannelTag.user_id == user.id, ChannelTag.channel_id == channel_id
)
).all()
)
tag_ids = reader_queries.user_tag_links(db, user, channel_id).get(channel_id, [])
return _channel_detail_dict(
channel,
subscribed=subscribed,
@@ -395,28 +310,15 @@ def explore_channel(
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
Returns the next page token so the page can "load more". Guarded against the shared
backfill quota reserve."""
channel = get_or_404(db, Channel, channel_id, what="channel")
if _is_blocked(db, user, channel_id):
channel = _channel_or_404(db, channel_id)
if queries.is_blocked(db, user, channel_id):
raise refused(403, "channel_blocked")
if quota.backfill_reserve_reached():
raise refused(429, "quota_reserve")
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
existing = db.execute(
select(ExploredChannel).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).scalar_one_or_none()
if existing is None:
db.add(ExploredChannel(user_id=user.id, channel_id=channel_id))
else:
existing.created_at = datetime.now(timezone.utc) # refresh the grace clock
followed = db.execute(
select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1)
).first()
if followed is None and not channel.from_explore:
channel.from_explore = True
service.explore(db, user, channel)
db.commit()
try:
@@ -430,7 +332,7 @@ def explore_channel(
if details_stale(channel, datetime.now(timezone.utc)):
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
db.commit()
channel = db.get(Channel, channel_id)
channel = _channel_or_404(db, channel_id)
result = explore_ingest_page(db, yt, channel, page_token)
except YouTubeError as exc:
raise youtube_refusal(exc) from exc
@@ -442,30 +344,20 @@ def explore_channel(
}
def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription:
sub = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
def _channel_or_404(db: Session, channel_id: str) -> "Channel":
channel = queries.channel(db, channel_id)
if channel is None:
raise not_found("channel")
return channel
def _user_subscription(db: Session, user: User, channel_id: str) -> "Subscription":
sub = queries.subscription(db, user, channel_id)
if sub is None:
raise not_found("subscription")
return sub
def _is_blocked(db: Session, user: User, channel_id: str) -> bool:
"""Whether this user has blocked this channel."""
return (
db.execute(
select(BlockedChannel.id).where(
BlockedChannel.user_id == user.id,
BlockedChannel.channel_id == channel_id,
)
).first()
is not None
)
class ChannelUpdateIn(BaseModel):
# PATCH semantics: a field left unset (None) is untouched. Every field is optional so the
# UI can send just the one control it changed. int/bool coercion + rejection of garbage
@@ -483,19 +375,14 @@ def update_channel(
db: Session = Depends(get_db),
) -> dict:
sub = _user_subscription(db, user, channel_id)
channel = db.get(Channel, channel_id)
deep_turned_on = False
if body.priority is not None:
sub.priority = body.priority
if body.hidden is not None:
sub.hidden = body.hidden
if body.deep_requested is not None:
# Opt this channel into (or out of) full-history backfill. The deep scheduler
# picks the flag up on its next run; turning it off won't undo already-fetched
# videos, it just stops further deep paging if nobody else still wants it.
want = body.deep_requested
deep_turned_on = want and not sub.deep_requested
sub.deep_requested = want
channel = queries.channel(db, channel_id)
deep_turned_on = service.update_subscription(
db,
sub,
priority=body.priority,
hidden=body.hidden,
deep_requested=body.deep_requested,
)
db.commit()
# Opting a channel into full history should give an immediate feed: if we haven't even
@@ -539,19 +426,8 @@ def reset_backfill(
state (a "reset" trigger). Clears the channel's backfill markers, opts it back into deep
backfill, and re-runs its recent pull immediately; the deep scheduler re-pages the full
back-catalog on its next run. Idempotent — videos upsert by id, so nothing duplicates."""
channel = get_or_404(db, Channel, channel_id, what="channel")
channel.backfill_done = False
channel.backfill_cursor = None
channel.recent_synced_at = None
# Re-arm the demand-driven deep scheduler for this channel. run_deep_backfill gates on an
# EXISTS over subscriptions with deep_requested=True, so flip it on EVERY subscriber rather
# than requiring the admin to be personally subscribed (they reset from the manager, not
# their own feed). No-op row count if the channel has no subscribers.
db.execute(
update(Subscription)
.where(Subscription.channel_id == channel_id)
.values(deep_requested=True)
)
channel = _channel_or_404(db, channel_id)
service.reset_backfill(db, channel)
db.commit()
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
run_recent_backfill(db, [channel], max_channels=1)
@@ -569,13 +445,8 @@ def subscribe(
local subscription with YouTube's returned resource id and enriches the channel's metadata
if it's still a stub, so the new row renders properly in the manager. It does NOT pull the
channel's videos — the scheduler picks those up on its next run, like any subscribed channel."""
channel = get_or_404(db, Channel, channel_id, what="channel")
existing = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
if existing is not None:
channel = _channel_or_404(db, channel_id)
if queries.subscription(db, user, channel_id) is not None:
raise refused(409, "already_subscribed")
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
@@ -615,24 +486,7 @@ def subscribe(
reason=exc.reason,
),
)
db.add(
Subscription(
user_id=user.id,
channel_id=channel_id,
yt_subscription_id=yt_sub_id or None,
subscribed_at=datetime.now(timezone.utc),
)
)
# Subscribing is the "keep" signal: a previously-explored channel becomes a normal organic
# one. Drop its ephemeral flags so the cleanup job leaves it alone and its videos stop being
# per-user private (they join the shared Library like any subscribed channel's uploads).
if channel.from_explore:
channel.from_explore = False
db.execute(
update(Video)
.where(Video.channel_id == channel_id, Video.via_explore.is_(True))
.values(via_explore=False)
)
service.record_subscription(db, user, channel, yt_sub_id)
db.commit()
return {"ok": True, "subscribed": channel_id}
@@ -659,7 +513,7 @@ def unsubscribe(
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
# a transient "connection lost" gateway blip.
raise youtube_refusal(exc) from exc
db.delete(sub)
service.drop_subscription(db, sub)
db.commit()
return {"ok": True, "unsubscribed": channel_id}
@@ -667,16 +521,7 @@ def unsubscribe(
@router.get("/blocked/list")
def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
"""The user's blocked channels (videos from these are excluded from their search/feed/explore)."""
rows = (
db.execute(
select(Channel)
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
.where(BlockedChannel.user_id == user.id)
.order_by(collated(Channel.title, lang_for_user(user)))
)
.scalars()
.all()
)
rows = queries.blocked_channels(db, user, lang_for_user(user))
return [
{"id": c.id, "title": c.title, "handle": c.handle, "thumbnail_url": c.thumbnail_url}
for c in rows
@@ -692,16 +537,9 @@ def block_channel(
"""Block a channel for this user: its videos won't appear in their live search (and won't be
ingested for them) and are hidden from their feed / explore. Idempotent. The now-hidden
un-kept search/explore videos are reclaimed by the discovery-cleanup job in due course."""
get_or_404(db, Channel, channel_id, what="channel")
if not _is_blocked(db, user, channel_id):
db.add(BlockedChannel(user_id=user.id, channel_id=channel_id))
# Stop any active exploration of it so it can be cleaned up.
db.execute(
ExploredChannel.__table__.delete().where(
(ExploredChannel.user_id == user.id) & (ExploredChannel.channel_id == channel_id)
)
)
db.commit()
_channel_or_404(db, channel_id)
service.block(db, user, channel_id)
db.commit()
return {"ok": True, "blocked": channel_id}
@@ -711,11 +549,7 @@ def unblock_channel(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
db.execute(
BlockedChannel.__table__.delete().where(
(BlockedChannel.user_id == user.id) & (BlockedChannel.channel_id == channel_id)
)
)
service.unblock(db, user, channel_id)
db.commit()
return {"ok": True, "unblocked": channel_id}
@@ -732,26 +566,12 @@ def attach_tag(
db: Session = Depends(get_db),
) -> dict:
_user_subscription(db, user, channel_id)
# Only the user's own tags may be attached as personal links (owned_or_404 → 404 if the tag
# is missing OR belongs to someone else, without leaking which).
owned_or_404(db, Tag, body.tag_id, user, what="tag")
exists = db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel_id,
ChannelTag.tag_id == body.tag_id,
ChannelTag.user_id == user.id,
)
).scalar_one_or_none()
if exists is None:
db.add(
ChannelTag(
channel_id=channel_id,
tag_id=body.tag_id,
user_id=user.id,
source="user",
)
)
db.commit()
# Only the user's own tags may be attached as personal links (a 404 if the tag is missing OR
# belongs to someone else, without leaking which).
if reader_queries.own_tag(db, user, body.tag_id) is None:
raise not_found("tag")
reader_service.attach_user_tag(db, user, channel_id, body.tag_id)
db.commit()
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
@@ -762,14 +582,6 @@ def detach_tag(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
row = db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel_id,
ChannelTag.tag_id == tag_id,
ChannelTag.user_id == user.id,
)
).scalar_one_or_none()
if row is not None:
db.delete(row)
db.commit()
reader_service.detach_user_tag(db, user, channel_id, tag_id)
db.commit()
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}