peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(catalog): the HTTP layer reads through queries and writes through the service

Browse Source 
Channels, search and sync do no database work in their routes any more: reads go through
catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search.
What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed
projection of results) it asks reader/queries and reader/service. The scheduler and the runs page
read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP
layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest
jobs commit only where they are named.
This commit is contained in:
peter
2026-10-19 02:32:20 +02:00
parent 238e6f8a61
commit 5229fd96be
21 changed files with 1223 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routes/channels.py
+69 -257
View File
@@ -4,28 +4,22 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w
from datetime import datetime, timezone
from functools import partial
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import Select, func, select, update
from sqlalchemy.orm import Session
from app.collation import collated, sorted_words
from app.collation import sorted_words
from app.mailtext import lang_for_user
from app import logs, quota, scheduler
from app.auth import admin_user, current_user, require_write_scope
from app.db import get_db
from app.models import User
from app.catalog.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
ExploredChannel,
Subscription,
Video,
)
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
from app.catalog import queries, service
from app.reader import queries as reader_queries
from app.reader import service as reader_service
from app.routes._common import not_found, youtube_refusal
from app.sync.explore import explore_ingest_page
from app.sync.runner import backfill_channel_now, run_recent_backfill
from app.sync.subscriptions import apply_channel_details, details_stale
@@ -33,35 +27,17 @@ from app.utils import iso
from app.youtube.client import YouTubeClient, YouTubeError
from app.refusal import refused
if TYPE_CHECKING:
from app.catalog.models import Channel, Subscription
router = APIRouter(prefix="/api/channels", tags=["channels"])
log = logs.get_logger(__name__)
def channel_aggregates(channel_ids: list[str]) -> Select:
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
alone; a test holds that (test_channel_list_index_only)."""
return (
select(
Video.channel_id,
func.count(Video.id),
func.max(Video.published_at),
func.coalesce(func.sum(Video.duration_seconds), 0),
)
.where(Video.channel_id.in_(channel_ids))
.group_by(Video.channel_id)
)
@router.get("")
def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = db.execute(
select(Subscription, Channel)
.join(Channel, Channel.id == Subscription.channel_id)
.where(Subscription.user_id == user.id)
.order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user)))
).all()
rows = queries.subscriptions(db, user, lang_for_user(user))
# Per-channel aggregates over the shared catalog (channel_aggregates), computed on read rather than
# denormalised: an Index Only Scan over a ~20 MB index, where it used to read every video's heap page
@@ -70,36 +46,21 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
# computed here — the manager table dropped its N/S/L column.)
channel_ids = [c.id for _, c in rows]
agg: dict[str, dict] = {}
if channel_ids:
for cid, n, last_up, total_dur in db.execute(channel_aggregates(channel_ids)).all():
agg[cid] = {
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
}
for cid, n, last_up, total_dur in queries.aggregates(db, channel_ids):
agg[cid] = {
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
}
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
# user) has requested full history and it isn't done yet. Their whole back-catalogue is
# coming for everyone (video data is shared), so the UI shouldn't offer "get full
# history" to a user who simply hasn't personally opted in.
deep_queued: set[str] = set()
if channel_ids:
for (cid,) in db.execute(
select(Subscription.channel_id)
.where(
Subscription.channel_id.in_(channel_ids),
Subscription.deep_requested.is_(True),
)
.distinct()
).all():
deep_queued.add(cid)
deep_queued = queries.deep_queued(db, channel_ids)
# The user's personal tag links, grouped by channel.
tags_by_channel: dict[str, list[int]] = {}
for cid, tag_id in db.execute(
select(ChannelTag.channel_id, ChannelTag.tag_id).where(ChannelTag.user_id == user.id)
).all():
tags_by_channel.setdefault(cid, []).append(tag_id)
tags_by_channel = reader_queries.user_tag_links(db, user)
return [
{
@@ -128,27 +89,7 @@ DISCOVERY_ENRICH_CAP = 200
def _discovery_rows(db: Session, user: User) -> list:
"""The user's playlist channels they don't subscribe to (and that aren't their own),
each with how many playlist videos come from it and across how many playlists."""
subscribed = select(Subscription.channel_id).where(Subscription.user_id == user.id)
q = (
select(
Channel,
func.count(func.distinct(PlaylistItem.video_id)),
func.count(func.distinct(Playlist.id)),
)
.select_from(PlaylistItem)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.join(Video, Video.id == PlaylistItem.video_id)
.join(Channel, Channel.id == Video.channel_id)
.where(Playlist.user_id == user.id, Channel.id.not_in(subscribed))
)
if user.yt_channel_id:
q = q.where(Channel.id != user.yt_channel_id)
return db.execute(
q.group_by(Channel.id).order_by(
func.count(func.distinct(PlaylistItem.video_id)).desc(),
collated(Channel.title, lang_for_user(user)),
)
).all()
return reader_queries.playlist_channels(db, user, user.yt_channel_id, lang_for_user(user))
@router.get("/discovery")
@@ -170,7 +111,7 @@ def discover_channels(
):
own = yt.get_my_channel_id()
if own:
user.yt_channel_id = own
service.remember_own_channel(user, own)
db.commit()
except YouTubeError as exc:
# No traceback anywhere in this file's YouTube handlers: for an upstream answer
@@ -226,7 +167,7 @@ def discover_channels(
]
def _channel_summary(ch: Channel) -> dict:
def _channel_summary(ch: "Channel") -> dict:
"""The channel fields common to the list and discovery projections."""
return {
"id": ch.id,
@@ -240,7 +181,7 @@ def _channel_summary(ch: Channel) -> dict:
def _channel_detail_dict(
channel: Channel,
channel: "Channel",
*,
subscribed: bool,
explored: bool,
@@ -309,7 +250,7 @@ def channel_detail(
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
cached after)."""
channel = get_or_404(db, Channel, channel_id, what="channel")
channel = _channel_or_404(db, channel_id)
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
stale = details_stale(channel, datetime.now(timezone.utc))
@@ -321,7 +262,7 @@ def channel_detail(
):
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
db.commit()
channel = db.get(Channel, channel_id)
channel = _channel_or_404(db, channel_id)
except YouTubeError as exc:
log.warning(
"channel detail enrich failed: %s",
@@ -331,41 +272,15 @@ def channel_detail(
),
)
db.rollback()
sub = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
sub = queries.subscription(db, user, channel_id)
subscribed = sub is not None
explored = (
db.execute(
select(ExploredChannel.id).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).first()
is not None
)
blocked = _is_blocked(db, user, channel_id)
explored = queries.exploration(db, user, channel_id) is not None
blocked = queries.is_blocked(db, user, channel_id)
# Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager
# list), so the channel page can show the per-type breakdown that moved off the table.
n_all, n_short, n_live = db.execute(
select(
func.count(Video.id),
func.count(Video.id).filter(Video.is_short.is_(True)),
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
).where(Video.channel_id == channel_id)
).one()
stored = int(n_all or 0)
count_short = int(n_short or 0)
count_live = int(n_live or 0)
stored, count_short, count_live = queries.video_counts(db, channel_id)
count_normal = max(0, stored - count_short - count_live)
tag_ids = list(
db.scalars(
select(ChannelTag.tag_id).where(
ChannelTag.user_id == user.id, ChannelTag.channel_id == channel_id
)
).all()
)
tag_ids = reader_queries.user_tag_links(db, user, channel_id).get(channel_id, [])
return _channel_detail_dict(
channel,
subscribed=subscribed,
@@ -395,28 +310,15 @@ def explore_channel(
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
Returns the next page token so the page can "load more". Guarded against the shared
backfill quota reserve."""
channel = get_or_404(db, Channel, channel_id, what="channel")
if _is_blocked(db, user, channel_id):
channel = _channel_or_404(db, channel_id)
if queries.is_blocked(db, user, channel_id):
raise refused(403, "channel_blocked")
if quota.backfill_reserve_reached():
raise refused(429, "quota_reserve")
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
existing = db.execute(
select(ExploredChannel).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).scalar_one_or_none()
if existing is None:
db.add(ExploredChannel(user_id=user.id, channel_id=channel_id))
else:
existing.created_at = datetime.now(timezone.utc) # refresh the grace clock
followed = db.execute(
select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1)
).first()
if followed is None and not channel.from_explore:
channel.from_explore = True
service.explore(db, user, channel)
db.commit()
try:
@@ -430,7 +332,7 @@ def explore_channel(
if details_stale(channel, datetime.now(timezone.utc)):
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
db.commit()
channel = db.get(Channel, channel_id)
channel = _channel_or_404(db, channel_id)
result = explore_ingest_page(db, yt, channel, page_token)
except YouTubeError as exc:
raise youtube_refusal(exc) from exc
@@ -442,30 +344,20 @@ def explore_channel(
}
def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription:
sub = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
def _channel_or_404(db: Session, channel_id: str) -> "Channel":
channel = queries.channel(db, channel_id)
if channel is None:
raise not_found("channel")
return channel
def _user_subscription(db: Session, user: User, channel_id: str) -> "Subscription":
sub = queries.subscription(db, user, channel_id)
if sub is None:
raise not_found("subscription")
return sub
def _is_blocked(db: Session, user: User, channel_id: str) -> bool:
"""Whether this user has blocked this channel."""
return (
db.execute(
select(BlockedChannel.id).where(
BlockedChannel.user_id == user.id,
BlockedChannel.channel_id == channel_id,
)
).first()
is not None
)
class ChannelUpdateIn(BaseModel):
# PATCH semantics: a field left unset (None) is untouched. Every field is optional so the
# UI can send just the one control it changed. int/bool coercion + rejection of garbage
@@ -483,19 +375,14 @@ def update_channel(
db: Session = Depends(get_db),
) -> dict:
sub = _user_subscription(db, user, channel_id)
channel = db.get(Channel, channel_id)
deep_turned_on = False
if body.priority is not None:
sub.priority = body.priority
if body.hidden is not None:
sub.hidden = body.hidden
if body.deep_requested is not None:
# Opt this channel into (or out of) full-history backfill. The deep scheduler
# picks the flag up on its next run; turning it off won't undo already-fetched
# videos, it just stops further deep paging if nobody else still wants it.
want = body.deep_requested
deep_turned_on = want and not sub.deep_requested
sub.deep_requested = want
channel = queries.channel(db, channel_id)
deep_turned_on = service.update_subscription(
db,
sub,
priority=body.priority,
hidden=body.hidden,
deep_requested=body.deep_requested,
)
db.commit()
# Opting a channel into full history should give an immediate feed: if we haven't even
@@ -539,19 +426,8 @@ def reset_backfill(
state (a "reset" trigger). Clears the channel's backfill markers, opts it back into deep
backfill, and re-runs its recent pull immediately; the deep scheduler re-pages the full
back-catalog on its next run. Idempotent — videos upsert by id, so nothing duplicates."""
channel = get_or_404(db, Channel, channel_id, what="channel")
channel.backfill_done = False
channel.backfill_cursor = None
channel.recent_synced_at = None
# Re-arm the demand-driven deep scheduler for this channel. run_deep_backfill gates on an
# EXISTS over subscriptions with deep_requested=True, so flip it on EVERY subscriber rather
# than requiring the admin to be personally subscribed (they reset from the manager, not
# their own feed). No-op row count if the channel has no subscribers.
db.execute(
update(Subscription)
.where(Subscription.channel_id == channel_id)
.values(deep_requested=True)
)
channel = _channel_or_404(db, channel_id)
service.reset_backfill(db, channel)
db.commit()
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
run_recent_backfill(db, [channel], max_channels=1)
@@ -569,13 +445,8 @@ def subscribe(
local subscription with YouTube's returned resource id and enriches the channel's metadata
if it's still a stub, so the new row renders properly in the manager. It does NOT pull the
channel's videos — the scheduler picks those up on its next run, like any subscribed channel."""
channel = get_or_404(db, Channel, channel_id, what="channel")
existing = db.execute(
select(Subscription).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).scalar_one_or_none()
if existing is not None:
channel = _channel_or_404(db, channel_id)
if queries.subscription(db, user, channel_id) is not None:
raise refused(409, "already_subscribed")
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
@@ -615,24 +486,7 @@ def subscribe(
reason=exc.reason,
),
)
db.add(
Subscription(
user_id=user.id,
channel_id=channel_id,
yt_subscription_id=yt_sub_id or None,
subscribed_at=datetime.now(timezone.utc),
)
)
# Subscribing is the "keep" signal: a previously-explored channel becomes a normal organic
# one. Drop its ephemeral flags so the cleanup job leaves it alone and its videos stop being
# per-user private (they join the shared Library like any subscribed channel's uploads).
if channel.from_explore:
channel.from_explore = False
db.execute(
update(Video)
.where(Video.channel_id == channel_id, Video.via_explore.is_(True))
.values(via_explore=False)
)
service.record_subscription(db, user, channel, yt_sub_id)
db.commit()
return {"ok": True, "subscribed": channel_id}
@@ -659,7 +513,7 @@ def unsubscribe(
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
# a transient "connection lost" gateway blip.
raise youtube_refusal(exc) from exc
db.delete(sub)
service.drop_subscription(db, sub)
db.commit()
return {"ok": True, "unsubscribed": channel_id}
@@ -667,16 +521,7 @@ def unsubscribe(
@router.get("/blocked/list")
def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
"""The user's blocked channels (videos from these are excluded from their search/feed/explore)."""
rows = (
db.execute(
select(Channel)
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
.where(BlockedChannel.user_id == user.id)
.order_by(collated(Channel.title, lang_for_user(user)))
)
.scalars()
.all()
)
rows = queries.blocked_channels(db, user, lang_for_user(user))
return [
{"id": c.id, "title": c.title, "handle": c.handle, "thumbnail_url": c.thumbnail_url}
for c in rows
@@ -692,16 +537,9 @@ def block_channel(
"""Block a channel for this user: its videos won't appear in their live search (and won't be
ingested for them) and are hidden from their feed / explore. Idempotent. The now-hidden
un-kept search/explore videos are reclaimed by the discovery-cleanup job in due course."""
get_or_404(db, Channel, channel_id, what="channel")
if not _is_blocked(db, user, channel_id):
db.add(BlockedChannel(user_id=user.id, channel_id=channel_id))
# Stop any active exploration of it so it can be cleaned up.
db.execute(
ExploredChannel.__table__.delete().where(
(ExploredChannel.user_id == user.id) & (ExploredChannel.channel_id == channel_id)
)
)
db.commit()
_channel_or_404(db, channel_id)
service.block(db, user, channel_id)
db.commit()
return {"ok": True, "blocked": channel_id}
@@ -711,11 +549,7 @@ def unblock_channel(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
db.execute(
BlockedChannel.__table__.delete().where(
(BlockedChannel.user_id == user.id) & (BlockedChannel.channel_id == channel_id)
)
)
service.unblock(db, user, channel_id)
db.commit()
return {"ok": True, "unblocked": channel_id}
@@ -732,26 +566,12 @@ def attach_tag(
db: Session = Depends(get_db),
) -> dict:
_user_subscription(db, user, channel_id)
# Only the user's own tags may be attached as personal links (owned_or_404 → 404 if the tag
# is missing OR belongs to someone else, without leaking which).
owned_or_404(db, Tag, body.tag_id, user, what="tag")
exists = db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel_id,
ChannelTag.tag_id == body.tag_id,
ChannelTag.user_id == user.id,
)
).scalar_one_or_none()
if exists is None:
db.add(
ChannelTag(
channel_id=channel_id,
tag_id=body.tag_id,
user_id=user.id,
source="user",
)
)
db.commit()
# Only the user's own tags may be attached as personal links (a 404 if the tag is missing OR
# belongs to someone else, without leaking which).
if reader_queries.own_tag(db, user, body.tag_id) is None:
raise not_found("tag")
reader_service.attach_user_tag(db, user, channel_id, body.tag_id)
db.commit()
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
@@ -762,14 +582,6 @@ def detach_tag(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
row = db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel_id,
ChannelTag.tag_id == tag_id,
ChannelTag.user_id == user.id,
)
).scalar_one_or_none()
if row is not None:
db.delete(row)
db.commit()
reader_service.detach_user_tag(db, user, channel_id, tag_id)
db.commit()
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}
backend/app/routes/runs.py
+2 -5
View File
@@ -19,7 +19,7 @@ from app import scheduler
from app.auth import admin_user
from app.db import get_db
from app.models import JobRun, LogEvent, User
from app.catalog.models import Channel
from app.catalog import queries as catalog_queries
from app.routes import _account as account
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
from app.runstatus import RunStatus
@@ -117,10 +117,7 @@ def list_runs(
lambda r: r.started_at,
)
channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target}
titles: dict[str, str] = {}
if channels:
rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(channels)))
titles = {cid: title for cid, title in rows.tuples() if title is not None}
titles = catalog_queries.channel_titles(db, channels)
lines: dict[int, int] = {}
if runs:
counted = db.execute(
backend/app/routes/scheduler.py
+9 -55
View File
@@ -4,7 +4,6 @@ and the shared quota picture."""
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import and_, func, or_, select
from sqlalchemy.orm import Session
from app import audit, quota, state, sysconfig
@@ -13,7 +12,7 @@ from app.auth import admin_user
from app.config import settings
from app.db import get_db
from app.models import User
from app.catalog.models import Channel, Subscription, Video
from app.catalog import queries as catalog_queries
from app.scheduler import (
JOB_INTERVALS,
MAX_INTERVAL,
@@ -24,7 +23,6 @@ from app.scheduler import (
trigger_all,
trigger_job,
)
from app.sync.explore import followed_channel
from app.sync.runner import estimate_deep_backfill
from app.routes._common import not_found
from app.refusal import refused
@@ -123,58 +121,14 @@ def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db))
snap = scheduler_snapshot(db)
# Work still queued (shared catalog, so these are instance-wide, not per-user).
deep_requested = (
select(Subscription.id)
.where(
Subscription.channel_id == Channel.id,
Subscription.deep_requested.is_(True),
)
.exists()
)
deep_channels = (
db.execute(select(Channel).where(Channel.backfill_done.is_(False), deep_requested))
.scalars()
.all()
)
deep_eta = estimate_deep_backfill(db, deep_channels)
queue = {
"channels_recent_pending": db.scalar(
# The same selection run_recent_backfill works through, or the card counts channels
# the job will never touch and never reaches zero.
select(func.count())
.select_from(Channel)
.where(Channel.recent_synced_at.is_(None), followed_channel())
)
or 0,
# The whole opted-in BACKLOG, deliberately not `run_deep_backfill`'s next-pass selection
# (which also wants `recent_synced_at`): the same figure /api/sync/my-status shows the user,
# and a channel missing its recent pull is one the same job tick fetches first, not one the
# job will never touch. Narrowing it here would under-report work that is genuinely queued.
"channels_deep_pending": deep_eta["channels_pending"],
"deep_eta_seconds": deep_eta["eta_seconds"],
"videos_pending_enrich": db.scalar(
select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
)
or 0,
"videos_pending_shorts": db.scalar(
select(func.count())
.select_from(Video)
.where(Video.shorts_probed.is_(False), Video.enriched_at.is_not(None))
)
or 0,
"videos_live_refresh": db.scalar(
select(func.count())
.select_from(Video)
.where(
or_(
Video.live_status.in_(("live", "upcoming")),
and_(Video.live_status == "was_live", Video.duration_seconds.is_(None)),
)
)
)
or 0,
}
deep_eta = estimate_deep_backfill(db, catalog_queries.deep_backlog(db))
queue = catalog_queries.work_queue(db)
# The whole opted-in BACKLOG, deliberately not `run_deep_backfill`'s next-pass selection
# (which also wants `recent_synced_at`): the same figure /api/sync/my-status shows the user,
# and a channel missing its recent pull is one the same job tick fetches first, not one the
# job will never touch. Narrowing it here would under-report work that is genuinely queued.
queue["channels_deep_pending"] = deep_eta["channels_pending"]
queue["deep_eta_seconds"] = deep_eta["eta_seconds"]
used = quota.units_used_today(db)
quota_info = {
backend/app/routes/search.py
+13 -242
View File
@@ -11,37 +11,24 @@ regardless of source.
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
dropped from the search response up front)."""
dropped from the search response up front).
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
catalog is `catalog/search`."""
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased
from sqlalchemy.orm import Session
from app import logs, quota, sysconfig
from app.auth import current_user
from app.catalog import queries
from app.catalog import search as catalog_search
from app.db import get_db
from app.models import User
from app.catalog.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
SearchFind,
Subscription,
Video,
)
from app.reader.models import Playlist, PlaylistItem, VideoState
from app.feedquery import feed_columns, serialize
from app.utils import like_escape
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import insert_stubs, apply_video_details, parse_dt
from app.reader import queries as reader_queries
from app.youtube import search_scrape
from app.youtube.client import YouTubeClient, YouTubeError
from app.youtube.shorts import make_client, probe_is_short
from app.refusal import refused
from app.routes._common import youtube_refusal
@@ -52,198 +39,6 @@ router = APIRouter(prefix="/api/search", tags=["search"])
SEARCH_COST = 100
def _classify_shorts(db: Session, videos: list[Video]) -> None:
"""Confirm/deny Shorts for the freshly-ingested videos so none slip into the feed.
Mirrors the scheduler's run_shorts_classification, but scoped to just these rows and run
synchronously (the search is an explicit, awaited action). Non-candidates (too long, no
duration, or live) are finalised without a probe; the rest hit youtube.com/shorts (no
quota). Undetermined probes are left for the scheduler to retry."""
probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
to_probe: list[Video] = []
for v in videos:
if v.shorts_probed:
continue
if v.enriched_at is None:
# Enrichment failed (proxy blip) or YouTube omitted the row — duration is unknown,
# so DON'T finalise shorts_probed off a null duration; leave it for the scheduler
# (which only classifies enriched rows) to retry. Otherwise a real Short would leak.
continue
if v.live_status != "none" or v.duration_seconds is None or v.duration_seconds > probe_max:
v.is_short = False
v.shorts_probed = True
else:
to_probe.append(v)
if to_probe:
with make_client() as client:
def work(v: Video):
return v, probe_is_short(client, v.id)
with ThreadPoolExecutor(max_workers=16) as pool:
for v, result in pool.map(work, to_probe):
if result is None:
continue
v.is_short = result
v.shorts_probed = True
db.commit()
def _serialize_ids(db: Session, user: User, ids: list[str]) -> list[dict]:
"""Serialize the given video ids with the shared feed projection (joined channel + this
user's watch state + saved), preserving the input (search relevance) order."""
if not ids:
return []
state = aliased(VideoState)
rows = db.execute(
select(*feed_columns(user, state))
.join(Channel, Channel.id == Video.channel_id)
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
.where(Video.id.in_(ids))
).all()
by_id = {r.id: serialize(r) for r in rows}
return [by_id[i] for i in ids if i in by_id]
def _ingest_candidates(
db: Session, yt: YouTubeClient, user: User, candidates: list[dict]
) -> list[str]:
"""Materialise one page of search candidates into the catalog (channel stubs + enrich,
video stubs flagged via_search, enrich, Shorts classification) and return the KEEPABLE
video ids — non-Short, non-live — in the input relevance order."""
ids = [it["id"] for it in candidates]
if not ids:
return []
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
# then enrich the new ones (channels.list, 1 unit). Stubs survive an enrich failure.
channel_ids = list({it["channel_id"] for it in candidates})
existing = set(db.scalars(select(Channel.id).where(Channel.id.in_(channel_ids))).all())
new_channel_ids = [c for c in channel_ids if c not in existing]
if new_channel_ids:
titles = {it["channel_id"]: it["channel_title"] for it in candidates}
for cid in new_channel_ids:
db.add(Channel(id=cid, title=titles.get(cid), from_search=True))
db.commit()
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
details = yt.get_channels(new_channel_ids)
apply_channel_details(db, details, new_channel_ids)
db.commit()
except YouTubeError:
# The stubs remain and the details can be backfilled later — but until they are, those
# channels sit in the results with no thumbnail and no subscriber count, looking like
# channels that have neither.
log.warning(
"the channels found by a search could not be enriched",
extra=logs.event("search.channel_enrich_failed", channels=len(new_channel_ids)),
)
db.rollback() # details can be backfilled later; the stubs remain
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
# NOTHING), so an organically-synced video stays organic.
# published_at is NOT NULL (migration 0068); the scrape source exposes no publish date
# (only a relative "3 years ago"), so fall back to now() to satisfy the constraint — the
# enrich in step 3 overwrites it with the video's real snippet date.
now = datetime.now(timezone.utc)
insert_stubs(
db,
[
{
"id": it["id"],
"channel_id": it["channel_id"],
"title": it["title"],
"published_at": parse_dt(it["published_at"]) or now,
"thumbnail_url": it["thumbnail_url"],
"via_search": True,
}
for it in candidates
],
)
# 3) Enrich (videos.list, 1 unit/50) so cards have duration/stats and we can judge live.
videos = db.scalars(select(Video).where(Video.id.in_(ids))).all()
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH):
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
except YouTubeError:
# Every candidate then stays unenriched, and an unenriched video cannot be judged live or
# Short — so it is dropped. A working search returns nothing and reads as "no results".
log.warning(
"the videos found by a search could not be enriched",
extra=logs.event("search.video_enrich_failed", candidates=len(ids)),
)
detail_by_id = {}
enriched_at = datetime.now(
timezone.utc
) # when THIS enrich ran (distinct from the stub placeholder)
for v in videos:
item = detail_by_id.get(v.id)
if item is None:
continue
apply_video_details(v, item)
v.enriched_at = enriched_at
db.commit()
# 4) Confirm/deny Shorts (no quota) so none enter via search.
_classify_shorts(db, videos)
keep = {
v.id
for v in videos
if v.enriched_at is not None and not v.is_short and v.live_status not in LIVE_OR_UPCOMING
}
return [vid for vid in ids if vid in keep]
def _new_first(db: Session, user: User, ids: list[str]) -> list[str]:
"""Reorder results so genuinely-new discoveries come first and things the user already has
(a subscribed channel, or a video they've watched / in-progress / saved / playlisted) sink
to the back — keeping the relevance order within each group."""
if not ids:
return ids
sub_ch = select(Subscription.channel_id).where(Subscription.user_id == user.id)
has_state = (
select(VideoState.id)
.where(VideoState.video_id == Video.id, VideoState.user_id == user.id)
.exists()
)
in_playlist = (
select(PlaylistItem.id)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.where(PlaylistItem.video_id == Video.id, Playlist.user_id == user.id)
.exists()
)
have = set(
db.scalars(
select(Video.id).where(
Video.id.in_(ids),
or_(Video.channel_id.in_(sub_ch), has_state, in_playlist),
)
).all()
)
return [i for i in ids if i not in have] + [i for i in ids if i in have]
def _fold_term(db: Session, video_ids: list[str], term: str) -> None:
"""Append `term` to each video's space-joined `search_terms`, unless it is already one of them.
"Already one of them" is a whole-term match, as a literal: a substring test skipped `cat` on a
video already found for `category`, and an unescaped one read the `_` in `my_channel` as a
wildcard — either way the term was never folded in, and local search lost that relevance."""
padded = func.concat(" ", func.coalesce(Video.search_terms, ""), " ")
db.execute(
update(Video)
.where(Video.id.in_(video_ids))
.where(padded.notilike(f"% {like_escape(term)} %"))
.values(
search_terms=func.trim(
func.coalesce(Video.search_terms, "").op("||")(" ").op("||")(term)
)
)
)
@router.get("/youtube")
def search_youtube(
q: str | None = None,
@@ -285,9 +80,7 @@ def search_youtube(
raise refused(429, "quota_spent")
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
blocked = set(
db.scalars(select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)).all()
)
blocked = queries.blocked_ids(db, user)
collected: list[str] = []
next_cursor = cursor
@@ -328,7 +121,7 @@ def search_youtube(
# write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
try:
collected.extend(_ingest_candidates(db, yt, user, candidates))
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
except Exception as exc: # any ingest failure, not just YouTubeError
db.rollback()
# The term is NOT in this line. A search string is exactly what the privacy
@@ -376,17 +169,12 @@ def search_youtube(
# Record these as THIS user's search finds (Mine feed / Source filter), and fold the
# query into each result's search_terms so local full-text search inherits YouTube's
# relevance. Idempotent; the generated search_vector updates automatically.
db.execute(
pg_insert(SearchFind)
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
)
_fold_term(db, ordered, term)
catalog_search.record_finds(db, user, ordered, term)
db.commit()
ordered = _new_first(db, user, ordered)
ordered = catalog_search.new_first(db, user, ordered)
return {
"items": _serialize_ids(db, user, ordered),
"items": reader_queries.feed_rows_by_ids(db, user, ordered),
"next_cursor": next_cursor,
"has_more": bool(next_cursor),
"source": source,
@@ -410,23 +198,6 @@ def clear_search(
ids = body.video_ids
if not ids:
return {"ok": True, "removed": 0}
db.execute(
delete(SearchFind).where(SearchFind.user_id == user.id, SearchFind.video_id.in_(ids))
)
db.commit()
has_state = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
in_playlist = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
other_find = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
removed = db.execute(
delete(Video).where(
Video.id.in_(ids),
Video.via_search.is_(True),
~has_state,
~in_playlist,
~subscribed,
~other_find,
)
).rowcount
removed = catalog_search.clear(db, user, ids)
db.commit()
return {"ok": True, "removed": removed}
backend/app/routes/sync.py
+10 -54
View File
@@ -1,5 +1,4 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, case, func, select, update
from sqlalchemy.orm import Session
from app import audit, quota, state
@@ -7,7 +6,7 @@ from app.audit import AuditAction
from app.auth import admin_user, current_user, has_read_scope
from app.db import get_db
from app.models import User
from app.catalog.models import Channel, Subscription, Video
from app.catalog import queries, service
from app.routes._common import youtube_refusal
from app.ratelimit import RateLimiter
from app.sync.runner import estimate_deep_backfill
@@ -48,19 +47,8 @@ def sync_subscriptions(user: User = Depends(current_user), db: Session = Depends
@router.get("/status")
def sync_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
sub_count = db.scalar(
select(func.count()).select_from(Subscription).where(Subscription.user_id == user.id)
)
return {
"subscriptions": sub_count,
"channels_total": db.scalar(select(func.count()).select_from(Channel)),
"channels_backfilling": db.scalar(
select(func.count()).select_from(Channel).where(Channel.backfill_done.is_(False))
),
"videos_total": db.scalar(select(func.count()).select_from(Video)),
"pending_enrich": db.scalar(
select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
),
**queries.catalog_totals(db, user),
"quota_used_today": quota.units_used_today(db),
"quota_remaining_today": quota.remaining_today(db),
"paused": state.is_sync_paused(db),
@@ -71,51 +59,21 @@ def sync_status(user: User = Depends(current_user), db: Session = Depends(get_db
@router.get("/my-status")
def my_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""Per-user sync progress: how far the channels *this* user subscribes to have synced."""
sub_join = and_(Subscription.channel_id == Channel.id, Subscription.user_id == user.id)
total, details_synced, recent_synced, all_deep_done = db.execute(
select(
func.count(Channel.id),
func.count(Channel.details_synced_at),
func.count(Channel.recent_synced_at),
func.sum(case((Channel.backfill_done.is_(True), 1), else_=0)),
)
.select_from(Channel)
.join(Subscription, sub_join)
).one()
total = total or 0
all_deep_done = int(all_deep_done or 0)
total, details_synced, recent_synced, all_deep_done = queries.my_progress(db, user)
# Per-user full-history (deep) backfill: channels this user has opted in, and an ETA
# for the still-pending ones (quota-bound estimate).
deep_requested_channels = (
db.execute(
select(Channel).join(
Subscription,
and_(sub_join, Subscription.deep_requested.is_(True)),
)
)
.scalars()
.all()
)
deep_requested_channels = queries.my_deep_requested(db, user)
deep_eta = estimate_deep_backfill(db, deep_requested_channels)
# "done / requested" is shown as one ratio, so both halves count the SAME channels: the ones this
# user asked full history for. Counting every subscribed channel whose back-catalog happens to be
# complete (another user asked, or it was short) gave "342/329".
deep_done = sum(1 for c in deep_requested_channels if c.backfill_done)
my_videos = db.scalar(
select(func.count(Video.id)).join(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
Subscription.hidden.is_(False),
),
)
)
my_videos = queries.my_video_count(db, user)
# Whole-catalog total (every video any user's subscriptions have pulled in), so the
# header can show "yours" against the shared library size.
total_videos = db.scalar(select(func.count(Video.id)))
total_videos = queries.video_total(db)
return {
"channels_total": total,
"channels_details_synced": details_synced,
@@ -126,8 +84,8 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
"channels_deep_requested": len(deep_requested_channels),
"deep_pending_count": deep_eta["channels_pending"],
"deep_eta_seconds": deep_eta["eta_seconds"],
"my_videos": my_videos or 0,
"total_videos": total_videos or 0,
"my_videos": my_videos,
"total_videos": total_videos,
"quota_used_today": quota.units_used_today(db),
"quota_remaining_today": quota.remaining_today(db),
"paused": state.is_sync_paused(db),
@@ -144,11 +102,9 @@ def deep_all(
on: bool = True,
) -> dict:
"""Opt every one of the user's channels into (or out of) full-history backfill."""
result = db.execute(
update(Subscription).where(Subscription.user_id == user.id).values(deep_requested=on)
)
updated = service.set_deep_all(db, user, on)
db.commit()
return {"ok": True, "updated": result.rowcount, "deep_requested": on}
return {"ok": True, "updated": updated, "deep_requested": on}
@router.post("/pause")