peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(feed): extract read-model to app.feedquery + admin_user sweep

Browse Source 
B-core-3 (2026-08-17 review): decouple two things from router modules.

1. Feed read-model → new app/feedquery.py: the pure query-building and
   row-serialization layer (feed_columns, serialize, _filtered_query,
   to_tsquery_str, and the keyset-pagination helpers) moves out of
   routes/feed.py, which keeps only the route handlers, _feed_params, and
   the request models. The three external consumers (playlists, search,
   plex/browse) now import from app.feedquery instead of reaching into the
   feed router.

2. admin_user sweep: 7 modules imported admin_user via app.routes.admin
   (an incidental re-export of app.auth.admin_user). Repoint them all to
   the definition in app.auth (audit, channels, config, quota, scheduler,
   plex/admin, plex/browse); routes.admin keeps its own import for its use.

Pure code motion — no behaviour change. Full DB-lane suite 287 passed /
2 skipped; ruff clean.
This commit is contained in:
peter
2026-08-17 17:42:01 +02:00
parent c361fc2581
commit 5f09259bbf
11 changed files with 459 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/feedquery.py
+437
View File
@@ -0,0 +1,437 @@
"""Feed read-model: the pure query-building and row-serialization layer behind the feed.
Extracted from `routes/feed.py` so the read-model has one home instead of living inside the
router module — the feed route handlers, the playlist-detail query, and the Plex search box all
share these helpers (the projection, the filter builder, `to_tsquery_str`, and keyset paging).
Nothing here defines a route or reaches for request state; it's all `(db, user, params) -> query`.
"""
import base64
import binascii
import json
import re
from datetime import date, datetime, timedelta, timezone
from fastapi import HTTPException
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
from sqlalchemy.orm import Session, aliased
from app.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
ChannelTag,
ExploredChannel,
Playlist,
PlaylistItem,
SearchFind,
Subscription,
Tag,
User,
Video,
VideoState,
)
from app.utils import iso
def _channel_url(channel_id: str, handle: str | None) -> str:
if handle and handle.startswith("@"):
return f"https://www.youtube.com/{handle}"
return f"https://www.youtube.com/channel/{channel_id}"
def _saved_expr(user: User):
"""A correlated EXISTS labelled `saved`: is this video in the user's Watch later list?
(Watch later is the built-in playlist that replaced the old per-video 'saved' status.)"""
return (
select(1)
.select_from(PlaylistItem)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.where(
Playlist.user_id == user.id,
Playlist.kind == "watch_later",
PlaylistItem.video_id == Video.id,
)
.exists()
.label("saved")
)
def feed_columns(user: User, state) -> list:
"""The shared feed-row projection (joined Channel fields + per-user watch state +
`saved`), consumed by both the feed query and the playlist-detail query so a new column
is added in exactly one place. `state` is the caller's aliased VideoState (outer-joined)."""
return [
Video.id,
Video.title,
Video.channel_id,
Channel.title.label("channel_title"),
Channel.thumbnail_url.label("channel_thumbnail"),
Channel.handle.label("channel_handle"),
Video.published_at,
Video.thumbnail_url,
Video.duration_seconds,
Video.view_count,
Video.is_short,
Video.live_status,
func.coalesce(state.status, "new").label("status"),
func.coalesce(state.position_seconds, 0).label("position_seconds"),
_saved_expr(user),
]
def serialize(row) -> dict:
return {
"id": row.id,
"title": row.title,
"channel_id": row.channel_id,
"channel_title": row.channel_title,
"channel_thumbnail": row.channel_thumbnail,
"channel_url": _channel_url(row.channel_id, row.channel_handle),
"published_at": iso(row.published_at),
"thumbnail_url": row.thumbnail_url,
"duration_seconds": row.duration_seconds,
"view_count": row.view_count,
"is_short": row.is_short,
"live_status": row.live_status,
"status": row.status or "new",
"position_seconds": row.position_seconds or 0,
"saved": bool(getattr(row, "saved", False)),
"watch_url": f"https://www.youtube.com/watch?v={row.id}",
}
def _filtered_query(
db: Session,
user: User,
*,
tags: list[int],
tag_mode: str,
channel_ids: list[str],
q: str | None,
min_duration: int | None,
max_duration: int | None,
max_age_days: int | None,
published_after: date | None,
published_before: date | None,
show_normal: bool,
include_shorts: bool,
include_live: bool,
show: str,
scope: str = "my",
library_source: str = "organic",
exclude_tag_category: str | None = None,
) -> tuple[Select, object]:
"""Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count.
Returns the column-bearing select plus the priority/relevance rank expression for sorting.
`scope="my"` (default) restricts the feed to the user's own non-hidden subscriptions.
`scope="all"` shows every video in the shared catalog (any user's ingested channels);
the subscription is then only LEFT-joined so per-channel priority still resolves for
channels the user happens to be subscribed to. Per-user watch state stays private in
either mode via the VideoState outer join."""
state = aliased(VideoState)
status_expr = func.coalesce(state.status, "new")
position_expr = func.coalesce(state.position_seconds, 0)
query = select(*feed_columns(user, state)).join(
Channel, Channel.id == Video.channel_id
)
if scope == "all":
# Whole shared catalog; subscription is optional (only for priority sort).
query = query.outerjoin(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
),
)
else:
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
# filter below can include either or both, and the subscription row stays available
# for the priority sort.
query = query.outerjoin(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
Subscription.hidden.is_(False),
),
).outerjoin(
SearchFind,
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
)
query = query.outerjoin(
state, and_(state.video_id == Video.id, state.user_id == user.id)
)
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
# abandoned). They're either deleted after a grace period or unhidden if they recover;
# either way they shouldn't show in any feed view meanwhile.
query = query.where(Video.unavailable_since.is_(None))
# Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone
# opened an un-subscribed channel's page) is visible only to users who explored or
# subscribe to its channel, so one user's curiosity never leaks into everyone else's
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
# asking for scope=all + channel_ids (the channel is in this accessible set).
accessible_explore = (
select(Subscription.channel_id).where(Subscription.user_id == user.id)
).union(
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
)
query = query.where(
or_(
Video.via_explore.is_(False),
Video.channel_id.in_(accessible_explore),
)
)
# Per-user channel blocklist: never show this user videos from a channel they blocked
# (applies to feed, Library, explore, and the channel page alike).
query = query.where(
Video.channel_id.notin_(
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
)
)
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
# "organic" = your subscriptions, "search" = videos you found via your own search,
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
if scope == "all":
if library_source == "organic":
query = query.where(Video.via_search.is_(False))
elif library_source == "search":
query = query.where(Video.via_search.is_(True))
# "all" → both organic and search-discovered videos
else:
subscribed = Subscription.channel_id.isnot(None)
searched = SearchFind.video_id.isnot(None)
if library_source == "search":
query = query.where(searched)
elif library_source == "all":
query = query.where(or_(subscribed, searched))
else: # "organic" (default): subscriptions only
query = query.where(subscribed)
if channel_ids:
# OR across the picked channels; every other filter still applies on top.
query = query.where(Video.channel_id.in_(channel_ids))
if min_duration is not None:
query = query.where(Video.duration_seconds >= min_duration)
if max_duration is not None:
query = query.where(Video.duration_seconds <= max_duration)
if max_age_days is not None:
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
query = query.where(
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
)
if published_after is not None:
start = datetime.combine(
published_after, datetime.min.time(), tzinfo=timezone.utc
)
query = query.where(Video.published_at >= start)
if published_before is not None:
end = datetime.combine(
published_before, datetime.min.time(), tzinfo=timezone.utc
) + timedelta(days=1)
query = query.where(Video.published_at < end)
# Full-text relevance search over the weighted search document (title > creator keywords +
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
rank_expr = None
if q:
ts_str = to_tsquery_str(q)
if ts_str:
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
query = query.where(
or_(
Video.search_vector.op("@@")(tsq),
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
)
)
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
# published_at then id.
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
else:
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
like = func.unaccent(f"%{q}%")
query = query.where(
or_(
func.unaccent(Video.title).ilike(like),
func.unaccent(Channel.title).ilike(like),
)
)
if tags:
# AND across tag categories (e.g. language AND topic narrows), OR within a
# category; the any/all toggle controls multiple topic tags.
cat_rows = db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tags))).all()
by_category: dict[str, list[int]] = {}
for tag_id, category in cat_rows:
by_category.setdefault(category, []).append(tag_id)
# Facet counting drops the category being counted so its own chips don't zero each
# other out (standard drill-down faceting: a category's count ignores its own
# selections but still honours the other categories' filters).
if exclude_tag_category:
by_category.pop(exclude_tag_category, None)
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
for category, ids in by_category.items():
if category == "topic" and tag_mode == "and" and len(ids) > 1:
sub = (
select(ChannelTag.channel_id)
.where(ChannelTag.tag_id.in_(ids), visible)
.group_by(ChannelTag.channel_id)
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
)
else:
sub = select(ChannelTag.channel_id).where(
ChannelTag.tag_id.in_(ids), visible
)
query = query.where(Video.channel_id.in_(sub))
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
# Explicit Watched/Saved/Hidden views show every type so nothing goes missing.
explicit_view = show in ("watched", "hidden")
if not explicit_view:
type_clauses = []
if show_normal:
type_clauses.append(
and_(Video.is_short.is_(False), Video.live_status.notin_(LIVE_OR_UPCOMING))
)
if include_shorts:
type_clauses.append(Video.is_short.is_(True))
if include_live:
type_clauses.append(Video.live_status.in_(LIVE_OR_UPCOMING))
query = query.where(or_(*type_clauses) if type_clauses else false())
if show == "unwatched":
query = query.where(status_expr.notin_(("watched", "hidden")))
elif show == "in_progress":
# Started but not finished: a stored resume position, not yet watched/hidden.
query = query.where(
and_(position_expr > 0, status_expr.notin_(("watched", "hidden")))
)
elif show == "watched":
query = query.where(status_expr == "watched")
elif show == "hidden":
query = query.where(status_expr == "hidden")
else: # all
query = query.where(status_expr != "hidden")
return query, rank_expr
def to_tsquery_str(q: str) -> str | None:
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
(the string is built from sanitised tokens, never the raw query)."""
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
if not tokens:
return None
tokens[-1] = tokens[-1] + ":*"
return " & ".join(tokens)
# --- Keyset (cursor) pagination ---------------------------------------------
#
# Each sort is described as an ordered list of key columns (expression + direction
# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
# they can never drift apart — the prerequisite for correct cursor paging. Keyset
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
# (OFFSET would shift and duplicate/skip rows).
def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
table: dict[str, list[dict]] = {
"newest": [pub],
"oldest": [{**pub, "asc": True}],
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
# Your per-channel priority (set in the channel manager), newest first within a tier.
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
# Deterministic per-seed shuffle: the same seed yields the same total order, so
# the cursor stays valid across pages of one shuffled run.
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
}
# Relevance (search-result ranking) only exists when there's a query to rank against;
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
if rank_expr is not None:
table["relevance"] = [
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
pub,
]
return table.get(sort) or table["newest"]
def _order_by(keys: list[dict]):
cols = []
for k in keys:
c = k["expr"].asc() if k["asc"] else k["expr"].desc()
cols.append(c.nulls_last() if k["nullable"] else c)
cols.append(Video.id.asc()) # unique final tiebreaker
return cols
def _keyset_predicate(keys: list[dict], cursor_id: str):
"""WHERE clause selecting rows strictly AFTER the cursor in the sort order.
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
also pull in the NULL group, and equality treats NULL == NULL."""
def eq(k):
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
def after(k):
if k["value"] is None:
return false() # nulls are last; nothing sorts strictly after them
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
branches = []
for j in range(len(keys)):
branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
return or_(*branches)
def _encode_cursor(keys: list[dict], row) -> str:
vals = []
for i, k in enumerate(keys):
v = getattr(row, f"_k{i}")
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
payload = json.dumps({"k": vals, "id": row.id})
return base64.urlsafe_b64encode(payload.encode()).decode()
def _decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
try:
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
raw = payload["k"]
if len(raw) != len(keys):
raise ValueError("key arity mismatch")
vals = [
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
for k, v in zip(keys, raw)
]
return vals, payload["id"]
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError):
raise HTTPException(status_code=400, detail="Invalid feed cursor")