peter
6116ad4bf4
Review round on the E2 stack (S1-S5). No correctness bugs; the surviving low/medium findings addressed: - altitude (#1): extract the shared `tags_by_category` helper used by BOTH filtered_query (SQL tag WHERE) and feed_summary (Python facet/count), removing the category-grouping duplication, and add bidirectional "keep in lockstep — equivalence-tested by test_feed_summary_matches_count_plus_facets" comments so the SQL<->Python tag rule can't silently drift. - test-coverage (#2): the /feed/summary equivalence test seeded only system tags, leaving the `ChannelTag.user_id == user.id` visibility branch untested. Add a user-owned Tag + link + two combos; the suite is now 346 (was 344) and the new cases pass (feed_summary agrees with /feed/count + /facets on user-visible tags). - dead-code (#3): annotate get_feed_count, get_facets, and the FE api.feedCount/ facets as "SUPERSEDED by /feed/summary — kept only as the equivalence oracle; retire together" so the intentionally-retained surface reads as deliberate. Not changed (confirmed no-op): the search channel-name resolve (#4) is once-per-request on a 633-row table and a deliberate pre-resolve for PERF-12's BitmapOr plan; the facet loop (#5) already computes passes() once per (channel, category) with a distinct exclude each, so there is no recomputation. Gate green: ruff + 346 pytest; frontend tsc/knip/prettier (FE change is a comment).
536 lines
25 KiB
Python
536 lines
25 KiB
Python
"""Feed read-model: the pure query-building and row-serialization layer behind the feed.
|
|
|
|
Extracted from `routes/feed.py` so the read-model has one home instead of living inside the
|
|
router module — the feed route handlers and the playlist-detail query share these helpers (the
|
|
projection, the filter builder, and keyset paging). Nothing here defines a route or reaches for
|
|
request state; it's all `(db, user, params) -> query`, and a malformed cursor raises a plain
|
|
`ValueError` for the route handler to translate (the tsquery sanitizer lives in `app.utils`).
|
|
"""
|
|
|
|
import base64
|
|
import binascii
|
|
import json
|
|
from datetime import date, datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
|
from sqlalchemy.orm import Session, aliased
|
|
|
|
from app.models import (
|
|
LIVE_OR_UPCOMING,
|
|
BlockedChannel,
|
|
Channel,
|
|
ChannelTag,
|
|
ExploredChannel,
|
|
Playlist,
|
|
PlaylistItem,
|
|
SearchFind,
|
|
Subscription,
|
|
Tag,
|
|
User,
|
|
Video,
|
|
VideoState,
|
|
)
|
|
from app.utils import iso, to_tsquery_str
|
|
|
|
|
|
def _channel_url(channel_id: str, handle: str | None) -> str:
|
|
if handle and handle.startswith("@"):
|
|
return f"https://www.youtube.com/{handle}"
|
|
return f"https://www.youtube.com/channel/{channel_id}"
|
|
|
|
|
|
def _saved_expr(user: User):
|
|
"""A correlated EXISTS labelled `saved`: is this video in the user's Watch later list?
|
|
(Watch later is the built-in playlist that replaced the old per-video 'saved' status.)"""
|
|
return (
|
|
select(1)
|
|
.select_from(PlaylistItem)
|
|
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
|
|
.where(
|
|
Playlist.user_id == user.id,
|
|
Playlist.kind == "watch_later",
|
|
PlaylistItem.video_id == Video.id,
|
|
)
|
|
.exists()
|
|
.label("saved")
|
|
)
|
|
|
|
|
|
def feed_columns(user: User, state) -> list:
|
|
"""The shared feed-row projection (joined Channel fields + per-user watch state +
|
|
`saved`), consumed by both the feed query and the playlist-detail query so a new column
|
|
is added in exactly one place. `state` is the caller's aliased VideoState (outer-joined)."""
|
|
return [
|
|
Video.id,
|
|
Video.title,
|
|
Video.channel_id,
|
|
Channel.title.label("channel_title"),
|
|
Channel.thumbnail_url.label("channel_thumbnail"),
|
|
Channel.handle.label("channel_handle"),
|
|
Video.published_at,
|
|
Video.thumbnail_url,
|
|
Video.duration_seconds,
|
|
Video.view_count,
|
|
Video.is_short,
|
|
Video.live_status,
|
|
func.coalesce(state.status, "new").label("status"),
|
|
func.coalesce(state.position_seconds, 0).label("position_seconds"),
|
|
_saved_expr(user),
|
|
]
|
|
|
|
|
|
def serialize(row) -> dict:
|
|
return {
|
|
"id": row.id,
|
|
"title": row.title,
|
|
"channel_id": row.channel_id,
|
|
"channel_title": row.channel_title,
|
|
"channel_thumbnail": row.channel_thumbnail,
|
|
"channel_url": _channel_url(row.channel_id, row.channel_handle),
|
|
"published_at": iso(row.published_at),
|
|
"thumbnail_url": row.thumbnail_url,
|
|
"duration_seconds": row.duration_seconds,
|
|
"view_count": row.view_count,
|
|
"is_short": row.is_short,
|
|
"live_status": row.live_status,
|
|
"status": row.status or "new",
|
|
"position_seconds": row.position_seconds or 0,
|
|
"saved": bool(getattr(row, "saved", False)),
|
|
"watch_url": f"https://www.youtube.com/watch?v={row.id}",
|
|
}
|
|
|
|
|
|
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
|
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
|
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
|
two can't disagree on which selected tag belongs to which category."""
|
|
out: dict[str, list[int]] = {}
|
|
for tid, cat in db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids))).all():
|
|
out.setdefault(cat, []).append(tid)
|
|
return out
|
|
|
|
|
|
def filtered_query(
|
|
db: Session,
|
|
user: User,
|
|
*,
|
|
tags: list[int],
|
|
tag_mode: str,
|
|
channel_ids: list[str],
|
|
q: str | None,
|
|
min_duration: int | None,
|
|
max_duration: int | None,
|
|
max_age_days: int | None,
|
|
published_after: date | None,
|
|
published_before: date | None,
|
|
show_normal: bool,
|
|
include_shorts: bool,
|
|
include_live: bool,
|
|
show: str,
|
|
scope: str = "my",
|
|
library_source: str = "organic",
|
|
exclude_tag_category: str | None = None,
|
|
) -> tuple[Select, object]:
|
|
"""Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count.
|
|
Returns the column-bearing select plus the priority/relevance rank expression for sorting.
|
|
|
|
`scope="my"` (default) restricts the feed to the user's own non-hidden subscriptions.
|
|
`scope="all"` shows every video in the shared catalog (any user's ingested channels);
|
|
the subscription is then only LEFT-joined so per-channel priority still resolves for
|
|
channels the user happens to be subscribed to. Per-user watch state stays private in
|
|
either mode via the VideoState outer join."""
|
|
state = aliased(VideoState)
|
|
status_expr = func.coalesce(state.status, "new")
|
|
|
|
query = select(*feed_columns(user, state)).join(
|
|
Channel, Channel.id == Video.channel_id
|
|
)
|
|
|
|
if scope == "all":
|
|
# Whole shared catalog; subscription is optional (only for priority sort).
|
|
query = query.outerjoin(
|
|
Subscription,
|
|
and_(
|
|
Subscription.channel_id == Video.channel_id,
|
|
Subscription.user_id == user.id,
|
|
),
|
|
)
|
|
else:
|
|
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
|
|
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
|
|
# filter below can include either or both, and the subscription row stays available
|
|
# for the priority sort.
|
|
query = query.outerjoin(
|
|
Subscription,
|
|
and_(
|
|
Subscription.channel_id == Video.channel_id,
|
|
Subscription.user_id == user.id,
|
|
Subscription.hidden.is_(False),
|
|
),
|
|
).outerjoin(
|
|
SearchFind,
|
|
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
|
)
|
|
|
|
# The state-gated views (watched/hidden/in_progress) only ever return videos that HAVE a
|
|
# video_states row, so an INNER join lets the planner drive from the user's own small video_states
|
|
# set (~100 rows) instead of scanning all videos and outer-joining (PERF-16). status is NOT NULL on
|
|
# a present row (server_default 'new'), so the raw-column filters below match the coalesce form
|
|
# exactly. 'unwatched'/'all' include row-less 'new' videos, so they keep the outer join.
|
|
state_required = show in ("watched", "hidden", "in_progress")
|
|
join_state = query.join if state_required else query.outerjoin
|
|
query = join_state(
|
|
state, and_(state.video_id == Video.id, state.user_id == user.id)
|
|
)
|
|
|
|
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
|
|
# abandoned). They're either deleted after a grace period or unhidden if they recover;
|
|
# either way they shouldn't show in any feed view meanwhile.
|
|
query = query.where(Video.unavailable_since.is_(None))
|
|
|
|
# Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone
|
|
# opened an un-subscribed channel's page) is visible only to users who explored or
|
|
# subscribe to its channel, so one user's curiosity never leaks into everyone else's
|
|
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
|
|
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
|
accessible_explore = (
|
|
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
|
).union(
|
|
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
|
|
)
|
|
query = query.where(
|
|
or_(
|
|
Video.via_explore.is_(False),
|
|
Video.channel_id.in_(accessible_explore),
|
|
)
|
|
)
|
|
|
|
# Per-user channel blocklist: never show this user videos from a channel they blocked
|
|
# (applies to feed, Library, explore, and the channel page alike).
|
|
query = query.where(
|
|
Video.channel_id.notin_(
|
|
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
|
)
|
|
)
|
|
|
|
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
|
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
|
# "organic" = your subscriptions, "search" = videos you found via your own search,
|
|
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
|
|
if scope == "all":
|
|
if library_source == "organic":
|
|
query = query.where(Video.via_search.is_(False))
|
|
elif library_source == "search":
|
|
query = query.where(Video.via_search.is_(True))
|
|
# "all" → both organic and search-discovered videos
|
|
else:
|
|
subscribed = Subscription.channel_id.isnot(None)
|
|
searched = SearchFind.video_id.isnot(None)
|
|
if library_source == "search":
|
|
query = query.where(searched)
|
|
elif library_source == "all":
|
|
query = query.where(or_(subscribed, searched))
|
|
else: # "organic" (default): subscriptions only
|
|
query = query.where(subscribed)
|
|
|
|
if channel_ids:
|
|
# OR across the picked channels; every other filter still applies on top.
|
|
query = query.where(Video.channel_id.in_(channel_ids))
|
|
if min_duration is not None:
|
|
query = query.where(Video.duration_seconds >= min_duration)
|
|
if max_duration is not None:
|
|
query = query.where(Video.duration_seconds <= max_duration)
|
|
if max_age_days is not None:
|
|
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
|
query = query.where(
|
|
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
|
|
)
|
|
if published_after is not None:
|
|
start = datetime.combine(
|
|
published_after, datetime.min.time(), tzinfo=timezone.utc
|
|
)
|
|
query = query.where(Video.published_at >= start)
|
|
if published_before is not None:
|
|
end = datetime.combine(
|
|
published_before, datetime.min.time(), tzinfo=timezone.utc
|
|
) + timedelta(days=1)
|
|
query = query.where(Video.published_at < end)
|
|
# Full-text relevance search over the weighted search document (title > creator keywords +
|
|
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
|
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
|
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
|
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
|
rank_expr = None
|
|
if q:
|
|
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
|
|
# this is a tiny standalone scan) and match videos by channel_id. ORing an unindexed
|
|
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
|
|
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
|
|
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
|
|
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
|
|
like = func.unaccent(f"%{q}%")
|
|
matching_channel_ids = (
|
|
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
channel_match = (
|
|
Video.channel_id.in_(matching_channel_ids) if matching_channel_ids else false()
|
|
)
|
|
ts_str = to_tsquery_str(q)
|
|
if ts_str:
|
|
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
|
query = query.where(or_(Video.search_vector.op("@@")(tsq), channel_match))
|
|
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
|
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
|
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
|
# published_at then id.
|
|
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
|
|
else:
|
|
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
|
query = query.where(or_(func.unaccent(Video.title).ilike(like), channel_match))
|
|
|
|
if tags:
|
|
# AND across tag categories (e.g. language AND topic narrows), OR within a category; the any/all
|
|
# toggle controls multiple topic tags. ⚠️ feed_summary.passes() reimplements this exact rule in
|
|
# Python for the sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
|
# test_feed_summary_matches_count_plus_facets).
|
|
by_category = tags_by_category(db, tags)
|
|
# Facet counting drops the category being counted so its own chips don't zero each
|
|
# other out (standard drill-down faceting: a category's count ignores its own
|
|
# selections but still honours the other categories' filters).
|
|
if exclude_tag_category:
|
|
by_category.pop(exclude_tag_category, None)
|
|
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
|
for category, ids in by_category.items():
|
|
if category == "topic" and tag_mode == "and" and len(ids) > 1:
|
|
sub = (
|
|
select(ChannelTag.channel_id)
|
|
.where(ChannelTag.tag_id.in_(ids), visible)
|
|
.group_by(ChannelTag.channel_id)
|
|
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
|
)
|
|
else:
|
|
sub = select(ChannelTag.channel_id).where(
|
|
ChannelTag.tag_id.in_(ids), visible
|
|
)
|
|
query = query.where(Video.channel_id.in_(sub))
|
|
|
|
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
|
# Explicit Watched/Saved/Hidden views show every type so nothing goes missing.
|
|
explicit_view = show in ("watched", "hidden")
|
|
if not explicit_view:
|
|
type_clauses = []
|
|
if show_normal:
|
|
type_clauses.append(
|
|
and_(Video.is_short.is_(False), Video.live_status.notin_(LIVE_OR_UPCOMING))
|
|
)
|
|
if include_shorts:
|
|
type_clauses.append(Video.is_short.is_(True))
|
|
if include_live:
|
|
type_clauses.append(Video.live_status.in_(LIVE_OR_UPCOMING))
|
|
query = query.where(or_(*type_clauses) if type_clauses else false())
|
|
|
|
if show == "unwatched":
|
|
query = query.where(status_expr.notin_(("watched", "hidden")))
|
|
elif show == "in_progress":
|
|
# Started but not finished: a stored resume position, not yet watched/hidden. INNER-joined
|
|
# (state_required) so the raw state columns are safe and the planner can use video_states.
|
|
query = query.where(
|
|
and_(state.position_seconds > 0, state.status.notin_(("watched", "hidden")))
|
|
)
|
|
elif show == "watched":
|
|
query = query.where(state.status == "watched")
|
|
elif show == "hidden":
|
|
query = query.where(state.status == "hidden")
|
|
else: # all
|
|
query = query.where(status_expr != "hidden")
|
|
|
|
return query, rank_expr
|
|
|
|
|
|
def feed_summary(db: Session, user: User, **params) -> dict:
|
|
"""Single-pass replacement for /feed/count + 3x /facets (PERF-13): the sidebar's total video count
|
|
and every tag's distinct-channel facet count, derived in Python from ONE grouped query instead of
|
|
four full passes over the subscribed set on every filter change.
|
|
|
|
The one query is the base filter with the TAG predicate removed, grouped by channel: it yields every
|
|
channel that has a matching video, with its video count. From that ~few-hundred-row map we compute:
|
|
- count = sum of the video counts of the channels that pass the SELECTED tag filter;
|
|
- facets = per tag, the number of distinct channels that carry it AND pass the drill-down filter
|
|
(the tag's own category dropped, unless it's a conjunctive topic-AND — mirroring the
|
|
old /facets exclude_tag_category logic exactly).
|
|
Tag matching is channel-based, identical to filtered_query's tag block (AND across categories, OR
|
|
within, topic-AND requires ALL selected topics)."""
|
|
tag_mode = params.get("tag_mode", "or")
|
|
|
|
# ONE pass: per-channel video counts for the base filter, tags stripped so every candidate channel
|
|
# is present to facet over.
|
|
base, _rank = filtered_query(db, user, **{**params, "tags": []})
|
|
ch_col = base.with_only_columns(Video.channel_id).subquery()
|
|
channel_counts: dict[str, int] = dict(
|
|
db.execute(select(ch_col.c.channel_id, func.count()).group_by(ch_col.c.channel_id)).all()
|
|
)
|
|
if not channel_counts:
|
|
return {"count": 0, "facets": {}}
|
|
|
|
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
|
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
|
channel_tags: dict[str, set[int]] = {}
|
|
tag_category: dict[int, str] = {}
|
|
for cid, tid, cat in db.execute(
|
|
select(ChannelTag.channel_id, ChannelTag.tag_id, Tag.category)
|
|
.join(Tag, Tag.id == ChannelTag.tag_id)
|
|
.where(visible, ChannelTag.channel_id.in_(channel_counts.keys()))
|
|
).all():
|
|
channel_tags.setdefault(cid, set()).add(tid)
|
|
tag_category[tid] = cat
|
|
|
|
# Selected tags grouped by category (read from Tag via the shared helper, so a selected tag that no
|
|
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
|
# behaviour).
|
|
selected_by_cat: dict[str, set[int]] = {
|
|
cat: set(ids) for cat, ids in tags_by_category(db, params.get("tags") or []).items()
|
|
}
|
|
|
|
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
|
# The Python mirror of filtered_query's SQL tag block (AND across categories, OR within,
|
|
# topic-AND requires all selected) — keep in lockstep; see the ⚠️ note there.
|
|
for cat, sel in selected_by_cat.items():
|
|
if cat == exclude_cat:
|
|
continue
|
|
if cat == "topic" and tag_mode == "and" and len(sel) > 1:
|
|
if not sel <= ch_tags: # conjunctive: channel must carry ALL selected topics
|
|
return False
|
|
elif not (sel & ch_tags): # disjunctive: channel must carry ANY selected tag
|
|
return False
|
|
return True
|
|
|
|
count = sum(
|
|
n for cid, n in channel_counts.items() if passes(channel_tags.get(cid, set()), None)
|
|
)
|
|
|
|
facets: dict[int, int] = {}
|
|
for category in ("language", "topic", "other"):
|
|
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
|
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
|
exclude = None if (category == "topic" and tag_mode == "and") else category
|
|
for cid, ch_tags in channel_tags.items():
|
|
if not passes(ch_tags, exclude):
|
|
continue
|
|
for tid in ch_tags:
|
|
if tag_category.get(tid) == category:
|
|
facets[tid] = facets.get(tid, 0) + 1
|
|
return {"count": count, "facets": facets}
|
|
|
|
|
|
# --- Keyset (cursor) pagination ---------------------------------------------
|
|
#
|
|
# Each sort is described as an ordered list of key columns (expression + direction
|
|
# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
|
|
# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
|
|
# they can never drift apart — the prerequisite for correct cursor paging. Keyset
|
|
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
|
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
|
# (OFFSET would shift and duplicate/skip rows).
|
|
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
|
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
|
table: dict[str, list[dict]] = {
|
|
"newest": [pub],
|
|
"oldest": [{**pub, "asc": True}],
|
|
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
|
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
|
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
|
|
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
|
|
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
|
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
|
|
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
|
|
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
|
|
# Your per-channel priority (set in the channel manager), newest first within a tier.
|
|
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
|
|
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
|
|
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
|
|
# Deterministic per-seed shuffle: the same seed yields the same total order, so
|
|
# the cursor stays valid across pages of one shuffled run.
|
|
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
|
|
}
|
|
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
|
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
|
if rank_expr is not None:
|
|
table["relevance"] = [
|
|
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
|
|
pub,
|
|
]
|
|
return table.get(sort) or table["newest"]
|
|
|
|
|
|
def order_by(keys: list[dict]):
|
|
cols = []
|
|
for k in keys:
|
|
c = k["expr"].asc() if k["asc"] else k["expr"].desc()
|
|
cols.append(c.nulls_last() if k["nullable"] else c)
|
|
cols.append(Video.id.asc()) # unique final tiebreaker
|
|
return cols
|
|
|
|
|
|
def keyset_predicate(keys: list[dict], cursor_id: str):
|
|
"""WHERE clause selecting rows strictly AFTER the cursor in the sort order.
|
|
|
|
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
|
|
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
|
|
also pull in the NULL group, and equality treats NULL == NULL.
|
|
|
|
Built recursively so that for a NON-nullable leading key we can prepend a non-strict
|
|
range bound (`col <= v` for desc / `col >= v` for asc) as an AND conjunct — logically
|
|
a no-op over the OR of strict-after branches, but it lets the planner bound the index
|
|
scan instead of scanning from the top and filtering every earlier row (PERF-15: deep
|
|
`newest` pages went 51.8k -> ~0.5k buffers). A NULLABLE key can't take that bound (the
|
|
NULL group sorts after every value and `col <= v` would drop it), so it keeps the pure
|
|
NULL-aware OR form."""
|
|
def eq(k):
|
|
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
|
|
|
|
def after(k):
|
|
if k["value"] is None:
|
|
return false() # nulls are last; nothing sorts strictly after them
|
|
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
|
|
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
|
|
|
|
def rec(i: int):
|
|
if i == len(keys):
|
|
return Video.id > cursor_id # all keys tied -> break by the unique id
|
|
k = keys[i]
|
|
tail = or_(after(k), and_(eq(k), rec(i + 1)))
|
|
if not k["nullable"] and k["value"] is not None:
|
|
bound = k["expr"] >= k["value"] if k["asc"] else k["expr"] <= k["value"]
|
|
return and_(bound, tail)
|
|
return tail
|
|
|
|
return rec(0)
|
|
|
|
|
|
def encode_cursor(keys: list[dict], row) -> str:
|
|
vals = []
|
|
for i, k in enumerate(keys):
|
|
v = getattr(row, f"_k{i}")
|
|
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
|
payload = json.dumps({"k": vals, "id": row.id})
|
|
return base64.urlsafe_b64encode(payload.encode()).decode()
|
|
|
|
|
|
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
|
|
try:
|
|
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
|
raw = payload["k"]
|
|
if len(raw) != len(keys):
|
|
raise ValueError("key arity mismatch")
|
|
vals = [
|
|
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
|
|
for k, v in zip(keys, raw)
|
|
]
|
|
return vals, payload["id"]
|
|
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
|
# Framework-free: the route handler translates this into an HTTP 400 so the
|
|
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
|
raise ValueError("Invalid feed cursor") from exc
|