peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(feed): un-block the search GIN + inner-join the state views (E2 S3, PERF-12/16)

Browse Source 
PERF-12: the search ORed `unaccent(channels.title) ILIKE '%q%'` straight into
the per-video predicate, which forced a full scan + detoast of every
search_vector and a per-candidate channel fetch. Pre-resolve the matching
channel ids (a few-hundred-row scan) and match `Video.channel_id.in_(ids)`
instead, so the planner keeps the search_vector GIN as a BitmapOr branch
(alongside ix_videos_channel_pub_desc). Empty match -> false() collapses the OR.

PERF-16: show=watched|hidden|in_progress used an outer join + coalesce(status,
'new'), so the planner couldn't start from the user's small video_states set.
Those views only return rows that HAVE a state, so inner-join video_states and
filter the raw column (status is NOT NULL on a present row, so it's identical).
'unwatched'/'all' keep the outer join. Dropped the now-dead position_expr.

New regression tests (test_feed_views): search returns both text- and channel-
name matches; the four watch-state views return the exact expected sets.

Measured on the prod-sized localdev reseed (prod planner, 0067/0068 applied):
- search /feed/count q=magyar: 663,463 -> 19,206 buffers (1,406 -> 31 ms; OLD
  seq-scan+detoast -> NEW BitmapOr GIN+channel index)
- search feed page: 2,908 -> 1,479 buffers (32.6 -> 1.4 ms)
- show=watched (user with 64 watched): 40,661 -> 264 buffers (~154x)
Gate green (336 pytest, incl. the 2 new equivalence tests).
This commit is contained in:
peter
2026-08-18 04:41:16 +02:00
parent 8c08ea3133
commit 6a35e83ded
2 changed files with 114 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/feedquery.py
+30 -19
View File
@@ -131,7 +131,6 @@ def filtered_query(
either mode via the VideoState outer join."""
state = aliased(VideoState)
status_expr = func.coalesce(state.status, "new")
position_expr = func.coalesce(state.position_seconds, 0)
query = select(*feed_columns(user, state)).join(
Channel, Channel.id == Video.channel_id
@@ -163,7 +162,14 @@ def filtered_query(
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
)
query = query.outerjoin(
# The state-gated views (watched/hidden/in_progress) only ever return videos that HAVE a
# video_states row, so an INNER join lets the planner drive from the user's own small video_states
# set (~100 rows) instead of scanning all videos and outer-joining (PERF-16). status is NOT NULL on
# a present row (server_default 'new'), so the raw-column filters below match the coalesce form
# exactly. 'unwatched'/'all' include row-less 'new' videos, so they keep the outer join.
state_required = show in ("watched", "hidden", "in_progress")
join_state = query.join if state_required else query.outerjoin
query = join_state(
state, and_(state.video_id == Video.id, state.user_id == user.id)
)
@@ -246,15 +252,25 @@ def filtered_query(
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
rank_expr = None
if q:
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
# this is a tiny standalone scan) and match videos by channel_id. ORing an unindexed
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
like = func.unaccent(f"%{q}%")
matching_channel_ids = (
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
.scalars()
.all()
)
channel_match = (
Video.channel_id.in_(matching_channel_ids) if matching_channel_ids else false()
)
ts_str = to_tsquery_str(q)
if ts_str:
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
query = query.where(
or_(
Video.search_vector.op("@@")(tsq),
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
)
)
query = query.where(or_(Video.search_vector.op("@@")(tsq), channel_match))
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
@@ -262,13 +278,7 @@ def filtered_query(
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
else:
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
like = func.unaccent(f"%{q}%")
query = query.where(
or_(
func.unaccent(Video.title).ilike(like),
func.unaccent(Channel.title).ilike(like),
)
)
query = query.where(or_(func.unaccent(Video.title).ilike(like), channel_match))
if tags:
# AND across tag categories (e.g. language AND topic narrows), OR within a
@@ -315,14 +325,15 @@ def filtered_query(
if show == "unwatched":
query = query.where(status_expr.notin_(("watched", "hidden")))
elif show == "in_progress":
# Started but not finished: a stored resume position, not yet watched/hidden.
# Started but not finished: a stored resume position, not yet watched/hidden. INNER-joined
# (state_required) so the raw state columns are safe and the planner can use video_states.
query = query.where(
and_(position_expr > 0, status_expr.notin_(("watched", "hidden")))
and_(state.position_seconds > 0, state.status.notin_(("watched", "hidden")))
)
elif show == "watched":
query = query.where(status_expr == "watched")
query = query.where(state.status == "watched")
elif show == "hidden":
query = query.where(status_expr == "hidden")
query = query.where(state.status == "hidden")
else: # all
query = query.where(status_expr != "hidden")