perf(db): the channel manager and the feed sidebar read an index, not the video heap

GET /api/channels summed every subscribed channel's videos by reading each video's heap page
(681 MB on prod): 51-61 ms warm, but 2.25 s on a first open and 6.4 s under EXPLAIN once the
table had left memory. /feed/summary and /feed/years did the same (~140 ms warm, ~800 ms cold).
Migration 0095 replaces ix_videos_channel_pub_desc with the same key INCLUDING the columns
those queries read, so they are Index Only Scans over a 25 MB index. A test holds the queries
and the index's column list together.

The admin Configuration page did a db.get per setting (two for an unset one): it now reads
every override in one query (~40 ms of ~55 on prod).
This commit is contained in:
2026-10-02 03:16:37 +02:00
parent 20010178b0
commit e995c664c6
7 changed files with 220 additions and 26 deletions
@@ -0,0 +1,83 @@
"""The channel manager's sums and the feed sidebar's counts read the index alone (E31 S4d).
`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary`
and `/feed/years` count the reader's videos per channel and per year under the default filters. They
used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else
had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every
column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries
and the index together: a column added to one and not the other fails it. DB lane.
"""
import re
from sqlalchemy import text
from sqlalchemy.dialects import postgresql
from app.feedquery import filtered_query
from app.models import Video
from app.routes.channels import channel_aggregates
INDEX = "ix_videos_channel_pub_cover"
# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open.
FEED_DEFAULTS = {
"tags": [],
"tag_modes": {"language": "or", "topic": "or", "other": "or"},
"untagged": False,
"channel_ids": [],
"q": None,
"min_duration": None,
"max_duration": None,
"max_age_days": None,
"published_after": None,
"published_before": None,
"show_normal": True,
"include_shorts": False,
"include_live": False,
"show": "unwatched",
"scope": "my",
"library_source": "organic",
}
def _index_columns(db) -> list[str]:
# Key columns first, then the INCLUDE ones, in the index's own order.
return list(
db.execute(
text(
"SELECT a.attname FROM pg_index i "
"JOIN pg_class c ON c.oid = i.indexrelid "
"CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) "
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum "
"WHERE c.relname = :name ORDER BY k.n"
),
{"name": INDEX},
).scalars()
)
def _video_columns(stmt) -> set[str]:
# The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree,
# which still carries the projection a with_only_columns replaced.
sql = str(stmt.compile(dialect=postgresql.dialect()))
return set(re.findall(r"\bvideos\.(\w+)", sql))
def _assert_covered(db, read: set[str]) -> None:
held = _index_columns(db)
assert held, f"{INDEX} is missing"
# It leads with channel_id, so it also serves every per-channel lookup the old index did.
assert held[0] == "channel_id"
assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}"
def test_the_channel_sums_read_only_columns_the_index_holds(db):
_assert_covered(db, _video_columns(channel_aggregates(["UC_x"])))
def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user):
# feed_summary groups the default filter by channel, feed_years by publish date.
base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS)
_assert_covered(
db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at))
)