perf(db): the channel manager and the feed sidebar read an index, not the video heap
GET /api/channels summed every subscribed channel's videos by reading each video's heap page (681 MB on prod): 51-61 ms warm, but 2.25 s on a first open and 6.4 s under EXPLAIN once the table had left memory. /feed/summary and /feed/years did the same (~140 ms warm, ~800 ms cold). Migration 0095 replaces ix_videos_channel_pub_desc with the same key INCLUDING the columns those queries read, so they are Index Only Scans over a 25 MB index. A test holds the queries and the index's column list together. The admin Configuration page did a db.get per setting (two for an unset one): it now reads every override in one query (~40 ms of ~55 on prod).
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
"""The channel manager's sums and the feed sidebar's counts read the index alone (E31 S4d).
|
||||
|
||||
`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary`
|
||||
and `/feed/years` count the reader's videos per channel and per year under the default filters. They
|
||||
used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else
|
||||
had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every
|
||||
column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries
|
||||
and the index together: a column added to one and not the other fails it. DB lane.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from app.feedquery import filtered_query
|
||||
from app.models import Video
|
||||
from app.routes.channels import channel_aggregates
|
||||
|
||||
INDEX = "ix_videos_channel_pub_cover"
|
||||
|
||||
# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open.
|
||||
FEED_DEFAULTS = {
|
||||
"tags": [],
|
||||
"tag_modes": {"language": "or", "topic": "or", "other": "or"},
|
||||
"untagged": False,
|
||||
"channel_ids": [],
|
||||
"q": None,
|
||||
"min_duration": None,
|
||||
"max_duration": None,
|
||||
"max_age_days": None,
|
||||
"published_after": None,
|
||||
"published_before": None,
|
||||
"show_normal": True,
|
||||
"include_shorts": False,
|
||||
"include_live": False,
|
||||
"show": "unwatched",
|
||||
"scope": "my",
|
||||
"library_source": "organic",
|
||||
}
|
||||
|
||||
|
||||
def _index_columns(db) -> list[str]:
|
||||
# Key columns first, then the INCLUDE ones, in the index's own order.
|
||||
return list(
|
||||
db.execute(
|
||||
text(
|
||||
"SELECT a.attname FROM pg_index i "
|
||||
"JOIN pg_class c ON c.oid = i.indexrelid "
|
||||
"CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) "
|
||||
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum "
|
||||
"WHERE c.relname = :name ORDER BY k.n"
|
||||
),
|
||||
{"name": INDEX},
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def _video_columns(stmt) -> set[str]:
|
||||
# The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree,
|
||||
# which still carries the projection a with_only_columns replaced.
|
||||
sql = str(stmt.compile(dialect=postgresql.dialect()))
|
||||
return set(re.findall(r"\bvideos\.(\w+)", sql))
|
||||
|
||||
|
||||
def _assert_covered(db, read: set[str]) -> None:
|
||||
held = _index_columns(db)
|
||||
assert held, f"{INDEX} is missing"
|
||||
# It leads with channel_id, so it also serves every per-channel lookup the old index did.
|
||||
assert held[0] == "channel_id"
|
||||
assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}"
|
||||
|
||||
|
||||
def test_the_channel_sums_read_only_columns_the_index_holds(db):
|
||||
_assert_covered(db, _video_columns(channel_aggregates(["UC_x"])))
|
||||
|
||||
|
||||
def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user):
|
||||
# feed_summary groups the default filter by channel, feed_years by publish date.
|
||||
base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS)
|
||||
_assert_covered(
|
||||
db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at))
|
||||
)
|
||||
Reference in New Issue
Block a user