peter
e995c664c6
GET /api/channels summed every subscribed channel's videos by reading each video's heap page (681 MB on prod): 51-61 ms warm, but 2.25 s on a first open and 6.4 s under EXPLAIN once the table had left memory. /feed/summary and /feed/years did the same (~140 ms warm, ~800 ms cold). Migration 0095 replaces ix_videos_channel_pub_desc with the same key INCLUDING the columns those queries read, so they are Index Only Scans over a 25 MB index. A test holds the queries and the index's column list together. The admin Configuration page did a db.get per setting (two for an unset one): it now reads every override in one query (~40 ms of ~55 on prod).
84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
"""The channel manager's sums and the feed sidebar's counts read the index alone (E31 S4d).
|
||
|
||
`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary`
|
||
and `/feed/years` count the reader's videos per channel and per year under the default filters. They
|
||
used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else
|
||
had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every
|
||
column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries
|
||
and the index together: a column added to one and not the other fails it. DB lane.
|
||
"""
|
||
|
||
import re
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.dialects import postgresql
|
||
|
||
from app.feedquery import filtered_query
|
||
from app.models import Video
|
||
from app.routes.channels import channel_aggregates
|
||
|
||
INDEX = "ix_videos_channel_pub_cover"
|
||
|
||
# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open.
|
||
FEED_DEFAULTS = {
|
||
"tags": [],
|
||
"tag_modes": {"language": "or", "topic": "or", "other": "or"},
|
||
"untagged": False,
|
||
"channel_ids": [],
|
||
"q": None,
|
||
"min_duration": None,
|
||
"max_duration": None,
|
||
"max_age_days": None,
|
||
"published_after": None,
|
||
"published_before": None,
|
||
"show_normal": True,
|
||
"include_shorts": False,
|
||
"include_live": False,
|
||
"show": "unwatched",
|
||
"scope": "my",
|
||
"library_source": "organic",
|
||
}
|
||
|
||
|
||
def _index_columns(db) -> list[str]:
|
||
# Key columns first, then the INCLUDE ones, in the index's own order.
|
||
return list(
|
||
db.execute(
|
||
text(
|
||
"SELECT a.attname FROM pg_index i "
|
||
"JOIN pg_class c ON c.oid = i.indexrelid "
|
||
"CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) "
|
||
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum "
|
||
"WHERE c.relname = :name ORDER BY k.n"
|
||
),
|
||
{"name": INDEX},
|
||
).scalars()
|
||
)
|
||
|
||
|
||
def _video_columns(stmt) -> set[str]:
|
||
# The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree,
|
||
# which still carries the projection a with_only_columns replaced.
|
||
sql = str(stmt.compile(dialect=postgresql.dialect()))
|
||
return set(re.findall(r"\bvideos\.(\w+)", sql))
|
||
|
||
|
||
def _assert_covered(db, read: set[str]) -> None:
|
||
held = _index_columns(db)
|
||
assert held, f"{INDEX} is missing"
|
||
# It leads with channel_id, so it also serves every per-channel lookup the old index did.
|
||
assert held[0] == "channel_id"
|
||
assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}"
|
||
|
||
|
||
def test_the_channel_sums_read_only_columns_the_index_holds(db):
|
||
_assert_covered(db, _video_columns(channel_aggregates(["UC_x"])))
|
||
|
||
|
||
def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user):
|
||
# feed_summary groups the default filter by channel, feed_years by publish date.
|
||
base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS)
|
||
_assert_covered(
|
||
db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at))
|
||
)
|