peter
5229fd96be
Channels, search and sync do no database work in their routes any more: reads go through catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search. What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed projection of results) it asks reader/queries and reader/service. The scheduler and the runs page read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest jobs commit only where they are named.
84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
"""The channel manager's sums and the feed sidebar's counts read the index alone.
|
||
|
||
`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary`
|
||
and `/feed/years` count the reader's videos per channel and per year under the default filters. They
|
||
used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else
|
||
had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every
|
||
column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries
|
||
and the index together: a column added to one and not the other fails it. DB lane.
|
||
"""
|
||
|
||
import re
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.dialects import postgresql
|
||
|
||
from app.feedquery import filtered_query
|
||
from app.catalog.models import Video
|
||
from app.catalog.queries import channel_aggregates
|
||
|
||
INDEX = "ix_videos_channel_pub_cover"
|
||
|
||
# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open.
|
||
FEED_DEFAULTS = {
|
||
"tags": [],
|
||
"tag_modes": {"language": "or", "topic": "or", "other": "or"},
|
||
"untagged": False,
|
||
"channel_ids": [],
|
||
"q": None,
|
||
"min_duration": None,
|
||
"max_duration": None,
|
||
"max_age_days": None,
|
||
"published_after": None,
|
||
"published_before": None,
|
||
"show_normal": True,
|
||
"include_shorts": False,
|
||
"include_live": False,
|
||
"show": "unwatched",
|
||
"scope": "my",
|
||
"library_source": "organic",
|
||
}
|
||
|
||
|
||
def _index_columns(db) -> list[str]:
|
||
# Key columns first, then the INCLUDE ones, in the index's own order.
|
||
return list(
|
||
db.execute(
|
||
text(
|
||
"SELECT a.attname FROM pg_index i "
|
||
"JOIN pg_class c ON c.oid = i.indexrelid "
|
||
"CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) "
|
||
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum "
|
||
"WHERE c.relname = :name ORDER BY k.n"
|
||
),
|
||
{"name": INDEX},
|
||
).scalars()
|
||
)
|
||
|
||
|
||
def _video_columns(stmt) -> set[str]:
|
||
# The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree,
|
||
# which still carries the projection a with_only_columns replaced.
|
||
sql = str(stmt.compile(dialect=postgresql.dialect()))
|
||
return set(re.findall(r"\bvideos\.(\w+)", sql))
|
||
|
||
|
||
def _assert_covered(db, read: set[str]) -> None:
|
||
held = _index_columns(db)
|
||
assert held, f"{INDEX} is missing"
|
||
# It leads with channel_id, so it also serves every per-channel lookup the old index did.
|
||
assert held[0] == "channel_id"
|
||
assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}"
|
||
|
||
|
||
def test_the_channel_sums_read_only_columns_the_index_holds(db):
|
||
_assert_covered(db, _video_columns(channel_aggregates(["UC_x"])))
|
||
|
||
|
||
def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user):
|
||
# feed_summary groups the default filter by channel, feed_years by publish date.
|
||
base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS)
|
||
_assert_covered(
|
||
db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at))
|
||
)
|