perf(db): each feed sort rides an index, and an ICU upgrade rebuilds the title ones
The views, duration and title sorts had no index: every page read the whole videos heap and kept a top-N (prod 350 ms a page, 550-780 for title). Migration 0096 adds one per direction and, for title, per reader collation, in the exact ORDER BY feedquery builds. Localdev prod copy: plain sorts 180-310 ms -> ~1 ms, page 20 3-5 ms, narrow filters unchanged. view_count/duration are only written by the enrich, which already rewrites indexed columns, so no HOT update is lost. The title indexes are the first on an ICU collation, and postgres:16-alpine floats: collation.heal_drift rebuilds them (CONCURRENTLY) at startup when Postgres reports the ICU version moved. test_feed_sort_indexes holds every videos sort to an index and checks the rebuild.
This commit is contained in:
@@ -7,17 +7,25 @@ sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds
|
||||
|
||||
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
||||
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
||||
list the same way through the database, so a list built in Python matches one built in SQL exactly."""
|
||||
list the same way through the database, so a list built in Python matches one built in SQL exactly.
|
||||
|
||||
The feed's title sorts ride indexes built in these collations (migration 0096), and an index keeps the
|
||||
order of the ICU that built it. `postgres:16-alpine` is a floating tag: an image update can bring a newer
|
||||
ICU, after which those indexes disagree with the comparisons a query makes (a keyset page skips or
|
||||
repeats a title). `heal_drift` rebuilds them at startup once Postgres reports the version moved."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import SQLColumnExpression, column, func, select, values
|
||||
from sqlalchemy import Engine, SQLColumnExpression, column, func, select, text, values
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.types import String
|
||||
|
||||
from app import logs
|
||||
from app.mailtext import normalize
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
||||
|
||||
|
||||
@@ -37,3 +45,39 @@ def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
||||
return []
|
||||
w = values(column("w", String), name="words").data([(x,) for x in words])
|
||||
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
||||
|
||||
|
||||
def heal_drift(engine: Engine) -> list[str]:
|
||||
"""Rebuild every index on an ICU collation whose library version moved, then record the new version.
|
||||
Returns the rebuilt indexes. CONCURRENTLY: the feed keeps reading while it runs."""
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
drifted = conn.execute(
|
||||
text(
|
||||
"SELECT oid, collname FROM pg_collation WHERE collname = ANY(:names) "
|
||||
"AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)"
|
||||
),
|
||||
{"names": list(ICU.values())},
|
||||
).all()
|
||||
if not drifted:
|
||||
return []
|
||||
indexes = list(
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT DISTINCT d.objid::regclass::text FROM pg_depend d "
|
||||
"JOIN pg_class c ON c.oid = d.objid AND c.relkind = 'i' "
|
||||
"WHERE d.classid = 'pg_class'::regclass "
|
||||
"AND d.refclassid = 'pg_collation'::regclass AND d.refobjid = ANY(:oids)"
|
||||
),
|
||||
{"oids": [row.oid for row in drifted]},
|
||||
).scalars()
|
||||
)
|
||||
log.warning(
|
||||
"ICU collation version changed (%s): rebuilding %d indexes",
|
||||
", ".join(row.collname for row in drifted),
|
||||
len(indexes),
|
||||
)
|
||||
for name in indexes:
|
||||
conn.execute(text(f"REINDEX INDEX CONCURRENTLY {name}"))
|
||||
for row in drifted:
|
||||
conn.execute(text(f'ALTER COLLATION "{row.collname}" REFRESH VERSION'))
|
||||
return indexes
|
||||
|
||||
Reference in New Issue
Block a user