peter
d928d50686
The views, duration and title sorts had no index: every page read the whole videos heap and kept a top-N (prod 350 ms a page, 550-780 for title). Migration 0096 adds one per direction and, for title, per reader collation, in the exact ORDER BY feedquery builds. Localdev prod copy: plain sorts 180-310 ms -> ~1 ms, page 20 3-5 ms, narrow filters unchanged. view_count/duration are only written by the enrich, which already rewrites indexed columns, so no HOT update is lost. The title indexes are the first on an ICU collation, and postgres:16-alpine floats: collation.heal_drift rebuilds them (CONCURRENTLY) at startup when Postgres reports the ICU version moved. test_feed_sort_indexes holds every videos sort to an index and checks the rebuild.
84 lines
3.8 KiB
Python
84 lines
3.8 KiB
Python
"""Alphabetical order in the reader's language (E30 S3).
|
|
|
|
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
|
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
|
server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text
|
|
sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in.
|
|
|
|
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
|
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
|
list the same way through the database, so a list built in Python matches one built in SQL exactly.
|
|
|
|
The feed's title sorts ride indexes built in these collations (migration 0096), and an index keeps the
|
|
order of the ICU that built it. `postgres:16-alpine` is a floating tag: an image update can bring a newer
|
|
ICU, after which those indexes disagree with the comparisons a query makes (a keyset page skips or
|
|
repeats a title). `heal_drift` rebuilds them at startup once Postgres reports the version moved."""
|
|
|
|
from typing import Any
|
|
|
|
from sqlalchemy import Engine, SQLColumnExpression, column, func, select, text, values
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.sql.elements import ColumnElement
|
|
from sqlalchemy.types import String
|
|
|
|
from app import logs
|
|
from app.mailtext import normalize
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
|
|
|
|
|
def collation(lang: str | None) -> str:
|
|
"""The ICU collation for a UI language; anything unknown sorts as English."""
|
|
return ICU[normalize(lang)]
|
|
|
|
|
|
def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]:
|
|
"""`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together."""
|
|
return func.lower(expr).collate(collation(lang))
|
|
|
|
|
|
def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
|
"""`words` in the reader's alphabetical order, by the same ICU collation as `collated`."""
|
|
if not words:
|
|
return []
|
|
w = values(column("w", String), name="words").data([(x,) for x in words])
|
|
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
|
|
|
|
|
def heal_drift(engine: Engine) -> list[str]:
|
|
"""Rebuild every index on an ICU collation whose library version moved, then record the new version.
|
|
Returns the rebuilt indexes. CONCURRENTLY: the feed keeps reading while it runs."""
|
|
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
|
drifted = conn.execute(
|
|
text(
|
|
"SELECT oid, collname FROM pg_collation WHERE collname = ANY(:names) "
|
|
"AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)"
|
|
),
|
|
{"names": list(ICU.values())},
|
|
).all()
|
|
if not drifted:
|
|
return []
|
|
indexes = list(
|
|
conn.execute(
|
|
text(
|
|
"SELECT DISTINCT d.objid::regclass::text FROM pg_depend d "
|
|
"JOIN pg_class c ON c.oid = d.objid AND c.relkind = 'i' "
|
|
"WHERE d.classid = 'pg_class'::regclass "
|
|
"AND d.refclassid = 'pg_collation'::regclass AND d.refobjid = ANY(:oids)"
|
|
),
|
|
{"oids": [row.oid for row in drifted]},
|
|
).scalars()
|
|
)
|
|
log.warning(
|
|
"ICU collation version changed (%s): rebuilding %d indexes",
|
|
", ".join(row.collname for row in drifted),
|
|
len(indexes),
|
|
)
|
|
for name in indexes:
|
|
conn.execute(text(f"REINDEX INDEX CONCURRENTLY {name}"))
|
|
for row in drifted:
|
|
conn.execute(text(f'ALTER COLLATION "{row.collname}" REFRESH VERSION'))
|
|
return indexes
|