From 3af3799b25a67b782713c54274cef354a198c090 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 18 Aug 2026 04:16:08 +0200 Subject: [PATCH] perf(feed): index the default published_at DESC NULLS LAST sort (E2 S1, PERF-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feed's default `newest` sort is `published_at DESC NULLS LAST, id ASC`, but the only published_at index was a plain ASC btree — read backwards it satisfies DESC NULLS FIRST, not NULLS LAST — so every feed page fell back to a full `videos` seq scan + top-N sort. Migration 0067 adds the two matching indexes (global + per-channel) CONCURRENTLY and drops the now-redundant single-column ix_videos_channel_id (the channel composite leads with channel_id, a strict superset). Measured on the prod-sized localdev reseed (257,312 videos, prod planner): - global newest: 44,747 buffers / 154 ms -> 60 buffers / 0.32 ms - per-channel(12.6k): 7,604 buffers / 96 ms -> 54 buffers / 0.14 ms CONCURRENTLY build of both on 257k rows: 1.7s total (~0 downtime). This is the likely root cause of the "prod feed is slow" symptom. Model: drop the inline index=True on Video.channel_id (the composite supersedes it); the composites are migration-defined like the search_vector GIN. Gate green (334 pytest incl. the migration drift + downgrade/re-upgrade round-trip). --- .../versions/0067_feed_sort_indexes.py | 55 +++++++++++++++++++ backend/app/models.py | 6 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/0067_feed_sort_indexes.py diff --git a/backend/alembic/versions/0067_feed_sort_indexes.py b/backend/alembic/versions/0067_feed_sort_indexes.py new file mode 100644 index 00000000..07f1e22b --- /dev/null +++ b/backend/alembic/versions/0067_feed_sort_indexes.py @@ -0,0 +1,55 @@ +"""feed sort indexes: published_at DESC NULLS LAST composite(s) for the default feed (PERF-1) + +The feed's default `newest` sort is `published_at DESC NULLS LAST, id ASC` (feedquery.order_by), but the +only published_at index was a plain ASC btree (`ix_videos_published_at`). A btree read backwards satisfies +`DESC NULLS FIRST`, not the `DESC NULLS LAST` the query asks for, so the planner could not use it for +ordering — every feed page fell back to a full `videos` scan + top-N sort. Measured on a prod-sized reseed +(257k videos): the default page went from ~44.8k shared buffers (dev planner) / ~169k (prod planner, the +per-channel nested loop that is the likely "prod feed is slow" root cause) down to ~624 buffers with the +matching index. + +This adds two indexes in the exact feed order: + - ix_videos_published_at_desc (published_at DESC NULLS LAST, id ASC) — the global feed / Library sort. + - ix_videos_channel_pub_desc (channel_id, published_at DESC NULLS LAST, id ASC) — the per-channel feed + (channel page, channel_ids filter). It leads with channel_id, so it is a strict superset of the old + single-column `ix_videos_channel_id` (FK cascade, sync upsert, autotag, channel-detail all still ride + it) — that redundant index is dropped. + +The plain ASC `ix_videos_published_at` stays: it already serves the `oldest` sort (ASC NULLS LAST = btree +default) as an Index Scan. + +CONCURRENTLY (in alembic's autocommit_block, since it can't run inside a transaction) so a live prod builds +these without blocking reads/writes; downtime ~0. IF (NOT) EXISTS keeps the drift test's +downgrade->re-upgrade round-trip idempotent. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0067_feed_sort_indexes" +down_revision: Union[str, None] = "0066_notif_created_at_notnull" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_published_at_desc " + "ON videos (published_at DESC NULLS LAST, id ASC)" + ) + op.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_channel_pub_desc " + "ON videos (channel_id, published_at DESC NULLS LAST, id ASC)" + ) + # Redundant now — ix_videos_channel_pub_desc leads with channel_id (a strict superset). + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_channel_id") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_channel_id ON videos (channel_id)" + ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_channel_pub_desc") + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_published_at_desc") diff --git a/backend/app/models.py b/backend/app/models.py index 65078e47..d6ddbbcd 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -251,8 +251,12 @@ class Video(Base, TimestampMixin): __tablename__ = "videos" id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id + # No standalone channel_id index: the composite `ix_videos_channel_pub_desc` + # (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it + # serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed + # sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`. channel_id: Mapped[str] = mapped_column( - ForeignKey("channels.id", ondelete="CASCADE"), index=True + ForeignKey("channels.id", ondelete="CASCADE") ) title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles) original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved