diff --git a/CHANGELOG.md b/CHANGELOG.md index e11557f7..216079a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.98.0 — 2026-10-03 + +### New + +- Search is much faster: a short or common word now answers in a fraction of a second, with the newest matches ranked by relevance. +- The feed, the channel list and the Plex pages load faster, and Plex cards fetch a picture sized to the card. +- While a new search counts its results, the count dims instead of showing the old number. + +### Fixed + +- A list whose next page failed to load says so and offers Retry, instead of asking again and again. +- The filter rail's tag counts follow the current search. +- Smaller fixes to layout, keyboard focus and wording. + ## 0.97.0 — 2026-10-01 ### New diff --git a/VERSION b/VERSION index 05e39cbb..95fce8ca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.97.0 +0.98.0 diff --git a/backend/alembic/versions/0095_videos_channel_cover_index.py b/backend/alembic/versions/0095_videos_channel_cover_index.py new file mode 100644 index 00000000..6a6839f5 --- /dev/null +++ b/backend/alembic/versions/0095_videos_channel_cover_index.py @@ -0,0 +1,83 @@ +"""videos: the per-channel index covers what the channel manager and the feed sidebar read (E31 S4d) + +The channel manager (`GET /api/channels`) sums each subscribed channel's videos: count, last upload, total +duration. The feed sidebar (`/feed/summary`, `/feed/years`) counts the reader's videos per channel and per +year under the default filters: available, not via explore, not a short, not live. Every one of them found +its rows through `ix_videos_channel_pub_desc` (channel_id, published_at DESC NULLS LAST, id) and then read +each video's heap page for the rest: ~270k rows at ~2.5 KB each, 681 MB on prod. That stays in cache only +while nothing else needs the memory. Measured on prod (2026-10-02): `/api/channels` 51–61 ms warm, but +2.25 s and 1.96 s on a first open in the browser and 6.4 s under EXPLAIN with 30k blocks read from disk; +`/feed/summary` and `/feed/years` ~140 ms warm, ~800 ms cold. + +The same key with those columns INCLUDEd makes them Index Only Scans. On a prod-copy localdev, after a +VACUUM, the manager's sums read 0 heap pages and the summary reads the 25 MB index instead of the heap. +The visibility map is what lets them skip the heap; prod's autovacuum kept it complete (relallvisible == +relpages). The key columns are unchanged, so every reader of the old index (the per-channel feed, the FK +cascade, the sync upsert, channel detail) rides this one. The old index is dropped. +tests/test_channel_list_index_only.py holds the queries and this column list together. + +CONCURRENTLY (autocommit_block) builds it without blocking a live prod. IF (NOT) EXISTS keeps the drift +test's downgrade → re-upgrade round trip idempotent. A re-run after a build that failed part-way rebuilds +the INVALID leftover, and the old index is dropped only once the new one is valid: a re-run that kept the +leftover and dropped the old one left `videos` with no usable channel_id index (E31 S5 review). +""" + +from alembic import op +from sqlalchemy import text + +revision: str = "0095_videos_channel_cover_index" +down_revision: str | None = "0094_plex_genres_hu_everywhere" +branch_labels: str | None = None +depends_on: str | None = None + + +def _valid(name: str) -> bool | None: + """The index's `indisvalid`, or None when there is no index of that name.""" + row = ( + op.get_bind() + .execute( + text( + "SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE c.relname = :name" + ), + {"name": name}, + ) + .first() + ) + return None if row is None else bool(row[0]) + + +def _build(name: str, ddl: str) -> None: + """CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that + failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the + planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built + again.""" + if _valid(name) is False: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}") + + +def _swap(build: str, ddl: str, drop: str) -> None: + _build(build, ddl) + if not _valid(build): + raise RuntimeError(f"{build} did not build valid; {drop} is kept") + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {drop}") + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + _swap( + "ix_videos_channel_pub_cover", + "ON videos (channel_id, published_at DESC NULLS LAST, id) " + "INCLUDE (duration_seconds, unavailable_since, via_explore, is_short, live_status)", + "ix_videos_channel_pub_desc", + ) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + _swap( + "ix_videos_channel_pub_desc", + "ON videos (channel_id, published_at DESC NULLS LAST, id ASC)", + "ix_videos_channel_pub_cover", + ) diff --git a/backend/alembic/versions/0096_videos_sort_indexes.py b/backend/alembic/versions/0096_videos_sort_indexes.py new file mode 100644 index 00000000..dfef1576 --- /dev/null +++ b/backend/alembic/versions/0096_videos_sort_indexes.py @@ -0,0 +1,86 @@ +"""videos: an index per feed sort, so a sorted page stops after its rows (E31 S4e) + +The feed's views, duration and title sorts had no index. Every page (the first and each next cursor) +read the whole videos heap, filtered it and kept a top-N: on prod ~350 ms a page for views/duration, +550–780 ms for title (the ICU comparison on every row), 2.3 s for page 2 when the copies of one request +queued. With an index in the exact ORDER BY (`feedquery.order_by`: the key, NULLS LAST, then id) the +planner walks it and stops after the page. On a prod-copy localdev every plain sort fell from 180–310 ms +to ~1 ms, deep pages (20) to 3–5 ms, and narrow filters (a tag, a channel, a year, nothing found) stayed +at 0.3–25 ms. + +One index per direction: a backward scan of `DESC NULLS LAST` gives `ASC NULLS FIRST`, not the +`ASC NULLS LAST` the asc sorts ask for. The title pair is per reader language (`app.collation.ICU`): +an index only serves an ORDER BY in its own collation. + +Cost: ~130 MB (localdev: 10 + 10 + 8 + 8 + 4 × 23). Writes stay cheap: view_count and duration are +written by the enrich, which rewrites indexed columns already (the revalidation job only touches +last_checked_at), so no HOT update is lost. A search combined with one of these sorts can now walk the +index too: a term matching ~2 % of the videos took 130–200 ms instead of 7 (still under the plain sort it +replaces); rarer and broader terms stay fast. Fencing the index off under a search made broad terms +(`m`, `the`) take 2 s, so it is left to the planner. + +The title indexes are the schema's first ones on an ICU collation: an ICU upgrade in the Postgres image +reorders them, and `app.collation.heal_drift` rebuilds them at startup when it sees one. +tests/test_feed_sort_indexes.py holds each sort and its index together. + +CONCURRENTLY (autocommit_block) builds them without blocking a live prod; IF (NOT) EXISTS keeps the drift +test's downgrade → re-upgrade round trip idempotent. +""" + +from alembic import op +from sqlalchemy import text + +revision: str = "0096_videos_sort_indexes" +down_revision: str | None = "0095_videos_channel_cover_index" +branch_labels: str | None = None +depends_on: str | None = None + +# name → the indexed ORDER BY (feedquery.sort_keys + order_by: key NULLS LAST, then id). +INDEXES = { + "ix_videos_views_desc": "view_count DESC NULLS LAST, id", + "ix_videos_views_asc": "view_count ASC NULLS LAST, id", + "ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id", + "ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id", + "ix_videos_title_en": '(lower(title) COLLATE "en-x-icu") ASC NULLS LAST, id', + "ix_videos_title_en_desc": '(lower(title) COLLATE "en-x-icu") DESC NULLS LAST, id', + "ix_videos_title_hu": '(lower(title) COLLATE "hu-x-icu") ASC NULLS LAST, id', + "ix_videos_title_hu_desc": '(lower(title) COLLATE "hu-x-icu") DESC NULLS LAST, id', +} + + +def _valid(name: str) -> bool | None: + """The index's `indisvalid`, or None when there is no index of that name.""" + row = ( + op.get_bind() + .execute( + text( + "SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE c.relname = :name" + ), + {"name": name}, + ) + .first() + ) + return None if row is None else bool(row[0]) + + +def _build(name: str, ddl: str) -> None: + """CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that + failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the + planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built + again.""" + if _valid(name) is False: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}") + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + for name, columns in INDEXES.items(): + _build(name, f"ON videos ({columns})") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + for name in INDEXES: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") diff --git a/backend/app/collation.py b/backend/app/collation.py index a69c6730..846049a2 100644 --- a/backend/app/collation.py +++ b/backend/app/collation.py @@ -7,17 +7,25 @@ sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds `collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression (`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python -list the same way through the database, so a list built in Python matches one built in SQL exactly.""" +list the same way through the database, so a list built in Python matches one built in SQL exactly. + +The feed's title sorts ride indexes built in these collations (migration 0096), and an index keeps the +order of the ICU that built it. `postgres:16-alpine` is a floating tag: an image update can bring a newer +ICU, after which those indexes disagree with the comparisons a query makes (a keyset page skips or +repeats a title). `heal_drift` rebuilds them at startup once Postgres reports the version moved.""" from typing import Any -from sqlalchemy import SQLColumnExpression, column, func, select, values +from sqlalchemy import Engine, SQLColumnExpression, column, func, select, text, values from sqlalchemy.orm import Session from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.types import String +from app import logs from app.mailtext import normalize +log = logs.get_logger(__name__) + ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"} @@ -37,3 +45,39 @@ def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]: return [] w = values(column("w", String), name="words").data([(x,) for x in words]) return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w))) + + +def heal_drift(engine: Engine) -> list[str]: + """Rebuild every index on an ICU collation whose library version moved, then record the new version. + Returns the rebuilt indexes. CONCURRENTLY: the feed keeps reading while it runs.""" + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + drifted = conn.execute( + text( + "SELECT oid, collname FROM pg_collation WHERE collname = ANY(:names) " + "AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)" + ), + {"names": list(ICU.values())}, + ).all() + if not drifted: + return [] + indexes = list( + conn.execute( + text( + "SELECT DISTINCT d.objid::regclass::text FROM pg_depend d " + "JOIN pg_class c ON c.oid = d.objid AND c.relkind = 'i' " + "WHERE d.classid = 'pg_class'::regclass " + "AND d.refclassid = 'pg_collation'::regclass AND d.refobjid = ANY(:oids)" + ), + {"oids": [row.oid for row in drifted]}, + ).scalars() + ) + log.warning( + "ICU collation version changed (%s): rebuilding %d indexes", + ", ".join(row.collname for row in drifted), + len(indexes), + ) + for name in indexes: + conn.execute(text(f"REINDEX INDEX CONCURRENTLY {name}")) + for row in drifted: + conn.execute(text(f'ALTER COLLATION "{row.collname}" REFRESH VERSION')) + return indexes diff --git a/backend/app/feedquery.py b/backend/app/feedquery.py index 1023ac2d..f612b8ca 100644 --- a/backend/app/feedquery.py +++ b/backend/app/feedquery.py @@ -12,7 +12,7 @@ import binascii import json from datetime import date, datetime, timedelta, timezone -from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select +from sqlalchemy import BigInteger, Select, and_, case, cast, false, func, or_, select from sqlalchemy.orm import Session, aliased from app.collation import collated @@ -265,7 +265,8 @@ def filtered_query( # the queries that surfaced the video > description; see Video.search_vector). YouTube-like # "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed, # accent-insensitive (unaccent_simple). The channel name still matches as a substring so - # channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort. + # channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort, over the + # newest RANK_WINDOW matches only (see there). rank_expr = None if q: # Resolve the channels whose NAME matches up front (there are only a few hundred channels, so @@ -543,6 +544,54 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> return table.get(sort) or table["newest"] +# Relevance ranks only the newest RANK_WINDOW matches (E31): ts_rank reads every match's tsvector, and +# a broad word matches most of the library ("a" ranked 193k rows in 6.8 s on prod, against 1.1 s to just +# find them). The rows past the window keep rank 0, so they follow the ranked ones newest first; a word +# that broad has no meaningful relevance order anyway, and a specific one fits the window whole. +RANK_WINDOW = 2000 + + +def rank_window_start(db: Session, query: Select) -> datetime | None: + """The published_at the ranked window starts at: the RANK_WINDOW-th newest match, None when fewer + match (rank them all). The newest-first walk stops after RANK_WINDOW matches (40 ms for "a").""" + return db.scalar( + query.with_only_columns(Video.published_at) + .order_by(Video.published_at.desc()) + .offset(RANK_WINDOW - 1) + .limit(1) + ) + + +def windowed_rank(rank_expr, since: datetime | None): + """`rank_expr` inside the window, 0 past it. CASE evaluates ts_rank only for the rows inside.""" + if since is None: + return rank_expr + return case((Video.published_at >= since, rank_expr), else_=0) + + +def windowed_relevance_rows( + db: Session, query: Select, keys: list[dict], window: datetime, cursor_id: str | None, n: int +) -> list: + """Up to `n` rows of the windowed relevance order, read as its two halves: the window ranked, then + the rest newest first (every window row sorts before every row past it). As one ORDER BY the + planner computes the CASE for every match and sorts them all (2.7 s for "a" on prod); split, each + half rides an index (80 ms + 5 ms). With a cursor, `keys` carry its values (keyset_predicate).""" + pub_key = keys[1] # relevance: [rank, published_at] + rows: list = [] + if cursor_id is None or pub_key["value"] >= window: + inside = query.where(Video.published_at >= window).order_by(*order_by(keys)) + if cursor_id is not None: + inside = inside.where(keyset_predicate(keys, cursor_id)) + rows = list(db.execute(inside.limit(n)).all()) + if len(rows) < n: + newest = [pub_key] # past the window every rank is 0: the order is newest first + rest = query.where(Video.published_at < window).order_by(*order_by(newest)) + if cursor_id is not None: + rest = rest.where(keyset_predicate(newest, cursor_id)) + rows += db.execute(rest.limit(n - len(rows))).all() + return rows + + def order_by(keys: list[dict]): cols = [] for k in keys: @@ -589,26 +638,45 @@ def keyset_predicate(keys: list[dict], cursor_id: str): return rec(0) -def encode_cursor(keys: list[dict], row) -> str: +def encode_cursor(keys: list[dict], row, window: datetime | None = None) -> str: + """`window` is the relevance window's start: the cursor carries it, so every page ranks against the + window page one did. Recomputed per page, a video ingested mid-scroll would push the window's oldest + row to rank 0, behind the cursor, and the reader would see it twice.""" vals = [] for i, k in enumerate(keys): v = getattr(row, f"_k{i}") vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v) - payload = json.dumps({"k": vals, "id": row.id}) + body: dict = {"k": vals, "id": row.id} + if window is not None: + body["w"] = window.isoformat() + payload = json.dumps(body) return base64.urlsafe_b64encode(payload.encode()).decode() -def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]: +def _cursor_dt(v): + """A cursor's timestamp: aware or refused. The server writes only aware ones (timestamptz), and + windowed_relevance_rows compares them in Python, where a naive one beside an aware one raises — a + 500 for a hand-made cursor instead of the 400 (E31 review 2).""" + if v is None: + return None + dt = datetime.fromisoformat(v) + if dt.tzinfo is None: + raise ValueError("naive cursor timestamp") + return dt + + +def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime | None]: try: payload = json.loads(base64.urlsafe_b64decode(cursor.encode())) raw = payload["k"] if len(raw) != len(keys): raise ValueError("key arity mismatch") - vals = [ - datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v - for k, v in zip(keys, raw) - ] - return vals, payload["id"] + vals = [_cursor_dt(v) if k["kind"] == "dt" else v for k, v in zip(keys, raw)] + for k, v in zip(keys, vals): + if v is None and not k["nullable"]: + raise ValueError("null value for a non-nullable key") + window = payload.get("w") + return vals, payload["id"], _cursor_dt(window) if window else None except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc: # Framework-free: the route handler translates this into an HTTP 400 so the # read-model stays web-layer-agnostic (see routes/feed.get_feed). diff --git a/backend/app/main.py b/backend/app/main.py index 68b6ddb0..8f52f23f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -24,11 +24,12 @@ from sqlalchemy.exc import DataError from starlette.middleware.sessions import SessionMiddleware from app import bodylimit +from app import collation from app import logsink from app import state from app import sysconfig from app.config import settings -from app.db import SessionLocal +from app.db import SessionLocal, engine from app.plex import stream as plex_stream from app.plex import subtitles as plex_subtitles from app.routes import ( @@ -97,6 +98,9 @@ async def lifespan(app: FastAPI): # owns, so racing it against early playback is safe. sweep = asyncio.create_task(_sweep_hls_orphans()) reaper = asyncio.create_task(_reap_hls_sessions()) + # Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve + # meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart. + heal = asyncio.create_task(_heal_collation_drift()) start_scheduler() try: yield @@ -106,7 +110,7 @@ async def lifespan(app: FastAPI): # pending ("Task was destroyed but it is pending!" — noise that reads like a bug during # triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own # bookkeeping honest. - for task in (sweep, reaper): + for task in (sweep, reaper, heal): task.cancel() with suppress(asyncio.CancelledError): await task @@ -139,6 +143,18 @@ async def _reap_hls_sessions() -> None: log.exception("HLS session reap failed") +async def _heal_collation_drift() -> None: + """Background startup check of the ICU collation the title-sort indexes are built in (see + `collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in + the old order.""" + try: + await asyncio.to_thread(collation.heal_drift, engine) + except asyncio.CancelledError: + raise + except Exception: + log.exception("ICU collation drift check failed") + + async def _sweep_hls_orphans() -> None: """Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here must never take the app down, and a cancelled shutdown is not one.""" diff --git a/backend/app/models.py b/backend/app/models.py index 91b268c0..2c958085 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -330,10 +330,12 @@ class Video(Base, TimestampMixin): __tablename__ = "videos" id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id - # No standalone channel_id index: the composite `ix_videos_channel_pub_desc` - # (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it - # serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed - # sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`. + # No standalone channel_id index: the composite `ix_videos_channel_pub_cover` + # (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it + # serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed + # sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only + # Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only). + # Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`. channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE")) title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles) original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved @@ -347,6 +349,9 @@ class Video(Base, TimestampMixin): DateTime(timezone=True), index=True, nullable=False, server_default=func.now() ) thumbnail_url: Mapped[str | None] = mapped_column(String(1024)) + # The feed's views / duration / title sorts each ride a migration-defined index in their exact + # ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to + # `feedquery.order_by` by test_feed_sort_indexes. duration_seconds: Mapped[int | None] = mapped_column(Integer) view_count: Mapped[int | None] = mapped_column(BigInteger) like_count: Mapped[int | None] = mapped_column(BigInteger) diff --git a/backend/app/plex/imgcache.py b/backend/app/plex/imgcache.py index 130c241d..8eed4e1b 100644 --- a/backend/app/plex/imgcache.py +++ b/backend/app/plex/imgcache.py @@ -23,9 +23,23 @@ log = logs.get_logger(__name__) _EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")] _CT_EXT = {ct: ext for ext, ct in _EXT_CT} -# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/ -# strips), so 400px covers 2× DPR; the faint backdrop art only needs ~1280px. Keeps images ~tens of KB. -SIZES = {"thumb": (400, 600), "art": (1280, 720)} +# Target box per variant — Plex resizes to these server-side. The full poster (400px) covers a detail +# page's ≤224px poster at 2× DPR, the full art (1280px) a hero backdrop. A card is much smaller: a grid +# poster is ~144px and a shelf's landscape art ~212px, so the `_sm` pair serves those at 1.33× DPR +# (E31 S4e: cards fetched the 400 poster, 42 KB, and the 1280 art for a 212px picture). The browser +# picks between a pair itself (`srcset`, built by the frontend's `lib/plexImage` cardImage, whose +# widths plexImage.contract.test holds to these). +SIZES = { + "thumb": (400, 600), + "thumb_sm": (240, 360), + "art": (1280, 720), + "art_sm": (480, 270), +} + + +def picture(variant: str) -> str: + """Which stored picture a size variant is cut from: the poster (`thumb`) or the backdrop (`art`).""" + return "art" if variant.startswith("art") else "thumb" def key_for(db: Session, rating_key: str, variant: str) -> str | None: diff --git a/backend/app/progress.py b/backend/app/progress.py index 7be04061..b6e3f900 100644 --- a/backend/app/progress.py +++ b/backend/app/progress.py @@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) -> sink(current, total, phase) +class JobFailed(RuntimeError): + """A job's own failure, said so the reader's language can name it (E31): the client reads + `scheduler.failure.` and interpolates `params` (keep them to values the reader may see — + numbers, ids, short codes). The message is the English line the log and `last_error` keep. An + upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised, + being diagnostic.""" + + def __init__(self, reason: str, message: str, **params: object): + super().__init__(message) + self.reason = reason + self.params = params + + class bind: """Context manager that binds a progress sink for the duration of a job run.""" diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 9c9c3a52..3f63dfa9 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from fastapi import APIRouter, BackgroundTasks, Depends from pydantic import BaseModel -from sqlalchemy import func, select, update +from sqlalchemy import Select, func, select, update from sqlalchemy.orm import Session from app.collation import collated, sorted_words @@ -40,6 +40,22 @@ router = APIRouter(prefix="/api/channels", tags=["channels"]) log = logs.get_logger(__name__) +def channel_aggregates(channel_ids: list[str]) -> Select: + """Per-channel stored count, last upload and total duration, in one grouped pass. Every column it + reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index + alone; a test holds that (test_channel_list_index_only).""" + return ( + select( + Video.channel_id, + func.count(Video.id), + func.max(Video.published_at), + func.coalesce(func.sum(Video.duration_seconds), 0), + ) + .where(Video.channel_id.in_(channel_ids)) + .group_by(Video.channel_id) + ) + + @router.get("") def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: rows = db.execute( @@ -49,24 +65,15 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_ .order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user))) ).all() - # Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last - # upload, total duration. Cheap — a single grouped scan, so computed on read rather than - # denormalised (measure-first: reads aren't frequent/hot enough to warrant cached columns). + # Per-channel aggregates over the shared catalog (channel_aggregates), computed on read rather than + # denormalised: an Index Only Scan over a ~20 MB index, where it used to read every video's heap page + # (681 MB on prod, a 2–6 s first open once something else had pushed it out of memory, E31 S4d). # (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer # computed here — the manager table dropped its N/S/L column.) channel_ids = [c.id for _, c in rows] agg: dict[str, dict] = {} if channel_ids: - for cid, n, last_up, total_dur in db.execute( - select( - Video.channel_id, - func.count(Video.id), - func.max(Video.published_at), - func.coalesce(func.sum(Video.duration_seconds), 0), - ) - .where(Video.channel_id.in_(channel_ids)) - .group_by(Video.channel_id) - ).all(): + for cid, n, last_up, total_dur in db.execute(channel_aggregates(channel_ids)).all(): agg[cid] = { "stored": n, "last_video_at": iso(last_up), diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 4ae00d95..4e905e0c 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -19,8 +19,11 @@ from app.feedquery import ( filtered_query, keyset_predicate, order_by, + rank_window_start, serialize, sort_keys, + windowed_rank, + windowed_relevance_rows, ) from app.models import ( POSITION_MAX_S, @@ -109,30 +112,45 @@ def get_feed( db: Session = Depends(get_db), ) -> dict: query, rank_expr = filtered_query(db, user, **params) - keys = sort_keys(sort, seed, rank_expr, lang_for_user(user)) - - # Expose each key column on the row so we can build the next cursor from the last item. - query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) - query = query.order_by(*order_by(keys)) + lang = lang_for_user(user) + keys = sort_keys(sort, seed, rank_expr, lang) + vals, cursor_id, window = None, None, None if cursor: # The read-model raises a plain ValueError on a malformed cursor; translate it to the # HTTP 400 here so feedquery stays web-framework-free. try: - vals, cursor_id = decode_cursor(keys, cursor) + vals, cursor_id, window = decode_cursor(keys, cursor) except ValueError: # `from None`: the ValueError is an expected bad-input signal, not an error to chain. raise refused(400, "invalid_cursor") from None + ranked = sort == "relevance" and rank_expr is not None + if ranked and not cursor: + window = rank_window_start(db, query) + if ranked and window is not None: + keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang) + + # Expose each key column on the row so we can build the next cursor from the last item. + query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) + if vals is not None: for k, v in zip(keys, vals): k["value"] = v - query = query.where(keyset_predicate(keys, cursor_id)) - - rows = db.execute(query.limit(limit + 1)).all() + if ranked and window is not None: + rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1) + else: + query = query.order_by(*order_by(keys)) + if cursor_id is not None: + query = query.where(keyset_predicate(keys, cursor_id)) + rows = db.execute(query.limit(limit + 1)).all() has_more = len(rows) > limit page = rows[:limit] return { "items": [serialize(r) for r in page], - "next_cursor": encode_cursor(keys, page[-1]) if (has_more and page) else None, + "next_cursor": ( + encode_cursor(keys, page[-1], window if ranked else None) + if (has_more and page) + else None + ), "has_more": has_more, "limit": limit, } diff --git a/backend/app/routes/messages.py b/backend/app/routes/messages.py index 342bda78..28270603 100644 --- a/backend/app/routes/messages.py +++ b/backend/app/routes/messages.py @@ -94,13 +94,31 @@ def _system_partner() -> dict: return {"id": SYSTEM_PARTNER_ID, "name": SYSTEM_NAME, "avatar_url": None} -def _serialize_msg(m: Message) -> dict: +# The welcome's opening line in every language — how a stored copy is recognised. It was rendered +# once, in the language of the day it was created, so an account that later switched language kept +# reading it in the old one (E31 S1: the owner's, in English). Recognised here, it is served in the +# reader's language instead. Derived from WELCOME itself, so a change to an OPENING line, or a +# language taken out of WELCOME, orphans the copies already stored: they are served as stored. The +# one retired language (German, 2026-07) left none on prod (checked 2026-10-02). +_WELCOME_OPENINGS = {text.partition("\n")[0] for text in WELCOME.values()} + + +def _system_body(body: str | None, lang: str | None) -> str | None: + """A system message's text for a reader of `lang`: the welcome in their language, any other as + stored.""" + if body is None or lang is None or body.partition("\n")[0] not in _WELCOME_OPENINGS: + return body + return WELCOME.get(lang, WELCOME["en"]) + + +def _serialize_msg(m: Message, lang: str | None = None) -> dict: + """`lang`: the reader's language, for a system message's text (a user message is ciphertext).""" return { "id": m.id, "kind": m.kind, "sender_id": m.sender_id, "recipient_id": m.recipient_id, - "body": m.body, # only set for system messages + "body": _system_body(m.body, lang) if m.kind == "system" else m.body, "ciphertext": m.ciphertext, "iv": m.iv, "read_at": iso(m.read_at), @@ -124,7 +142,7 @@ def ensure_welcome(db: Session, user: User) -> None: db.add(m) db.commit() db.refresh(m) - manager.push(user.id, {"type": "message", "message": _serialize_msg(m)}) + manager.push(user.id, {"type": "message", "message": _serialize_msg(m, lang_for_user(user))}) # --- E2EE key directory / blob store ----------------------------------------- @@ -313,7 +331,7 @@ def list_conversations(user: User = Depends(require_human), db: Session = Depend items.append( { "partner": partner, - "last_message": _serialize_msg(m), + "last_message": _serialize_msg(m, lang_for_user(user)), "unread": int(unread.get(pid, 0)), } ) @@ -366,7 +384,7 @@ def get_thread( # refreshes live instead of waiting for the next 20s poll. The client re-fetches the # count on this signal, so no number is carried here. manager.push(user.id, {"type": "unread"}) - return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]} + return {"partner": partner, "items": [_serialize_msg(m, lang_for_user(user)) for m in msgs]} @router.post("") diff --git a/backend/app/routes/plex/browse.py b/backend/app/routes/plex/browse.py index ab56d020..16acda1b 100644 --- a/backend/app/routes/plex/browse.py +++ b/backend/app/routes/plex/browse.py @@ -358,6 +358,24 @@ def facets( uid = user.id + # The shows that pass the watch-state scope, asked ONCE. The per-user episode rollup behind it is a + # full pass over the user's episodes (~10 ms on a prod-sized library), and each of the show groups + # (genres, ratings, studios, years, rating max) asked it again: half of a filtered facets call + # (E31 S4e). The same filter on the same rows, so the counts do not change + # (test_plex_facets_grid_parity). + shows_in_scope: list[int] | None = None + if show_ids and show in ("watched", "in_progress", "unwatched"): + agg = rollup.agg_subquery(db, uid).subquery() + in_scope = select(PlexShow.id).outerjoin(agg, agg.c.show_id == PlexShow.id) + in_scope = apply_show_state_filter( + in_scope, + func.coalesce(agg.c.total, 0), + func.coalesce(agg.c.watched, 0), + func.coalesce(agg.c.inprog, 0), + show, + ) + shows_in_scope = list(db.scalars(in_scope.where(PlexShow.library_id.in_(show_ids)))) + def wsq(query, model, ids, pp, kind_movie, with_dur): """Apply the (self-excluded) metadata filters + library scope + the per-user watch-state to a facet aggregation query, so the facets reflect exactly the visible result set.""" @@ -376,14 +394,8 @@ def facets( st = aliased(PlexState) query = query.outerjoin(st, and_(st.item_id == model.id, st.user_id == uid)) query = apply_movie_state_filter(query, st, show) - else: - if show in ("watched", "in_progress", "unwatched"): - agg = rollup.agg_subquery(db, uid).subquery() - query = query.outerjoin(agg, agg.c.show_id == model.id) - tot = func.coalesce(agg.c.total, 0) - wat = func.coalesce(agg.c.watched, 0) - inp = func.coalesce(agg.c.inprog, 0) - query = apply_show_state_filter(query, tot, wat, inp, show) + elif shows_in_scope is not None: + query = query.where(model.id.in_(shows_in_scope)) return query genre_counts: dict[str, int] = {} diff --git a/backend/app/routes/plex/playback.py b/backend/app/routes/plex/playback.py index 562973a4..f819f043 100644 --- a/backend/app/routes/plex/playback.py +++ b/backend/app/routes/plex/playback.py @@ -65,7 +65,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response brief session only to resolve the image key + read the Plex config, then RELEASES it before the fetch. Auth is the signed-session check (no DB user-load).""" _require_session(request) - v = "art" if variant == "art" else "thumb" + v = variant if variant in imgcache.SIZES else "thumb" w, h = imgcache.SIZES[v] # Size is part of the cache key: Plex resizes server-side (see PlexClient.image_bytes), so we cache # the small variant. The width tag also means changing a target size just re-caches, never serves a @@ -77,7 +77,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response # Cold miss: short-lived session for the key + Plex client config, released before the slow fetch. try: with SessionLocal() as db: - key = imgcache.key_for(db, str(rating_key), v) + key = imgcache.key_for(db, str(rating_key), imgcache.picture(v)) if not key: raise not_found("image") plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index feaf43fd..f54dad64 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -231,17 +231,22 @@ def _job(name: str, fn) -> None: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) err = str(exc) or exc.__class__.__name__ + # A failure the job named (progress.JobFailed) rides along as data, as a result does, so the + # client says it in the reader's language; `err` stays the English line. + failure = ( + {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None + ) _record( name, status="error", last_finished=_now_iso(), last_error=err, - result=None, + result=failure, progress=None, ) if actor_id is not None: - _notify_done(db, actor_id, name, "error", err) - _audit_run(db, actor_id, name, "error", err, None) + _notify_done(db, actor_id, name, "error", err, failure) + _audit_run(db, actor_id, name, "error", err, failure) finally: # ALWAYS release the claim, even if the body raised before its own status update — so a # failure can never wedge the job as permanently "running". diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 28fa2cb4..b495fbdf 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -83,6 +83,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # outage from one permanently dead feed id — which is the whole difference on an instance with # a single subscription, where BOTH render as 100% failed. first_error: str | None = None + # The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key). + first_failure: tuple[str, str] | None = None # What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples — # the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage # of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now. @@ -92,18 +94,19 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # hand the same rows to the same broken write) — the feed fallback's candidates. unread: list[str] = [] - def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None: + def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None: """Count a channel that did not get polled. `unexpected` is the message for a failure that is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks tell nobody anything the first one didn't. Call it from inside the `except` block.""" - nonlocal failed, first_error + nonlocal failed, first_error, first_failure if unexpected is not None: level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG log.log(level, unexpected, exc_info=True) failed += 1 reasons[reason] += 1 first_error = first_error or detail + first_failure = first_failure or (channel_id, reason) if len(examples) < RSS_FAILURE_EXAMPLES: examples.append(detail) @@ -125,13 +128,16 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # bug on our side and earns a traceback (the first of its kind per poll). except RssError as exc: log.debug("RSS poll skipped for channel %s: %s", channel.id, exc) - fail(exc.reason, f"{channel.id}: {exc}") + fail(exc.reason, channel.id, f"{channel.id}: {exc}") unread.append(channel.id) # silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the # traceback, once per reason per poll, which is the flood control this loop needs. except Exception as exc: # noqa: BLE001 — one channel must not stop the poll fail( - "error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}" + "error", + channel.id, + f"{channel.id}: {exc}", + unexpected=f"RSS fetch failed for {channel.id}", ) else: try: @@ -144,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str db.rollback() fail( "apply", + channel.id, f"{channel.id} (apply): {exc}", unexpected=f"RSS apply failed for {channel.id}", ) @@ -180,9 +187,17 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # "fix this subscription"; `Connection refused` says "the egress is down". # The fallback's count rides along: a red card during an RSS outage should still say # whether the uploads are arriving another way. - raise RuntimeError( + # As data too (E31): the audit log and the inbox say it in the reader's language, the first + # failure as its channel and its reason code (`http_404` vs `transport` is that difference). + first_channel, first_reason = first_failure or ("", "error") + raise progress.JobFailed( + "rss_all_failed", f"RSS poll failed for all {total} channels — first: {first_error}" - f" (read through the API instead: {fallback['read']})" + f" (read through the API instead: {fallback['read']})", + total=total, + channel=first_channel, + cause=first_reason, + read=fallback["read"], ) return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]} diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index d8e3f407..cef6ee07 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -10,6 +10,7 @@ stored in the DB and stay env-only). from dataclasses import dataclass +from sqlalchemy import select from sqlalchemy.orm import Session from app import logs, security @@ -45,8 +46,8 @@ class ConfigSpec: # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it -# off env into the admin UI; the admin page renders it automatically from this list. The ORDER is the -# admin page's tab order (first appearance of each group), so keep a group's keys together. +# off env into the admin UI; the admin page renders it automatically from this list. A group's keys +# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS. SPECS: tuple[ConfigSpec, ...] = ( # --- Access / registration --- ConfigSpec("allow_registration", "bool", "access", "allow_registration"), @@ -219,6 +220,16 @@ SPECS: tuple[ConfigSpec, ...] = ( # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) ) +# The admin page's sections (E31, the owner's pick): sixteen groups as tabs wrapped to three rows at 133% +# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in +# exactly one section (test-held), so a new group cannot fall off the page. +SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("access", ("access", "google", "email")), + ("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")), + ("media", ("downloads", "plex", "tmdb")), + ("system", ("maintenance", "audit", "diagnostics", "issues")), +) + _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} @@ -297,7 +308,11 @@ def _raw_override(db: Session, key: str) -> str | None: per key so the operator gets a real signal without a per-request traceback. Recovery is documented in docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2) """ - row = db.get(SystemConfig, key) + return _raw_of(db.get(SystemConfig, key), key) + + +def _raw_of(row: SystemConfig | None, key: str) -> str | None: + """`_raw_override` over a row already read (describe() reads them all in one query).""" if row is None or row.value is None: return None if not row.encrypted: @@ -317,8 +332,11 @@ def _raw_override(db: Session, key: str) -> str | None: def get(db: Session, key: str): """Effective value: the DB override (typed) if set, else the env/config default.""" - s = _BY_KEY[key] - raw = _raw_override(db, key) + return _effective(_BY_KEY[key], _raw_override(db, key)) + + +def _effective(s: ConfigSpec, raw: str | None): + """`get` over an override already read.""" if raw is None: return _default(s) if raw == "" and not s.allow_empty: @@ -410,8 +428,12 @@ def describe(db: Session) -> dict: """Admin-UI metadata + current values, grouped. Secret values are never echoed — only whether an override is set and whether an env default exists.""" groups: dict[str, list[dict]] = {} + # Every override in one query: a db.get per key was one round trip each (two for a key with no + # row, which the session cannot cache), ~40 ms of the page's ~55 on prod (E31 S4d). + rows = {r.key: r for r in db.scalars(select(SystemConfig))} for s in SPECS: - is_set = db.get(SystemConfig, s.key) is not None + row = rows.get(s.key) + is_set = row is not None item: dict = { "key": s.key, "type": s.type, @@ -426,7 +448,10 @@ def describe(db: Session) -> dict: if s.secret: item["default_is_set"] = bool(_default(s)) else: - item["value"] = get(db, s.key) + item["value"] = _effective(s, _raw_of(row, s.key)) item["default"] = _default(s) groups.setdefault(s.group, []).append(item) - return {"groups": groups, "secrets_manageable": secrets_manageable()} + sections = [ + {"id": sid, "groups": [g for g in members if g in groups]} for sid, members in SECTIONS + ] + return {"groups": groups, "sections": sections, "secrets_manageable": secrets_manageable()} diff --git a/backend/app/utils.py b/backend/app/utils.py index 248fb389..9dc90251 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -60,16 +60,24 @@ def now_utc() -> datetime: return datetime.now(timezone.utc) +# The shortest last word that is prefix-matched. A 1–2 letter prefix matches a large share of the +# library and ts_rank then scores every match: "m" took ~5–8 s on prod against ~100 ms for a whole +# word (E31 S4e). A shorter last word is matched whole instead ("tv", "4k" still find themselves). +MIN_PREFIX_LEN = 3 + + def to_tsquery_str(q: str) -> str | None: """Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word - prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*". + prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*" — + once it has MIN_PREFIX_LEN characters ("ti amo m" → "ti & amo & m"). Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in (the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed search and the Plex library search, hence this neutral home rather than either domain module.""" tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)] if not tokens: return None - tokens[-1] = tokens[-1] + ":*" + if len(tokens[-1]) >= MIN_PREFIX_LEN: + tokens[-1] = tokens[-1] + ":*" return " & ".join(tokens) diff --git a/backend/tests/test_channel_list_index_only.py b/backend/tests/test_channel_list_index_only.py new file mode 100644 index 00000000..cdcbd72b --- /dev/null +++ b/backend/tests/test_channel_list_index_only.py @@ -0,0 +1,83 @@ +"""The channel manager's sums and the feed sidebar's counts read the index alone (E31 S4d). + +`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary` +and `/feed/years` count the reader's videos per channel and per year under the default filters. They +used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else +had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every +column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries +and the index together: a column added to one and not the other fails it. DB lane. +""" + +import re + +from sqlalchemy import text +from sqlalchemy.dialects import postgresql + +from app.feedquery import filtered_query +from app.models import Video +from app.routes.channels import channel_aggregates + +INDEX = "ix_videos_channel_pub_cover" + +# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open. +FEED_DEFAULTS = { + "tags": [], + "tag_modes": {"language": "or", "topic": "or", "other": "or"}, + "untagged": False, + "channel_ids": [], + "q": None, + "min_duration": None, + "max_duration": None, + "max_age_days": None, + "published_after": None, + "published_before": None, + "show_normal": True, + "include_shorts": False, + "include_live": False, + "show": "unwatched", + "scope": "my", + "library_source": "organic", +} + + +def _index_columns(db) -> list[str]: + # Key columns first, then the INCLUDE ones, in the index's own order. + return list( + db.execute( + text( + "SELECT a.attname FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indexrelid " + "CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) " + "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum " + "WHERE c.relname = :name ORDER BY k.n" + ), + {"name": INDEX}, + ).scalars() + ) + + +def _video_columns(stmt) -> set[str]: + # The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree, + # which still carries the projection a with_only_columns replaced. + sql = str(stmt.compile(dialect=postgresql.dialect())) + return set(re.findall(r"\bvideos\.(\w+)", sql)) + + +def _assert_covered(db, read: set[str]) -> None: + held = _index_columns(db) + assert held, f"{INDEX} is missing" + # It leads with channel_id, so it also serves every per-channel lookup the old index did. + assert held[0] == "channel_id" + assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}" + + +def test_the_channel_sums_read_only_columns_the_index_holds(db): + _assert_covered(db, _video_columns(channel_aggregates(["UC_x"]))) + + +def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user): + # feed_summary groups the default filter by channel, feed_years by publish date. + base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS) + _assert_covered( + db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at)) + ) diff --git a/backend/tests/test_feed_relevance_window.py b/backend/tests/test_feed_relevance_window.py new file mode 100644 index 00000000..396d37b6 --- /dev/null +++ b/backend/tests/test_feed_relevance_window.py @@ -0,0 +1,133 @@ +"""Relevance ranks only the newest RANK_WINDOW matches (E31), DB-lane. + +A broad word ("a") matched most of the prod library and ts_rank read every match's tsvector: 6.8 s a page. +The ranking now covers the newest RANK_WINDOW matches and the rest follow newest first. The window's start +rides the cursor, so a video ingested mid-scroll cannot push a row behind the cursor and repeat it. +""" + +import base64 +import json +from datetime import datetime, timezone + +import pytest + +from app import feedquery +from app.models import Channel, Subscription, User, Video + + +def _video(vid: str, day: int, title: str) -> Video: + return Video( + id=vid, + channel_id="UCw", + published_at=datetime(2024, 1, day, tzinfo=timezone.utc), + title=title, + is_short=False, + live_status="none", + via_search=False, + via_explore=False, + ) + + +def _seed(db) -> User: + db.add(Channel(id="UCw", title="Window")) + user = User(email="window@example.com", email_verified=True) + db.add(user) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UCw", priority=0, hidden=False)) + # m1 is the oldest and the strongest match, m4 the strongest of the newest three; the rest tie on one + # title "foo" each (ts_rank counts occurrences, it does not normalise by length). + db.add_all( + [ + _video("m1", 1, "foo foo foo foo foo"), + _video("m2", 2, "foo and some other words here"), + _video("m3", 3, "foo plus some filler words"), + _video("m4", 4, "foo foo foo"), + _video("m5", 5, "foo with a longer title"), + _video("m6", 6, "foo with yet another title"), + ] + ) + db.commit() + return user + + +def _ids(client, cursor=None, limit=50): + url = f"/api/feed?q=foo&sort=relevance&limit={limit}" + if cursor: + url += f"&cursor={cursor}" + resp = client.get(url) + assert resp.status_code == 200, resp.text + body = resp.json() + return [it["id"] for it in body["items"]], body["next_cursor"] + + +def test_a_specific_word_ranks_every_match(db, as_user): + client = as_user(_seed(db)) + ids, _ = _ids(client) + assert ids[:2] == ["m1", "m4"], ids + + +def test_a_broad_word_ranks_the_newest_window_and_the_rest_follow_newest_first( + db, as_user, monkeypatch +): + monkeypatch.setattr(feedquery, "RANK_WINDOW", 3) + client = as_user(_seed(db)) + ids, _ = _ids(client) + assert ids[0] == "m4", ids # the strongest of the newest three + assert sorted(ids[:3]) == ["m4", "m5", "m6"], ids + assert ids[3:] == ["m3", "m2", "m1"], ids # past the window: newest first, m1 not ranked + + +def test_paging_keeps_the_window_page_one_ranked_against(db, as_user, monkeypatch): + monkeypatch.setattr(feedquery, "RANK_WINDOW", 3) + client = as_user(_seed(db)) + full, _ = _ids(client) + + first, cursor = _ids(client, limit=2) + assert first == full[:2] + # A newer match lands mid-scroll. Ranked against a recomputed window, m4 (shown on page one) would + # fall out of it to rank 0, sort behind the cursor and come back on page two. + db.add(_video("m7", 7, "foo arriving while the reader scrolls")) + db.commit() + + walked = list(first) + while cursor: + page, cursor = _ids(client, cursor, limit=2) + walked.extend(page) + assert len(walked) == len(set(walked)), walked + assert set(full) <= set(walked), walked + + +@pytest.mark.parametrize( + ("window", "limit"), + # 3: the page is read as its two halves (the window ranked, the rest newest first), and pages of 2 + # and 3 cross the window's edge mid-page and exactly at it. 2000: one ranked query. + [(3, 2), (3, 3), (2000, 2)], +) +def test_cursor_walk_matches_the_unpaged_order(db, as_user, monkeypatch, window, limit): + monkeypatch.setattr(feedquery, "RANK_WINDOW", window) + client = as_user(_seed(db)) + full, _ = _ids(client) + if window == 3: + assert full == ["m4", "m6", "m5", "m3", "m2", "m1"] + walked, cursor = _ids(client, limit=limit) + while cursor: + page, cursor = _ids(client, cursor, limit=limit) + walked.extend(page) + assert walked == full + + +@pytest.mark.parametrize( + "payload", + [ + {"k": [5, None], "id": "m1", "w": "2024-01-01T00:00:00+00:00"}, # null published_at + {"k": [5, "2024-01-03T00:00:00+00:00"], "id": "m1", "w": "2024-01-01T00:00:00"}, # naive w + {"k": [5, "2024-01-03T00:00:00"], "id": "m1", "w": "2024-01-01T00:00:00+00:00"}, # naive k + ], +) +def test_a_hand_made_relevance_cursor_is_refused_not_a_crash(db, as_user, payload): + # The page halves compare the cursor's timestamp with the window's in Python: a null or a naive + # one raised there, past the route's 400 — a 500 (E31 review 2). + client = as_user(_seed(db)) + cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + resp = client.get(f"/api/feed?q=foo&sort=relevance&limit=2&cursor={cursor}") + assert resp.status_code == 400, resp.text diff --git a/backend/tests/test_feed_sort_indexes.py b/backend/tests/test_feed_sort_indexes.py new file mode 100644 index 00000000..1184c68e --- /dev/null +++ b/backend/tests/test_feed_sort_indexes.py @@ -0,0 +1,100 @@ +"""Every feed sort on a videos column rides an index, and an ICU upgrade rebuilds the title ones (E31 S4e). + +Without an index in the exact ORDER BY, each feed page read the whole videos heap and kept a top-N +(prod: 350 ms a page for views/duration, 550–780 ms for title). Migration 0096 adds one per direction +and per reader language. The planner is told it may not sort or scan the table, so a plan that still +sorts it whole means no index matches the ORDER BY `feedquery.order_by` builds: a key, direction, NULLS +rule or collation changed on one side only. An Incremental Sort is fine: the index gives the key's order +and only its ties are sorted by id (`newest`/`oldest` ride the published_at indexes that way). DB lane. +""" + +import re + +import pytest +from sqlalchemy import select, text +from sqlalchemy.dialects import postgresql + +from app import collation +from app.db import engine +from app.feedquery import order_by, sort_keys +from app.models import Video + +VIDEO_SORTS = [ + "newest", + "oldest", + "views", + "views_asc", + "duration_desc", + "duration_asc", + "title", + "title_desc", +] + + +def _plan(db, sort: str, lang: str) -> str: + stmt = select(Video.id).order_by(*order_by(sort_keys(sort, 0, None, lang))).limit(60) + sql = str(stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) + db.execute(text("SET LOCAL enable_seqscan = off")) + db.execute(text("SET LOCAL enable_sort = off")) + db.execute(text("SET LOCAL enable_bitmapscan = off")) + return "\n".join(db.execute(text(f"EXPLAIN {sql}")).scalars()) + + +@pytest.mark.parametrize("lang", sorted(collation.ICU)) +@pytest.mark.parametrize("sort", VIDEO_SORTS) +def test_the_sort_walks_an_index(db, sort, lang): + plan = _plan(db, sort, lang) + db.rollback() + full_sort = re.search(r"(? list[str]: + return list( + db.execute( + text( + "SELECT collname FROM pg_collation WHERE collname = ANY(:names) " + "AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)" + ), + {"names": list(collation.ICU.values())}, + ).scalars() + ) + + +def _files(db) -> dict[str, int]: + return dict( + db.execute( + text("SELECT relname, relfilenode FROM pg_class WHERE relname LIKE 'ix_videos_title_%'") + ).all() + ) + + +def test_an_icu_upgrade_rebuilds_the_title_indexes(db): + # What an image update with a newer ICU looks like from inside: the recorded version no longer + # matches the library's (the test database runs as the bootstrap superuser, as the app's does). + db.execute(text("UPDATE pg_collation SET collversion = '0.0' WHERE collname = 'hu-x-icu'")) + db.commit() + try: + assert _drifted(db) == ["hu-x-icu"] + before = _files(db) + # A concurrent rebuild waits for every open transaction, this session's too. + db.rollback() + rebuilt = collation.heal_drift(engine) + assert sorted(rebuilt) == ["ix_videos_title_hu", "ix_videos_title_hu_desc"] + after = _files(db) + # Rebuilt = written to new files; the English pair, on an unchanged ICU, is left alone. + assert {n for n in before if before[n] != after[n]} == set(rebuilt) + assert _drifted(db) == [] + finally: + db.execute( + text( + "UPDATE pg_collation SET collversion = pg_collation_actual_version(oid) " + "WHERE collname = 'hu-x-icu'" + ) + ) + db.commit() + + +def test_no_drift_rebuilds_nothing(db): + assert collation.heal_drift(engine) == [] diff --git a/backend/tests/test_messages_welcome.py b/backend/tests/test_messages_welcome.py new file mode 100644 index 00000000..8d279450 --- /dev/null +++ b/backend/tests/test_messages_welcome.py @@ -0,0 +1,53 @@ +"""E31 S1: the welcome message is rendered once, when the account is created, in the language of that day. +An account that switched language afterwards kept reading it in the old one (the owner's: English under a +Hungarian UI). It is served in the reader's language now; any other system message stays as stored. +Needs the DB.""" + +from fastapi.testclient import TestClient + +import app.state as state +from app.auth import current_user +from app.db import get_db +from app.main import app +from app.models import Message, User +from app.routes.messages import WELCOME + + +def _client(db, user, monkeypatch, override_dep) -> TestClient: + monkeypatch.setattr(state, "_configured_cache", True) + override_dep(get_db, db.info["request"]) + override_dep(current_user, lambda: user) + return TestClient(app) + + +def _user_with_english_welcome(db) -> User: + """An account whose welcome was written in English, then switched to Hungarian.""" + user = User(email="welcome-hu@example.com", preferences={"language": "hu"}) + db.add(user) + db.flush() + db.add(Message(kind="system", sender_id=None, recipient_id=user.id, body=WELCOME["en"])) + db.commit() + return user + + +def test_the_welcome_is_read_in_the_readers_language(db, monkeypatch, override_dep): + user = _user_with_english_welcome(db) + client = _client(db, user, monkeypatch, override_dep) + + thread = client.get("/api/messages/conversations/0").json()["items"] + assert [m["body"] for m in thread] == [WELCOME["hu"]] + + convs = client.get("/api/messages/conversations").json()["items"] + assert [c["last_message"]["body"] for c in convs if c["partner"]["id"] == 0] == [WELCOME["hu"]] + + +def test_another_system_message_is_served_as_stored(db, monkeypatch, override_dep): + user = _user_with_english_welcome(db) + db.add( + Message(kind="system", sender_id=None, recipient_id=user.id, body="Maintenance tonight.") + ) + db.commit() + client = _client(db, user, monkeypatch, override_dep) + + bodies = [m["body"] for m in client.get("/api/messages/conversations/0").json()["items"]] + assert bodies == [WELCOME["hu"], "Maintenance tonight."] diff --git a/backend/tests/test_migration_rerun_invalid_index.py b/backend/tests/test_migration_rerun_invalid_index.py new file mode 100644 index 00000000..6d419755 --- /dev/null +++ b/backend/tests/test_migration_rerun_invalid_index.py @@ -0,0 +1,53 @@ +"""A concurrent index build that failed part-way, then re-run (E31 S5 review), DB-lane. + +CREATE INDEX CONCURRENTLY that fails leaves an INVALID index under its name, and alembic_version is not +stamped, so the next start runs the migration again. `IF NOT EXISTS` matched the name and kept the +leftover, which the planner never uses; 0095 then dropped the old channel_id index, leaving `videos` +with no usable one. A re-run now rebuilds the leftover and drops the old index only once the new one is +valid. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def _index(name: str) -> tuple[bool, str] | None: + with engine.connect() as conn: + row = conn.execute( + text( + "SELECT i.indisvalid, pg_get_indexdef(i.indexrelid) FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indexrelid WHERE c.relname = :name" + ), + {"name": name}, + ).first() + return None if row is None else (bool(row[0]), row[1]) + + +def _leave_invalid(name: str) -> None: + """What a build that failed part-way leaves behind: the name taken, the index INVALID.""" + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + conn.execute(text(f"CREATE INDEX {name} ON videos (id)")) + conn.execute( + text(f"UPDATE pg_index SET indisvalid = false WHERE indexrelid = '{name}'::regclass") + ) + + +def test_a_rerun_rebuilds_an_invalid_leftover_before_dropping_the_old_index(db): + db.rollback() # CONCURRENTLY waits for every open transaction, this session's too + cfg = Config("alembic.ini") + command.downgrade(cfg, "0094_plex_genres_hu_everywhere") + try: + _leave_invalid("ix_videos_channel_pub_cover") + _leave_invalid("ix_videos_views_desc") + finally: + command.upgrade(cfg, "head") + + cover = _index("ix_videos_channel_pub_cover") + assert cover is not None and cover[0], cover + assert "INCLUDE (duration_seconds" in cover[1] + assert _index("ix_videos_channel_pub_desc") is None + views = _index("ix_videos_views_desc") + assert views is not None and views[0] and "view_count DESC NULLS LAST" in views[1], views diff --git a/backend/tests/test_plex_facets_grid_parity.py b/backend/tests/test_plex_facets_grid_parity.py index 69780f4e..151bbc55 100644 --- a/backend/tests/test_plex_facets_grid_parity.py +++ b/backend/tests/test_plex_facets_grid_parity.py @@ -214,3 +214,28 @@ def test_hidden_movie_absent_from_grid_and_facets(db, as_user): facets = client.get("/api/plex/facets?scope=movie&show=all") crs = {c["value"]: c["count"] for c in facets.json()["content_ratings"]} assert crs.get("R") == 1, crs + + +def test_a_watch_state_scope_rolls_the_episodes_up_once(db, as_user): + """The per-user episode rollup behind a show's watch state is a pass over every episode; each show + facet group asked it again (E31 S4e: half of a filtered facets call). It is asked once now.""" + from sqlalchemy import event + + from app.db import engine + + seed = _seed(db) + client = as_user(seed["user"]) + rollups: list[str] = [] + + def count(conn, cursor, statement, *_): + if "GROUP BY plex_items.show_id" in statement: + rollups.append(statement) + + event.listen(engine, "before_cursor_execute", count) + try: + for show in ("watched", "in_progress", "unwatched"): + rollups.clear() + assert client.get(f"/api/plex/facets?scope=both&show={show}").status_code == 200 + assert len(rollups) == 1, show + finally: + event.remove(engine, "before_cursor_execute", count) diff --git a/backend/tests/test_plex_image_sizes.py b/backend/tests/test_plex_image_sizes.py new file mode 100644 index 00000000..6e3cdaad --- /dev/null +++ b/backend/tests/test_plex_image_sizes.py @@ -0,0 +1,77 @@ +"""A card asks the Plex image proxy for a small cut; a detail page keeps the full one (E31 S4e). + +Cards fetched the 400×600 poster for a ~144px picture and the 1280×720 backdrop for a ~212px shelf +picture. The `_sm` variants are cut by Plex at a card's size and cached apart from the full ones; an +unknown variant still gets the full poster. DB lane (the proxy opens its own short session). +""" + +from fastapi.testclient import TestClient + +import app.state as app_state +from app.main import app +from app.models import PlexItem, PlexLibrary +from app.plex import imgcache +from app.routes.plex import playback + + +class _FakePlex: + calls: list[tuple[str, int | None, int | None]] = [] + + def __init__(self, db) -> None: + pass + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + return None + + def image_bytes(self, key, width=None, height=None): + _FakePlex.calls.append((key, width, height)) + return b"\xff\xd8jpeg", "image/jpeg" + + +def _setup(db, make_user, monkeypatch, tmp_path) -> TestClient: + user = make_user() + lib = PlexLibrary(plex_key="90", title="Movies", kind="movie", enabled=True) + db.add(lib) + db.flush() + db.add( + PlexItem( + rating_key="pm1", + library_id=lib.id, + kind="movie", + title="A Movie", + thumb_key="/library/metadata/pm1/thumb/1", + art_key="/library/metadata/pm1/art/1", + ) + ) + db.commit() + monkeypatch.setattr(app_state, "_configured_cache", True) + monkeypatch.setattr(playback, "resolved_user_id", lambda request: (user.id, True)) + monkeypatch.setattr(playback, "PlexClient", _FakePlex) + monkeypatch.setattr(imgcache, "CACHE_DIR", tmp_path) + _FakePlex.calls = [] + return TestClient(app) + + +def test_each_variant_is_cut_at_its_own_size(db, make_user, monkeypatch, tmp_path): + client = _setup(db, make_user, monkeypatch, tmp_path) + for variant in ("thumb", "thumb_sm", "art", "art_sm"): + assert client.get(f"/api/plex/image/pm1?variant={variant}").status_code == 200 + assert _FakePlex.calls == [ + ("/library/metadata/pm1/thumb/1", 400, 600), + ("/library/metadata/pm1/thumb/1", 240, 360), + ("/library/metadata/pm1/art/1", 1280, 720), + ("/library/metadata/pm1/art/1", 480, 270), + ] + # Each cut is cached under its own name: asking again reaches Plex no more. + for variant in ("thumb", "thumb_sm", "art", "art_sm"): + client.get(f"/api/plex/image/pm1?variant={variant}") + assert len(_FakePlex.calls) == 4 + + +def test_an_unknown_variant_gets_the_full_poster(db, make_user, monkeypatch, tmp_path): + client = _setup(db, make_user, monkeypatch, tmp_path) + assert client.get("/api/plex/image/pm1?variant=huge").status_code == 200 + assert _FakePlex.calls == [("/library/metadata/pm1/thumb/1", 400, 600)] diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index f77cd6c8..abeac71f 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -86,3 +86,31 @@ def test_a_job_runs_audit_row_carries_its_result_as_data(db): {"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}}, {"status": "error", "summary": "boom"}, ] + + +def test_a_failure_the_job_names_rides_along_as_data(db, make_user): + """E31: a failed RSS poll reached the Hungarian audit log as the server's English sentence. A + failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the + dashboard, the manual-run notice and the audit row — so the client says it in the reader's + language. Any other failure keeps only its own text (the test above).""" + from app import progress + from app.models import AuditLog + + def outage(_db): + raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport") + + admin = make_user("failure@example.com", role="admin") + token = scheduler._manual_actor.set(admin.id) + try: + scheduler._job("rss_poll", outage) + finally: + scheduler._manual_actor.reset(token) + + named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"} + rss = _snapshot_job("rss_poll") + assert rss["last_error"] == "RSS poll failed" + assert rss["result"] == named + notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one() + assert notice.data["result"] == named + row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first() + assert row is not None and row.after["result"] == named diff --git a/backend/tests/test_sync_guards.py b/backend/tests/test_sync_guards.py index 824f62d9..8408cbc7 100644 --- a/backend/tests/test_sync_guards.py +++ b/backend/tests/test_sync_guards.py @@ -6,6 +6,7 @@ import logging import httpx import pytest +from app import progress from app.sync import runner from app.sync.playlists import FAILED, _Breaker from app.sync.subscriptions import should_prune @@ -224,8 +225,15 @@ class TestRssPollResult: def dead(cid): raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404") - with pytest.raises(RuntimeError, match="HTTP 404"): + with pytest.raises(RuntimeError, match="HTTP 404") as err: self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",)) + # E31: and as data, for the reader's language — whose feed, and the reason code that tells a + # dead feed (`http_404`) from a dead egress (`transport`). + assert isinstance(err.value, progress.JobFailed) + assert (err.value.reason, err.value.params) == ( + "rss_all_failed", + {"total": 1, "channel": "UC_dead", "cause": "http_404", "read": 0}, + ) def test_a_read_only_database_is_a_total_outage_too(self, monkeypatch): # Every feed reads fine, nothing can be stored: just as un-polled, just as red. diff --git a/backend/tests/test_sysconfig_describe_queries.py b/backend/tests/test_sysconfig_describe_queries.py new file mode 100644 index 00000000..9c1aff47 --- /dev/null +++ b/backend/tests/test_sysconfig_describe_queries.py @@ -0,0 +1,39 @@ +"""The admin Configuration page reads every override in one query (E31 S4d). + +`sysconfig.describe()` did a `db.get` per key to tell whether an override was set, and `get()` did one +more for its value: ~55 keys, and two round trips for a key with no row, since the session cannot cache +an absence. That was ~40 ms of the page's ~55 on prod. It now reads the table once. DB lane. +""" + +from sqlalchemy import event + +from app import sysconfig + + +def _statements(db, fn) -> int: + seen: list[str] = [] + engine = db.get_bind() + listener = lambda *a, **k: seen.append(a[2]) # noqa: E731 — (conn, cursor, statement, …) + event.listen(engine, "before_cursor_execute", listener) + try: + fn() + finally: + event.remove(engine, "before_cursor_execute", listener) + return len(seen) + + +def test_describe_reads_the_overrides_once(db): + sysconfig.set_value(db, "quota_daily_budget", 5000) + db.commit() + assert _statements(db, lambda: sysconfig.describe(db)) == 1 + + +def test_describe_shows_the_stored_value_and_the_default(db): + sysconfig.set_value(db, "quota_daily_budget", 5000) + db.commit() + items = {i["key"]: i for g in sysconfig.describe(db)["groups"].values() for i in g} + assert items["quota_daily_budget"]["is_set"] is True + assert items["quota_daily_budget"]["value"] == 5000 + other = next(i for k, i in items.items() if k != "quota_daily_budget" and not i["secret"]) + assert other["is_set"] is False + assert other["value"] == other["default"] diff --git a/backend/tests/test_sysconfig_registry.py b/backend/tests/test_sysconfig_registry.py index 7a25e5d6..4190f71e 100644 --- a/backend/tests/test_sysconfig_registry.py +++ b/backend/tests/test_sysconfig_registry.py @@ -1,8 +1,10 @@ -"""The sysconfig registry: its order is the admin page's tab order, and a closed-set knob is a CHOICE. +"""The sysconfig registry: a group's keys stay together, every group has its section, and a closed-set +knob is a CHOICE. The admin Configuration page used to keep its own hand-written group order, which had already drifted -(`explore` was missing, so its tab sorted last). The page now follows SPECS, so SPECS has to read in -tab order: one run of keys per group. +(`explore` was missing, so its tab sorted last). The page now follows the registry: SECTIONS orders the +groups under their headings (E31), and a group missing there would fall off the page — so each group +of SPECS sits in exactly one section. A closed-set admin knob is a CHOICE, not free text (E22 S2 follow-up). @@ -22,7 +24,7 @@ from app.config import Settings, choices_of CHOICE_KEYS = {s.key for s in sysconfig.SPECS if s.type == "choice"} -def test_each_group_is_one_run_of_keys_so_the_registry_reads_in_tab_order(): +def test_each_group_is_one_run_of_keys(): runs = [ s.group for i, s in enumerate(sysconfig.SPECS) @@ -32,6 +34,13 @@ def test_each_group_is_one_run_of_keys_so_the_registry_reads_in_tab_order(): assert runs[:2] == ["access", "email"] +def test_every_group_sits_in_exactly_one_section(): + listed = [g for _, members in sysconfig.SECTIONS for g in members] + assert len(listed) == len(set(listed)), listed + assert set(listed) == {s.group for s in sysconfig.SPECS} + assert [sid for sid, _ in sysconfig.SECTIONS] == ["access", "youtube", "media", "system"] + + def test_every_closed_set_knob_is_a_choice_whose_options_come_from_config(): assert CHOICE_KEYS == { "search_source", @@ -58,19 +67,23 @@ def test_the_env_accepts_any_letter_case_and_rejects_an_unknown_value(): class _Row: - def __init__(self, value): - self.value, self.encrypted = value, False + def __init__(self, key, value): + self.key, self.value, self.encrypted = key, value, False class _FakeDb: """Just enough of a Session for sysconfig's reads: one stored override row per key.""" def __init__(self, **rows): - self.rows = {k: _Row(v) for k, v in rows.items()} + self.rows = {k: _Row(k, v) for k, v in rows.items()} def get(self, _model, key): return self.rows.get(key) + def scalars(self, _stmt): + # describe() reads every override at once (E31 S4d). + return list(self.rows.values()) + def test_a_stored_value_reads_back_canonical_and_a_stale_one_falls_back_to_the_default(): assert sysconfig.get_str(_FakeDb(plex_transcode_hw="OFF"), "plex_transcode_hw") == "off" @@ -85,6 +98,14 @@ def test_describe_hands_the_picker_its_options(): assert items["quota_daily_budget"]["choices"] == [] +def test_describe_lists_every_group_under_its_section_in_order(): + d = sysconfig.describe(_FakeDb()) + assert [s["id"] for s in d["sections"]] == ["access", "youtube", "media", "system"] + assert d["sections"][0]["groups"] == ["access", "google", "email"] + listed = [g for s in d["sections"] for g in s["groups"]] + assert sorted(listed) == sorted(d["groups"]) + + def test_set_value_stores_the_canonical_spelling_and_refuses_anything_else(db): sysconfig.set_value(db, "search_source", "API") assert sysconfig.get_str(db, "search_source") == "api" diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 1bd47d65..091f402a 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -1,8 +1,8 @@ -"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a).""" +"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a); to_tsquery_str().""" from datetime import date, datetime, timezone -from app.utils import iso, normalize_description +from app.utils import iso, normalize_description, to_tsquery_str def test_none_maps_to_none(): @@ -56,3 +56,19 @@ def test_normalize_description_single_line_unchanged(): def test_normalize_description_folds_crlf_line_endings(): # splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization. assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc" + + +def test_tsquery_prefix_matches_a_last_word_of_three_or_more(): + assert to_tsquery_str("ti amo magyar") == "ti & amo & magyar:*" + assert to_tsquery_str("min") == "min:*" + + +def test_tsquery_matches_a_short_last_word_whole(): + # A 1-2 letter prefix made ts_rank score most of the library (5-8 s on prod, E31 S4e). + assert to_tsquery_str("m") == "m" + assert to_tsquery_str("minecraft 4k") == "minecraft & 4k" + assert to_tsquery_str("ti amo m") == "ti & amo & m" + + +def test_tsquery_without_a_word_is_none(): + assert to_tsquery_str("?! --") is None diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 2350d0d1..606120b0 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -39,8 +39,8 @@ runs this check before it is called done (E28 S3g): 4. **Seen.** A screenshot of every changed state, thin edges at 1:1; the tablet widths (1024 × 768, a narrow split screen) when the layout can reflow. Phones are the E29 epic's, not this check's. 5. **Judged as a whole** (E30 S6). For each screenshot, one written line on its composition: where the empty - space is, what hugs one side, what the eye lands on first. Then the screen at 200% browser zoom and at - the largest text size. Both are held to [Composition and the reader](#composition-and-the-reader--decided-e30-s6). + space is, what hugs one side, what the eye lands on first. Then the screen at 125% and 133% browser + zoom and at the largest text size. All are held to [Composition and the reader](#composition-and-the-reader--decided-e30-s6). A rule that holds is not the same as a screen that works: the person page passed every rule above with half its panel empty. @@ -465,8 +465,9 @@ of a full-width panel and nothing on the right. (TMDB, YouTube) is fetched ahead of time by a job, not on the click, so a page never waits on it. Late content keeps its place (B8). [Nielsen #1, RAIL] - **Zoom and text size are first-class.** The owner reads with the browser zoomed and a large text size. - Every screen holds at 200% browser zoom on a desktop window and at the largest Settings text size at - 1024 wide: nothing clipped or overlapping, no sideways page scroll, every control reachable and at + Every screen holds at 125% and 133% browser zoom on a 1080p desktop window (what the owner uses: the + layout of a 1536 × 729 and a 1444 × 685 viewport) and at the largest Settings text size at 1024 wide: + nothing clipped or overlapping, no sideways page scroll, every control reachable and at least its rung, text wrapping rather than cut. Measured, not assumed, on every UI change. [WCAG 1.4.4, 1.4.10, 1.4.12] @@ -688,3 +689,16 @@ of a full-width panel and nothing on the right. | 2026-09-27 | Match switch on the panel | `ui/MatchToggle` above the picker from two picks, everywhere; one wording (Any / All) | The picker's popover footer (Feed, Plex collection) | A closed picker hid an active "all"; six pickers had drawn it two ways | | 2026-09-27 | Missing picture, bare link | A glyph of the kind for no picture; a download with no name is titled by its short link, once | An empty box; the URL as title, channel and link | The owner's picks 4A and 3A: an empty box read as loading, the URL showed three times | | 2026-09-27 | Copy on a selection | Right-click on selected words keeps the browser's menu; no app Copy row | An app menu with Copy first | A real browser's menu has copy, search and translate; only the desktop pane's is thin | +| 2026-10-01 | Zoom target 125–133% | Screens are held at 125% and 133% browser zoom (1536 × 729, 1444 × 685 on a 1080p window) and the largest text size | 200% browser zoom | The owner: 200% was a guess; 125–133% is what they read Siftlode and most sites at, so 200% would be an upper bound nobody uses | +| 2026-10-01 | E31 Nav rail at short heights | One "Admin" rail item opening a panel of the admin pages | A denser rail when short; a visible "+4" hint | The rail fits at 133–150% without a hidden scroll; the admin pages are rare and admin-only | +| 2026-10-01 | E31 Configuration sections | A vertical, grouped section list, as Settings has | Group tabs + sub-tabs; one long page with a contents list | Sixteen tabs wrapped to three rows at 133%; one pattern for both settings pages | +| 2026-10-01 | E31 Statistics overview | Number tiles, and the per-operation counts as horizontal bars | Three narrow cards; a 30-day chart | The label sits by its value; what spends the quota is seen at a glance | +| 2026-10-01 | E31 Channel page header | The full banner, folding to one row once the grid scrolls | Always compact; no banner | The header is fixed and took 40% of a 685px window; the channel keeps its face at first sight | +| 2026-10-01 | E31 Channel page watch state | A fresh channel page shows the unwatched videos; a picked state is remembered | Show all, forgotten on reload | The owner: open a channel to see what is new; a choice made there should stick | +| 2026-10-01 | E31 Audit log runs | A job's successful runs fold into one row that opens, whatever ran between; only that job's failure (never folded, danger colour) breaks it | A filter defaulting to admin actions; both | Nothing hidden, the noise collapsed, the failure seen | +| 2026-10-01 | E31 Users row | Role and state as badges, the actions in the row's one menu | An inline role toggle; icon-only buttons | Action placement: the row shows state, rare and destructive actions live in the menu | +| 2026-10-01 | E31 YouTube player | Video beside an info panel at 1280px and wider (the video as large as the height allows); stacked below | Today's stack with a one-line bar; full-window like Plex | The owner's eyesight: about 1460px of video on 1080p instead of 1008, the actions still visible | +| 2026-10-01 | E31 Plex info and show pages | A second column for the facts, keywords and crew, as the person page; a collection's hero sized to its content | Full-width text; a larger poster and fanart | The right ~40% was empty; one composition for every media page | +| 2026-10-01 | E31 Nav rail rows are links | Each page row is a real link: a middle-click or Ctrl-click opens the page in a new tab | A button | The owner: open a page in a tab of its own, as on any site | +| 2026-10-02 | E31 Plex detail second column (refined) | The facts over the key crew beside the text; the keywords, chips that filter like the genres, under the text; the show page gets all three too | All three in the column, as the mockup drew it | At 20 keywords and 12 crew the column ran to twice the text's height (849 vs ~400 px at 133%): a hole under the text instead of beside it | +| 2026-10-02 | E31 Plex detail poster and crew | The movie and show poster keeps its 2:3 and grows a rung in a wide hero (w-56 at a 64rem panel); the crew shows its first 6 names and "Show N more" (a list of 8 or fewer shows whole) | Poster stretched to the row; a "More" on the keywords | The row is as tall as the crew column, so the poster drew ~175×590 and cropped; the crew, not the keywords, set the hero's height | diff --git a/frontend/e2e/channel.spec.ts b/frontend/e2e/channel.spec.ts index b3b46db9..2b9a3c23 100644 --- a/frontend/e2e/channel.spec.ts +++ b/frontend/e2e/channel.spec.ts @@ -106,28 +106,34 @@ test.describe("channel view", () => { await expectContentType(page, "includeLive", false); }); - // Platform S4b: the whole channel chrome (banner, identity, tabs, toolbar) is fixed — it stays - // put while only the video grid scrolls. - test("the channel chrome stays fixed while the video grid scrolls", async ({ page }) => { + // Platform S4b + E31 S4c: the channel chrome sits outside the scroller. Scrolling into the videos + // folds the banner and identity once (the name moves into the tab row); from then on the folded + // chrome stays put while only the video grid scrolls. + test("the channel chrome folds once, then stays fixed while the grid scrolls", async ({ + page, + }) => { await openApp(page); await page.getByTestId("video-card-channel").first().click(); const count = page.getByTestId("item-count"); - const title = page.getByTestId("channel-title"); await expect(count).toBeVisible(); await expect(page.getByTestId("video-card").first()).toBeVisible(); - const titleTop = await title.evaluate((el) => el.getBoundingClientRect().top); - const countTop = await count.evaluate((el) => el.getBoundingClientRect().top); + await expect(page.getByTestId("channel-folded")).toHaveCount(0); // Keep nudging until the grid is tall enough to actually scroll (it may still be filling in). await expect .poll(async () => { - await page.getByRole("main").evaluate((el) => el.scrollBy(0, 1500)); + await page.getByRole("main").evaluate((el) => el.scrollBy(0, 600)); return page.getByRole("main").evaluate((el) => el.scrollTop); }) .toBeGreaterThan(400); + const folded = page.getByTestId("channel-folded"); + await expect(folded).toBeVisible(); + await expect(folded.getByTestId("channel-title")).toBeVisible(); + const titleTop = await folded.evaluate((el) => el.getBoundingClientRect().top); + const countTop = await count.evaluate((el) => el.getBoundingClientRect().top); - // The chrome didn't move — banner title and toolbar are in exactly the same place. - expect(await title.evaluate((el) => el.getBoundingClientRect().top)).toBe(titleTop); + await page.getByRole("main").evaluate((el) => el.scrollBy(0, 400)); + expect(await folded.evaluate((el) => el.getBoundingClientRect().top)).toBe(titleTop); expect(await count.evaluate((el) => el.getBoundingClientRect().top)).toBe(countTop); }); }); diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 4500b12e..3d1b998a 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -547,6 +547,15 @@ export default tseslint.config( // lib/*.contract.test.ts). "no-restricted-syntax": [ "error", + // A next-page fetch called bare CANCELS the one in flight and sends the same cursor again; + // our requests ignore the abort, so the server ran both (E31 S4e: every feed page 2–3×). + // lib/nextPage.ts NEXT_PAGE makes the second call join the first. + { + selector: + "CallExpression[arguments.length=0]:matches([callee.name='fetchNextPage'], [callee.property.name='fetchNextPage'])", + message: + "fetchNextPage() cancels and re-sends a page already loading: call fetchNextPage(NEXT_PAGE) (lib/nextPage.ts).", + }, // `enabled:` sees only the native attribute; a FocusSafeButton that holds the focus is // disabled by aria-disabled, so a bare `enabled:hover:` lit it up on hover as if it worked. { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index deac0a4c..2b0db8f3 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -16,7 +16,7 @@ import { type PanelLayout, } from "./lib/panelLayout"; import { notify, removeByMetaKind } from "./lib/notifications"; -import { LS, setAccountRaw } from "./lib/storage"; +import { LS, setTabRaw } from "./lib/storage"; import { usePref } from "./lib/usePref"; import { useVerifyEntry } from "./lib/useVerifyEntry"; import { useNavigation, useNavigationActions } from "./components/navigationContext"; @@ -249,7 +249,7 @@ export default function App() { if (result === "ok") { notify({ level: "success", message: t("settings.account.googleLink.linked") }); void refetchMe(); - setAccountRaw(LS.settingsTab, "account"); + setTabRaw(LS.settingsTab, "account"); setPage("settings"); } else { const key = result === "conflict" || result === "mismatch" ? result : "error"; diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index bfc0a8e0..7c4a13d7 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -13,6 +13,7 @@ import { api, type LogFacet, type LogQuery, type ServerLogRow } from "../lib/api import { copyText } from "../lib/clipboard"; import { formatDateTimeSec } from "../lib/format"; import { nextNavId, notifyNav, rememberNavPosition } from "../lib/history"; +import { nextPageOnClick } from "../lib/nextPage"; import { LEVEL_GROUPS, type LevelGroup } from "../lib/log"; import { DEFAULT_LOGS_STATE, @@ -156,7 +157,8 @@ export default function AdminLogs() { ? "countsFailed" : null; const retryFailed = () => { - if (logs.isFetchNextPageError) void logs.fetchNextPage(); + if (logs.isFetchNextPageError) + void logs.fetchNextPage(nextPageOnClick(logs.isFetchingNextPage)); else if (logs.isError) void logs.refetch(); if (facets.isError) void facets.refetch(); }; @@ -190,7 +192,7 @@ export default function AdminLogs() { const loadOlder = () => { setLive(false); - void logs.fetchNextPage(); + void logs.fetchNextPage(nextPageOnClick(logs.isFetchingNextPage)); }; const copy = async () => { diff --git a/frontend/src/components/AdminUsers.dom.test.tsx b/frontend/src/components/AdminUsers.dom.test.tsx index 81c87530..c6eb94b3 100644 --- a/frontend/src/components/AdminUsers.dom.test.tsx +++ b/frontend/src/components/AdminUsers.dom.test.tsx @@ -116,13 +116,63 @@ describe("a user row", () => { }); }); +/** The open menu's rows (a closed popover's are hidden from the accessibility tree in jsdom). */ +const menuRows = () => screen.queryAllByRole("menuitem", { hidden: true }); + +/** The row's actions live in its "…" menu (E31): open it, then pick the row. */ +async function chooseInRowMenu(label: string) { + fireEvent.click(await screen.findByRole("button", { name: "users.rowMenu" })); + const item = await waitFor(() => { + const found = menuRows().find((m) => m.textContent?.startsWith(label)); + if (!found) throw new Error(`no menu row "${label}"`); + return found; + }); + fireEvent.click(item); +} + +// E31 (DESIGN § Action placement): the role, the suspension and the delete moved off the row into +// ONE menu — the row's "…", which a right-click on the row opens too. A locked one stays listed with +// its reason. +describe("a user row's menu", () => { + it("holds the role, the suspension and the delete; the row shows no action button", async () => { + mount(); + const open = await screen.findByTitle("users.account.open"); + const card = open.parentElement!; + const buttons = [...card.querySelectorAll("button")].filter((b) => b !== open); + expect(buttons.map((b) => b.getAttribute("aria-label"))).toEqual(["users.rowMenu"]); + fireEvent.contextMenu(card, { clientX: 5, clientY: 5 }); + await waitFor(() => expect(menuRows().length).toBe(3)); + expect(menuRows().map((i) => i.textContent)).toEqual([ + "users.roles.makeAdmin", + "users.suspend.action", + "users.delete.action", + ]); + }); + + it("lists your own row's actions as locked, with the reason", async () => { + vi.spyOn(api, "adminUsers").mockResolvedValue([{ ...row, id: 1, email: "admin@x" }] as never); + render( + + + , + ); + fireEvent.click(await screen.findByRole("button", { name: "users.rowMenu" })); + await waitFor(() => expect(menuRows().length).toBe(3)); + const [role, suspend, del] = menuRows(); + expect(role!.hasAttribute("disabled")).toBe(true); + expect(role!.textContent).toBe("users.roles.makeAdminusers.roles.selfLocked"); + expect(suspend!.textContent).toBe("users.suspend.actionusers.suspend.selfLocked"); + expect(del!.textContent).toBe("users.delete.actionusers.delete.locked"); + }); +}); + describe("a user action that fails", () => { it("says a user who is gone, and refetches the list", async () => { const users = mount(); vi.spyOn(globalThis, "fetch").mockResolvedValue( answer(404, { code: "not_found", reason: "user" }), ); - fireEvent.click(await screen.findByText("users.roles.makeAdmin")); + await chooseInRowMenu("users.roles.makeAdmin"); await waitFor(() => expect(notify).toHaveBeenCalledWith({ level: "error", @@ -136,7 +186,7 @@ describe("a user action that fails", () => { it("is not toasted on top of the error dialog the request already opened", async () => { mount(); vi.spyOn(globalThis, "fetch").mockResolvedValue(answer(409, "Cannot demote the last admin")); - fireEvent.click(await screen.findByText("users.roles.makeAdmin")); + await chooseInRowMenu("users.roles.makeAdmin"); const { reportError } = await import("../lib/errorDialog"); await waitFor(() => expect(reportError).toHaveBeenCalled()); expect(notify.mock.calls.filter(([n]) => n.level === "error")).toEqual([]); diff --git a/frontend/src/components/AdminUsers.tsx b/frontend/src/components/AdminUsers.tsx index 13d9e541..7717cf5e 100644 --- a/frontend/src/components/AdminUsers.tsx +++ b/frontend/src/components/AdminUsers.tsx @@ -41,11 +41,15 @@ import AccountPanel from "./AccountPanel"; import { useConfirm } from "./confirmContext"; import Tabs from "./ui/Tabs"; import { useTabPanel } from "./ui/tabPanel"; -import { useAccountPersistedState } from "../lib/storage"; +import { useTabPersistedState } from "../lib/storage"; import { PageToolbar } from "./ui/PageShell"; import { useMe } from "../lib/useMe"; import { notifyActionError, notifyKnownFailure } from "../lib/actionErrors"; import { Pressable } from "./ui/Pressable"; +import { useContextMenu } from "./ui/useContextMenu"; +import { ContextMenu } from "./ui/ContextMenu"; +import { MoreButton } from "./ui/MoreButton"; +import type { ActionMenuItem } from "./ui/ActionMenu"; import { Input } from "./ui/Input"; import { SURFACE } from "./ui/surfaces"; @@ -65,7 +69,7 @@ const ROW_OPEN = export default function AdminUsers() { const me = useMe(); const { t } = useTranslation(); - const [tab, setTab] = useAccountPersistedState(ADMIN_USERS_TAB_KEY, "roles"); + const [tab, setTab] = useTabPersistedState(ADMIN_USERS_TAB_KEY, "roles"); const panel = useTabPanel(); const tabs = [ { id: "roles", label: t("users.tabs.roles") }, @@ -206,10 +210,58 @@ function UsersRoles({ me }: { me: Me }) { if (ok) del.mutate({ id: u.id, email: u.email }); }; + // The row's actions are ONE menu (DESIGN § Action placement, E31): the role, the suspension and + // the delete — rare and two of them destructive — open from the row's "…", a right-click on the row, + // a long-press or Shift+F10. The row keeps what the admin reads: the name, the role and state + // badges, the last sign-in. A locked action (the demo, yourself) stays listed with its reason. + const rowMenu = useContextMenu({ keyOf: (u) => u.id }); + const menuItems = (u: AdminUserRow): ActionMenuItem[] => { + const self = u.id === me.id; + const pending = pendingIds.has(u.id); + const locked = (demoNote: string, selfNote: string) => + u.is_demo ? demoNote : self ? selfNote : undefined; + const roleNote = locked(t("users.roles.demoLocked"), t("users.roles.selfLocked")); + const suspendNote = locked(t("users.suspend.demoLocked"), t("users.suspend.selfLocked")); + const deleteNote = u.is_demo || self ? t("users.delete.locked") : undefined; + return [ + { + id: "role", + label: u.role === "admin" ? t("users.roles.makeUser") : t("users.roles.makeAdmin"), + icon: , + disabled: pending || !!roleNote, + note: roleNote, + onSelect: () => onToggle(u), + }, + { + id: "suspend", + label: u.is_suspended ? t("users.suspend.undoAction") : t("users.suspend.action"), + icon: u.is_suspended ? : , + disabled: pending || !!suspendNote, + note: suspendNote, + onSelect: () => onSuspend(u), + }, + { + id: "delete", + label: t("users.delete.action"), + icon: , + danger: true, + separatorBefore: true, + disabled: pending || !!deleteNote, + note: deleteNote, + onSelect: () => onDelete(u), + }, + ]; + }; + const rows = q.data ?? []; return (
+ {panelFor && setPanelFor(null)} />}

{t("users.roles.intro")}

{q.isError && !q.data ? ( @@ -232,6 +284,7 @@ function UsersRoles({ me }: { me: Me }) { // stretched layer. The hover used to be the name button's own and ended at the first badge.
{t("users.roles.unverified")} )} - - - - - onSuspend(u)} - aria-disabled={u.is_demo || self} - disabled={pendingIds.has(u.id)} - > - {u.is_suspended ? ( - - ) : ( - - )} - - - - onDelete(u)} - aria-disabled={u.is_demo || self} - disabled={pendingIds.has(u.id)} - > - - - - {/* How long ago this account last signed in. LAST in the row on purpose: the badges and - the role button before it vary in width row to row, so only the row's own right - edge lines the values up into one column. The exact moment is the hint here and a + {/* How long ago this account last signed in. At the row's right end on purpose (only the + fixed-width "…" after it): the badges before it vary in width row to row, so only + the right edge lines the values up into one column. The exact moment is the hint here and a line in the account panel; below `sm` the row has no room, and the panel still says it. */} +
); })} diff --git a/frontend/src/components/AuditEarlierRuns.tsx b/frontend/src/components/AuditEarlierRuns.tsx new file mode 100644 index 00000000..62036b7e --- /dev/null +++ b/frontend/src/components/AuditEarlierRuns.tsx @@ -0,0 +1,48 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import clsx from "clsx"; +import { ChevronRight } from "lucide-react"; +import type { AuditRow } from "../lib/api"; +import { Button } from "./ui/Button"; +import { RelativeTime } from "./ui/RelativeTime"; + +/** A folded run of one job's successful runs (lib/auditGroups, E31): the row shows the newest, this + * opens the earlier ones, each with its moment and its result (`detailOf`, the row's own wording). */ +export default function AuditEarlierRuns({ + runs, + detailOf, +}: { + runs: AuditRow[]; + detailOf: (run: AuditRow) => string | null; +}) { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + const earlier = runs.slice(1); + return ( +
+ + {open && ( +
    + {earlier.map((run) => ( +
  • + + {detailOf(run)} +
    • + ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/AuditLog.tsx b/frontend/src/components/AuditLog.tsx index b034905b..b2a5c97c 100644 --- a/frontend/src/components/AuditLog.tsx +++ b/frontend/src/components/AuditLog.tsx @@ -14,6 +14,7 @@ import { LoadingState, StateMessage } from "./ui/QueryState"; import { auditColumns } from "./auditColumns"; import { notifyActionError } from "../lib/actionErrors"; import { compareText } from "../lib/collate"; +import { foldJobRuns } from "../lib/auditGroups"; // Admin-only audit trail: an append-only who/what/when log of admin actions + scheduler run // summaries (never per-row). Read-only apart from Clear. Client-side sort/filter/paginate via @@ -25,7 +26,8 @@ export default function AuditLog() { const confirm = useConfirm(); const q = useQuery({ queryKey: qk.adminAudit(), queryFn: () => api.adminAudit() }); const data = q.data; - const rows = useMemo(() => data?.items ?? [], [data]); + // Back-to-back successful runs of one job fold into one row (lib/auditGroups, E31). + const rows = useMemo(() => foldJobRuns(data?.items ?? []), [data]); const actionOptions = useMemo(() => { // By what the reader sees — the named action, in their alphabet — not by the code. diff --git a/frontend/src/components/ChannelDiscovery.tsx b/frontend/src/components/ChannelDiscovery.tsx index 3e0896f8..96164322 100644 --- a/frontend/src/components/ChannelDiscovery.tsx +++ b/frontend/src/components/ChannelDiscovery.tsx @@ -242,6 +242,7 @@ export default function ChannelDiscovery({ onSortChange={setSort} emptyText={t("channels.discovery.empty")} keyboardNav + narrowedBy={q} /> ) : ( ({ + ...(await importOriginal()), + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), +})); +vi.mock("./navigationContext", () => ({ + useNavigationActions: () => ({ closeChannel: vi.fn(), openChannel: vi.fn() }), +})); +vi.mock("./confirmContext", () => ({ useConfirm: () => async () => true })); +vi.mock("./feedFiltersContext", () => ({ + useFeedFilters: () => ({ includeNormal: true, includeShorts: false, includeLive: false }), +})); +vi.mock("./Feed", () => ({ default: () => null })); +vi.mock("./ChannelAbout", () => ({ default: () => null })); +vi.mock("./ui/PageShell", async (importOriginal) => ({ + ...(await importOriginal()), + PageToolbar: ({ children }: { children?: ReactNode }) => <>{children}, +})); + +import ChannelPage from "./ChannelPage"; +import { CONTENT_ID } from "./ui/landmarks"; +import { api, type ChannelDetail, type Me } from "../lib/api"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + document.getElementById(CONTENT_ID)?.remove(); + vi.restoreAllMocks(); +}); + +const CH = "UCqoAEDirJPjEUFcF2FklnBA"; +const BANNER = 'img[src^="https://yt3."]'; + +// The page's one scroller (PageScroller's
), with a scroll position jsdom does not lay out. +function scroller() { + const main = document.createElement("main"); + main.id = CONTENT_ID; + let top = 0; + Object.defineProperty(main, "scrollTop", { get: () => top, configurable: true }); + Object.defineProperty(main, "scrollHeight", { value: 6000, configurable: true }); + Object.defineProperty(main, "clientHeight", { value: 600, configurable: true }); + document.body.appendChild(main); + return (to: number) => + act(() => { + top = to; + main.dispatchEvent(new Event("scroll")); + }); +} + +// Renders the page and waits for the detail to land (the banner shows). +async function renderPage() { + vi.spyOn(api, "channelDetail").mockResolvedValue({ + id: CH, + title: "StarTalk", + banner_url: "https://yt3.googleusercontent.com/banner", + subscribed: true, + blocked: false, + hidden: false, + deep_requested: false, + backfill_done: true, + stored_videos: 10, + tag_ids: [], + } as unknown as ChannelDetail); + vi.spyOn(api, "tags").mockResolvedValue([] as never); + const view = render( + + {}} + onShowInFeed={() => {}} + /> + , + ); + await waitFor(() => expect(view.container.querySelector(BANNER)).not.toBeNull()); + return view; +} + +describe("ChannelPage — the header folds while the grid scrolls", () => { + it("drops the banner and puts the name in the tab row once scrolled, and brings them back at the top", async () => { + const scrollTo = scroller(); + const { container } = await renderPage(); + const banner = () => container.querySelector(BANNER); + expect(banner()).not.toBeNull(); + expect(screen.queryByTestId("channel-folded")).toBeNull(); + + scrollTo(400); + expect(banner()).toBeNull(); + expect(screen.getByTestId("channel-folded").textContent).toContain("StarTalk"); + + scrollTo(10); + expect(screen.queryByTestId("channel-folded")).not.toBeNull(); + scrollTo(0); + expect(banner()).not.toBeNull(); + expect(screen.queryByTestId("channel-folded")).toBeNull(); + }); + + it("keeps a focused Back focused through the fold", async () => { + const scrollTo = scroller(); + await renderPage(); + const back = screen.getByTestId("channel-back"); + back.focus(); + scrollTo(400); + expect(screen.getByTestId("channel-folded")).toBeTruthy(); + expect(document.activeElement).toBe(back); + }); +}); diff --git a/frontend/src/components/ChannelPage.tsx b/frontend/src/components/ChannelPage.tsx index a15c2ca3..758a3b78 100644 --- a/frontend/src/components/ChannelPage.tsx +++ b/frontend/src/components/ChannelPage.tsx @@ -24,11 +24,15 @@ import { import Avatar from "./ui/Avatar"; import Tabs from "./ui/Tabs"; import { useTabPanel } from "./ui/tabPanel"; +import { useScrollFold } from "./ui/scrollFold"; import Feed from "./Feed"; import ChannelAboutContent from "./ChannelAbout"; import { PageToolbar } from "./ui/PageShell"; import { PageToolbarSlot } from "./ui/pageToolbarSlot"; import { useFeedFilters } from "./feedFiltersContext"; +import { SHOW_IDS } from "./feedFiltersInit"; +import { usePref } from "../lib/usePref"; +import { LS } from "../lib/storage"; import { useConfirm } from "./confirmContext"; import { notify } from "../lib/notifications"; import { notifyYouTubeActionError } from "../lib/youtubeErrors"; @@ -86,6 +90,10 @@ export default function ChannelPage({ // where the channelScoped Feed portals ITS toolbar (via the overridden PageToolbarSlot below), so // the toolbar lands under the tabs inside the fixed band rather than in the scroller. const [chromeSlot, setChromeSlot] = useState(null); + // The banner and the identity block fold away once the grid scrolls (E31, the owner's pick): the chrome + // is fixed, so unfolded it kept ~40% of a 133% screen for good. Folded, the name rides in the tab row. + const foldable = useRef(null); + const folded = useScrollFold(foldable); // The banner comes from Google's image CDN, which refuses any request carrying a Referer from our // origin — every time, not only in a burst (measured 2026-09-26) — so the sends none // (`referrerPolicy`, as ui/Avatar). A load can still drop, and the browser NEGATIVE-CACHES the miss, @@ -260,7 +268,13 @@ export default function ChannelPage({ // Channel-scoped feed: the whole catalog filtered to this channel (scope=all + source=all so // the explored, per-user-private videos show — the backend gates them to this explorer). - const [filters, setFilters] = useState(() => ({ + // The watch state is the one filter a channel page remembers (the owner, E31): a fresh page shows + // what is still unwatched, and a change sticks — across channels, reloads and devices. + const [show, setShow] = usePref("channelShow", "unwatched", { + localKey: LS.channelShow, + parse: (v) => (typeof v === "string" && SHOW_IDS.includes(v) ? v : "unwatched"), + }); + const [ownFilters, setOwnFilters] = useState(() => ({ tags: [], tagModes: DEFAULT_TAG_MODES, q: "", @@ -271,9 +285,14 @@ export default function ChannelPage({ includeNormal: globalFilters.includeNormal, includeShorts: globalFilters.includeShorts, includeLive: globalFilters.includeLive, - show: "all", + show: "unwatched", channelIds: [channelId], })); + const filters: FeedFilters = { ...ownFilters, show }; + const setFilters = (next: FeedFilters) => { + if (next.show !== show) setShow(next.show); + setOwnFilters(next); + }; const name = ch?.title ?? initialName ?? channelId; const ytUrl = channelYouTubeUrl(channelId, ch?.handle); @@ -306,251 +325,292 @@ export default function ChannelPage({ : null, joined ? t("channel.joined", { date: joined }) : null, ].filter(Boolean) as string[]; + // The page's name and its state, shown in the identity block or, folded, in the tab row. + const heading = ( + + {name} + + ); + const stateBadge = ch?.blocked ? ( + + {t("channel.blockedBadge")} + + ) : ( + ch?.explored && + !ch?.subscribed && ( + + {t("channel.exploringBadge")} + + ) + ); // The channel page owns no scroller of its own (PageShell provides the one app scroller). Its // whole chrome is fixed — wrapped in it portals into the shell's fixed band, so the - // banner/identity/tabs/toolbar stay put and only the video grid scrolls. + // tabs/toolbar stay put and only the video grid scrolls; the banner/identity fold away meanwhile. const bannerSrc = ch?.banner_url ? `${ch.banner_url}=w1707${bannerAttempt ? `?r=${bannerAttempt}` : ""}` : null; return ( <> - {/* Banner + back — OPTION C: inset, rounded card (not full-bleed) */} -
- {bannerSrc && bannerAttempt <= MAX_BANNER_RETRIES ? ( - // Match YouTube's banner: the stored bannerExternalUrl is the full 16:9 template at a - // low default size, so request a crisp wide version (=w1707) and object-cover the - // desktop "safe area" — the centre 2560×423 (~6:1) band. -
- per attempt; the ?r= cache-buster on a retry bypasses the - // browser's negative cache (googleusercontent accepts an extra query param). - key={bannerAttempt} - src={bannerSrc} - referrerPolicy="no-referrer" - alt="" - className={`w-full h-full object-cover object-center ${ - bannerLoaded === bannerSrc ? "" : "opacity-0" - }`} - onLoad={() => setBannerLoaded(bannerSrc)} - onError={() => { - if (bannerAttempt <= MAX_BANNER_RETRIES) { - window.setTimeout( - () => setBannerAttempt((a) => a + 1), - 500 * (bannerAttempt + 1), - ); - } - }} - /> -
- ) : ( - // No banner (or a dead one): a neutral bar the SAME height as a real banner, so the - // overlapping avatar + the Back button have the same room and don't collide (a short bar - // let the -mt-8 avatar ride up into the Back button). -
- )} - -
- -
- {/* Identity + actions. relative+z-base so the avatar that overlaps up into the banner - paints ABOVE it (the banner's positioned container would otherwise cover it). */} -
- -
-
- - {name} - - {ch?.blocked ? ( - - {t("channel.blockedBadge")} - +
+ {!folded && ( +
+ {/* Banner — OPTION C: inset, rounded card (not full-bleed) */} +
+ {bannerSrc && bannerAttempt <= MAX_BANNER_RETRIES ? ( + // Match YouTube's banner: the stored bannerExternalUrl is the full 16:9 template at a + // low default size, so request a crisp wide version (=w1707) and object-cover the + // desktop "safe area" — the centre 2560×423 (~6:1) band. +
+ per attempt; the ?r= cache-buster on a retry bypasses the + // browser's negative cache (googleusercontent accepts an extra query param). + key={bannerAttempt} + src={bannerSrc} + referrerPolicy="no-referrer" + alt="" + className={`w-full h-full object-cover object-center ${ + bannerLoaded === bannerSrc ? "" : "opacity-0" + }`} + onLoad={() => setBannerLoaded(bannerSrc)} + onError={() => { + if (bannerAttempt <= MAX_BANNER_RETRIES) { + window.setTimeout( + () => setBannerAttempt((a) => a + 1), + 500 * (bannerAttempt + 1), + ); + } + }} + /> +
) : ( - ch?.explored && - !ch?.subscribed && ( - - {t("channel.exploringBadge")} - - ) + // No banner (or a dead one): a neutral bar the SAME height as a real banner, so the + // overlapping avatar + the Back button have the same room and don't collide (a short bar + // let the -mt-8 avatar ride up into the Back button). +
)}
-
- {metaParts.map((part, i) => ( - - {i > 0 && ·} - {part} - - ))} -
-
-
- {exploring && ( -
- - {t("channel.loadingVideos")} +
+ {/* Identity. relative+z-base so the avatar that overlaps up into the banner paints + ABOVE it (the banner's positioned container would otherwise cover it). */} +
+ +
+
+ {heading} + {stateBadge} +
+
+ {metaParts.map((part, i) => ( + + {i > 0 && ·} + {part} + + ))} +
+
+
+
)} - {/* Tabs — the channel actions ride along here rather than off in the header's top-right +
+ {exploring && ( +
+ + {t("channel.loadingVideos")} +
+ )} + + {/* Tabs — the channel actions ride along here rather than off in the header's top-right corner: there are only ever two tabs, so there's room, and it keeps the things you click within a few pixels of each other instead of across the page. */} -
- setTab(id as "videos" | "about")} - tabs={[ - { id: "videos", label: t("channel.tabVideos") }, - { id: "about", label: t("channel.tabAbout") }, - ]} - /> -
- + {/* One Back button in both states, so a focused Back keeps its focus when the header folds: + over the banner's corner (absolute to the chrome) while unfolded, first in this row folded. */} + - )} - {!me.is_demo && ( - block.mutate()} - loading={block.isPending} - label={ch?.blocked ? t("channel.unblock") : t("channel.block")} - > - - - )} - {/* Hide-from-feed toggle — subscription-scoped, so offered only on a followed channel + {ch?.subscribed && !ch.backfill_done && !me.is_demo && ( + + )} + {!me.is_demo && ( + block.mutate()} + loading={block.isPending} + label={ch?.blocked ? t("channel.unblock") : t("channel.block")} + > + + + )} + {/* Hide-from-feed toggle — subscription-scoped, so offered only on a followed channel (mirrors the manager's Eye/EyeOff). Keeps the subscription; only drops its videos from the feed. EyeOff = currently hidden (click to unhide). */} - {ch?.subscribed && !me.is_demo && ( - toggleHidden.mutate()} - loading={toggleHidden.isPending} - label={ch.hidden ? t("channel.unhide") : t("channel.hide")} - title={ch.hidden ? t("channel.hiddenHint") : t("channel.hideHint")} - > - {ch.hidden ? : } - - )} - {!me.is_demo && - !ch?.blocked && - (ch?.subscribed ? ( - - ) : ( - - ))} - {/* Tagging is a subscription-scoped action (attach_tag → _user_subscription 404s + {ch.hidden ? : } + + )} + {!me.is_demo && + !ch?.blocked && + (ch?.subscribed ? ( + + ) : ( + + ))} + {/* Tagging is a subscription-scoped action (attach_tag → _user_subscription 404s otherwise), and personal tags are for your subscriptions — so only offer it on a channel you actually follow. */} - {ch?.subscribed && !me.is_demo && ( - toggleTag(channelId, tagId, ch.tag_ids.includes(tagId))} - onCreate={(name) => createAndAttachTag(channelId, name)} - // The header row has free width — show tags inline instead of collapsing to "+N". - bounded={false} - /> - )} + {ch?.subscribed && !me.is_demo && ( +
+ toggleTag(channelId, tagId, ch.tag_ids.includes(tagId))} + onCreate={(name) => createAndAttachTag(channelId, name)} + // The header row's leftover width: tags inline, "+N" only when they do not fit. + bounded={false} + /> +
+ )} +
diff --git a/frontend/src/components/ChannelPage.watchState.dom.test.tsx b/frontend/src/components/ChannelPage.watchState.dom.test.tsx new file mode 100644 index 00000000..1533fa0b --- /dev/null +++ b/frontend/src/components/ChannelPage.watchState.dom.test.tsx @@ -0,0 +1,109 @@ +// @vitest-environment jsdom +// +// E31 (the owner): a freshly opened channel page shows what is still unwatched, and a watch state the +// viewer picks there sticks — a reload (a fresh mount, seeded from the stored preference) keeps it. +import type { ReactNode } from "react"; +import { describe, it, expect, vi, beforeAll, afterEach } from "vitest"; +import { render, cleanup, act } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { FeedFilters } from "../lib/api"; + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), +})); +vi.mock("./navigationContext", () => ({ + useNavigationActions: () => ({ closeChannel: vi.fn(), openChannel: vi.fn() }), +})); +vi.mock("./confirmContext", () => ({ useConfirm: () => async () => true })); +vi.mock("./feedFiltersContext", () => ({ + useFeedFilters: () => ({ includeNormal: true, includeShorts: false, includeLive: false }), +})); +const feed = vi.hoisted(() => ({ + last: null as null | { filters: FeedFilters; setFilters: (f: FeedFilters) => void }, +})); +vi.mock("./Feed", () => ({ + default: (p: { filters: FeedFilters; setFilters: (f: FeedFilters) => void }) => { + feed.last = p; + return null; + }, +})); +vi.mock("./ChannelAbout", () => ({ default: () => null })); +vi.mock("./ui/PageShell", async (importOriginal) => ({ + ...(await importOriginal()), + PageToolbar: ({ children }: { children?: ReactNode }) => <>{children}, +})); + +import ChannelPage from "./ChannelPage"; +import { api, type ChannelDetail, type Me } from "../lib/api"; +import { qk } from "../lib/queryKeys"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + feed.last = null; +}); + +const CH = "UCqoAEDirJPjEUFcF2FklnBA"; + +function mount(preferences: Record = {}) { + vi.spyOn(api, "channelDetail").mockResolvedValue({ + id: CH, + title: "StarTalk", + subscribed: true, + stored_videos: 10, + } as unknown as ChannelDetail); + vi.spyOn(api, "tags").mockResolvedValue([] as never); + const save = vi.spyOn(api, "savePrefs").mockResolvedValue({} as never); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), { id: 1, preferences } as unknown as Me); + render( + + {}} + onShowInFeed={() => {}} + /> + , + ); + return save; +} + +describe("ChannelPage — the watch state", () => { + it("shows only the unwatched videos on a freshly opened page", () => { + mount(); + expect(feed.last?.filters.show).toBe("unwatched"); + }); + + it("stores a picked watch state", () => { + const save = mount(); + act(() => feed.last!.setFilters({ ...feed.last!.filters, show: "all" })); + expect(feed.last?.filters.show).toBe("all"); + expect(save).toHaveBeenCalledWith({ channelShow: "all" }); + }); + + it("keeps the stored watch state after a reload", () => { + mount({ channelShow: "watched" }); + expect(feed.last?.filters.show).toBe("watched"); + }); + + it("falls back to unwatched for a stored value it does not know", () => { + mount({ channelShow: "bogus" }); + expect(feed.last?.filters.show).toBe("unwatched"); + }); +}); diff --git a/frontend/src/components/Channels.tsx b/frontend/src/components/Channels.tsx index 58cd49e9..59730f94 100644 --- a/frontend/src/components/Channels.tsx +++ b/frontend/src/components/Channels.tsx @@ -754,6 +754,7 @@ export default function Channels() { keyboardNav reveal={follow} surfacedFilters={SURFACED_FILTERS} + narrowedBy={q} /> ) : ( <> diff --git a/frontend/src/components/ConfigPanel.choice.dom.test.tsx b/frontend/src/components/ConfigPanel.choice.dom.test.tsx index d22dab5f..cc7b8ff1 100644 --- a/frontend/src/components/ConfigPanel.choice.dom.test.tsx +++ b/frontend/src/components/ConfigPanel.choice.dom.test.tsx @@ -72,6 +72,7 @@ vi.mock("../lib/api", async (importOriginal) => { choice("log_level", LEVELS, "INFO"), ], }, + sections: [{ id: "s", groups: ["g"] }], secrets_manageable: true, }), setConfig: (key: string, value: unknown) => setConfig(key, value), diff --git a/frontend/src/components/ConfigPanel.field.dom.test.tsx b/frontend/src/components/ConfigPanel.field.dom.test.tsx index 9d697748..49e1c407 100644 --- a/frontend/src/components/ConfigPanel.field.dom.test.tsx +++ b/frontend/src/components/ConfigPanel.field.dom.test.tsx @@ -46,6 +46,7 @@ vi.mock("../lib/api", async (importOriginal) => { groups: { g: [text("site_name"), { ...text("signups_open"), type: "bool", value: "false" }], }, + sections: [{ id: "s", groups: ["g"] }], secrets_manageable: true, }), }, diff --git a/frontend/src/components/ConfigPanel.sections.dom.test.tsx b/frontend/src/components/ConfigPanel.sections.dom.test.tsx new file mode 100644 index 00000000..388634cf --- /dev/null +++ b/frontend/src/components/ConfigPanel.sections.dom.test.tsx @@ -0,0 +1,92 @@ +// @vitest-environment jsdom +// +// E31 (the owner's pick): Configuration's sixteen group tabs wrapped to three rows at 133% zoom, so they +// are a side list under four headings, as Settings has. The registry owns the order: the groups follow +// its sections, not the order the groups object happens to arrive in. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { ConfigItem } from "../lib/api"; + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), + }; +}); + +const text = (key: string, group: string): ConfigItem => ({ + key, + type: "str", + group, + secret: false, + min: null, + max: null, + is_set: false, + allow_empty: true, + choices: [], + value: "", + default: "", +}); + +vi.mock("../lib/api", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + api: { + ...actual.api, + adminConfig: () => + Promise.resolve({ + groups: { b: [text("kb", "b")], c: [text("kc", "c")], a: [text("ka", "a")] }, + sections: [ + { id: "one", groups: ["a", "b"] }, + { id: "two", groups: ["c"] }, + ], + secrets_manageable: true, + }), + }, + }; +}); +vi.mock("../lib/useMe", () => ({ useMe: () => ({ email_configured: true }) })); + +import ConfigPanel from "./ConfigPanel"; + +afterEach(() => { + cleanup(); + localStorage.clear(); +}); + +const renderPanel = () => + render( + + + , + ); + +describe("Configuration sections (E31)", () => { + it("lists the groups in the registry's sections, each heading once above its run", async () => { + renderPanel(); + const list = await screen.findByRole("tablist"); + expect(list.getAttribute("aria-orientation")).toBe("vertical"); + // The list's own lines, top to bottom: headings and tabs. + const lines = Array.from(list.children).map((el) => + el.getAttribute("role") === "tab" ? `tab:${el.textContent}` : `head:${el.textContent}`, + ); + expect(lines).toEqual([ + "head:config.sections.one", + "tab:config.groups.a", + "tab:config.groups.b", + "head:config.sections.two", + "tab:config.groups.c", + ]); + }); + + it("opens on the first group and switches to the one picked", async () => { + renderPanel(); + expect(await screen.findByText("config.fields.ka.label")).toBeTruthy(); + fireEvent.click(screen.getByRole("tab", { name: "config.groups.c" })); + expect(screen.getByText("config.fields.kc.label")).toBeTruthy(); + expect(screen.queryByText("config.fields.ka.label")).toBeNull(); + }); +}); diff --git a/frontend/src/components/ConfigPanel.tsx b/frontend/src/components/ConfigPanel.tsx index ddbdf971..57c8d81d 100644 --- a/frontend/src/components/ConfigPanel.tsx +++ b/frontend/src/components/ConfigPanel.tsx @@ -11,12 +11,11 @@ import { useMe } from "../lib/useMe"; import Tabs from "./ui/Tabs"; import { useTabPanel } from "./ui/tabPanel"; import { LoadingState, StateMessage } from "./ui/QueryState"; -import { PageToolbar } from "./ui/PageShell"; import { Switch } from "./ui/form"; import SegmentedToggle from "./ui/SegmentedToggle"; import FilterSelect from "./ui/FilterSelect"; import { DraftSaveBar, type SaveState } from "./ui/DraftSaveBar"; -import { LS, useAccountPersistedState } from "../lib/storage"; +import { LS, useTabPersistedState } from "../lib/storage"; import { notifyActionError } from "../lib/actionErrors"; import { Button } from "./ui/Button"; import { CheckRow } from "./ui/NativeControls"; @@ -57,7 +56,7 @@ export default function ConfigPanel() { const [reseedToken, setReseedToken] = useState(0); // One tab per config group (persisted). Called before the loading guard so hook order is // stable; the active id is clamped to a real group at render time once data has loaded. - const [tab, setTab] = useAccountPersistedState(LS.configTab); + const [tab, setTab] = useTabPersistedState(LS.configTab); // A stable signature of the config SHAPE — changes only when keys are added/removed, not when an // unchanged refetch produces a new baseline object. Re-seed the draft on a shape change (initial @@ -176,146 +175,152 @@ export default function ConfigPanel() { return ; } - const groupKeys = Object.keys(data.groups); + // The groups in the registry's sections, in order (E31: sixteen tabs wrapped to three rows at 133% + // zoom, so they are a side list under four headings, as Settings has). + const groupKeys = data.sections.flatMap((s) => s.groups); // Clamp the persisted tab to a real group (the registry can change between sessions). The // tab pill carries a dot when that group has unsaved edits, so a draft in a hidden tab is // never silently lost behind the global Save bar. const activeGroup = groupKeys.includes(tab) ? tab : groupKeys[0]; const dirtyByGroup = new Set(dirtyKeys.map((k) => byKey[k]?.group).filter(Boolean)); - const groupTabs = groupKeys.map((g) => ({ - id: g, - label: `${t(`config.groups.${g}`, g)}${dirtyByGroup.has(g) ? " •" : ""}`, - })); + const groupTabs = data.sections.flatMap((s) => + s.groups.map((g) => ({ + id: g, + group: t(`config.sections.${s.id}`, s.id), + label: `${t(`config.groups.${g}`, g)}${dirtyByGroup.has(g) ? " •" : ""}`, + testId: `config-tab-${g}`, + })), + ); return ( <> - {/* The tab bar is the page's band (B4); the intro scrolls with the fields — in the band it - took three lines of a short window for text read once. */} - -
+
+

{t("config.intro")}

+ {/* The groups as a side list under their headings, the fields beside it: Settings' own + layout, one pattern for both settings pages (E31). */} +
-
- -
-

{t("config.intro")}

- {activeGroup && ( -
-
- {(data.groups[activeGroup] ?? []).map((item) => ( - setDraft((d) => ({ ...d, [item.key]: v }))} - onResetField={() => { - if (item.secret) setSecretReset((s) => ({ ...s, [item.key]: !s[item.key] })); - else setDraft((d) => ({ ...d, [item.key]: "" })); - }} - /> - ))} -
+ {activeGroup && ( +
+
+ {(data.groups[activeGroup] ?? []).map((item) => ( + setDraft((d) => ({ ...d, [item.key]: v }))} + onResetField={() => { + if (item.secret) setSecretReset((s) => ({ ...s, [item.key]: !s[item.key] })); + else setDraft((d) => ({ ...d, [item.key]: "" })); + }} + /> + ))} +
- {activeGroup === "email" && ( -
- {/* The one place an operator can see that mail is off. Sending is fail-soft, so a + {activeGroup === "email" && ( +
+ {/* The one place an operator can see that mail is off. Sending is fail-soft, so a half-filled config costs verification links, resets and access-request notices with no error anywhere — say so next to the fields that fix it. */} - {!me.email_configured && {t("config.emailOff")}} - - - - {dirty &&

{t("config.testDirty")}

} -
- )} + {!me.email_configured && {t("config.emailOff")}} + + + + {dirty &&

{t("config.testDirty")}

} +
+ )} - {activeGroup === "issues" && ( -
- - - - {dirty &&

{t("config.testDirty")}

} -
- )} + {activeGroup === "issues" && ( +
+ + + + {dirty &&

{t("config.testDirty")}

} +
+ )} - {activeGroup === "plex" && ( -
- - - - {dirty &&

{t("config.testDirty")}

} - {plexResult && ( -
-

- {t("config.plexConnected", { - name: plexResult.server_name, - version: plexResult.version ?? "", - })} -

- {plexResult.media_check?.checked && - (plexResult.media_check.ok ? ( -

{t("config.plexMediaOk")}

- ) : ( -

- {t("config.plexMediaMissing", { - path: plexResult.media_check.local_path ?? "", - })} -

- ))} -

{t("config.plexPickLibraries")}

-
- {plexResult.sections.map((s) => ( - togglePlexLib(s.key)} - className="text-sm" - > - {s.title} - - {s.type === "movie" ? t("config.plexMovies") : t("config.plexShows")} - {s.count != null ? ` · ${s.count}` : ""} - - - ))} + {activeGroup === "plex" && ( +
+ + + + {dirty &&

{t("config.testDirty")}

} + {plexResult && ( +
+

+ {t("config.plexConnected", { + name: plexResult.server_name, + version: plexResult.version ?? "", + })} +

+ {plexResult.media_check?.checked && + (plexResult.media_check.ok ? ( +

{t("config.plexMediaOk")}

+ ) : ( +

+ {t("config.plexMediaMissing", { + path: plexResult.media_check.local_path ?? "", + })} +

+ ))} +

{t("config.plexPickLibraries")}

+
+ {plexResult.sections.map((s) => ( + togglePlexLib(s.key)} + className="text-sm" + > + {s.title} + + {s.type === "movie" ? t("config.plexMovies") : t("config.plexShows")} + {s.count != null ? ` · ${s.count}` : ""} + + + ))} +
+

{t("config.plexPickHint")}

-

{t("config.plexPickHint")}

-
- )} -
- )} -
- )} + )} +
+ )} +
+ )} +
({ + ...(await importOriginal()), + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), +})); +vi.mock("./wizardContext", () => ({ useWizard: () => ({ openWizard: vi.fn() }) })); +vi.mock("./VirtualFeed", () => ({ default: () => null })); +vi.mock("./SelectedViewChip", () => ({ default: () => null })); +vi.mock("./ActiveFilterChips", () => ({ default: () => null })); + +import Feed from "./Feed"; +import { api, DEFAULT_TAG_MODES, type FeedFilters } from "../lib/api"; +import { qk } from "../lib/queryKeys"; +import type { ReactElement } from "react"; +import { PageToolbarSlot } from "./ui/pageToolbarSlot"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +const FILTERS: FeedFilters = { + tags: [], + tagModes: DEFAULT_TAG_MODES, + channelIds: [], + q: "", + sort: "newest", + scope: "my", + includeNormal: true, + includeShorts: false, + includeLive: false, + show: "unwatched", +}; + +// The pill lives in the toolbar, which portals into the shell's band: give it one. +const slot = document.body.appendChild(document.createElement("div")); +const feed = (q: string, qc: QueryClient): ReactElement => ( + + + {}} + view="cards" + setView={() => {}} + canRead + ytSearch="" + onYtSearch={() => {}} + onExitYtSearch={() => {}} + /> + + +); + +describe("Feed — the count while a new search's count loads", () => { + it("dims the previous count and marks it busy until the new one lands", async () => { + vi.spyOn(api, "feed").mockResolvedValue({ items: [], next_cursor: null } as never); + let settle: (v: unknown) => void = () => {}; + vi.spyOn(api, "feedSummary").mockImplementation(((f: FeedFilters) => + f.q === "kitchen" + ? Promise.resolve({ count: 1937, facets: {} }) + : new Promise((r) => (settle = r))) as never); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), { id: 1, can_write: true, prefs: {} }); + const pill = () => screen.getByTestId("item-count"); + + const view = render(feed("kitchen", qc)); + await waitFor(() => expect(pill().getAttribute("title")).toBe("1,937")); + expect(pill().getAttribute("aria-busy")).toBeNull(); + + view.rerender(feed("a", qc)); + await waitFor(() => expect(pill().getAttribute("aria-busy")).toBe("true")); + expect(pill().getAttribute("title")).toBe("1,937"); // still the old figure, now marked as such + + settle({ count: 178991, facets: {} }); + await waitFor(() => expect(pill().getAttribute("title")).toBe("178,991")); + expect(pill().getAttribute("aria-busy")).toBeNull(); + }); + + it("says the count failed instead of showing the empty box of a first load", async () => { + vi.spyOn(api, "feed").mockResolvedValue({ items: [], next_cursor: null } as never); + vi.spyOn(api, "feedSummary").mockImplementation(((f: FeedFilters) => + f.q === "kitchen" + ? Promise.resolve({ count: 1937, facets: {} }) + : Promise.reject(new Error("boom"))) as never); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), { id: 1, can_write: true, prefs: {} }); + const pill = () => screen.getByTestId("item-count"); + + const view = render(feed("kitchen", qc)); + await waitFor(() => expect(pill().getAttribute("title")).toBe("1,937")); + view.rerender(feed("a", qc)); + await waitFor(() => expect(pill().getAttribute("title")).toBe("common.countFailed")); + expect(pill().getAttribute("aria-busy")).toBeNull(); + }); +}); diff --git a/frontend/src/components/Feed.nextPageFailed.dom.test.tsx b/frontend/src/components/Feed.nextPageFailed.dom.test.tsx new file mode 100644 index 00000000..ee75c06d --- /dev/null +++ b/frontend/src/components/Feed.nextPageFailed.dom.test.tsx @@ -0,0 +1,114 @@ +// @vitest-environment jsdom +// +// A next page that fails. Retry is off and the read-ahead re-runs whenever a fetch settles, so a failed +// page was requested again once a round-trip, forever, while the end of the list was in view, and +// nothing said it had failed (E31 review 2). The read-ahead now stops at a failed page, and a row says +// so and retries on a click. VirtualFeed is replaced by what Feed hands it: the read-ahead's gate and a +// button standing in for the read-ahead itself. +import { describe, it, expect, vi, beforeAll, afterEach } from "vitest"; +import { render, screen, waitFor, cleanup, fireEvent } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), +})); +vi.mock("./wizardContext", () => ({ useWizard: () => ({ openWizard: vi.fn() }) })); +vi.mock("./VirtualFeed", () => ({ + default: ({ + hasNextPage, + fetchNextPage, + }: { + hasNextPage: boolean; + fetchNextPage: (o: { cancelRefetch: boolean }) => unknown; + }) => ( +
+ {String(hasNextPage)} + +
+ ), +})); +vi.mock("./SelectedViewChip", () => ({ default: () => null })); +vi.mock("./ActiveFilterChips", () => ({ default: () => null })); + +import Feed from "./Feed"; +import { api, DEFAULT_TAG_MODES, type FeedFilters } from "../lib/api"; +import { qk } from "../lib/queryKeys"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +const FILTERS: FeedFilters = { + tags: [], + tagModes: DEFAULT_TAG_MODES, + channelIds: [], + q: "", + sort: "newest", + scope: "my", + includeNormal: true, + includeShorts: false, + includeLive: false, + show: "unwatched", +}; + +describe("Feed — a next page that fails", () => { + it("stops the read-ahead, says so, and loads the page on a retry", async () => { + const page = (n: number, next: string | null) => ({ + items: [ + { id: `v${n}`, title: `v${n}`, channel_id: "UCx", published_at: "2026-01-01T00:00:00Z" }, + ], + next_cursor: next, + }); + let failNext = true; + const feed = vi + .spyOn(api, "feed") + .mockImplementation(((_f: FeedFilters, cursor: string | null) => + cursor === null + ? Promise.resolve(page(1, "c2")) + : failNext + ? Promise.reject(new Error("boom")) + : Promise.resolve(page(2, null))) as never); + vi.spyOn(api, "feedSummary").mockResolvedValue({ count: 2, facets: {} } as never); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), { id: 1, can_write: true, prefs: {} }); + render( + + {}} + view="cards" + setView={() => {}} + canRead + ytSearch="" + onYtSearch={() => {}} + onExitYtSearch={() => {}} + /> + , + ); + await waitFor(() => expect(screen.getByTestId("has-next").textContent).toBe("true")); + + fireEvent.click(screen.getByText("read ahead")); + await waitFor(() => expect(screen.getByText("common.loadMoreFailed")).toBeTruthy()); + expect(screen.getByTestId("has-next").textContent).toBe("false"); // the read-ahead stands down + const calls = feed.mock.calls.length; + + failNext = false; + fireEvent.click(screen.getByRole("button", { name: /retry/i })); + await waitFor(() => expect(feed.mock.calls.length).toBe(calls + 1)); + await waitFor(() => expect(screen.queryByText("common.loadMoreFailed")).toBeNull()); + }); +}); diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index 37a82723..bd10d7e1 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -7,6 +7,7 @@ import { useQueryClient, } from "@tanstack/react-query"; import { qk } from "../lib/queryKeys"; +import { nextPageOnClick } from "../lib/nextPage"; import { ArrowLeft, Columns2, @@ -37,6 +38,7 @@ import { } from "../lib/feedSort"; import { type SortOption, type SortState } from "../lib/columnSort"; import { usePref } from "../lib/usePref"; +import { SHOW_IDS } from "./feedFiltersInit"; import { FEED_VIEWS, type FeedView } from "../lib/feedView"; import { clearedFilters, useActiveFilters } from "../lib/useActiveFilters"; import ActiveFilterChips from "./ActiveFilterChips"; @@ -64,7 +66,6 @@ const PAGE = 60; // The "Show" view filter, content-type filter and ordering live in the feed toolbar // (above the cards), not the filter sidebar. -const SHOW_IDS = ["unwatched", "in_progress", "all", "watched", "hidden"]; const CONTENT = [ { key: "includeNormal", label: "sidebar.content.normal" }, { key: "includeShorts", label: "sidebar.content.shorts" }, @@ -549,14 +550,16 @@ export default function Feed({ onResetState={onResetState} onOpen={openVideo} hasNextPage={false} - isFetchingNextPage={false} + isFetching={false} fetchNextPage={() => {}} /> {ytQuery.hasNextPage && (