From 67c9f101537f84d5b58a65179ebb966a822f224c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 29 Sep 2026 22:39:57 +0200 Subject: [PATCH] =?UTF-8?q?feat(i18n):=20every=20server=20A=E2=86=92Z=20li?= =?UTF-8?q?st=20in=20the=20reader's=20alphabet=20(E30=20S3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Postgres image is Alpine, whose en_US.utf8 is byte order: "Életrajzi" and "Ötödik" sorted after "Zorró" in the feed, the channel lists, the Plex grid, its genres and collections. `app/collation.py` names the ICU collation of the reader's language (hu-x-icu puts "Ö" after "O" and "Cs" after "Cu"), wraps every text ORDER BY with it - the feed's keyset cursor compares with the same expression, so paging walks the order it sorted in - and orders the lists built in Python (genre and studio facets, collection members, the discovery re-sort) through the same collation. Plex title tiebreaks use the shown (HU) title. --- CLAUDE.md | 2 +- backend/app/README.md | 3 +- backend/app/collation.py | 39 ++++ backend/app/feedquery.py | 11 +- backend/app/routes/channels.py | 18 +- backend/app/routes/downloads.py | 4 +- backend/app/routes/feed.py | 3 +- backend/app/routes/messages.py | 4 +- backend/app/routes/plex/browse.py | 20 +- backend/app/routes/plex/collections.py | 27 ++- backend/app/routes/plex/detail.py | 11 +- backend/app/routes/plex/discover.py | 4 +- backend/app/routes/plex/people.py | 11 +- backend/tests/test_collation.py | 209 ++++++++++++++++++ ....test.ts => localeFormat.contract.test.ts} | 0 15 files changed, 328 insertions(+), 38 deletions(-) create mode 100644 backend/app/collation.py create mode 100644 backend/tests/test_collation.py rename frontend/src/lib/{dateFormat.contract.test.ts => localeFormat.contract.test.ts} (100%) diff --git a/CLAUDE.md b/CLAUDE.md index 9f1f4240..5a0f01ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,7 +60,7 @@ how it gets built. Concretely: `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, `downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`, -`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, +`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`, `security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and diff --git a/backend/app/README.md b/backend/app/README.md index 9fb2cac1..ec7ba9ef 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -23,7 +23,8 @@ Three layers, by *who* runs the code — not by feature: 3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log - level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core + level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the + reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core module must never import a `routes/…` module). **Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a diff --git a/backend/app/collation.py b/backend/app/collation.py new file mode 100644 index 00000000..a69c6730 --- /dev/null +++ b/backend/app/collation.py @@ -0,0 +1,39 @@ +"""Alphabetical order in the reader's language (E30 S3). + +Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE +order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the +server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text +sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in. + +`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression +(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python +list the same way through the database, so a list built in Python matches one built in SQL exactly.""" + +from typing import Any + +from sqlalchemy import SQLColumnExpression, column, func, select, values +from sqlalchemy.orm import Session +from sqlalchemy.sql.elements import ColumnElement +from sqlalchemy.types import String + +from app.mailtext import normalize + +ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"} + + +def collation(lang: str | None) -> str: + """The ICU collation for a UI language; anything unknown sorts as English.""" + return ICU[normalize(lang)] + + +def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]: + """`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together.""" + return func.lower(expr).collate(collation(lang)) + + +def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]: + """`words` in the reader's alphabetical order, by the same ICU collation as `collated`.""" + if not words: + return [] + w = values(column("w", String), name="words").data([(x,) for x in words]) + return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w))) diff --git a/backend/app/feedquery.py b/backend/app/feedquery.py index b4e17c9d..1023ac2d 100644 --- a/backend/app/feedquery.py +++ b/backend/app/feedquery.py @@ -15,6 +15,7 @@ from datetime import date, datetime, timedelta, timezone from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select from sqlalchemy.orm import Session, aliased +from app.collation import collated from app.models import ( LIVE_OR_UPCOMING, BlockedChannel, @@ -478,8 +479,10 @@ def feed_years(db: Session, user: User, **params) -> list[dict]: # replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows, # and the page boundary is stable even as the scheduler ingests new videos mid-scroll # (OFFSET would shift and duplicate/skip rows). -def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]: +def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> list[dict]: + """`lang` orders the title sorts in the reader's alphabet (`collation`); the other keys ignore it.""" pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"} + title = collated(Video.title, lang) table: dict[str, list[dict]] = { "newest": [pub], "oldest": [{**pub, "asc": True}], @@ -491,10 +494,8 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]: "duration_asc": [ {"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"} ], - "title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}], - "title_desc": [ - {"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"} - ], + "title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}], + "title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}], "subscribers": [ {"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"} ], diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index f28a9d20..9c9c3a52 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -9,6 +9,8 @@ from pydantic import BaseModel from sqlalchemy import func, select, update from sqlalchemy.orm import Session +from app.collation import collated, sorted_words +from app.mailtext import lang_for_user from app import logs, quota from app.auth import admin_user, current_user, require_human, require_write_scope from app.db import get_db @@ -44,7 +46,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_ select(Subscription, Channel) .join(Channel, Channel.id == Subscription.channel_id) .where(Subscription.user_id == user.id) - .order_by(Subscription.priority.desc(), func.lower(Channel.title)) + .order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user))) ).all() # Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last @@ -139,7 +141,7 @@ def _discovery_rows(db: Session, user: User) -> list: return db.execute( q.group_by(Channel.id).order_by( func.count(func.distinct(PlaylistItem.video_id)).desc(), - func.lower(Channel.title), + collated(Channel.title, lang_for_user(user)), ) ).all() @@ -192,10 +194,14 @@ def discover_channels( ): apply_channel_details(db, yt.get_channels(need), need) db.commit() - # Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST, - # as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up). + # Match _discovery_rows' ORDER BY exactly: video-count DESC, then title in the reader's + # alphabet (NULLs LAST, as Postgres orders them — coercing NULL→"" would wrongly float + # untitled channels up). + titles = [r[0].title for r in rows if r[0].title is not None] + alpha = {w: i for i, w in enumerate(sorted_words(db, titles, lang_for_user(user)))} rows = sorted( - rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower()) + rows, + key=lambda r: (-int(r[1]), r[0].title is None, alpha.get(r[0].title or "", 0)), ) except YouTubeError as exc: log.warning( @@ -659,7 +665,7 @@ def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_d select(Channel) .join(BlockedChannel, BlockedChannel.channel_id == Channel.id) .where(BlockedChannel.user_id == user.id) - .order_by(func.lower(Channel.title)) + .order_by(collated(Channel.title, lang_for_user(user))) ) .scalars() .all() diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 99d2affa..af75086e 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -21,6 +21,8 @@ from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.orm import Session +from app.mailtext import lang_for_user +from app.collation import collated from app import audit, sysconfig from app.auth import admin_user, require_human from app.config import settings @@ -988,7 +990,7 @@ def share_recipients( db.execute( select(User) .where(User.id != user.id, *messageable_clauses()) - .order_by(func.lower(User.email)) + .order_by(collated(User.email, lang_for_user(user))) ) .scalars() .all() diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index fb5edcb3..4ae00d95 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError from app import quota from app.auth import current_user from app.db import get_db +from app.mailtext import lang_for_user from app.feedquery import ( decode_cursor, encode_cursor, @@ -108,7 +109,7 @@ def get_feed( db: Session = Depends(get_db), ) -> dict: query, rank_expr = filtered_query(db, user, **params) - keys = sort_keys(sort, seed, rank_expr) + keys = sort_keys(sort, seed, rank_expr, lang_for_user(user)) # Expose each key column on the row so we can build the next cursor from the last item. query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) diff --git a/backend/app/routes/messages.py b/backend/app/routes/messages.py index 1cbe96fd..342bda78 100644 --- a/backend/app/routes/messages.py +++ b/backend/app/routes/messages.py @@ -21,6 +21,8 @@ from pydantic import BaseModel from sqlalchemy import and_, case, delete, func, or_, select from sqlalchemy.orm import Session +from app.mailtext import lang_for_user +from app.collation import collated from app.auth import require_human, resolved_user_id from app.db import SessionLocal, get_db from app.models import Message, MessageKey, User @@ -246,7 +248,7 @@ def list_recipients( select(User) .join(MessageKey, MessageKey.user_id == User.id) .where(User.id != user.id, *messageable_clauses()) - .order_by(func.lower(func.coalesce(User.display_name, User.email))) + .order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user))) ) .scalars() .all() diff --git a/backend/app/routes/plex/browse.py b/backend/app/routes/plex/browse.py index 61fc25f7..ab56d020 100644 --- a/backend/app/routes/plex/browse.py +++ b/backend/app/routes/plex/browse.py @@ -15,6 +15,8 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Session, aliased +from app.mailtext import lang_for_user +from app.collation import collated, sorted_words from app import sysconfig from app.auth import current_user from app.db import get_db @@ -55,14 +57,15 @@ from app.utils import to_tsquery_str router = APIRouter(prefix="/api/plex", tags=["plex"]) -def sort_column(u, sort: str): +def sort_column(u, sort: str, lang: str = "en"): """The browse grid's sort key -> orderable column on the union subquery. Module-level (not inline in the handler) so a test can pin its vocabulary to `PLEX_SORT_KEYS` — a key the Sort control offers but this cannot order would quietly fall back to `added_at` and look like a broken control. - An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback.""" + An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback. The title sorts + in the reader's alphabet (`collation`).""" return { "added": u.c.added_at, - "title": func.lower(u.c.title), + "title": collated(u.c.title, lang), "year": u.c.year, "rating": u.c.rating, "duration": u.c.duration_s, @@ -79,7 +82,7 @@ def list_libraries(user: User = Depends(current_user), db: Session = Depends(get db.execute( select(PlexLibrary) .where(PlexLibrary.overlay_of.is_(None)) - .order_by(PlexLibrary.kind.desc(), PlexLibrary.title) + .order_by(PlexLibrary.kind.desc(), collated(PlexLibrary.title, lang_for_user(user))) ) .scalars() .all() @@ -238,7 +241,7 @@ def unified_library( if tsq is not None: order = [u.c.rank.desc()] else: - col = sort_column(u, sort) + col = sort_column(u, sort, lang) order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()] order.append(u.c.rk.desc()) rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all() @@ -530,16 +533,19 @@ def facets( for c in split_csv(f.playable): if c in PLAYBACK_CLASSES: playable_counts.setdefault(c, 0) + # Names in the reader's alphabet: the genres are Hungarian words for a Hungarian reader (E30 S1). + alpha = {w: i for i, w in enumerate(sorted_words(db, [*genre_counts, *studio_counts], lang))} return { "genres": [ - {"value": g, "count": c} for g, c in sorted(genre_counts.items(), key=lambda kv: kv[0]) + {"value": g, "count": c} + for g, c in sorted(genre_counts.items(), key=lambda kv: alpha[kv[0]]) ], "content_ratings": [ {"value": cr, "count": c} for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1]) ], "studios": [ {"value": s, "count": c} - for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], kv[0])) + for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]])) ], # Fixed good→heavy order (not by count), so the class list reads as a severity scale. "playable": [ diff --git a/backend/app/routes/plex/collections.py b/backend/app/routes/plex/collections.py index 0f315677..239c2ed3 100644 --- a/backend/app/routes/plex/collections.py +++ b/backend/app/routes/plex/collections.py @@ -9,6 +9,8 @@ from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.orm import Session +from app.mailtext import lang_for_user +from app.collation import collated, sorted_words from app import logs from app.auth import admin_user, current_user from app.db import get_db @@ -25,7 +27,7 @@ from app.plex import sync as plex_sync from app.plex.cards import display_title, movie_card, norm_lang, show_card from app.plex.client import PlexClient, PlexError, PlexNotConfigured from app.plex.lookups import by_rating_key -from app.plex.query import content_libs, runtime_by_key, top_art_keys +from app.plex.query import content_libs, runtime_by_key, title_col, top_art_keys from app.utils import like_escape from app.refusal import refused from app.routes._common import not_found @@ -142,7 +144,7 @@ def collections( db.execute( query.order_by( PlexCollection.child_count.desc().nullslast(), - func.lower(PlexCollection.title), + collated(PlexCollection.title, lang_for_user(user)), ) ) .scalars() @@ -208,16 +210,24 @@ MEMBER_SORT_KEYS: dict[str, Callable[[Member], Any]] = { DEFAULT_MEMBER_SORT = "year" -def sort_members(members: list[Member], sort: str, sort_dir: str) -> list[Member]: +def sort_members( + members: list[Member], sort: str, sort_dir: str, alpha: dict[str, int] | None = None +) -> list[Member]: """Order a collection's members by one of `PLEX_SORT_KEYS`. Two properties worth stating because they are easy to get wrong with a plain `reverse=True`: members with no value for the key sink to the END in BOTH directions (matching the browse grid's `nullslast()` — a movie with no rating is not "the worst rated"), and ties always resolve title-ascending, so flipping the direction reverses the ranking rather than the alphabet inside - it. An unknown key falls back to the page's default instead of returning an arbitrary order.""" + it. An unknown key falls back to the page's default instead of returning an arbitrary order. + + `alpha` ranks each title in the reader's alphabet (`collation.sorted_words`); without it, titles + compare as Python strings, which puts every accented initial after "z".""" + by_title = (lambda m: alpha[m.title]) if alpha is not None else (lambda m: m.title) key = MEMBER_SORT_KEYS.get(sort, MEMBER_SORT_KEYS[DEFAULT_MEMBER_SORT]) - ordered = sorted(members, key=lambda m: m.title) # stable secondary, applied first + if key is MEMBER_SORT_KEYS["title"]: + key = by_title + ordered = sorted(members, key=by_title) # stable secondary, applied first have = [m for m in ordered if key(m) is not None] missing = [m for m in ordered if key(m) is None] have.sort(key=key, reverse=sort_dir != "asc") @@ -246,7 +256,7 @@ def collection_detail( db.execute( select(PlexItem) .where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key])) - .order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title)) + .order_by(PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang)) ) .scalars() .all() @@ -255,7 +265,7 @@ def collection_detail( db.execute( select(PlexShow) .where(PlexShow.collection_keys.contains([rating_key])) - .order_by(PlexShow.year.asc().nullslast(), func.lower(PlexShow.title)) + .order_by(PlexShow.year.asc().nullslast(), collated(title_col(PlexShow, lang), lang)) ) .scalars() .all() @@ -310,7 +320,8 @@ def collection_detail( ) for sh in shows ] - members = sort_members(members, sort, sort_dir) + alpha = {w: i for i, w in enumerate(sorted_words(db, [m.title for m in members], lang))} + members = sort_members(members, sort, sort_dir, alpha) items = [ movie_card(m.obj, mstate.get(m.obj.id), lang=lang) if m.kind == "movie" diff --git a/backend/app/routes/plex/detail.py b/backend/app/routes/plex/detail.py index db6cadb9..fe2a9aee 100644 --- a/backend/app/routes/plex/detail.py +++ b/backend/app/routes/plex/detail.py @@ -7,9 +7,10 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from fastapi import APIRouter, Depends -from sqlalchemy import func, select +from sqlalchemy import select from sqlalchemy.orm import Session +from app.collation import collated, sorted_words from app import logs from app.auth import current_user from app.db import get_db @@ -25,6 +26,7 @@ from app.models import ( from app.plex import detail as plex_detail from app.plex import people as plex_people from app.plex import rollup, subtitles +from app.plex.query import title_col from app.plex import tmdb as plex_tmdb from app.plex.cards import episode_card, movie_card, show_card from app.plex.client import PlexClient, PlexError, PlexNotConfigured @@ -277,7 +279,8 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int, lang: str) -> li ).scalars() } # Smallest (most specific — e.g. a franchise) first; the big auto-lists (IMDb/TMDb Popular) last. - ordered = sorted((c for c in cols.values()), key=lambda c: (c.child_count or 0, c.title)) + alpha = {w: i for i, w in enumerate(sorted_words(db, [c.title for c in cols.values()], lang))} + ordered = sorted(cols.values(), key=lambda c: (c.child_count or 0, alpha[c.title])) out = [] for col in ordered: rk = col.rating_key @@ -289,7 +292,9 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int, lang: str) -> li PlexItem.kind == "movie", PlexItem.collection_keys.contains([rk]), ) - .order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title)) + .order_by( + PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang) + ) .limit(60) ) .scalars() diff --git a/backend/app/routes/plex/discover.py b/backend/app/routes/plex/discover.py index 39b91033..05d6936b 100644 --- a/backend/app/routes/plex/discover.py +++ b/backend/app/routes/plex/discover.py @@ -18,6 +18,7 @@ from fastapi import APIRouter, Depends, Query from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session +from app.collation import sorted_words from app.auth import current_user from app.db import get_db from app.models import PlexItem, PlexShow, PlexState, User @@ -264,7 +265,8 @@ def _genre_tiles( expanded = base.subquery() for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all(): counts[g] = counts.get(g, 0) + c - top = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:GENRE_TILES] + alpha = {w: i for i, w in enumerate(sorted_words(db, list(counts), lang))} + top = sorted(counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]]))[:GENRE_TILES] # A 2×2 poster collage per tile: the top movies in that genre (color hash is the client fallback). art = top_art_keys( db, diff --git a/backend/app/routes/plex/people.py b/backend/app/routes/plex/people.py index 053d2dec..3c4b4c38 100644 --- a/backend/app/routes/plex/people.py +++ b/backend/app/routes/plex/people.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends from sqlalchemy import func, or_, select from sqlalchemy.orm import Session +from app.collation import collated from app import logs from app.auth import current_user from app.db import SessionLocal, get_db @@ -19,7 +20,7 @@ from app.models import ( from app.plex import rollup from app.plex import tmdb as plex_tmdb from app.plex.cards import movie_card, norm_lang, show_card -from app.plex.query import content_libs, split_csv +from app.plex.query import content_libs, split_csv, title_col router = APIRouter(prefix="/api/plex", tags=["plex"]) log = logs.get_logger(__name__) @@ -202,7 +203,9 @@ def person_detail( PlexItem.directors.contains([name]), ), ) - .order_by(PlexItem.year.desc().nullslast(), func.lower(PlexItem.title)) + .order_by( + PlexItem.year.desc().nullslast(), collated(title_col(PlexItem, lang), lang) + ) .limit(_PERSON_SECTION_MAX) ) .scalars() @@ -223,7 +226,9 @@ def person_detail( PlexShow.directors.contains([name]), ), ) - .order_by(PlexShow.year.desc().nullslast(), func.lower(PlexShow.title)) + .order_by( + PlexShow.year.desc().nullslast(), collated(title_col(PlexShow, lang), lang) + ) .limit(_PERSON_SECTION_MAX) ) .scalars() diff --git a/backend/tests/test_collation.py b/backend/tests/test_collation.py new file mode 100644 index 00000000..f1d0ba42 --- /dev/null +++ b/backend/tests/test_collation.py @@ -0,0 +1,209 @@ +"""E30 S3: every A→Z list the server orders is in the reader's alphabet. The database image is Alpine, +whose `en_US.utf8` is BYTE order: "Életrajzi" and "Ötödik" sorted after "Zorró" in the feed, the channel +list, the Plex grid, the genre list. Hungarian puts "Ö" after "O" and "Cs" after "Cu". Needs the DB.""" + +from fastapi.testclient import TestClient + +import app.state as state +from app.auth import current_user +from app.collation import sorted_words +from app.db import get_db +from app.main import app +from app.models import ( + Channel, + Playlist, + PlaylistItem, + PlexCollection, + PlexItem, + PlexLibrary, + Subscription, + User, + Video, +) +from datetime import datetime, timezone + +WORDS = ["Zorró", "Életrajzi", "Ötödik", "oroszlán", "Csillag", "Cukor"] +HU = ["Cukor", "Csillag", "Életrajzi", "oroszlán", "Ötödik", "Zorró"] +EN = ["Csillag", "Cukor", "Életrajzi", "oroszlán", "Ötödik", "Zorró"] + + +def _user(db, lang: str) -> User: + user = User(email=f"collate-{lang}@example.com", preferences={"language": lang}) + db.add(user) + db.flush() + return user + + +def _client(db, user, monkeypatch, override_dep) -> TestClient: + monkeypatch.setattr(state, "_configured_cache", True) + override_dep(get_db, db.info["request"]) + override_dep(current_user, lambda: user) + return TestClient(app) + + +def test_words_sort_in_the_readers_alphabet(db): + assert sorted_words(db, WORDS, "hu") == HU + assert sorted_words(db, WORDS, "en") == EN + assert sorted_words(db, WORDS, "xx") == EN # an unknown language sorts as English + assert sorted_words(db, [], "hu") == [] + + +def _seed_feed(db, user): + db.add(Channel(id="UCc", title="Channel")) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UCc", priority=0, hidden=False)) + db.add_all( + Video( + id=f"v{i}", + channel_id="UCc", + published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc), + title=w, + is_short=False, + live_status="none", + via_search=False, + via_explore=False, + ) + for i, w in enumerate(WORDS) + ) + db.commit() + + +def test_the_feed_title_sort_pages_in_the_readers_alphabet(db, monkeypatch, override_dep): + user = _user(db, "hu") + _seed_feed(db, user) + client = _client(db, user, monkeypatch, override_dep) + + full = [it["title"] for it in client.get("/api/feed?sort=title&limit=50").json()["items"]] + assert full == HU + # The cursor compares with the same collated expression, so paging walks the same order. + walked: list[str] = [] + cursor = None + for _ in range(len(WORDS)): + url = "/api/feed?sort=title&limit=2" + (f"&cursor={cursor}" if cursor else "") + body = client.get(url).json() + walked += [it["title"] for it in body["items"]] + cursor = body["next_cursor"] + if not body["has_more"]: + break + assert walked == HU + desc = client.get("/api/feed?sort=title_desc&limit=50").json()["items"] + assert [it["title"] for it in desc] == HU[::-1] + + +def test_the_channel_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep): + user = _user(db, "hu") + db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS)) + db.flush() + db.add_all( + Subscription(user_id=user.id, channel_id=f"UC{i}", priority=0, hidden=False) + for i in range(len(WORDS)) + ) + db.commit() + r = _client(db, user, monkeypatch, override_dep).get("/api/channels") + assert [c["title"] for c in r.json()] == HU + + +def _seed_plex(db): + lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True) + db.add(lib) + db.flush() + col = PlexCollection(rating_key="col1", library_id=lib.id, title="Set", child_count=len(WORDS)) + db.add(col) + db.add_all( + PlexItem( + rating_key=f"m{i}", + library_id=lib.id, + kind="movie", + title=f"EN {i}", + title_hu=w, + year=2000, + genres_hu=[w], + collection_keys=["col1"], + ) + for i, w in enumerate(WORDS) + ) + db.commit() + + +def _plex(db, monkeypatch, override_dep) -> TestClient: + user = _user(db, "hu") + _seed_plex(db) + return _client(db, user, monkeypatch, override_dep) + + +def test_the_plex_grid_sorts_by_the_hungarian_title(db, monkeypatch, override_dep): + client = _plex(db, monkeypatch, override_dep) + grid = client.get("/api/plex/library?scope=movie&sort=title&sort_dir=asc&lang=hu").json() + assert [it["title"] for it in grid["items"]] == HU + + +def test_the_plex_genre_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep): + client = _plex(db, monkeypatch, override_dep) + facets = client.get("/api/plex/facets?scope=movie&lang=hu").json() + assert [g["value"] for g in facets["genres"]] == HU + + +def test_a_collection_sorts_by_the_hungarian_title(db, monkeypatch, override_dep): + client = _plex(db, monkeypatch, override_dep) + members = client.get("/api/plex/collections/col1?sort=title&sort_dir=asc&lang=hu").json() + assert [it["title"] for it in members["items"]] == HU + # year ties everywhere, so the default (year) order falls through to the title tiebreak + default = client.get("/api/plex/collections/col1?lang=hu").json() + assert [it["title"] for it in default["items"]] == HU + + +def _seed_discovery(db, user, synced: bool): + """Channels the user's playlist holds one video from each, not subscribed: discovery lists them, + count-tied, so the title decides. `synced=False` leaves them stubs, which sends the endpoint down + its enrich path — the one that re-sorts in Python and has to agree with the ORDER BY.""" + pl = Playlist(user_id=user.id, name="Mix") + db.add(pl) + db.flush() + stamp = datetime(2024, 1, 1, tzinfo=timezone.utc) if synced else None + for i, w in enumerate(WORDS): + db.add(Channel(id=f"UCd{i}", title=w, details_synced_at=stamp)) + db.flush() + db.add( + Video( + id=f"d{i}", + channel_id=f"UCd{i}", + published_at=datetime(2024, 1, 1, tzinfo=timezone.utc), + title="x", + is_short=False, + live_status="none", + via_search=False, + via_explore=False, + ) + ) + db.flush() + db.add(PlaylistItem(playlist_id=pl.id, video_id=f"d{i}", position=i)) + db.commit() + + +def test_channel_discovery_is_in_the_readers_alphabet(db, monkeypatch, override_dep): + user = _user(db, "hu") + _seed_discovery(db, user, synced=True) + r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery") + assert [c["title"] for c in r.json()] == HU + + +def test_channel_discovery_keeps_that_order_after_enriching(db, monkeypatch, override_dep): + import app.routes.channels as channels + + class FakeYouTube: + def __init__(self, *a, **k): ... + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get_channels(self, ids): + return [] + + monkeypatch.setattr(channels, "YouTubeClient", FakeYouTube) + monkeypatch.setattr(channels, "apply_channel_details", lambda *a, **k: None) + user = _user(db, "hu") + _seed_discovery(db, user, synced=False) + r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery") + assert [c["title"] for c in r.json()] == HU diff --git a/frontend/src/lib/dateFormat.contract.test.ts b/frontend/src/lib/localeFormat.contract.test.ts similarity index 100% rename from frontend/src/lib/dateFormat.contract.test.ts rename to frontend/src/lib/localeFormat.contract.test.ts