feat(discover): Surprise me leans on what you watch, and stays a surprise

The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They
are now weighted draws (`app.draw`, Efraimidis-Spirakis over the same md5 shuffle): a title's
weight is 1 + its affinity. Plex: the genres of the latest watched titles (half-life 180 days, a
show counts once) and the TMDB recommendations stored on them (`plex.affinity`). YouTube: the
channels the user watches (half-life 90 days). One slot of three stays a blind draw, and with no
history the order is exactly the old one. No visible change.

The YouTube draw builds its pool once (a materialized CTE read twice): the scope is ~200k videos on
a prod-sized catalog, and a second ordered pass doubled the cost. Measured on localdev: a reroll
210 -> 292 ms.

backend/app/README.md

@@ -25,8 +25,8 @@ Three layers, by *who* runs the code — not by feature:
    background run's durable row, its words in `runstatus`),
    `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
    configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
-   level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
-   reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
+   level), `debugsession` (a recording that tags its user's lines, E16), `collation` (A→Z in the
+   reader's alphabet, E30) and `draw` (the weighted, seeded draw "Surprise me" deals from), … are single-file primitives many layers import. They depend on nothing above them (a core
    module must never import a `routes/…` module).
 
 **Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a

backend/app/discover/queries.py

@@ -17,9 +17,25 @@ otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand st
 here takes the feed's seeded-md5 idiom instead of `random()`.
 """
 
-from sqlalchemy import and_, func, or_, select
+import json
+from datetime import datetime, timezone
+
+from sqlalchemy import (
+    Float,
+    Select,
+    and_,
+    cast,
+    func,
+    literal,
+    literal_column,
+    or_,
+    select,
+    union_all,
+)
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Session, aliased
 
+from app import draw
 from app.discover.budget import band
 from app.feedquery import (
     feed_columns,
@@ -33,7 +49,7 @@ from app.models import User
 from app.plex.models import PlexItem, PlexShow, PlexState
 from app.catalog.models import Channel, Video
 from app.reader.models import PlaylistItem, VideoState
-from app.plex import rollup
+from app.plex import affinity, rollup
 from app.plex.cards import display_title, leaf_card, movie_card, show_card
 from app.plex.query import apply_show_state_filter, content_libs
 from app.userscope import owned_by
@@ -115,14 +131,93 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
     params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
     query, rank_expr = filtered_query(db, user, **params)
     keys = sort_keys("shuffle", seed, rank_expr)
+
     # Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
     # hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
     # go: the resume tier has already had its chance at them, and it may have passed precisely because
     # the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
     # runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
-    rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all()
-    out = [serialize(r) for r in rows]
-    return [r for r in out if not r["position_seconds"]][:limit]
+    take = limit * 4
+
+    def unstarted(rows) -> list[dict]:
+        return [r for r in map(serialize, rows) if not r["position_seconds"]]
+
+    channels = _channel_affinity(db, user)
+    if not channels:
+        return unstarted(db.execute(query.order_by(*order_by(keys)).limit(take)).all())[:limit]
+
+    # Both orders from ONE pass over the scope. The pool is the whole feed scope (~200k videos on a
+    # prod-sized catalog, ~390 ms a pass), so a second ordered query for the weighted draw doubled
+    # the cost; a materialized pool of (id, shuffle key, weighted key) is read twice instead.
+    weight = literal_column("1.0") + CHANNEL_WEIGHT * func.coalesce(
+        cast(cast(literal(json.dumps(channels)), JSONB).op("->>")(Video.channel_id), Float), 0.0
+    )
+    shuffle = func.md5(func.concat(Video.id, str(seed)))
+    pool = (
+        query.with_only_columns(
+            Video.id.label("vid"),
+            shuffle.label("plain"),
+            draw.weighted_key(draw.seeded_u(Video.id, seed), weight).label("leaning"),
+        )
+        .cte("surprise_pool")
+        .prefix_with("MATERIALIZED")
+    )
+
+    def top(side: int, *order) -> Select:
+        part = (
+            select(pool.c.vid, pool.c.plain, pool.c.leaning, literal(side).label("side"))
+            .order_by(*order, pool.c.vid)
+            .limit(take)
+            .subquery()
+        )
+        return select(part)
+
+    # One statement, so the pool is built once; each side's order is restored from its keys.
+    got = db.execute(union_all(top(0, pool.c.plain), top(1, pool.c.leaning, pool.c.plain))).all()
+    plain_ids = [
+        r.vid for r in sorted((r for r in got if r.side == 0), key=lambda r: (r.plain, r.vid))
+    ]
+    leaning_ids = [
+        r.vid
+        for r in sorted((r for r in got if r.side == 1), key=lambda r: (r.leaning, r.plain, r.vid))
+    ]
+    rows = {
+        r["id"]: r
+        for r in unstarted(db.execute(query.where(Video.id.in_({*plain_ids, *leaning_ids}))).all())
+    }
+    blind = [rows[i] for i in plain_ids if i in rows]
+    leaning = [rows[i] for i in leaning_ids if i in rows]
+    return draw.blend(leaning, blind, seed, limit, key=lambda r: r["id"])
+
+
+# How much a channel the user watches most outweighs one they never watch (1 + this), and how far back
+# the watched history counts: the latest WATCHED_RECENT watches, the recent ones weighing more.
+CHANNEL_WEIGHT = 3.0
+WATCHED_RECENT = 300
+WATCHED_HALF_LIFE_DAYS = 90
+
+
+def _channel_affinity(db: Session, user: User) -> dict[str, float]:
+    """Each channel the user watched lately, scored 0..1 against the most watched: the sum of its
+    watches, each weighing half as much every WATCHED_HALF_LIFE_DAYS. Empty with no history."""
+    rows = db.execute(
+        select(Video.channel_id, VideoState.watched_at)
+        .join(Video, Video.id == VideoState.video_id)
+        .where(owned_by(VideoState.user_id, user), VideoState.status == "watched")
+        .order_by(VideoState.watched_at.desc().nullslast())
+        .limit(WATCHED_RECENT)
+    ).all()
+    now = datetime.now(timezone.utc)
+    scores: dict[str, float] = {}
+    for channel_id, watched_at in rows:
+        age = (
+            (now - watched_at).total_seconds() / 86400 if watched_at else 2 * WATCHED_HALF_LIFE_DAYS
+        )
+        scores[channel_id] = scores.get(channel_id, 0.0) + 0.5 ** (
+            max(0.0, age) / WATCHED_HALF_LIFE_DAYS
+        )
+    top = max(scores.values(), default=0.0)
+    return {k: v / top for k, v in scores.items()} if top else {}
 
 
 def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
@@ -289,21 +384,24 @@ def plex_surprise(
     if not movie_libs:
         return []
     lo, hi = band(budget)
-    q = (
-        select(PlexItem)
-        .where(
-            PlexItem.kind == "movie",
-            PlexItem.library_id.in_(movie_libs),
-            _unseen_movie_conds(uid),
-        )
-        .order_by(_seeded(PlexItem.rating_key, seed))
-        .limit(limit)
+    q = select(PlexItem).where(
+        PlexItem.kind == "movie",
+        PlexItem.library_id.in_(movie_libs),
+        _unseen_movie_conds(uid),
     )
     if lo is not None:
         q = q.where(PlexItem.duration_s >= lo)
     if hi is not None:
         q = q.where(PlexItem.duration_s <= hi)
-    items = db.execute(q).scalars().all()
+    shuffled = _seeded(PlexItem.rating_key, seed)
+    blind = db.execute(q.order_by(shuffled, PlexItem.id).limit(limit)).scalars().all()
+    leaning = affinity.for_user(db, uid)
+    if leaning:
+        key = draw.weighted_key(draw.seeded_u(PlexItem.rating_key, seed), leaning.weight(PlexItem))
+        weighted = db.execute(q.order_by(key, shuffled, PlexItem.id).limit(limit)).scalars().all()
+        items = draw.blend(list(weighted), list(blind), seed, limit, key=lambda it: it.id)
+    else:
+        items = list(blind)
     states = rollup.item_state_map(db, uid, [it.id for it in items])
     return [movie_card(it, states.get(it.id), lang=lang) for it in items]
 

backend/app/draw.py

@@ -0,0 +1,72 @@
+"""A weighted draw that stays a surprise: what "Surprise me" deals from.
+
+A title's chance to come first is its weight over the pool's (Efraimidis–Spirakis: order by
+-ln(1-u)/w, u uniform per row). The weight is 1 + the title's affinity to what the user watches
+(`plex.affinity`, `discover.queries`), so a title nobody leans towards still comes up — only less
+often. Two things keep it a surprise rather than a ranking:
+
+- **u is the seed's md5 of the row's key**, the feed's shuffle idiom. With every weight 1 the order
+  is exactly the md5 order the draw had before it was weighted, so a user with no history sees the
+  old draw, and one seed still moves both sources of the Home hero together. A draw that is not
+  seeded (the Plex Discover rail) takes u from random().
+- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few
+  favourites cannot fill the rail.
+"""
+
+from collections.abc import Callable, Hashable
+from typing import Any
+
+from sqlalchemy import BigInteger, Float, cast, func, literal
+from sqlalchemy.dialects.postgresql import BIT
+from sqlalchemy.orm import InstrumentedAttribute
+from sqlalchemy.sql.elements import ColumnElement
+
+# 2**32: the first 8 hex digits of an md5 as a fraction of one.
+_SPAN = 4294967296.0
+
+
+def seeded_u(
+    key: ColumnElement[Any] | InstrumentedAttribute[Any], seed: int
+) -> ColumnElement[float]:
+    """A uniform u in [0, 1) per row, from md5(key || seed) — the order `md5(...)` ascending gives."""
+    digest = func.md5(func.concat(key, str(seed)))
+    head = cast(cast(literal("x") + func.substr(digest, 1, 8), BIT(32)), BigInteger)
+    return cast(head, Float) / _SPAN
+
+
+def weighted_key(u: ColumnElement[float], weight: ColumnElement[float]) -> ColumnElement[float]:
+    """The row's place in a weighted draw, ascending: -ln(1-u)/w. Monotonic in u when every w is 1,
+    so the unweighted order is unchanged."""
+    return -func.ln(1.0 - u) / weight
+
+
+def random_u() -> ColumnElement[float]:
+    return func.random()
+
+
+def blend(
+    weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x
+) -> list:
+    """`n` picks: slot i comes from `blind` when (i + offset) % 3 == 0, else from `weighted`, each
+    skipping what an earlier slot took (`key` tells two picks apart). A side that runs out leaves the
+    rest to the other."""
+    out: list = []
+    taken: set = set()
+    sources = (iter(weighted), iter(blind))
+
+    def nxt(source):
+        for item in source:
+            if key(item) not in taken:
+                return item
+        return None
+
+    for i in range(n):
+        first, second = (sources[1], sources[0]) if (i + offset) % 3 == 0 else sources
+        item = nxt(first)
+        if item is None:
+            item = nxt(second)
+        if item is None:
+            break
+        taken.add(key(item))
+        out.append(item)
+    return out

backend/app/plex/affinity.py

@@ -0,0 +1,120 @@
+"""What a user's Plex history leans towards, as a weight for a surprise draw (`app.draw`).
+
+Two signals, both already in the database (no TMDB call):
+- **genres** of the titles the user watched, the recent ones weighing more (a half-life of
+  HALF_LIFE_DAYS), each scored 0..1 against the strongest;
+- the **TMDB recommendations** stored on those titles (`recommendations`), each scored by the most
+  recent title that recommends it.
+
+A title counts once however many of its episodes were watched — a binge of one show would otherwise
+outvote every film — and only the RECENT_TITLES latest titles count. A candidate's weight is
+1 + GENRE_WEIGHT × (the mean score of its genres) + REC_WEIGHT × (its recommendation score), so a
+title with no affinity keeps weight 1 and still comes up.
+"""
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from sqlalchemy import Float, cast, func, literal, literal_column, select
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.elements import ColumnElement
+
+from app.plex.models import PlexItem, PlexShow, PlexState
+from app.userscope import owned_by
+
+HALF_LIFE_DAYS = 180
+RECENT_TITLES = 200
+GENRE_WEIGHT = 2.0
+REC_WEIGHT = 3.0
+# A watch with no date (an old import) counts as one this many half-lives ago.
+_UNDATED_HALF_LIVES = 2
+
+
+@dataclass(frozen=True)
+class Affinity:
+    genres: dict[str, float] = field(default_factory=dict)
+    movie_recs: dict[str, float] = field(default_factory=dict)
+    show_recs: dict[str, float] = field(default_factory=dict)
+
+    def __bool__(self) -> bool:
+        return bool(self.genres or self.movie_recs or self.show_recs)
+
+    def weight(self, model: type[PlexItem] | type[PlexShow]) -> ColumnElement[float]:
+        """The draw weight of a `model` row (a movie or a show)."""
+        scores = cast(literal(json.dumps(self.genres)), JSONB)
+        recs = self.show_recs if model is PlexShow else self.movie_recs
+        genre = func.jsonb_array_elements_text(model.genres).table_valued("value").alias("g")
+        genre_aff = (
+            select(
+                func.coalesce(
+                    func.avg(func.coalesce(cast(scores.op("->>")(genre.c.value), Float), 0.0)),
+                    0.0,
+                )
+            )
+            .select_from(genre)
+            .scalar_subquery()
+        )
+        rec_aff = func.coalesce(
+            cast(cast(literal(json.dumps(recs)), JSONB).op("->>")(model.guid), Float), 0.0
+        )
+        return literal_column("1.0") + GENRE_WEIGHT * genre_aff + REC_WEIGHT * rec_aff
+
+
+def for_user(db: Session, uid: int) -> Affinity:
+    """The user's affinity from their latest watched titles; empty (`not aff`) with no history."""
+    # One key per title: a show for an episode, the item for a film. concat() of a NULL show_id is
+    # just "s", which nullif turns back into NULL so the film's key takes over.
+    title = func.coalesce(
+        func.nullif(func.concat("s", PlexItem.show_id), "s"), func.concat("m", PlexItem.id)
+    ).label("title")
+    latest = (
+        select(title, PlexState.watched_at, PlexItem.id.label("item_id"), PlexItem.show_id)
+        .join(PlexItem, PlexItem.id == PlexState.item_id)
+        .where(owned_by(PlexState.user_id, uid), PlexState.status == "watched")
+        .distinct(title)
+        .order_by(title, PlexState.watched_at.desc().nullslast())
+        .subquery()
+    )
+    rows = db.execute(
+        select(
+            latest.c.watched_at,
+            PlexItem.genres,
+            PlexItem.recommendations,
+            PlexShow.genres,
+            PlexShow.recommendations,
+        )
+        .select_from(latest)
+        .join(PlexItem, PlexItem.id == latest.c.item_id)
+        .outerjoin(PlexShow, PlexShow.id == latest.c.show_id)
+        .order_by(latest.c.watched_at.desc().nullslast())
+        .limit(RECENT_TITLES)
+    ).all()
+    if not rows:
+        return Affinity()
+
+    now = datetime.now(timezone.utc)
+    genres: dict[str, float] = {}
+    movie_recs: dict[str, float] = {}
+    show_recs: dict[str, float] = {}
+    for watched_at, item_genres, item_recs, show_genres, show_recs_json in rows:
+        if watched_at is None:
+            r = 0.5**_UNDATED_HALF_LIVES
+        else:
+            r = 0.5 ** (max(0.0, (now - watched_at).total_seconds() / 86400) / HALF_LIFE_DAYS)
+        for g in (show_genres if show_genres is not None else item_genres) or []:
+            genres[g] = genres.get(g, 0.0) + r
+        for rec in (show_recs_json if show_recs_json is not None else item_recs) or []:
+            if not rec.get("tmdb_id"):
+                continue
+            bucket = show_recs if rec.get("media_type") == "tv" else movie_recs
+            guid = f"tmdb:{rec['tmdb_id']}"
+            bucket[guid] = max(bucket.get(guid, 0.0), r)
+    return Affinity(_scaled(genres), _scaled(movie_recs), _scaled(show_recs))
+
+
+def _scaled(scores: dict[str, float]) -> dict[str, float]:
+    """Each score over the strongest, so the strongest is 1."""
+    top = max(scores.values(), default=0.0)
+    return {k: v / top for k, v in scores.items()} if top else {}

backend/app/plex/discover.py

@@ -22,7 +22,8 @@ from sqlalchemy.orm import Session
 from app.collation import sorted_words
 from app.discover.queries import RAIL_LIMIT
 from app.models import User
-from app.plex import rollup
+from app import draw
+from app.plex import affinity, rollup
 from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card
 from app.plex.models import PlexItem, PlexShow, PlexState
 from app.plex.query import (
@@ -399,22 +400,25 @@ def discover_page(db: Session, user: User, scope: str, lang: str, f: LibraryFilt
     }
 
 
-def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list:
+def _draw(db: Session, stmt, key_col, per: int, exclude: list[str], weight=None) -> list:
     """`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a
     reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave
-    short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail."""
-    fresh = (
-        db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per))
-        .scalars()
-        .all()
-        if exclude
-        else []
-    )
+    short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail.
+    With a `weight` (`plex.affinity`) the draw leans on it, every third slot left blind (`app.draw`)."""
+
+    def pick(q, n: int) -> list:
+        blind = db.execute(q.order_by(func.random()).limit(n)).scalars().all()
+        if weight is None:
+            return list(blind)
+        key = draw.weighted_key(draw.random_u(), weight)
+        leaning = db.execute(q.order_by(key).limit(n)).scalars().all()
+        return draw.blend(list(leaning), list(blind), 1, n, key=lambda r: r.rating_key)
+
+    fresh = pick(stmt.where(key_col.not_in(exclude)), per) if exclude else []
     if len(fresh) == per:
-        return list(fresh)
+        return fresh
     taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else []
-    rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars()
-    return [*fresh, *rest.all()]
+    return [*fresh, *pick(stmt.where(*taken), per - len(fresh))]
 
 
 def _kept(
@@ -474,6 +478,8 @@ def surprise(
     if keep:
         return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)}
     per = min(limit, SURPRISE_PER_KIND)
+    # What the user's history leans towards; with none, every title weighs the same (a plain draw).
+    leaning = affinity.for_user(db, user.id)
     movies = []
     if movie_ids:
         # Exclude movies this user has watched.
@@ -501,6 +507,7 @@ def surprise(
             PlexItem.rating_key,
             per,
             exclude,
+            leaning.weight(PlexItem) if leaning else None,
         )
     shows = []
     if show_ids:
@@ -523,6 +530,7 @@ def surprise(
             PlexShow.rating_key,
             per,
             exclude,
+            leaning.weight(PlexShow) if leaning else None,
         )
     mcards = _movie_cards(db, user.id, movies, lang)
     scards = _show_cards(db, user.id, shows, lang)