From 9310ff85bfd21ed6c3bb77d9c5b1b3553c61df2d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 7 Nov 2026 02:41:30 +0100 Subject: [PATCH] feat(discover): Surprise me leans on what you watch, and stays a surprise The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are now weighted draws (`app.draw`, Efraimidis-Spirakis over the same md5 shuffle): a title's weight is 1 + its affinity. Plex: the genres of the latest watched titles (half-life 180 days, a show counts once) and the TMDB recommendations stored on them (`plex.affinity`). YouTube: the channels the user watches (half-life 90 days). One slot of three stays a blind draw, and with no history the order is exactly the old one. No visible change. The YouTube draw builds its pool once (a materialized CTE read twice): the scope is ~200k videos on a prod-sized catalog, and a second ordered pass doubled the cost. Measured on localdev: a reroll 210 -> 292 ms. --- CLAUDE.md | 2 +- backend/app/README.md | 4 +- backend/app/discover/queries.py | 128 +++++++++-- backend/app/draw.py | 72 +++++++ backend/app/plex/affinity.py | 120 +++++++++++ backend/app/plex/discover.py | 34 +-- backend/tests/test_surprise_affinity.py | 269 ++++++++++++++++++++++++ 7 files changed, 598 insertions(+), 31 deletions(-) create mode 100644 backend/app/draw.py create mode 100644 backend/app/plex/affinity.py create mode 100644 backend/tests/test_surprise_affinity.py diff --git a/CLAUDE.md b/CLAUDE.md index 4b6b62a3..0d16a1f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ how it gets built. Concretely: `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, `downloads/`, `reader/`, `discover/`, `catalog/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`, `sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, -`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** +`security`, `logs`, `logsink`, `diagnostics`, `debugsession`, `draw`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** The domain is four **units**, each owning its tables: Capture (`downloads/`), Reader (`reader/`, `discover/`, the playlist sync), Catalog (`catalog/`, the `sync/` ingests) and Plex (`plex/`). A unit's route does no database work: it reads through the unit's `queries`, writes through its `service` (which flushes) and commits. A unit's code reads another unit's tables diff --git a/backend/app/README.md b/backend/app/README.md index 66c583cc..d79b0c11 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -25,8 +25,8 @@ Three layers, by *who* runs the code — not by feature: background run's durable row, its words in `runstatus`), `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log - level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the - reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core + level), `debugsession` (a recording that tags its user's lines, E16), `collation` (A→Z in the + reader's alphabet, E30) and `draw` (the weighted, seeded draw "Surprise me" deals from), … are single-file primitives many layers import. They depend on nothing above them (a core module must never import a `routes/…` module). **Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a diff --git a/backend/app/discover/queries.py b/backend/app/discover/queries.py index c26e8342..26a4cadb 100644 --- a/backend/app/discover/queries.py +++ b/backend/app/discover/queries.py @@ -17,9 +17,25 @@ otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand st here takes the feed's seeded-md5 idiom instead of `random()`. """ -from sqlalchemy import and_, func, or_, select +import json +from datetime import datetime, timezone + +from sqlalchemy import ( + Float, + Select, + and_, + cast, + func, + literal, + literal_column, + or_, + select, + union_all, +) +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session, aliased +from app import draw from app.discover.budget import band from app.feedquery import ( feed_columns, @@ -33,7 +49,7 @@ from app.models import User from app.plex.models import PlexItem, PlexShow, PlexState from app.catalog.models import Channel, Video from app.reader.models import PlaylistItem, VideoState -from app.plex import rollup +from app.plex import affinity, rollup from app.plex.cards import display_title, leaf_card, movie_card, show_card from app.plex.query import apply_show_state_filter, content_libs from app.userscope import owned_by @@ -115,14 +131,93 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) -> params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short") query, rank_expr = filtered_query(db, user, **params) keys = sort_keys("shuffle", seed, rank_expr) + # Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and # hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must # go: the resume tier has already had its chance at them, and it may have passed precisely because # the REMAINING time did not fit the budget — offering the same item back as a surprise, whose # runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question. - rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all() - out = [serialize(r) for r in rows] - return [r for r in out if not r["position_seconds"]][:limit] + take = limit * 4 + + def unstarted(rows) -> list[dict]: + return [r for r in map(serialize, rows) if not r["position_seconds"]] + + channels = _channel_affinity(db, user) + if not channels: + return unstarted(db.execute(query.order_by(*order_by(keys)).limit(take)).all())[:limit] + + # Both orders from ONE pass over the scope. The pool is the whole feed scope (~200k videos on a + # prod-sized catalog, ~390 ms a pass), so a second ordered query for the weighted draw doubled + # the cost; a materialized pool of (id, shuffle key, weighted key) is read twice instead. + weight = literal_column("1.0") + CHANNEL_WEIGHT * func.coalesce( + cast(cast(literal(json.dumps(channels)), JSONB).op("->>")(Video.channel_id), Float), 0.0 + ) + shuffle = func.md5(func.concat(Video.id, str(seed))) + pool = ( + query.with_only_columns( + Video.id.label("vid"), + shuffle.label("plain"), + draw.weighted_key(draw.seeded_u(Video.id, seed), weight).label("leaning"), + ) + .cte("surprise_pool") + .prefix_with("MATERIALIZED") + ) + + def top(side: int, *order) -> Select: + part = ( + select(pool.c.vid, pool.c.plain, pool.c.leaning, literal(side).label("side")) + .order_by(*order, pool.c.vid) + .limit(take) + .subquery() + ) + return select(part) + + # One statement, so the pool is built once; each side's order is restored from its keys. + got = db.execute(union_all(top(0, pool.c.plain), top(1, pool.c.leaning, pool.c.plain))).all() + plain_ids = [ + r.vid for r in sorted((r for r in got if r.side == 0), key=lambda r: (r.plain, r.vid)) + ] + leaning_ids = [ + r.vid + for r in sorted((r for r in got if r.side == 1), key=lambda r: (r.leaning, r.plain, r.vid)) + ] + rows = { + r["id"]: r + for r in unstarted(db.execute(query.where(Video.id.in_({*plain_ids, *leaning_ids}))).all()) + } + blind = [rows[i] for i in plain_ids if i in rows] + leaning = [rows[i] for i in leaning_ids if i in rows] + return draw.blend(leaning, blind, seed, limit, key=lambda r: r["id"]) + + +# How much a channel the user watches most outweighs one they never watch (1 + this), and how far back +# the watched history counts: the latest WATCHED_RECENT watches, the recent ones weighing more. +CHANNEL_WEIGHT = 3.0 +WATCHED_RECENT = 300 +WATCHED_HALF_LIFE_DAYS = 90 + + +def _channel_affinity(db: Session, user: User) -> dict[str, float]: + """Each channel the user watched lately, scored 0..1 against the most watched: the sum of its + watches, each weighing half as much every WATCHED_HALF_LIFE_DAYS. Empty with no history.""" + rows = db.execute( + select(Video.channel_id, VideoState.watched_at) + .join(Video, Video.id == VideoState.video_id) + .where(owned_by(VideoState.user_id, user), VideoState.status == "watched") + .order_by(VideoState.watched_at.desc().nullslast()) + .limit(WATCHED_RECENT) + ).all() + now = datetime.now(timezone.utc) + scores: dict[str, float] = {} + for channel_id, watched_at in rows: + age = ( + (now - watched_at).total_seconds() / 86400 if watched_at else 2 * WATCHED_HALF_LIFE_DAYS + ) + scores[channel_id] = scores.get(channel_id, 0.0) + 0.5 ** ( + max(0.0, age) / WATCHED_HALF_LIFE_DAYS + ) + top = max(scores.values(), default=0.0) + return {k: v / top for k, v in scores.items()} if top else {} def yt_video_row(db: Session, user: User, video_id: str) -> dict | None: @@ -289,21 +384,24 @@ def plex_surprise( if not movie_libs: return [] lo, hi = band(budget) - q = ( - select(PlexItem) - .where( - PlexItem.kind == "movie", - PlexItem.library_id.in_(movie_libs), - _unseen_movie_conds(uid), - ) - .order_by(_seeded(PlexItem.rating_key, seed)) - .limit(limit) + q = select(PlexItem).where( + PlexItem.kind == "movie", + PlexItem.library_id.in_(movie_libs), + _unseen_movie_conds(uid), ) if lo is not None: q = q.where(PlexItem.duration_s >= lo) if hi is not None: q = q.where(PlexItem.duration_s <= hi) - items = db.execute(q).scalars().all() + shuffled = _seeded(PlexItem.rating_key, seed) + blind = db.execute(q.order_by(shuffled, PlexItem.id).limit(limit)).scalars().all() + leaning = affinity.for_user(db, uid) + if leaning: + key = draw.weighted_key(draw.seeded_u(PlexItem.rating_key, seed), leaning.weight(PlexItem)) + weighted = db.execute(q.order_by(key, shuffled, PlexItem.id).limit(limit)).scalars().all() + items = draw.blend(list(weighted), list(blind), seed, limit, key=lambda it: it.id) + else: + items = list(blind) states = rollup.item_state_map(db, uid, [it.id for it in items]) return [movie_card(it, states.get(it.id), lang=lang) for it in items] diff --git a/backend/app/draw.py b/backend/app/draw.py new file mode 100644 index 00000000..25ce8747 --- /dev/null +++ b/backend/app/draw.py @@ -0,0 +1,72 @@ +"""A weighted draw that stays a surprise: what "Surprise me" deals from. + +A title's chance to come first is its weight over the pool's (Efraimidis–Spirakis: order by +-ln(1-u)/w, u uniform per row). The weight is 1 + the title's affinity to what the user watches +(`plex.affinity`, `discover.queries`), so a title nobody leans towards still comes up — only less +often. Two things keep it a surprise rather than a ranking: + +- **u is the seed's md5 of the row's key**, the feed's shuffle idiom. With every weight 1 the order + is exactly the md5 order the draw had before it was weighted, so a user with no history sees the + old draw, and one seed still moves both sources of the Home hero together. A draw that is not + seeded (the Plex Discover rail) takes u from random(). +- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few + favourites cannot fill the rail. +""" + +from collections.abc import Callable, Hashable +from typing import Any + +from sqlalchemy import BigInteger, Float, cast, func, literal +from sqlalchemy.dialects.postgresql import BIT +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.elements import ColumnElement + +# 2**32: the first 8 hex digits of an md5 as a fraction of one. +_SPAN = 4294967296.0 + + +def seeded_u( + key: ColumnElement[Any] | InstrumentedAttribute[Any], seed: int +) -> ColumnElement[float]: + """A uniform u in [0, 1) per row, from md5(key || seed) — the order `md5(...)` ascending gives.""" + digest = func.md5(func.concat(key, str(seed))) + head = cast(cast(literal("x") + func.substr(digest, 1, 8), BIT(32)), BigInteger) + return cast(head, Float) / _SPAN + + +def weighted_key(u: ColumnElement[float], weight: ColumnElement[float]) -> ColumnElement[float]: + """The row's place in a weighted draw, ascending: -ln(1-u)/w. Monotonic in u when every w is 1, + so the unweighted order is unchanged.""" + return -func.ln(1.0 - u) / weight + + +def random_u() -> ColumnElement[float]: + return func.random() + + +def blend( + weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x +) -> list: + """`n` picks: slot i comes from `blind` when (i + offset) % 3 == 0, else from `weighted`, each + skipping what an earlier slot took (`key` tells two picks apart). A side that runs out leaves the + rest to the other.""" + out: list = [] + taken: set = set() + sources = (iter(weighted), iter(blind)) + + def nxt(source): + for item in source: + if key(item) not in taken: + return item + return None + + for i in range(n): + first, second = (sources[1], sources[0]) if (i + offset) % 3 == 0 else sources + item = nxt(first) + if item is None: + item = nxt(second) + if item is None: + break + taken.add(key(item)) + out.append(item) + return out diff --git a/backend/app/plex/affinity.py b/backend/app/plex/affinity.py new file mode 100644 index 00000000..4d25fbd3 --- /dev/null +++ b/backend/app/plex/affinity.py @@ -0,0 +1,120 @@ +"""What a user's Plex history leans towards, as a weight for a surprise draw (`app.draw`). + +Two signals, both already in the database (no TMDB call): +- **genres** of the titles the user watched, the recent ones weighing more (a half-life of + HALF_LIFE_DAYS), each scored 0..1 against the strongest; +- the **TMDB recommendations** stored on those titles (`recommendations`), each scored by the most + recent title that recommends it. + +A title counts once however many of its episodes were watched — a binge of one show would otherwise +outvote every film — and only the RECENT_TITLES latest titles count. A candidate's weight is +1 + GENRE_WEIGHT × (the mean score of its genres) + REC_WEIGHT × (its recommendation score), so a +title with no affinity keeps weight 1 and still comes up. +""" + +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from sqlalchemy import Float, cast, func, literal, literal_column, select +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Session +from sqlalchemy.sql.elements import ColumnElement + +from app.plex.models import PlexItem, PlexShow, PlexState +from app.userscope import owned_by + +HALF_LIFE_DAYS = 180 +RECENT_TITLES = 200 +GENRE_WEIGHT = 2.0 +REC_WEIGHT = 3.0 +# A watch with no date (an old import) counts as one this many half-lives ago. +_UNDATED_HALF_LIVES = 2 + + +@dataclass(frozen=True) +class Affinity: + genres: dict[str, float] = field(default_factory=dict) + movie_recs: dict[str, float] = field(default_factory=dict) + show_recs: dict[str, float] = field(default_factory=dict) + + def __bool__(self) -> bool: + return bool(self.genres or self.movie_recs or self.show_recs) + + def weight(self, model: type[PlexItem] | type[PlexShow]) -> ColumnElement[float]: + """The draw weight of a `model` row (a movie or a show).""" + scores = cast(literal(json.dumps(self.genres)), JSONB) + recs = self.show_recs if model is PlexShow else self.movie_recs + genre = func.jsonb_array_elements_text(model.genres).table_valued("value").alias("g") + genre_aff = ( + select( + func.coalesce( + func.avg(func.coalesce(cast(scores.op("->>")(genre.c.value), Float), 0.0)), + 0.0, + ) + ) + .select_from(genre) + .scalar_subquery() + ) + rec_aff = func.coalesce( + cast(cast(literal(json.dumps(recs)), JSONB).op("->>")(model.guid), Float), 0.0 + ) + return literal_column("1.0") + GENRE_WEIGHT * genre_aff + REC_WEIGHT * rec_aff + + +def for_user(db: Session, uid: int) -> Affinity: + """The user's affinity from their latest watched titles; empty (`not aff`) with no history.""" + # One key per title: a show for an episode, the item for a film. concat() of a NULL show_id is + # just "s", which nullif turns back into NULL so the film's key takes over. + title = func.coalesce( + func.nullif(func.concat("s", PlexItem.show_id), "s"), func.concat("m", PlexItem.id) + ).label("title") + latest = ( + select(title, PlexState.watched_at, PlexItem.id.label("item_id"), PlexItem.show_id) + .join(PlexItem, PlexItem.id == PlexState.item_id) + .where(owned_by(PlexState.user_id, uid), PlexState.status == "watched") + .distinct(title) + .order_by(title, PlexState.watched_at.desc().nullslast()) + .subquery() + ) + rows = db.execute( + select( + latest.c.watched_at, + PlexItem.genres, + PlexItem.recommendations, + PlexShow.genres, + PlexShow.recommendations, + ) + .select_from(latest) + .join(PlexItem, PlexItem.id == latest.c.item_id) + .outerjoin(PlexShow, PlexShow.id == latest.c.show_id) + .order_by(latest.c.watched_at.desc().nullslast()) + .limit(RECENT_TITLES) + ).all() + if not rows: + return Affinity() + + now = datetime.now(timezone.utc) + genres: dict[str, float] = {} + movie_recs: dict[str, float] = {} + show_recs: dict[str, float] = {} + for watched_at, item_genres, item_recs, show_genres, show_recs_json in rows: + if watched_at is None: + r = 0.5**_UNDATED_HALF_LIVES + else: + r = 0.5 ** (max(0.0, (now - watched_at).total_seconds() / 86400) / HALF_LIFE_DAYS) + for g in (show_genres if show_genres is not None else item_genres) or []: + genres[g] = genres.get(g, 0.0) + r + for rec in (show_recs_json if show_recs_json is not None else item_recs) or []: + if not rec.get("tmdb_id"): + continue + bucket = show_recs if rec.get("media_type") == "tv" else movie_recs + guid = f"tmdb:{rec['tmdb_id']}" + bucket[guid] = max(bucket.get(guid, 0.0), r) + return Affinity(_scaled(genres), _scaled(movie_recs), _scaled(show_recs)) + + +def _scaled(scores: dict[str, float]) -> dict[str, float]: + """Each score over the strongest, so the strongest is 1.""" + top = max(scores.values(), default=0.0) + return {k: v / top for k, v in scores.items()} if top else {} diff --git a/backend/app/plex/discover.py b/backend/app/plex/discover.py index ac9ed64a..11461e4e 100644 --- a/backend/app/plex/discover.py +++ b/backend/app/plex/discover.py @@ -22,7 +22,8 @@ from sqlalchemy.orm import Session from app.collation import sorted_words from app.discover.queries import RAIL_LIMIT from app.models import User -from app.plex import rollup +from app import draw +from app.plex import affinity, rollup from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card from app.plex.models import PlexItem, PlexShow, PlexState from app.plex.query import ( @@ -399,22 +400,25 @@ def discover_page(db: Session, user: User, scope: str, lang: str, f: LibraryFilt } -def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list: +def _draw(db: Session, stmt, key_col, per: int, exclude: list[str], weight=None) -> list: """`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave - short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail.""" - fresh = ( - db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per)) - .scalars() - .all() - if exclude - else [] - ) + short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail. + With a `weight` (`plex.affinity`) the draw leans on it, every third slot left blind (`app.draw`).""" + + def pick(q, n: int) -> list: + blind = db.execute(q.order_by(func.random()).limit(n)).scalars().all() + if weight is None: + return list(blind) + key = draw.weighted_key(draw.random_u(), weight) + leaning = db.execute(q.order_by(key).limit(n)).scalars().all() + return draw.blend(list(leaning), list(blind), 1, n, key=lambda r: r.rating_key) + + fresh = pick(stmt.where(key_col.not_in(exclude)), per) if exclude else [] if len(fresh) == per: - return list(fresh) + return fresh taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else [] - rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars() - return [*fresh, *rest.all()] + return [*fresh, *pick(stmt.where(*taken), per - len(fresh))] def _kept( @@ -474,6 +478,8 @@ def surprise( if keep: return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)} per = min(limit, SURPRISE_PER_KIND) + # What the user's history leans towards; with none, every title weighs the same (a plain draw). + leaning = affinity.for_user(db, user.id) movies = [] if movie_ids: # Exclude movies this user has watched. @@ -501,6 +507,7 @@ def surprise( PlexItem.rating_key, per, exclude, + leaning.weight(PlexItem) if leaning else None, ) shows = [] if show_ids: @@ -523,6 +530,7 @@ def surprise( PlexShow.rating_key, per, exclude, + leaning.weight(PlexShow) if leaning else None, ) mcards = _movie_cards(db, user.id, movies, lang) scards = _show_cards(db, user.id, shows, lang) diff --git a/backend/tests/test_surprise_affinity.py b/backend/tests/test_surprise_affinity.py new file mode 100644 index 00000000..0921c02c --- /dev/null +++ b/backend/tests/test_surprise_affinity.py @@ -0,0 +1,269 @@ +"""Surprise me leans on what the user watches, and stays a surprise. + +The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are +now weighted draws (`app.draw`): a title's weight is 1 + its affinity — for Plex the genres of the +films and shows the user watched (recent ones weigh more) and the TMDB recommendations stored on them; +for YouTube the channels the user watches. One slot of three stays a blind draw, and with no watch +history the order is exactly the old one. DB lane. +""" + +import hashlib +from datetime import datetime, timedelta, timezone + +from sqlalchemy import text + +from app import draw +from app.catalog.models import Channel, Subscription, Video +from app.discover import queries +from app.plex import affinity, discover as plex_discover +from app.plex.models import PlexItem, PlexLibrary, PlexShow, PlexState +from app.plex.query import LibraryFilters +from app.reader.models import VideoState + +NOW = datetime.now(timezone.utc) +SEEDS = range(90) + + +# ── the draw itself ────────────────────────────────────────────────────────────────────────────── + + +def test_every_third_slot_is_blind_and_nothing_repeats(): + weighted = ["a", "b", "c", "d", "e", "f"] + blind = ["f", "e", "d", "c", "b", "a"] + # offset 0: slots 0 and 3 are blind + assert draw.blend(weighted, blind, 0, 6) == ["f", "a", "b", "e", "c", "d"] + # offset 1: slot 2 and 5 + assert draw.blend(weighted, blind, 1, 6) == ["a", "b", "f", "c", "d", "e"] + assert draw.blend(weighted, blind, 2, 3) == ["a", "f", "b"] + + +def test_a_short_side_is_topped_up_from_the_other(): + assert draw.blend(["a"], ["a", "b", "c"], 0, 3) == ["a", "b", "c"] + assert draw.blend(["a", "b", "c"], [], 0, 3) == ["a", "b", "c"] + + +# ── YouTube: the channels the user watches ─────────────────────────────────────────────────────── + + +def _md5_order(ids, seed): + return sorted(ids, key=lambda i: hashlib.md5(f"{i}{seed}".encode()).hexdigest()) + + +def _seed_yt(db, make_user, *, watched_in_fav: int): + user = make_user() + fav, other = Channel(id="UCfav", title="Fav"), Channel(id="UCother", title="Other") + db.add_all([fav, other]) + db.flush() + db.add_all( + [ + Subscription(user_id=user.id, channel_id="UCfav"), + Subscription(user_id=user.id, channel_id="UCother"), + ] + ) + + def video(vid, ch, days): + return Video( + id=vid, + channel_id=ch, + title=vid, + duration_seconds=20 * 60, + published_at=NOW - timedelta(days=days), + ) + + # One unseen video per channel, plus the user's watched history on the favourite. + db.add_all([video("fav_new", "UCfav", 1), video("other_new", "UCother", 1)]) + for i in range(watched_in_fav): + db.add(video(f"fav_seen{i}", "UCfav", 10 + i)) + db.flush() + for i in range(watched_in_fav): + db.add( + VideoState( + user_id=user.id, + video_id=f"fav_seen{i}", + status="watched", + watched_at=NOW - timedelta(days=i), + ) + ) + db.commit() + return user + + +def _first(rows): + return rows[0]["id"] if rows else None + + +def test_a_watched_channel_wins_most_draws_but_not_all(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + leads = [_first(queries.yt_surprise(db, user, "any", s, 2)) for s in SEEDS] + fav = leads.count("fav_new") / len(leads) + # weight 1 + 3 against 1: 0.8 in a weighted slot, 0.5 in the blind one (a third of the seeds) + assert 0.6 < fav < 0.9, fav + + +def test_the_blind_slot_deals_the_plain_shuffle_however_strong_the_lean(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + for s in range(0, 30, 3): # (0 + seed) % 3 == 0: the first slot is the blind one + got = _first(queries.yt_surprise(db, user, "any", s, 2)) + assert got == _md5_order(["fav_new", "other_new"], s)[0] + + +def test_with_no_history_the_youtube_order_is_the_old_shuffle(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=0) + for s in range(12): + got = [r["id"] for r in queries.yt_surprise(db, user, "any", s, 2)] + assert got == _md5_order(["fav_new", "other_new"], s) + + +def test_a_watched_video_is_never_drawn(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + drawn = {r["id"] for s in SEEDS for r in queries.yt_surprise(db, user, "any", s, 5)} + assert drawn == {"fav_new", "other_new"} + + +# ── Plex: genres and recommendations of what the user watched ──────────────────────────────────── + + +def _seed_plex(db, make_user, *, history: bool): + user = make_user() + lib = PlexLibrary(plex_key="77", title="Movies", kind="movie", enabled=True) + db.add(lib) + db.flush() + tmdb_ids = iter(range(5000, 6000)) + + def movie(key, genres, recs=None): + m = PlexItem( + rating_key=key, + library_id=lib.id, + kind="movie", + title=key, + duration_s=100 * 60, + genres=genres, + guid=f"tmdb:{next(tmdb_ids)}", + recommendations=recs, + ) + db.add(m) + return m + + horror = movie("horror_new", ["Horror"]) + comedy = movie("comedy_new", ["Comedy"]) + tipped = movie("tipped_new", ["Western"]) + db.flush() + tip = [{"tmdb_id": int(tipped.guid.split(":")[1]), "media_type": "movie", "title": "t"}] + seen = [movie(f"horror_seen{i}", ["Horror"], tip if i == 0 else None) for i in range(3)] + db.flush() + if history: + for i, m in enumerate(seen): + db.add( + PlexState( + user_id=user.id, + item_id=m.id, + status="watched", + watched_at=NOW - timedelta(days=i), + ) + ) + db.commit() + from app import sysconfig + + sysconfig.set_value(db, "plex_enabled", True) + db.commit() + return user, horror, comedy, tipped + + +def test_the_genre_and_the_recommendation_of_what_was_watched_win_most_draws(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + leads = [_first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) for s in SEEDS] + # Weights: Horror 1 + 2 (its genre), the recommended Western 1 + 3, the Comedy nobody chose 1. In + # a weighted slot the Comedy leads 1 time in 8, in the blind third 1 in 3: about 0.19 against + # 0.81 for the other two together (a uniform draw: 1 in 3 each). It still leads now and then — + # a surprise, not a ranking. + comedy = leads.count("comedy_new") + assert comedy > 0 + assert leads.count("horror_new") > 1.5 * comedy and leads.count("tipped_new") > 1.5 * comedy + assert leads.count("horror_new") + leads.count("tipped_new") > 3 * comedy + + +def test_with_no_history_the_plex_order_is_the_old_shuffle(db, make_user): + user, *_ = _seed_plex(db, make_user, history=False) + keys = ["horror_new", "comedy_new", "tipped_new"] + [f"horror_seen{i}" for i in range(3)] + for s in range(12): + got = [c["id"] for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")] + assert got == _md5_order(keys, s) + + +def test_a_watched_film_is_never_drawn(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + drawn = { + c["id"] for s in SEEDS for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en") + } + assert drawn == {"horror_new", "comedy_new", "tipped_new"} + + +def test_the_plex_blind_slot_deals_the_plain_shuffle(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + unseen = ["horror_new", "comedy_new", "tipped_new"] + for s in range(0, 30, 3): + got = _first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) + assert got == _md5_order(unseen, s)[0] + + +def test_a_binged_show_counts_once_and_recent_watches_weigh_more(db, make_user): + user = make_user() + lib = PlexLibrary(plex_key="78", title="Shows", kind="show", enabled=True) + mlib = PlexLibrary(plex_key="79", title="Movies", kind="movie", enabled=True) + db.add_all([lib, mlib]) + db.flush() + show = PlexShow(rating_key="sitcom", library_id=lib.id, title="Sitcom", genres=["Comedy"]) + db.add(show) + db.flush() + eps = [ + PlexItem( + rating_key=f"ep{i}", + library_id=lib.id, + kind="episode", + title=f"e{i}", + show_id=show.id, + season_number=1, + episode_number=i, + ) + for i in range(10) + ] + films = [ + PlexItem( + rating_key=f"h{i}", library_id=mlib.id, kind="movie", title=f"h{i}", genres=["Horror"] + ) + for i in range(2) + ] + old = PlexItem( + rating_key="w0", library_id=mlib.id, kind="movie", title="w0", genres=["Western"] + ) + db.add_all([*eps, *films, old]) + db.flush() + for it in [*eps, *films]: + db.add(PlexState(user_id=user.id, item_id=it.id, status="watched", watched_at=NOW)) + db.add( + PlexState( + user_id=user.id, item_id=old.id, status="watched", watched_at=NOW - timedelta(days=720) + ) + ) + db.commit() + + genres = affinity.for_user(db, user.id).genres + # Ten episodes of one show are one title: two horror films outweigh it. + assert genres["Horror"] == 1.0 + assert genres["Comedy"] == 0.5 + # Four half-lives ago weighs a sixteenth of today. + assert abs(genres["Western"] - 0.5 / 16) < 0.01 + + +def test_the_discover_rail_leans_the_same_way(db, make_user): + # The rail draws with random(), not a seed; setseed makes this session's draws repeatable. + user, *_ = _seed_plex(db, make_user, history=True) + db.execute(text("SELECT setseed(0.42)")) + leads = [ + plex_discover.surprise(db, user, "movie", "en", 3, [], [], LibraryFilters())["items"][0][ + "id" + ] + for _ in SEEDS + ] + assert leads.count("comedy_new") > 0 + assert leads.count("horror_new") + leads.count("tipped_new") > 3 * leads.count("comedy_new")