feat(discover): Surprise me leans on what you watch, and stays a surprise
The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are now weighted draws (`app.draw`, Efraimidis-Spirakis over the same md5 shuffle): a title's weight is 1 + its affinity. Plex: the genres of the latest watched titles (half-life 180 days, a show counts once) and the TMDB recommendations stored on them (`plex.affinity`). YouTube: the channels the user watches (half-life 90 days). One slot of three stays a blind draw, and with no history the order is exactly the old one. No visible change. The YouTube draw builds its pool once (a materialized CTE read twice): the scope is ~200k videos on a prod-sized catalog, and a second ordered pass doubled the cost. Measured on localdev: a reroll 210 -> 292 ms.
This commit is contained in:
@@ -65,7 +65,7 @@ how it gets built. Concretely:
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `reader/`, `discover/`, `catalog/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`, `draw`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
The domain is four **units**, each owning its tables: Capture (`downloads/`), Reader (`reader/`, `discover/`, the playlist
|
||||
sync), Catalog (`catalog/`, the `sync/` ingests) and Plex (`plex/`). A unit's route does no database work: it reads through
|
||||
the unit's `queries`, writes through its `service` (which flushes) and commits. A unit's code reads another unit's tables
|
||||
|
||||
@@ -25,8 +25,8 @@ Three layers, by *who* runs the code — not by feature:
|
||||
background run's durable row, its words in `runstatus`),
|
||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
level), `debugsession` (a recording that tags its user's lines, E16), `collation` (A→Z in the
|
||||
reader's alphabet, E30) and `draw` (the weighted, seeded draw "Surprise me" deals from), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
|
||||
+113
-15
@@ -17,9 +17,25 @@ otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand st
|
||||
here takes the feed's seeded-md5 idiom instead of `random()`.
|
||||
"""
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
Float,
|
||||
Select,
|
||||
and_,
|
||||
cast,
|
||||
func,
|
||||
literal,
|
||||
literal_column,
|
||||
or_,
|
||||
select,
|
||||
union_all,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import draw
|
||||
from app.discover.budget import band
|
||||
from app.feedquery import (
|
||||
feed_columns,
|
||||
@@ -33,7 +49,7 @@ from app.models import User
|
||||
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.reader.models import PlaylistItem, VideoState
|
||||
from app.plex import rollup
|
||||
from app.plex import affinity, rollup
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
from app.userscope import owned_by
|
||||
@@ -115,14 +131,93 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
|
||||
params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("shuffle", seed, rank_expr)
|
||||
|
||||
# Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
|
||||
# hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
|
||||
# go: the resume tier has already had its chance at them, and it may have passed precisely because
|
||||
# the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
|
||||
# runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
|
||||
rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all()
|
||||
out = [serialize(r) for r in rows]
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
take = limit * 4
|
||||
|
||||
def unstarted(rows) -> list[dict]:
|
||||
return [r for r in map(serialize, rows) if not r["position_seconds"]]
|
||||
|
||||
channels = _channel_affinity(db, user)
|
||||
if not channels:
|
||||
return unstarted(db.execute(query.order_by(*order_by(keys)).limit(take)).all())[:limit]
|
||||
|
||||
# Both orders from ONE pass over the scope. The pool is the whole feed scope (~200k videos on a
|
||||
# prod-sized catalog, ~390 ms a pass), so a second ordered query for the weighted draw doubled
|
||||
# the cost; a materialized pool of (id, shuffle key, weighted key) is read twice instead.
|
||||
weight = literal_column("1.0") + CHANNEL_WEIGHT * func.coalesce(
|
||||
cast(cast(literal(json.dumps(channels)), JSONB).op("->>")(Video.channel_id), Float), 0.0
|
||||
)
|
||||
shuffle = func.md5(func.concat(Video.id, str(seed)))
|
||||
pool = (
|
||||
query.with_only_columns(
|
||||
Video.id.label("vid"),
|
||||
shuffle.label("plain"),
|
||||
draw.weighted_key(draw.seeded_u(Video.id, seed), weight).label("leaning"),
|
||||
)
|
||||
.cte("surprise_pool")
|
||||
.prefix_with("MATERIALIZED")
|
||||
)
|
||||
|
||||
def top(side: int, *order) -> Select:
|
||||
part = (
|
||||
select(pool.c.vid, pool.c.plain, pool.c.leaning, literal(side).label("side"))
|
||||
.order_by(*order, pool.c.vid)
|
||||
.limit(take)
|
||||
.subquery()
|
||||
)
|
||||
return select(part)
|
||||
|
||||
# One statement, so the pool is built once; each side's order is restored from its keys.
|
||||
got = db.execute(union_all(top(0, pool.c.plain), top(1, pool.c.leaning, pool.c.plain))).all()
|
||||
plain_ids = [
|
||||
r.vid for r in sorted((r for r in got if r.side == 0), key=lambda r: (r.plain, r.vid))
|
||||
]
|
||||
leaning_ids = [
|
||||
r.vid
|
||||
for r in sorted((r for r in got if r.side == 1), key=lambda r: (r.leaning, r.plain, r.vid))
|
||||
]
|
||||
rows = {
|
||||
r["id"]: r
|
||||
for r in unstarted(db.execute(query.where(Video.id.in_({*plain_ids, *leaning_ids}))).all())
|
||||
}
|
||||
blind = [rows[i] for i in plain_ids if i in rows]
|
||||
leaning = [rows[i] for i in leaning_ids if i in rows]
|
||||
return draw.blend(leaning, blind, seed, limit, key=lambda r: r["id"])
|
||||
|
||||
|
||||
# How much a channel the user watches most outweighs one they never watch (1 + this), and how far back
|
||||
# the watched history counts: the latest WATCHED_RECENT watches, the recent ones weighing more.
|
||||
CHANNEL_WEIGHT = 3.0
|
||||
WATCHED_RECENT = 300
|
||||
WATCHED_HALF_LIFE_DAYS = 90
|
||||
|
||||
|
||||
def _channel_affinity(db: Session, user: User) -> dict[str, float]:
|
||||
"""Each channel the user watched lately, scored 0..1 against the most watched: the sum of its
|
||||
watches, each weighing half as much every WATCHED_HALF_LIFE_DAYS. Empty with no history."""
|
||||
rows = db.execute(
|
||||
select(Video.channel_id, VideoState.watched_at)
|
||||
.join(Video, Video.id == VideoState.video_id)
|
||||
.where(owned_by(VideoState.user_id, user), VideoState.status == "watched")
|
||||
.order_by(VideoState.watched_at.desc().nullslast())
|
||||
.limit(WATCHED_RECENT)
|
||||
).all()
|
||||
now = datetime.now(timezone.utc)
|
||||
scores: dict[str, float] = {}
|
||||
for channel_id, watched_at in rows:
|
||||
age = (
|
||||
(now - watched_at).total_seconds() / 86400 if watched_at else 2 * WATCHED_HALF_LIFE_DAYS
|
||||
)
|
||||
scores[channel_id] = scores.get(channel_id, 0.0) + 0.5 ** (
|
||||
max(0.0, age) / WATCHED_HALF_LIFE_DAYS
|
||||
)
|
||||
top = max(scores.values(), default=0.0)
|
||||
return {k: v / top for k, v in scores.items()} if top else {}
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
@@ -289,21 +384,24 @@ def plex_surprise(
|
||||
if not movie_libs:
|
||||
return []
|
||||
lo, hi = band(budget)
|
||||
q = (
|
||||
select(PlexItem)
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
_unseen_movie_conds(uid),
|
||||
)
|
||||
.order_by(_seeded(PlexItem.rating_key, seed))
|
||||
.limit(limit)
|
||||
q = select(PlexItem).where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
_unseen_movie_conds(uid),
|
||||
)
|
||||
if lo is not None:
|
||||
q = q.where(PlexItem.duration_s >= lo)
|
||||
if hi is not None:
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
shuffled = _seeded(PlexItem.rating_key, seed)
|
||||
blind = db.execute(q.order_by(shuffled, PlexItem.id).limit(limit)).scalars().all()
|
||||
leaning = affinity.for_user(db, uid)
|
||||
if leaning:
|
||||
key = draw.weighted_key(draw.seeded_u(PlexItem.rating_key, seed), leaning.weight(PlexItem))
|
||||
weighted = db.execute(q.order_by(key, shuffled, PlexItem.id).limit(limit)).scalars().all()
|
||||
items = draw.blend(list(weighted), list(blind), seed, limit, key=lambda it: it.id)
|
||||
else:
|
||||
items = list(blind)
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""A weighted draw that stays a surprise: what "Surprise me" deals from.
|
||||
|
||||
A title's chance to come first is its weight over the pool's (Efraimidis–Spirakis: order by
|
||||
-ln(1-u)/w, u uniform per row). The weight is 1 + the title's affinity to what the user watches
|
||||
(`plex.affinity`, `discover.queries`), so a title nobody leans towards still comes up — only less
|
||||
often. Two things keep it a surprise rather than a ranking:
|
||||
|
||||
- **u is the seed's md5 of the row's key**, the feed's shuffle idiom. With every weight 1 the order
|
||||
is exactly the md5 order the draw had before it was weighted, so a user with no history sees the
|
||||
old draw, and one seed still moves both sources of the Home hero together. A draw that is not
|
||||
seeded (the Plex Discover rail) takes u from random().
|
||||
- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few
|
||||
favourites cannot fill the rail.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Hashable
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import BigInteger, Float, cast, func, literal
|
||||
from sqlalchemy.dialects.postgresql import BIT
|
||||
from sqlalchemy.orm import InstrumentedAttribute
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
# 2**32: the first 8 hex digits of an md5 as a fraction of one.
|
||||
_SPAN = 4294967296.0
|
||||
|
||||
|
||||
def seeded_u(
|
||||
key: ColumnElement[Any] | InstrumentedAttribute[Any], seed: int
|
||||
) -> ColumnElement[float]:
|
||||
"""A uniform u in [0, 1) per row, from md5(key || seed) — the order `md5(...)` ascending gives."""
|
||||
digest = func.md5(func.concat(key, str(seed)))
|
||||
head = cast(cast(literal("x") + func.substr(digest, 1, 8), BIT(32)), BigInteger)
|
||||
return cast(head, Float) / _SPAN
|
||||
|
||||
|
||||
def weighted_key(u: ColumnElement[float], weight: ColumnElement[float]) -> ColumnElement[float]:
|
||||
"""The row's place in a weighted draw, ascending: -ln(1-u)/w. Monotonic in u when every w is 1,
|
||||
so the unweighted order is unchanged."""
|
||||
return -func.ln(1.0 - u) / weight
|
||||
|
||||
|
||||
def random_u() -> ColumnElement[float]:
|
||||
return func.random()
|
||||
|
||||
|
||||
def blend(
|
||||
weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x
|
||||
) -> list:
|
||||
"""`n` picks: slot i comes from `blind` when (i + offset) % 3 == 0, else from `weighted`, each
|
||||
skipping what an earlier slot took (`key` tells two picks apart). A side that runs out leaves the
|
||||
rest to the other."""
|
||||
out: list = []
|
||||
taken: set = set()
|
||||
sources = (iter(weighted), iter(blind))
|
||||
|
||||
def nxt(source):
|
||||
for item in source:
|
||||
if key(item) not in taken:
|
||||
return item
|
||||
return None
|
||||
|
||||
for i in range(n):
|
||||
first, second = (sources[1], sources[0]) if (i + offset) % 3 == 0 else sources
|
||||
item = nxt(first)
|
||||
if item is None:
|
||||
item = nxt(second)
|
||||
if item is None:
|
||||
break
|
||||
taken.add(key(item))
|
||||
out.append(item)
|
||||
return out
|
||||
@@ -0,0 +1,120 @@
|
||||
"""What a user's Plex history leans towards, as a weight for a surprise draw (`app.draw`).
|
||||
|
||||
Two signals, both already in the database (no TMDB call):
|
||||
- **genres** of the titles the user watched, the recent ones weighing more (a half-life of
|
||||
HALF_LIFE_DAYS), each scored 0..1 against the strongest;
|
||||
- the **TMDB recommendations** stored on those titles (`recommendations`), each scored by the most
|
||||
recent title that recommends it.
|
||||
|
||||
A title counts once however many of its episodes were watched — a binge of one show would otherwise
|
||||
outvote every film — and only the RECENT_TITLES latest titles count. A candidate's weight is
|
||||
1 + GENRE_WEIGHT × (the mean score of its genres) + REC_WEIGHT × (its recommendation score), so a
|
||||
title with no affinity keeps weight 1 and still comes up.
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import Float, cast, func, literal, literal_column, select
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||
from app.userscope import owned_by
|
||||
|
||||
HALF_LIFE_DAYS = 180
|
||||
RECENT_TITLES = 200
|
||||
GENRE_WEIGHT = 2.0
|
||||
REC_WEIGHT = 3.0
|
||||
# A watch with no date (an old import) counts as one this many half-lives ago.
|
||||
_UNDATED_HALF_LIVES = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Affinity:
|
||||
genres: dict[str, float] = field(default_factory=dict)
|
||||
movie_recs: dict[str, float] = field(default_factory=dict)
|
||||
show_recs: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return bool(self.genres or self.movie_recs or self.show_recs)
|
||||
|
||||
def weight(self, model: type[PlexItem] | type[PlexShow]) -> ColumnElement[float]:
|
||||
"""The draw weight of a `model` row (a movie or a show)."""
|
||||
scores = cast(literal(json.dumps(self.genres)), JSONB)
|
||||
recs = self.show_recs if model is PlexShow else self.movie_recs
|
||||
genre = func.jsonb_array_elements_text(model.genres).table_valued("value").alias("g")
|
||||
genre_aff = (
|
||||
select(
|
||||
func.coalesce(
|
||||
func.avg(func.coalesce(cast(scores.op("->>")(genre.c.value), Float), 0.0)),
|
||||
0.0,
|
||||
)
|
||||
)
|
||||
.select_from(genre)
|
||||
.scalar_subquery()
|
||||
)
|
||||
rec_aff = func.coalesce(
|
||||
cast(cast(literal(json.dumps(recs)), JSONB).op("->>")(model.guid), Float), 0.0
|
||||
)
|
||||
return literal_column("1.0") + GENRE_WEIGHT * genre_aff + REC_WEIGHT * rec_aff
|
||||
|
||||
|
||||
def for_user(db: Session, uid: int) -> Affinity:
|
||||
"""The user's affinity from their latest watched titles; empty (`not aff`) with no history."""
|
||||
# One key per title: a show for an episode, the item for a film. concat() of a NULL show_id is
|
||||
# just "s", which nullif turns back into NULL so the film's key takes over.
|
||||
title = func.coalesce(
|
||||
func.nullif(func.concat("s", PlexItem.show_id), "s"), func.concat("m", PlexItem.id)
|
||||
).label("title")
|
||||
latest = (
|
||||
select(title, PlexState.watched_at, PlexItem.id.label("item_id"), PlexItem.show_id)
|
||||
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||
.where(owned_by(PlexState.user_id, uid), PlexState.status == "watched")
|
||||
.distinct(title)
|
||||
.order_by(title, PlexState.watched_at.desc().nullslast())
|
||||
.subquery()
|
||||
)
|
||||
rows = db.execute(
|
||||
select(
|
||||
latest.c.watched_at,
|
||||
PlexItem.genres,
|
||||
PlexItem.recommendations,
|
||||
PlexShow.genres,
|
||||
PlexShow.recommendations,
|
||||
)
|
||||
.select_from(latest)
|
||||
.join(PlexItem, PlexItem.id == latest.c.item_id)
|
||||
.outerjoin(PlexShow, PlexShow.id == latest.c.show_id)
|
||||
.order_by(latest.c.watched_at.desc().nullslast())
|
||||
.limit(RECENT_TITLES)
|
||||
).all()
|
||||
if not rows:
|
||||
return Affinity()
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
genres: dict[str, float] = {}
|
||||
movie_recs: dict[str, float] = {}
|
||||
show_recs: dict[str, float] = {}
|
||||
for watched_at, item_genres, item_recs, show_genres, show_recs_json in rows:
|
||||
if watched_at is None:
|
||||
r = 0.5**_UNDATED_HALF_LIVES
|
||||
else:
|
||||
r = 0.5 ** (max(0.0, (now - watched_at).total_seconds() / 86400) / HALF_LIFE_DAYS)
|
||||
for g in (show_genres if show_genres is not None else item_genres) or []:
|
||||
genres[g] = genres.get(g, 0.0) + r
|
||||
for rec in (show_recs_json if show_recs_json is not None else item_recs) or []:
|
||||
if not rec.get("tmdb_id"):
|
||||
continue
|
||||
bucket = show_recs if rec.get("media_type") == "tv" else movie_recs
|
||||
guid = f"tmdb:{rec['tmdb_id']}"
|
||||
bucket[guid] = max(bucket.get(guid, 0.0), r)
|
||||
return Affinity(_scaled(genres), _scaled(movie_recs), _scaled(show_recs))
|
||||
|
||||
|
||||
def _scaled(scores: dict[str, float]) -> dict[str, float]:
|
||||
"""Each score over the strongest, so the strongest is 1."""
|
||||
top = max(scores.values(), default=0.0)
|
||||
return {k: v / top for k, v in scores.items()} if top else {}
|
||||
@@ -22,7 +22,8 @@ from sqlalchemy.orm import Session
|
||||
from app.collation import sorted_words
|
||||
from app.discover.queries import RAIL_LIMIT
|
||||
from app.models import User
|
||||
from app.plex import rollup
|
||||
from app import draw
|
||||
from app.plex import affinity, rollup
|
||||
from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||
from app.plex.query import (
|
||||
@@ -399,22 +400,25 @@ def discover_page(db: Session, user: User, scope: str, lang: str, f: LibraryFilt
|
||||
}
|
||||
|
||||
|
||||
def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list:
|
||||
def _draw(db: Session, stmt, key_col, per: int, exclude: list[str], weight=None) -> list:
|
||||
"""`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a
|
||||
reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave
|
||||
short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail."""
|
||||
fresh = (
|
||||
db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per))
|
||||
.scalars()
|
||||
.all()
|
||||
if exclude
|
||||
else []
|
||||
)
|
||||
short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail.
|
||||
With a `weight` (`plex.affinity`) the draw leans on it, every third slot left blind (`app.draw`)."""
|
||||
|
||||
def pick(q, n: int) -> list:
|
||||
blind = db.execute(q.order_by(func.random()).limit(n)).scalars().all()
|
||||
if weight is None:
|
||||
return list(blind)
|
||||
key = draw.weighted_key(draw.random_u(), weight)
|
||||
leaning = db.execute(q.order_by(key).limit(n)).scalars().all()
|
||||
return draw.blend(list(leaning), list(blind), 1, n, key=lambda r: r.rating_key)
|
||||
|
||||
fresh = pick(stmt.where(key_col.not_in(exclude)), per) if exclude else []
|
||||
if len(fresh) == per:
|
||||
return list(fresh)
|
||||
return fresh
|
||||
taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else []
|
||||
rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars()
|
||||
return [*fresh, *rest.all()]
|
||||
return [*fresh, *pick(stmt.where(*taken), per - len(fresh))]
|
||||
|
||||
|
||||
def _kept(
|
||||
@@ -474,6 +478,8 @@ def surprise(
|
||||
if keep:
|
||||
return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)}
|
||||
per = min(limit, SURPRISE_PER_KIND)
|
||||
# What the user's history leans towards; with none, every title weighs the same (a plain draw).
|
||||
leaning = affinity.for_user(db, user.id)
|
||||
movies = []
|
||||
if movie_ids:
|
||||
# Exclude movies this user has watched.
|
||||
@@ -501,6 +507,7 @@ def surprise(
|
||||
PlexItem.rating_key,
|
||||
per,
|
||||
exclude,
|
||||
leaning.weight(PlexItem) if leaning else None,
|
||||
)
|
||||
shows = []
|
||||
if show_ids:
|
||||
@@ -523,6 +530,7 @@ def surprise(
|
||||
PlexShow.rating_key,
|
||||
per,
|
||||
exclude,
|
||||
leaning.weight(PlexShow) if leaning else None,
|
||||
)
|
||||
mcards = _movie_cards(db, user.id, movies, lang)
|
||||
scards = _show_cards(db, user.id, shows, lang)
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Surprise me leans on what the user watches, and stays a surprise.
|
||||
|
||||
The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are
|
||||
now weighted draws (`app.draw`): a title's weight is 1 + its affinity — for Plex the genres of the
|
||||
films and shows the user watched (recent ones weigh more) and the TMDB recommendations stored on them;
|
||||
for YouTube the channels the user watches. One slot of three stays a blind draw, and with no watch
|
||||
history the order is exactly the old one. DB lane.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import draw
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.discover import queries
|
||||
from app.plex import affinity, discover as plex_discover
|
||||
from app.plex.models import PlexItem, PlexLibrary, PlexShow, PlexState
|
||||
from app.plex.query import LibraryFilters
|
||||
from app.reader.models import VideoState
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
SEEDS = range(90)
|
||||
|
||||
|
||||
# ── the draw itself ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_every_third_slot_is_blind_and_nothing_repeats():
|
||||
weighted = ["a", "b", "c", "d", "e", "f"]
|
||||
blind = ["f", "e", "d", "c", "b", "a"]
|
||||
# offset 0: slots 0 and 3 are blind
|
||||
assert draw.blend(weighted, blind, 0, 6) == ["f", "a", "b", "e", "c", "d"]
|
||||
# offset 1: slot 2 and 5
|
||||
assert draw.blend(weighted, blind, 1, 6) == ["a", "b", "f", "c", "d", "e"]
|
||||
assert draw.blend(weighted, blind, 2, 3) == ["a", "f", "b"]
|
||||
|
||||
|
||||
def test_a_short_side_is_topped_up_from_the_other():
|
||||
assert draw.blend(["a"], ["a", "b", "c"], 0, 3) == ["a", "b", "c"]
|
||||
assert draw.blend(["a", "b", "c"], [], 0, 3) == ["a", "b", "c"]
|
||||
|
||||
|
||||
# ── YouTube: the channels the user watches ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _md5_order(ids, seed):
|
||||
return sorted(ids, key=lambda i: hashlib.md5(f"{i}{seed}".encode()).hexdigest())
|
||||
|
||||
|
||||
def _seed_yt(db, make_user, *, watched_in_fav: int):
|
||||
user = make_user()
|
||||
fav, other = Channel(id="UCfav", title="Fav"), Channel(id="UCother", title="Other")
|
||||
db.add_all([fav, other])
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
Subscription(user_id=user.id, channel_id="UCfav"),
|
||||
Subscription(user_id=user.id, channel_id="UCother"),
|
||||
]
|
||||
)
|
||||
|
||||
def video(vid, ch, days):
|
||||
return Video(
|
||||
id=vid,
|
||||
channel_id=ch,
|
||||
title=vid,
|
||||
duration_seconds=20 * 60,
|
||||
published_at=NOW - timedelta(days=days),
|
||||
)
|
||||
|
||||
# One unseen video per channel, plus the user's watched history on the favourite.
|
||||
db.add_all([video("fav_new", "UCfav", 1), video("other_new", "UCother", 1)])
|
||||
for i in range(watched_in_fav):
|
||||
db.add(video(f"fav_seen{i}", "UCfav", 10 + i))
|
||||
db.flush()
|
||||
for i in range(watched_in_fav):
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id=f"fav_seen{i}",
|
||||
status="watched",
|
||||
watched_at=NOW - timedelta(days=i),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
|
||||
def _first(rows):
|
||||
return rows[0]["id"] if rows else None
|
||||
|
||||
|
||||
def test_a_watched_channel_wins_most_draws_but_not_all(db, make_user):
|
||||
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||
leads = [_first(queries.yt_surprise(db, user, "any", s, 2)) for s in SEEDS]
|
||||
fav = leads.count("fav_new") / len(leads)
|
||||
# weight 1 + 3 against 1: 0.8 in a weighted slot, 0.5 in the blind one (a third of the seeds)
|
||||
assert 0.6 < fav < 0.9, fav
|
||||
|
||||
|
||||
def test_the_blind_slot_deals_the_plain_shuffle_however_strong_the_lean(db, make_user):
|
||||
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||
for s in range(0, 30, 3): # (0 + seed) % 3 == 0: the first slot is the blind one
|
||||
got = _first(queries.yt_surprise(db, user, "any", s, 2))
|
||||
assert got == _md5_order(["fav_new", "other_new"], s)[0]
|
||||
|
||||
|
||||
def test_with_no_history_the_youtube_order_is_the_old_shuffle(db, make_user):
|
||||
user = _seed_yt(db, make_user, watched_in_fav=0)
|
||||
for s in range(12):
|
||||
got = [r["id"] for r in queries.yt_surprise(db, user, "any", s, 2)]
|
||||
assert got == _md5_order(["fav_new", "other_new"], s)
|
||||
|
||||
|
||||
def test_a_watched_video_is_never_drawn(db, make_user):
|
||||
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||
drawn = {r["id"] for s in SEEDS for r in queries.yt_surprise(db, user, "any", s, 5)}
|
||||
assert drawn == {"fav_new", "other_new"}
|
||||
|
||||
|
||||
# ── Plex: genres and recommendations of what the user watched ────────────────────────────────────
|
||||
|
||||
|
||||
def _seed_plex(db, make_user, *, history: bool):
|
||||
user = make_user()
|
||||
lib = PlexLibrary(plex_key="77", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
tmdb_ids = iter(range(5000, 6000))
|
||||
|
||||
def movie(key, genres, recs=None):
|
||||
m = PlexItem(
|
||||
rating_key=key,
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title=key,
|
||||
duration_s=100 * 60,
|
||||
genres=genres,
|
||||
guid=f"tmdb:{next(tmdb_ids)}",
|
||||
recommendations=recs,
|
||||
)
|
||||
db.add(m)
|
||||
return m
|
||||
|
||||
horror = movie("horror_new", ["Horror"])
|
||||
comedy = movie("comedy_new", ["Comedy"])
|
||||
tipped = movie("tipped_new", ["Western"])
|
||||
db.flush()
|
||||
tip = [{"tmdb_id": int(tipped.guid.split(":")[1]), "media_type": "movie", "title": "t"}]
|
||||
seen = [movie(f"horror_seen{i}", ["Horror"], tip if i == 0 else None) for i in range(3)]
|
||||
db.flush()
|
||||
if history:
|
||||
for i, m in enumerate(seen):
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=m.id,
|
||||
status="watched",
|
||||
watched_at=NOW - timedelta(days=i),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
from app import sysconfig
|
||||
|
||||
sysconfig.set_value(db, "plex_enabled", True)
|
||||
db.commit()
|
||||
return user, horror, comedy, tipped
|
||||
|
||||
|
||||
def test_the_genre_and_the_recommendation_of_what_was_watched_win_most_draws(db, make_user):
|
||||
user, *_ = _seed_plex(db, make_user, history=True)
|
||||
leads = [_first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) for s in SEEDS]
|
||||
# Weights: Horror 1 + 2 (its genre), the recommended Western 1 + 3, the Comedy nobody chose 1. In
|
||||
# a weighted slot the Comedy leads 1 time in 8, in the blind third 1 in 3: about 0.19 against
|
||||
# 0.81 for the other two together (a uniform draw: 1 in 3 each). It still leads now and then —
|
||||
# a surprise, not a ranking.
|
||||
comedy = leads.count("comedy_new")
|
||||
assert comedy > 0
|
||||
assert leads.count("horror_new") > 1.5 * comedy and leads.count("tipped_new") > 1.5 * comedy
|
||||
assert leads.count("horror_new") + leads.count("tipped_new") > 3 * comedy
|
||||
|
||||
|
||||
def test_with_no_history_the_plex_order_is_the_old_shuffle(db, make_user):
|
||||
user, *_ = _seed_plex(db, make_user, history=False)
|
||||
keys = ["horror_new", "comedy_new", "tipped_new"] + [f"horror_seen{i}" for i in range(3)]
|
||||
for s in range(12):
|
||||
got = [c["id"] for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")]
|
||||
assert got == _md5_order(keys, s)
|
||||
|
||||
|
||||
def test_a_watched_film_is_never_drawn(db, make_user):
|
||||
user, *_ = _seed_plex(db, make_user, history=True)
|
||||
drawn = {
|
||||
c["id"] for s in SEEDS for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")
|
||||
}
|
||||
assert drawn == {"horror_new", "comedy_new", "tipped_new"}
|
||||
|
||||
|
||||
def test_the_plex_blind_slot_deals_the_plain_shuffle(db, make_user):
|
||||
user, *_ = _seed_plex(db, make_user, history=True)
|
||||
unseen = ["horror_new", "comedy_new", "tipped_new"]
|
||||
for s in range(0, 30, 3):
|
||||
got = _first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en"))
|
||||
assert got == _md5_order(unseen, s)[0]
|
||||
|
||||
|
||||
def test_a_binged_show_counts_once_and_recent_watches_weigh_more(db, make_user):
|
||||
user = make_user()
|
||||
lib = PlexLibrary(plex_key="78", title="Shows", kind="show", enabled=True)
|
||||
mlib = PlexLibrary(plex_key="79", title="Movies", kind="movie", enabled=True)
|
||||
db.add_all([lib, mlib])
|
||||
db.flush()
|
||||
show = PlexShow(rating_key="sitcom", library_id=lib.id, title="Sitcom", genres=["Comedy"])
|
||||
db.add(show)
|
||||
db.flush()
|
||||
eps = [
|
||||
PlexItem(
|
||||
rating_key=f"ep{i}",
|
||||
library_id=lib.id,
|
||||
kind="episode",
|
||||
title=f"e{i}",
|
||||
show_id=show.id,
|
||||
season_number=1,
|
||||
episode_number=i,
|
||||
)
|
||||
for i in range(10)
|
||||
]
|
||||
films = [
|
||||
PlexItem(
|
||||
rating_key=f"h{i}", library_id=mlib.id, kind="movie", title=f"h{i}", genres=["Horror"]
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
old = PlexItem(
|
||||
rating_key="w0", library_id=mlib.id, kind="movie", title="w0", genres=["Western"]
|
||||
)
|
||||
db.add_all([*eps, *films, old])
|
||||
db.flush()
|
||||
for it in [*eps, *films]:
|
||||
db.add(PlexState(user_id=user.id, item_id=it.id, status="watched", watched_at=NOW))
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id, item_id=old.id, status="watched", watched_at=NOW - timedelta(days=720)
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
genres = affinity.for_user(db, user.id).genres
|
||||
# Ten episodes of one show are one title: two horror films outweigh it.
|
||||
assert genres["Horror"] == 1.0
|
||||
assert genres["Comedy"] == 0.5
|
||||
# Four half-lives ago weighs a sixteenth of today.
|
||||
assert abs(genres["Western"] - 0.5 / 16) < 0.01
|
||||
|
||||
|
||||
def test_the_discover_rail_leans_the_same_way(db, make_user):
|
||||
# The rail draws with random(), not a seed; setseed makes this session's draws repeatable.
|
||||
user, *_ = _seed_plex(db, make_user, history=True)
|
||||
db.execute(text("SELECT setseed(0.42)"))
|
||||
leads = [
|
||||
plex_discover.surprise(db, user, "movie", "en", 3, [], [], LibraryFilters())["items"][0][
|
||||
"id"
|
||||
]
|
||||
for _ in SEEDS
|
||||
]
|
||||
assert leads.count("comedy_new") > 0
|
||||
assert leads.count("horror_new") + leads.count("tipped_new") > 3 * leads.count("comedy_new")
|
||||
Reference in New Issue
Block a user