feat(discover): Surprise me leans on what you watch, and stays a surprise
The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are now weighted draws (`app.draw`, Efraimidis-Spirakis over the same md5 shuffle): a title's weight is 1 + its affinity. Plex: the genres of the latest watched titles (half-life 180 days, a show counts once) and the TMDB recommendations stored on them (`plex.affinity`). YouTube: the channels the user watches (half-life 90 days). One slot of three stays a blind draw, and with no history the order is exactly the old one. No visible change. The YouTube draw builds its pool once (a materialized CTE read twice): the scope is ~200k videos on a prod-sized catalog, and a second ordered pass doubled the cost. Measured on localdev: a reroll 210 -> 292 ms.
This commit is contained in:
@@ -65,7 +65,7 @@ how it gets built. Concretely:
|
|||||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||||
`downloads/`, `reader/`, `discover/`, `catalog/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
|
`downloads/`, `reader/`, `discover/`, `catalog/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
|
||||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`, `draw`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||||
The domain is four **units**, each owning its tables: Capture (`downloads/`), Reader (`reader/`, `discover/`, the playlist
|
The domain is four **units**, each owning its tables: Capture (`downloads/`), Reader (`reader/`, `discover/`, the playlist
|
||||||
sync), Catalog (`catalog/`, the `sync/` ingests) and Plex (`plex/`). A unit's route does no database work: it reads through
|
sync), Catalog (`catalog/`, the `sync/` ingests) and Plex (`plex/`). A unit's route does no database work: it reads through
|
||||||
the unit's `queries`, writes through its `service` (which flushes) and commits. A unit's code reads another unit's tables
|
the unit's `queries`, writes through its `service` (which flushes) and commits. A unit's code reads another unit's tables
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ Three layers, by *who* runs the code — not by feature:
|
|||||||
background run's durable row, its words in `runstatus`),
|
background run's durable row, its words in `runstatus`),
|
||||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
level), `debugsession` (a recording that tags its user's lines, E16), `collation` (A→Z in the
|
||||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
reader's alphabet, E30) and `draw` (the weighted, seeded draw "Surprise me" deals from), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||||
module must never import a `routes/…` module).
|
module must never import a `routes/…` module).
|
||||||
|
|
||||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||||
|
|||||||
+113
-15
@@ -17,9 +17,25 @@ otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand st
|
|||||||
here takes the feed's seeded-md5 idiom instead of `random()`.
|
here takes the feed's seeded-md5 idiom instead of `random()`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import and_, func, or_, select
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Float,
|
||||||
|
Select,
|
||||||
|
and_,
|
||||||
|
cast,
|
||||||
|
func,
|
||||||
|
literal,
|
||||||
|
literal_column,
|
||||||
|
or_,
|
||||||
|
select,
|
||||||
|
union_all,
|
||||||
|
)
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Session, aliased
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
|
from app import draw
|
||||||
from app.discover.budget import band
|
from app.discover.budget import band
|
||||||
from app.feedquery import (
|
from app.feedquery import (
|
||||||
feed_columns,
|
feed_columns,
|
||||||
@@ -33,7 +49,7 @@ from app.models import User
|
|||||||
from app.plex.models import PlexItem, PlexShow, PlexState
|
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||||
from app.catalog.models import Channel, Video
|
from app.catalog.models import Channel, Video
|
||||||
from app.reader.models import PlaylistItem, VideoState
|
from app.reader.models import PlaylistItem, VideoState
|
||||||
from app.plex import rollup
|
from app.plex import affinity, rollup
|
||||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||||
from app.plex.query import apply_show_state_filter, content_libs
|
from app.plex.query import apply_show_state_filter, content_libs
|
||||||
from app.userscope import owned_by
|
from app.userscope import owned_by
|
||||||
@@ -115,14 +131,93 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
|
|||||||
params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
|
params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
|
||||||
query, rank_expr = filtered_query(db, user, **params)
|
query, rank_expr = filtered_query(db, user, **params)
|
||||||
keys = sort_keys("shuffle", seed, rank_expr)
|
keys = sort_keys("shuffle", seed, rank_expr)
|
||||||
|
|
||||||
# Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
|
# Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
|
||||||
# hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
|
# hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
|
||||||
# go: the resume tier has already had its chance at them, and it may have passed precisely because
|
# go: the resume tier has already had its chance at them, and it may have passed precisely because
|
||||||
# the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
|
# the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
|
||||||
# runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
|
# runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
|
||||||
rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all()
|
take = limit * 4
|
||||||
out = [serialize(r) for r in rows]
|
|
||||||
return [r for r in out if not r["position_seconds"]][:limit]
|
def unstarted(rows) -> list[dict]:
|
||||||
|
return [r for r in map(serialize, rows) if not r["position_seconds"]]
|
||||||
|
|
||||||
|
channels = _channel_affinity(db, user)
|
||||||
|
if not channels:
|
||||||
|
return unstarted(db.execute(query.order_by(*order_by(keys)).limit(take)).all())[:limit]
|
||||||
|
|
||||||
|
# Both orders from ONE pass over the scope. The pool is the whole feed scope (~200k videos on a
|
||||||
|
# prod-sized catalog, ~390 ms a pass), so a second ordered query for the weighted draw doubled
|
||||||
|
# the cost; a materialized pool of (id, shuffle key, weighted key) is read twice instead.
|
||||||
|
weight = literal_column("1.0") + CHANNEL_WEIGHT * func.coalesce(
|
||||||
|
cast(cast(literal(json.dumps(channels)), JSONB).op("->>")(Video.channel_id), Float), 0.0
|
||||||
|
)
|
||||||
|
shuffle = func.md5(func.concat(Video.id, str(seed)))
|
||||||
|
pool = (
|
||||||
|
query.with_only_columns(
|
||||||
|
Video.id.label("vid"),
|
||||||
|
shuffle.label("plain"),
|
||||||
|
draw.weighted_key(draw.seeded_u(Video.id, seed), weight).label("leaning"),
|
||||||
|
)
|
||||||
|
.cte("surprise_pool")
|
||||||
|
.prefix_with("MATERIALIZED")
|
||||||
|
)
|
||||||
|
|
||||||
|
def top(side: int, *order) -> Select:
|
||||||
|
part = (
|
||||||
|
select(pool.c.vid, pool.c.plain, pool.c.leaning, literal(side).label("side"))
|
||||||
|
.order_by(*order, pool.c.vid)
|
||||||
|
.limit(take)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
return select(part)
|
||||||
|
|
||||||
|
# One statement, so the pool is built once; each side's order is restored from its keys.
|
||||||
|
got = db.execute(union_all(top(0, pool.c.plain), top(1, pool.c.leaning, pool.c.plain))).all()
|
||||||
|
plain_ids = [
|
||||||
|
r.vid for r in sorted((r for r in got if r.side == 0), key=lambda r: (r.plain, r.vid))
|
||||||
|
]
|
||||||
|
leaning_ids = [
|
||||||
|
r.vid
|
||||||
|
for r in sorted((r for r in got if r.side == 1), key=lambda r: (r.leaning, r.plain, r.vid))
|
||||||
|
]
|
||||||
|
rows = {
|
||||||
|
r["id"]: r
|
||||||
|
for r in unstarted(db.execute(query.where(Video.id.in_({*plain_ids, *leaning_ids}))).all())
|
||||||
|
}
|
||||||
|
blind = [rows[i] for i in plain_ids if i in rows]
|
||||||
|
leaning = [rows[i] for i in leaning_ids if i in rows]
|
||||||
|
return draw.blend(leaning, blind, seed, limit, key=lambda r: r["id"])
|
||||||
|
|
||||||
|
|
||||||
|
# How much a channel the user watches most outweighs one they never watch (1 + this), and how far back
|
||||||
|
# the watched history counts: the latest WATCHED_RECENT watches, the recent ones weighing more.
|
||||||
|
CHANNEL_WEIGHT = 3.0
|
||||||
|
WATCHED_RECENT = 300
|
||||||
|
WATCHED_HALF_LIFE_DAYS = 90
|
||||||
|
|
||||||
|
|
||||||
|
def _channel_affinity(db: Session, user: User) -> dict[str, float]:
|
||||||
|
"""Each channel the user watched lately, scored 0..1 against the most watched: the sum of its
|
||||||
|
watches, each weighing half as much every WATCHED_HALF_LIFE_DAYS. Empty with no history."""
|
||||||
|
rows = db.execute(
|
||||||
|
select(Video.channel_id, VideoState.watched_at)
|
||||||
|
.join(Video, Video.id == VideoState.video_id)
|
||||||
|
.where(owned_by(VideoState.user_id, user), VideoState.status == "watched")
|
||||||
|
.order_by(VideoState.watched_at.desc().nullslast())
|
||||||
|
.limit(WATCHED_RECENT)
|
||||||
|
).all()
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
scores: dict[str, float] = {}
|
||||||
|
for channel_id, watched_at in rows:
|
||||||
|
age = (
|
||||||
|
(now - watched_at).total_seconds() / 86400 if watched_at else 2 * WATCHED_HALF_LIFE_DAYS
|
||||||
|
)
|
||||||
|
scores[channel_id] = scores.get(channel_id, 0.0) + 0.5 ** (
|
||||||
|
max(0.0, age) / WATCHED_HALF_LIFE_DAYS
|
||||||
|
)
|
||||||
|
top = max(scores.values(), default=0.0)
|
||||||
|
return {k: v / top for k, v in scores.items()} if top else {}
|
||||||
|
|
||||||
|
|
||||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||||
@@ -289,21 +384,24 @@ def plex_surprise(
|
|||||||
if not movie_libs:
|
if not movie_libs:
|
||||||
return []
|
return []
|
||||||
lo, hi = band(budget)
|
lo, hi = band(budget)
|
||||||
q = (
|
q = select(PlexItem).where(
|
||||||
select(PlexItem)
|
PlexItem.kind == "movie",
|
||||||
.where(
|
PlexItem.library_id.in_(movie_libs),
|
||||||
PlexItem.kind == "movie",
|
_unseen_movie_conds(uid),
|
||||||
PlexItem.library_id.in_(movie_libs),
|
|
||||||
_unseen_movie_conds(uid),
|
|
||||||
)
|
|
||||||
.order_by(_seeded(PlexItem.rating_key, seed))
|
|
||||||
.limit(limit)
|
|
||||||
)
|
)
|
||||||
if lo is not None:
|
if lo is not None:
|
||||||
q = q.where(PlexItem.duration_s >= lo)
|
q = q.where(PlexItem.duration_s >= lo)
|
||||||
if hi is not None:
|
if hi is not None:
|
||||||
q = q.where(PlexItem.duration_s <= hi)
|
q = q.where(PlexItem.duration_s <= hi)
|
||||||
items = db.execute(q).scalars().all()
|
shuffled = _seeded(PlexItem.rating_key, seed)
|
||||||
|
blind = db.execute(q.order_by(shuffled, PlexItem.id).limit(limit)).scalars().all()
|
||||||
|
leaning = affinity.for_user(db, uid)
|
||||||
|
if leaning:
|
||||||
|
key = draw.weighted_key(draw.seeded_u(PlexItem.rating_key, seed), leaning.weight(PlexItem))
|
||||||
|
weighted = db.execute(q.order_by(key, shuffled, PlexItem.id).limit(limit)).scalars().all()
|
||||||
|
items = draw.blend(list(weighted), list(blind), seed, limit, key=lambda it: it.id)
|
||||||
|
else:
|
||||||
|
items = list(blind)
|
||||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||||
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
"""A weighted draw that stays a surprise: what "Surprise me" deals from.
|
||||||
|
|
||||||
|
A title's chance to come first is its weight over the pool's (Efraimidis–Spirakis: order by
|
||||||
|
-ln(1-u)/w, u uniform per row). The weight is 1 + the title's affinity to what the user watches
|
||||||
|
(`plex.affinity`, `discover.queries`), so a title nobody leans towards still comes up — only less
|
||||||
|
often. Two things keep it a surprise rather than a ranking:
|
||||||
|
|
||||||
|
- **u is the seed's md5 of the row's key**, the feed's shuffle idiom. With every weight 1 the order
|
||||||
|
is exactly the md5 order the draw had before it was weighted, so a user with no history sees the
|
||||||
|
old draw, and one seed still moves both sources of the Home hero together. A draw that is not
|
||||||
|
seeded (the Plex Discover rail) takes u from random().
|
||||||
|
- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few
|
||||||
|
favourites cannot fill the rail.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Callable, Hashable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, Float, cast, func, literal
|
||||||
|
from sqlalchemy.dialects.postgresql import BIT
|
||||||
|
from sqlalchemy.orm import InstrumentedAttribute
|
||||||
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
|
|
||||||
|
# 2**32: the first 8 hex digits of an md5 as a fraction of one.
|
||||||
|
_SPAN = 4294967296.0
|
||||||
|
|
||||||
|
|
||||||
|
def seeded_u(
|
||||||
|
key: ColumnElement[Any] | InstrumentedAttribute[Any], seed: int
|
||||||
|
) -> ColumnElement[float]:
|
||||||
|
"""A uniform u in [0, 1) per row, from md5(key || seed) — the order `md5(...)` ascending gives."""
|
||||||
|
digest = func.md5(func.concat(key, str(seed)))
|
||||||
|
head = cast(cast(literal("x") + func.substr(digest, 1, 8), BIT(32)), BigInteger)
|
||||||
|
return cast(head, Float) / _SPAN
|
||||||
|
|
||||||
|
|
||||||
|
def weighted_key(u: ColumnElement[float], weight: ColumnElement[float]) -> ColumnElement[float]:
|
||||||
|
"""The row's place in a weighted draw, ascending: -ln(1-u)/w. Monotonic in u when every w is 1,
|
||||||
|
so the unweighted order is unchanged."""
|
||||||
|
return -func.ln(1.0 - u) / weight
|
||||||
|
|
||||||
|
|
||||||
|
def random_u() -> ColumnElement[float]:
|
||||||
|
return func.random()
|
||||||
|
|
||||||
|
|
||||||
|
def blend(
|
||||||
|
weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x
|
||||||
|
) -> list:
|
||||||
|
"""`n` picks: slot i comes from `blind` when (i + offset) % 3 == 0, else from `weighted`, each
|
||||||
|
skipping what an earlier slot took (`key` tells two picks apart). A side that runs out leaves the
|
||||||
|
rest to the other."""
|
||||||
|
out: list = []
|
||||||
|
taken: set = set()
|
||||||
|
sources = (iter(weighted), iter(blind))
|
||||||
|
|
||||||
|
def nxt(source):
|
||||||
|
for item in source:
|
||||||
|
if key(item) not in taken:
|
||||||
|
return item
|
||||||
|
return None
|
||||||
|
|
||||||
|
for i in range(n):
|
||||||
|
first, second = (sources[1], sources[0]) if (i + offset) % 3 == 0 else sources
|
||||||
|
item = nxt(first)
|
||||||
|
if item is None:
|
||||||
|
item = nxt(second)
|
||||||
|
if item is None:
|
||||||
|
break
|
||||||
|
taken.add(key(item))
|
||||||
|
out.append(item)
|
||||||
|
return out
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""What a user's Plex history leans towards, as a weight for a surprise draw (`app.draw`).
|
||||||
|
|
||||||
|
Two signals, both already in the database (no TMDB call):
|
||||||
|
- **genres** of the titles the user watched, the recent ones weighing more (a half-life of
|
||||||
|
HALF_LIFE_DAYS), each scored 0..1 against the strongest;
|
||||||
|
- the **TMDB recommendations** stored on those titles (`recommendations`), each scored by the most
|
||||||
|
recent title that recommends it.
|
||||||
|
|
||||||
|
A title counts once however many of its episodes were watched — a binge of one show would otherwise
|
||||||
|
outvote every film — and only the RECENT_TITLES latest titles count. A candidate's weight is
|
||||||
|
1 + GENRE_WEIGHT × (the mean score of its genres) + REC_WEIGHT × (its recommendation score), so a
|
||||||
|
title with no affinity keeps weight 1 and still comes up.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import Float, cast, func, literal, literal_column, select
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
|
|
||||||
|
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||||
|
from app.userscope import owned_by
|
||||||
|
|
||||||
|
HALF_LIFE_DAYS = 180
|
||||||
|
RECENT_TITLES = 200
|
||||||
|
GENRE_WEIGHT = 2.0
|
||||||
|
REC_WEIGHT = 3.0
|
||||||
|
# A watch with no date (an old import) counts as one this many half-lives ago.
|
||||||
|
_UNDATED_HALF_LIVES = 2
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Affinity:
|
||||||
|
genres: dict[str, float] = field(default_factory=dict)
|
||||||
|
movie_recs: dict[str, float] = field(default_factory=dict)
|
||||||
|
show_recs: dict[str, float] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def __bool__(self) -> bool:
|
||||||
|
return bool(self.genres or self.movie_recs or self.show_recs)
|
||||||
|
|
||||||
|
def weight(self, model: type[PlexItem] | type[PlexShow]) -> ColumnElement[float]:
|
||||||
|
"""The draw weight of a `model` row (a movie or a show)."""
|
||||||
|
scores = cast(literal(json.dumps(self.genres)), JSONB)
|
||||||
|
recs = self.show_recs if model is PlexShow else self.movie_recs
|
||||||
|
genre = func.jsonb_array_elements_text(model.genres).table_valued("value").alias("g")
|
||||||
|
genre_aff = (
|
||||||
|
select(
|
||||||
|
func.coalesce(
|
||||||
|
func.avg(func.coalesce(cast(scores.op("->>")(genre.c.value), Float), 0.0)),
|
||||||
|
0.0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.select_from(genre)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
rec_aff = func.coalesce(
|
||||||
|
cast(cast(literal(json.dumps(recs)), JSONB).op("->>")(model.guid), Float), 0.0
|
||||||
|
)
|
||||||
|
return literal_column("1.0") + GENRE_WEIGHT * genre_aff + REC_WEIGHT * rec_aff
|
||||||
|
|
||||||
|
|
||||||
|
def for_user(db: Session, uid: int) -> Affinity:
|
||||||
|
"""The user's affinity from their latest watched titles; empty (`not aff`) with no history."""
|
||||||
|
# One key per title: a show for an episode, the item for a film. concat() of a NULL show_id is
|
||||||
|
# just "s", which nullif turns back into NULL so the film's key takes over.
|
||||||
|
title = func.coalesce(
|
||||||
|
func.nullif(func.concat("s", PlexItem.show_id), "s"), func.concat("m", PlexItem.id)
|
||||||
|
).label("title")
|
||||||
|
latest = (
|
||||||
|
select(title, PlexState.watched_at, PlexItem.id.label("item_id"), PlexItem.show_id)
|
||||||
|
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||||
|
.where(owned_by(PlexState.user_id, uid), PlexState.status == "watched")
|
||||||
|
.distinct(title)
|
||||||
|
.order_by(title, PlexState.watched_at.desc().nullslast())
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
rows = db.execute(
|
||||||
|
select(
|
||||||
|
latest.c.watched_at,
|
||||||
|
PlexItem.genres,
|
||||||
|
PlexItem.recommendations,
|
||||||
|
PlexShow.genres,
|
||||||
|
PlexShow.recommendations,
|
||||||
|
)
|
||||||
|
.select_from(latest)
|
||||||
|
.join(PlexItem, PlexItem.id == latest.c.item_id)
|
||||||
|
.outerjoin(PlexShow, PlexShow.id == latest.c.show_id)
|
||||||
|
.order_by(latest.c.watched_at.desc().nullslast())
|
||||||
|
.limit(RECENT_TITLES)
|
||||||
|
).all()
|
||||||
|
if not rows:
|
||||||
|
return Affinity()
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
genres: dict[str, float] = {}
|
||||||
|
movie_recs: dict[str, float] = {}
|
||||||
|
show_recs: dict[str, float] = {}
|
||||||
|
for watched_at, item_genres, item_recs, show_genres, show_recs_json in rows:
|
||||||
|
if watched_at is None:
|
||||||
|
r = 0.5**_UNDATED_HALF_LIVES
|
||||||
|
else:
|
||||||
|
r = 0.5 ** (max(0.0, (now - watched_at).total_seconds() / 86400) / HALF_LIFE_DAYS)
|
||||||
|
for g in (show_genres if show_genres is not None else item_genres) or []:
|
||||||
|
genres[g] = genres.get(g, 0.0) + r
|
||||||
|
for rec in (show_recs_json if show_recs_json is not None else item_recs) or []:
|
||||||
|
if not rec.get("tmdb_id"):
|
||||||
|
continue
|
||||||
|
bucket = show_recs if rec.get("media_type") == "tv" else movie_recs
|
||||||
|
guid = f"tmdb:{rec['tmdb_id']}"
|
||||||
|
bucket[guid] = max(bucket.get(guid, 0.0), r)
|
||||||
|
return Affinity(_scaled(genres), _scaled(movie_recs), _scaled(show_recs))
|
||||||
|
|
||||||
|
|
||||||
|
def _scaled(scores: dict[str, float]) -> dict[str, float]:
|
||||||
|
"""Each score over the strongest, so the strongest is 1."""
|
||||||
|
top = max(scores.values(), default=0.0)
|
||||||
|
return {k: v / top for k, v in scores.items()} if top else {}
|
||||||
@@ -22,7 +22,8 @@ from sqlalchemy.orm import Session
|
|||||||
from app.collation import sorted_words
|
from app.collation import sorted_words
|
||||||
from app.discover.queries import RAIL_LIMIT
|
from app.discover.queries import RAIL_LIMIT
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.plex import rollup
|
from app import draw
|
||||||
|
from app.plex import affinity, rollup
|
||||||
from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card
|
from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card
|
||||||
from app.plex.models import PlexItem, PlexShow, PlexState
|
from app.plex.models import PlexItem, PlexShow, PlexState
|
||||||
from app.plex.query import (
|
from app.plex.query import (
|
||||||
@@ -399,22 +400,25 @@ def discover_page(db: Session, user: User, scope: str, lang: str, f: LibraryFilt
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list:
|
def _draw(db: Session, stmt, key_col, per: int, exclude: list[str], weight=None) -> list:
|
||||||
"""`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a
|
"""`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a
|
||||||
reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave
|
reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave
|
||||||
short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail."""
|
short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail.
|
||||||
fresh = (
|
With a `weight` (`plex.affinity`) the draw leans on it, every third slot left blind (`app.draw`)."""
|
||||||
db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per))
|
|
||||||
.scalars()
|
def pick(q, n: int) -> list:
|
||||||
.all()
|
blind = db.execute(q.order_by(func.random()).limit(n)).scalars().all()
|
||||||
if exclude
|
if weight is None:
|
||||||
else []
|
return list(blind)
|
||||||
)
|
key = draw.weighted_key(draw.random_u(), weight)
|
||||||
|
leaning = db.execute(q.order_by(key).limit(n)).scalars().all()
|
||||||
|
return draw.blend(list(leaning), list(blind), 1, n, key=lambda r: r.rating_key)
|
||||||
|
|
||||||
|
fresh = pick(stmt.where(key_col.not_in(exclude)), per) if exclude else []
|
||||||
if len(fresh) == per:
|
if len(fresh) == per:
|
||||||
return list(fresh)
|
return fresh
|
||||||
taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else []
|
taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else []
|
||||||
rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars()
|
return [*fresh, *pick(stmt.where(*taken), per - len(fresh))]
|
||||||
return [*fresh, *rest.all()]
|
|
||||||
|
|
||||||
|
|
||||||
def _kept(
|
def _kept(
|
||||||
@@ -474,6 +478,8 @@ def surprise(
|
|||||||
if keep:
|
if keep:
|
||||||
return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)}
|
return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)}
|
||||||
per = min(limit, SURPRISE_PER_KIND)
|
per = min(limit, SURPRISE_PER_KIND)
|
||||||
|
# What the user's history leans towards; with none, every title weighs the same (a plain draw).
|
||||||
|
leaning = affinity.for_user(db, user.id)
|
||||||
movies = []
|
movies = []
|
||||||
if movie_ids:
|
if movie_ids:
|
||||||
# Exclude movies this user has watched.
|
# Exclude movies this user has watched.
|
||||||
@@ -501,6 +507,7 @@ def surprise(
|
|||||||
PlexItem.rating_key,
|
PlexItem.rating_key,
|
||||||
per,
|
per,
|
||||||
exclude,
|
exclude,
|
||||||
|
leaning.weight(PlexItem) if leaning else None,
|
||||||
)
|
)
|
||||||
shows = []
|
shows = []
|
||||||
if show_ids:
|
if show_ids:
|
||||||
@@ -523,6 +530,7 @@ def surprise(
|
|||||||
PlexShow.rating_key,
|
PlexShow.rating_key,
|
||||||
per,
|
per,
|
||||||
exclude,
|
exclude,
|
||||||
|
leaning.weight(PlexShow) if leaning else None,
|
||||||
)
|
)
|
||||||
mcards = _movie_cards(db, user.id, movies, lang)
|
mcards = _movie_cards(db, user.id, movies, lang)
|
||||||
scards = _show_cards(db, user.id, shows, lang)
|
scards = _show_cards(db, user.id, shows, lang)
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
"""Surprise me leans on what the user watches, and stays a surprise.
|
||||||
|
|
||||||
|
The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are
|
||||||
|
now weighted draws (`app.draw`): a title's weight is 1 + its affinity — for Plex the genres of the
|
||||||
|
films and shows the user watched (recent ones weigh more) and the TMDB recommendations stored on them;
|
||||||
|
for YouTube the channels the user watches. One slot of three stays a blind draw, and with no watch
|
||||||
|
history the order is exactly the old one. DB lane.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app import draw
|
||||||
|
from app.catalog.models import Channel, Subscription, Video
|
||||||
|
from app.discover import queries
|
||||||
|
from app.plex import affinity, discover as plex_discover
|
||||||
|
from app.plex.models import PlexItem, PlexLibrary, PlexShow, PlexState
|
||||||
|
from app.plex.query import LibraryFilters
|
||||||
|
from app.reader.models import VideoState
|
||||||
|
|
||||||
|
NOW = datetime.now(timezone.utc)
|
||||||
|
SEEDS = range(90)
|
||||||
|
|
||||||
|
|
||||||
|
# ── the draw itself ──────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_third_slot_is_blind_and_nothing_repeats():
|
||||||
|
weighted = ["a", "b", "c", "d", "e", "f"]
|
||||||
|
blind = ["f", "e", "d", "c", "b", "a"]
|
||||||
|
# offset 0: slots 0 and 3 are blind
|
||||||
|
assert draw.blend(weighted, blind, 0, 6) == ["f", "a", "b", "e", "c", "d"]
|
||||||
|
# offset 1: slot 2 and 5
|
||||||
|
assert draw.blend(weighted, blind, 1, 6) == ["a", "b", "f", "c", "d", "e"]
|
||||||
|
assert draw.blend(weighted, blind, 2, 3) == ["a", "f", "b"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_short_side_is_topped_up_from_the_other():
|
||||||
|
assert draw.blend(["a"], ["a", "b", "c"], 0, 3) == ["a", "b", "c"]
|
||||||
|
assert draw.blend(["a", "b", "c"], [], 0, 3) == ["a", "b", "c"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── YouTube: the channels the user watches ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _md5_order(ids, seed):
|
||||||
|
return sorted(ids, key=lambda i: hashlib.md5(f"{i}{seed}".encode()).hexdigest())
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_yt(db, make_user, *, watched_in_fav: int):
|
||||||
|
user = make_user()
|
||||||
|
fav, other = Channel(id="UCfav", title="Fav"), Channel(id="UCother", title="Other")
|
||||||
|
db.add_all([fav, other])
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
Subscription(user_id=user.id, channel_id="UCfav"),
|
||||||
|
Subscription(user_id=user.id, channel_id="UCother"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
def video(vid, ch, days):
|
||||||
|
return Video(
|
||||||
|
id=vid,
|
||||||
|
channel_id=ch,
|
||||||
|
title=vid,
|
||||||
|
duration_seconds=20 * 60,
|
||||||
|
published_at=NOW - timedelta(days=days),
|
||||||
|
)
|
||||||
|
|
||||||
|
# One unseen video per channel, plus the user's watched history on the favourite.
|
||||||
|
db.add_all([video("fav_new", "UCfav", 1), video("other_new", "UCother", 1)])
|
||||||
|
for i in range(watched_in_fav):
|
||||||
|
db.add(video(f"fav_seen{i}", "UCfav", 10 + i))
|
||||||
|
db.flush()
|
||||||
|
for i in range(watched_in_fav):
|
||||||
|
db.add(
|
||||||
|
VideoState(
|
||||||
|
user_id=user.id,
|
||||||
|
video_id=f"fav_seen{i}",
|
||||||
|
status="watched",
|
||||||
|
watched_at=NOW - timedelta(days=i),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
def _first(rows):
|
||||||
|
return rows[0]["id"] if rows else None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_watched_channel_wins_most_draws_but_not_all(db, make_user):
|
||||||
|
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||||
|
leads = [_first(queries.yt_surprise(db, user, "any", s, 2)) for s in SEEDS]
|
||||||
|
fav = leads.count("fav_new") / len(leads)
|
||||||
|
# weight 1 + 3 against 1: 0.8 in a weighted slot, 0.5 in the blind one (a third of the seeds)
|
||||||
|
assert 0.6 < fav < 0.9, fav
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_blind_slot_deals_the_plain_shuffle_however_strong_the_lean(db, make_user):
|
||||||
|
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||||
|
for s in range(0, 30, 3): # (0 + seed) % 3 == 0: the first slot is the blind one
|
||||||
|
got = _first(queries.yt_surprise(db, user, "any", s, 2))
|
||||||
|
assert got == _md5_order(["fav_new", "other_new"], s)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_with_no_history_the_youtube_order_is_the_old_shuffle(db, make_user):
|
||||||
|
user = _seed_yt(db, make_user, watched_in_fav=0)
|
||||||
|
for s in range(12):
|
||||||
|
got = [r["id"] for r in queries.yt_surprise(db, user, "any", s, 2)]
|
||||||
|
assert got == _md5_order(["fav_new", "other_new"], s)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_watched_video_is_never_drawn(db, make_user):
|
||||||
|
user = _seed_yt(db, make_user, watched_in_fav=12)
|
||||||
|
drawn = {r["id"] for s in SEEDS for r in queries.yt_surprise(db, user, "any", s, 5)}
|
||||||
|
assert drawn == {"fav_new", "other_new"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Plex: genres and recommendations of what the user watched ────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_plex(db, make_user, *, history: bool):
|
||||||
|
user = make_user()
|
||||||
|
lib = PlexLibrary(plex_key="77", title="Movies", kind="movie", enabled=True)
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
tmdb_ids = iter(range(5000, 6000))
|
||||||
|
|
||||||
|
def movie(key, genres, recs=None):
|
||||||
|
m = PlexItem(
|
||||||
|
rating_key=key,
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title=key,
|
||||||
|
duration_s=100 * 60,
|
||||||
|
genres=genres,
|
||||||
|
guid=f"tmdb:{next(tmdb_ids)}",
|
||||||
|
recommendations=recs,
|
||||||
|
)
|
||||||
|
db.add(m)
|
||||||
|
return m
|
||||||
|
|
||||||
|
horror = movie("horror_new", ["Horror"])
|
||||||
|
comedy = movie("comedy_new", ["Comedy"])
|
||||||
|
tipped = movie("tipped_new", ["Western"])
|
||||||
|
db.flush()
|
||||||
|
tip = [{"tmdb_id": int(tipped.guid.split(":")[1]), "media_type": "movie", "title": "t"}]
|
||||||
|
seen = [movie(f"horror_seen{i}", ["Horror"], tip if i == 0 else None) for i in range(3)]
|
||||||
|
db.flush()
|
||||||
|
if history:
|
||||||
|
for i, m in enumerate(seen):
|
||||||
|
db.add(
|
||||||
|
PlexState(
|
||||||
|
user_id=user.id,
|
||||||
|
item_id=m.id,
|
||||||
|
status="watched",
|
||||||
|
watched_at=NOW - timedelta(days=i),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
from app import sysconfig
|
||||||
|
|
||||||
|
sysconfig.set_value(db, "plex_enabled", True)
|
||||||
|
db.commit()
|
||||||
|
return user, horror, comedy, tipped
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_genre_and_the_recommendation_of_what_was_watched_win_most_draws(db, make_user):
|
||||||
|
user, *_ = _seed_plex(db, make_user, history=True)
|
||||||
|
leads = [_first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) for s in SEEDS]
|
||||||
|
# Weights: Horror 1 + 2 (its genre), the recommended Western 1 + 3, the Comedy nobody chose 1. In
|
||||||
|
# a weighted slot the Comedy leads 1 time in 8, in the blind third 1 in 3: about 0.19 against
|
||||||
|
# 0.81 for the other two together (a uniform draw: 1 in 3 each). It still leads now and then —
|
||||||
|
# a surprise, not a ranking.
|
||||||
|
comedy = leads.count("comedy_new")
|
||||||
|
assert comedy > 0
|
||||||
|
assert leads.count("horror_new") > 1.5 * comedy and leads.count("tipped_new") > 1.5 * comedy
|
||||||
|
assert leads.count("horror_new") + leads.count("tipped_new") > 3 * comedy
|
||||||
|
|
||||||
|
|
||||||
|
def test_with_no_history_the_plex_order_is_the_old_shuffle(db, make_user):
|
||||||
|
user, *_ = _seed_plex(db, make_user, history=False)
|
||||||
|
keys = ["horror_new", "comedy_new", "tipped_new"] + [f"horror_seen{i}" for i in range(3)]
|
||||||
|
for s in range(12):
|
||||||
|
got = [c["id"] for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")]
|
||||||
|
assert got == _md5_order(keys, s)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_watched_film_is_never_drawn(db, make_user):
|
||||||
|
user, *_ = _seed_plex(db, make_user, history=True)
|
||||||
|
drawn = {
|
||||||
|
c["id"] for s in SEEDS for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")
|
||||||
|
}
|
||||||
|
assert drawn == {"horror_new", "comedy_new", "tipped_new"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_plex_blind_slot_deals_the_plain_shuffle(db, make_user):
|
||||||
|
user, *_ = _seed_plex(db, make_user, history=True)
|
||||||
|
unseen = ["horror_new", "comedy_new", "tipped_new"]
|
||||||
|
for s in range(0, 30, 3):
|
||||||
|
got = _first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en"))
|
||||||
|
assert got == _md5_order(unseen, s)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_binged_show_counts_once_and_recent_watches_weigh_more(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
lib = PlexLibrary(plex_key="78", title="Shows", kind="show", enabled=True)
|
||||||
|
mlib = PlexLibrary(plex_key="79", title="Movies", kind="movie", enabled=True)
|
||||||
|
db.add_all([lib, mlib])
|
||||||
|
db.flush()
|
||||||
|
show = PlexShow(rating_key="sitcom", library_id=lib.id, title="Sitcom", genres=["Comedy"])
|
||||||
|
db.add(show)
|
||||||
|
db.flush()
|
||||||
|
eps = [
|
||||||
|
PlexItem(
|
||||||
|
rating_key=f"ep{i}",
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="episode",
|
||||||
|
title=f"e{i}",
|
||||||
|
show_id=show.id,
|
||||||
|
season_number=1,
|
||||||
|
episode_number=i,
|
||||||
|
)
|
||||||
|
for i in range(10)
|
||||||
|
]
|
||||||
|
films = [
|
||||||
|
PlexItem(
|
||||||
|
rating_key=f"h{i}", library_id=mlib.id, kind="movie", title=f"h{i}", genres=["Horror"]
|
||||||
|
)
|
||||||
|
for i in range(2)
|
||||||
|
]
|
||||||
|
old = PlexItem(
|
||||||
|
rating_key="w0", library_id=mlib.id, kind="movie", title="w0", genres=["Western"]
|
||||||
|
)
|
||||||
|
db.add_all([*eps, *films, old])
|
||||||
|
db.flush()
|
||||||
|
for it in [*eps, *films]:
|
||||||
|
db.add(PlexState(user_id=user.id, item_id=it.id, status="watched", watched_at=NOW))
|
||||||
|
db.add(
|
||||||
|
PlexState(
|
||||||
|
user_id=user.id, item_id=old.id, status="watched", watched_at=NOW - timedelta(days=720)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
genres = affinity.for_user(db, user.id).genres
|
||||||
|
# Ten episodes of one show are one title: two horror films outweigh it.
|
||||||
|
assert genres["Horror"] == 1.0
|
||||||
|
assert genres["Comedy"] == 0.5
|
||||||
|
# Four half-lives ago weighs a sixteenth of today.
|
||||||
|
assert abs(genres["Western"] - 0.5 / 16) < 0.01
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_discover_rail_leans_the_same_way(db, make_user):
|
||||||
|
# The rail draws with random(), not a seed; setseed makes this session's draws repeatable.
|
||||||
|
user, *_ = _seed_plex(db, make_user, history=True)
|
||||||
|
db.execute(text("SELECT setseed(0.42)"))
|
||||||
|
leads = [
|
||||||
|
plex_discover.surprise(db, user, "movie", "en", 3, [], [], LibraryFilters())["items"][0][
|
||||||
|
"id"
|
||||||
|
]
|
||||||
|
for _ in SEEDS
|
||||||
|
]
|
||||||
|
assert leads.count("comedy_new") > 0
|
||||||
|
assert leads.count("horror_new") + leads.count("tipped_new") > 3 * leads.count("comedy_new")
|
||||||
Reference in New Issue
Block a user