perf(feed): a shuffle walks a stored random key instead of sorting the whole scope
Every video carries a random number drawn once (videos.shuffle_key, migration 0118, indexed with id). A seed picks a point on that ring: the feed's Shuffle sort pages along it and wraps, and the Home surprise reads a 500-video window of it and shuffles that window by the seed, so no video is favoured for sitting after a wide gap. The md5(id || seed) order read and sorted the whole feed scope on every call: ~350 ms on prod per surprise, 381-767 ms per Shuffle page on localdev, now 14 ms and 4 ms.
This commit is contained in:
@@ -11,8 +11,13 @@ often. Two things keep it a surprise rather than a ranking:
|
||||
seeded (the Plex Discover rail) takes u from random().
|
||||
- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few
|
||||
favourites cannot fill the rail.
|
||||
|
||||
The YouTube surprise draws over a window of the feed scope rather than all of it (`yt_surprise`):
|
||||
the window is read in SQL, and `shuffled` / `leaning` order it here with the same u.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
from collections.abc import Callable, Hashable
|
||||
from typing import Any
|
||||
|
||||
@@ -44,6 +49,27 @@ def random_u() -> ColumnElement[float]:
|
||||
return func.random()
|
||||
|
||||
|
||||
def u_of(key: str, seed: int) -> float:
|
||||
"""`seeded_u` in Python: the same u for the same key and seed."""
|
||||
return int(hashlib.md5(f"{key}{seed}".encode()).hexdigest()[:8], 16) / _SPAN
|
||||
|
||||
|
||||
def shuffled(items: list, seed: int, key: Callable[[Any], str]) -> list:
|
||||
"""`items` in the seed's md5 order — the order `ORDER BY md5(key || seed)` gives."""
|
||||
return sorted(items, key=lambda it: hashlib.md5(f"{key(it)}{seed}".encode()).hexdigest())
|
||||
|
||||
|
||||
def leaning(
|
||||
items: list, seed: int, key: Callable[[Any], str], weight: Callable[[Any], float]
|
||||
) -> list:
|
||||
"""`items` in the weighted draw's order (`weighted_key` in Python), ties in the md5 order."""
|
||||
order = {key(it): i for i, it in enumerate(shuffled(items, seed, key))}
|
||||
return sorted(
|
||||
items,
|
||||
key=lambda it: (-math.log(1.0 - u_of(key(it), seed)) / weight(it), order[key(it)]),
|
||||
)
|
||||
|
||||
|
||||
def blend(
|
||||
weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x
|
||||
) -> list:
|
||||
|
||||
Reference in New Issue
Block a user