fix(feed): an out-of-range seed or cursor is refused, not a 500

A seed past a float overflowed the shuffle's start point, and a cursor holding Infinity or a 400-digit
key overflowed its int/float read: both answered 500. The routes bound the seed (422), the start
point folds it to 32 bits, and the cursor read counts an overflow as a bad cursor (400). A plan test
holds that the walk reads ix_videos_shuffle_key with no sort over a realistic scope.
This commit is contained in:
2026-11-07 21:48:15 +01:00
parent 9778bdf351
commit 26ed8855c9
5 changed files with 90 additions and 10 deletions
+73 -1
View File
@@ -6,10 +6,12 @@ a window of it and shuffles that window by the seed (`discover.queries.yt_surpri
scope by md5(id || seed) cost ~350 ms per call on prod and grew with every Shuffle page.
"""
import base64
import random
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from sqlalchemy import event
from sqlalchemy import event, insert, text
from app import feedquery
from app.catalog.models import Channel, Subscription, Video
@@ -125,6 +127,51 @@ def test_neither_draw_orders_the_scope_by_a_hash(db, make_user):
assert "ORDER BY videos.shuffle_key, videos.id" in s.replace("\n", " "), s
def test_the_walk_reads_the_ring_index_and_sorts_nothing(db, make_user):
# The text of the query is not the plan: each arc must be served by ix_videos_shuffle_key in key
# order (an index on another shape, or none, leaves a Sort of the whole scope). A scope of a few
# thousand videos, analysed, so the planner weighs it as it does a real one.
user = _seed(db, make_user)
rng = random.Random(7)
db.execute(
insert(Video),
[
{
"id": f"bulk{i:05}",
"channel_id": "UCring",
"title": "b",
"duration_seconds": 600,
"published_at": NOW - timedelta(minutes=i),
"shuffle_key": rng.random(),
}
for i in range(4000)
],
)
db.commit()
db.execute(text("ANALYZE videos"))
query, _ = feedquery.filtered_query(db, user, **_FEED_BASE)
captured: list[tuple[str, object]] = []
def grab(conn, cursor, statement, params, *args):
if "FROM videos" in statement and "LIMIT" in statement:
captured.append((statement, params))
engine = db.get_bind()
event.listen(engine, "before_cursor_execute", grab)
try:
seed = _seed_landing_between(0.45, 0.55)
feedquery.shuffle_walk(db, query, seed, 6) # the first arc, from the seed's point
feedquery.shuffle_walk(db, query, seed, 6, (1, 0.3, "bulk00001")) # the wrap, resumed
finally:
event.remove(engine, "before_cursor_execute", grab)
assert len(captured) == 2
conn = db.connection()
for statement, params in captured:
plan = "\n".join(r[0] for r in conn.exec_driver_sql("EXPLAIN " + statement, params))
assert "ix_videos_shuffle_key" in plan, plan
assert "Sort Key: videos.shuffle_key" not in plan, plan
def test_the_surprise_reads_a_window_of_the_ring(db, make_user, monkeypatch):
user = _seed(db, make_user)
monkeypatch.setattr(queries, "SURPRISE_WINDOW", 3)
@@ -143,6 +190,31 @@ def test_a_wide_gap_on_the_ring_does_not_make_a_favourite(db, make_user):
assert leads.count("lonely") < 15, leads.count("lonely")
def test_any_seed_lands_on_the_ring():
# An int too large for a float overflowed the multiplication: a 500 for a hand-typed seed.
for seed in (10**400, -(10**400), 2**53 + 1, -7):
assert 0 <= feedquery.shuffle_start(seed) < 1, seed
def test_a_seed_out_of_range_is_refused_not_a_500(db, make_user, as_user):
client = as_user(_seed(db, make_user))
huge = "9" * 400
for url in (
f"/api/feed?sort=shuffle&seed={huge}",
f"/api/discover?seed={huge}",
f"/api/discover/pick?seed={huge}",
):
assert client.get(url).status_code == 422, url
def test_a_shuffle_cursor_past_a_float_is_a_bad_cursor(db, make_user, as_user):
client = as_user(_seed(db, make_user))
for k in ("[Infinity, 0.5]", f"[0, {'9' * 400}]"):
raw = '{"k": %s, "id": "k20"}' % k
cursor = base64.urlsafe_b64encode(raw.encode()).decode()
assert client.get(f"/api/feed?sort=shuffle&seed=1&cursor={cursor}").status_code == 400, k
def test_a_new_video_gets_its_own_key(db, make_user):
_seed(db, make_user, {})
db.add_all([Video(id=f"n{i}", channel_id="UCring", title="n") for i in range(3)])