peter
b04b51c909
A related list's cursor cost nothing, and a cursor is a free string: a replayed one read and enriched without end. Every request now takes a slot of the user's daily cap — taken under a lock before the work (`quota.claim_action`), so requests sent together cannot all pass the last free one, and given back when the request fails outright (`quota.refund_action`). The live search's scrape source takes its slots the same way. A limit of 0 turns the feature off: a distinct 409 `related_disabled` / `search_disabled`, and `/api/me` says `related_enabled` / `search_enabled` so the UI can leave it out. A videos.list that fails no longer reads as an empty page: `ingest_candidates` raises, and the search and the related list answer the refusal (the shared budget spent says "quota") instead of "no results". The two routes' page loops are one (`_gather`). `clear` deletes only what the caller found: a related list's rows carry no find, and any user could name them.
416 lines
16 KiB
Python
416 lines
16 KiB
Python
"""`GET /api/search/related/{video_id}` — a video's related videos, without what the user has seen.
|
|
|
|
The scrape is replaced by pages the test hands out, and YouTube's videos.list by an offline enrich
|
|
(30-minute videos, so no Shorts probe goes out). What is held: the list leaves out what this user
|
|
watched or hid — here or in their Takeout history, unless taken back to "new" since — blocked
|
|
channels and the video itself; it pages on until it has enough; it lands in the catalog the way
|
|
search finds do; and a failure, or the day's cap, says so instead of answering an empty list."""
|
|
|
|
import threading
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select, text
|
|
|
|
from app import quota, sysconfig
|
|
from app.catalog.models import BlockedChannel, Channel, Subscription, Video
|
|
from app.models import QuotaEvent
|
|
from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark
|
|
from app.routes import search as search_routes
|
|
from app.youtube.client import YouTubeError
|
|
from app.youtube.innertube import ScrapeError
|
|
|
|
T0 = datetime(2026, 10, 1, tzinfo=timezone.utc)
|
|
SOURCE = "srcvideo001"
|
|
|
|
|
|
def _vid(n: int) -> str:
|
|
return f"rel{n:08d}"
|
|
|
|
|
|
def _item(n: int, channel: str = "UCrelated01") -> dict:
|
|
return {
|
|
"id": _vid(n),
|
|
"channel_id": channel,
|
|
"channel_title": f"Chan {channel}",
|
|
"title": f"Related {n}",
|
|
"published_at": None,
|
|
"thumbnail_url": None,
|
|
"live_broadcast": "none",
|
|
}
|
|
|
|
|
|
class _FakeYT:
|
|
def __init__(self, *_a, **_kw):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def get_channels(self, ids):
|
|
return []
|
|
|
|
def get_videos(self, ids):
|
|
return [
|
|
{
|
|
"id": i,
|
|
"snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"},
|
|
"contentDetails": {"duration": "PT30M"},
|
|
"statistics": {},
|
|
"status": {},
|
|
}
|
|
for i in ids
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def pages(monkeypatch, db):
|
|
"""The scrape answers with the pages in `state["pages"]`, one per call, recording each call."""
|
|
sysconfig.set_value(db, "shorts_probe_max_seconds", 60)
|
|
state: dict = {"pages": [], "calls": []}
|
|
|
|
def related(video_id, continuation=None, *, hl="en", gl="US"):
|
|
state["calls"].append((video_id, continuation, hl, gl))
|
|
page = state["pages"].pop(0)
|
|
if isinstance(page, Exception):
|
|
raise page
|
|
return page
|
|
|
|
monkeypatch.setattr(search_routes.related_scrape, "related", related)
|
|
monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT)
|
|
return state
|
|
|
|
|
|
def _page(items: list[dict], token: str | None = "NEXT") -> dict:
|
|
return {"items": items, "next_page_token": token}
|
|
|
|
|
|
def _ids(resp) -> list[str]:
|
|
assert resp.status_code == 200, resp.text
|
|
return [it["id"] for it in resp.json()["items"]]
|
|
|
|
|
|
def _known(db, *ns: int, channel: str = "UCrelated01") -> None:
|
|
"""These related videos already in the catalog (a state needs the row)."""
|
|
if db.get(Channel, channel) is None:
|
|
db.add(Channel(id=channel, title="Chan"))
|
|
for n in ns:
|
|
db.add(Video(id=_vid(n), channel_id=channel, title=f"Related {n}", published_at=T0))
|
|
db.commit()
|
|
|
|
|
|
def test_a_related_list_comes_into_the_catalog_as_search_finds_do(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [_page([_item(n) for n in range(1, 21)])]
|
|
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
|
|
assert _ids(resp) == [_vid(n) for n in range(1, 21)]
|
|
assert resp.json()["next_cursor"] == "NEXT" and resp.json()["has_more"] is True
|
|
video = db.get(Video, _vid(1))
|
|
# Flagged like a search find: hidden from the shared Library, reclaimed when nobody keeps it.
|
|
assert video.via_search is True and video.enriched_at is not None
|
|
assert pages["calls"] == [(SOURCE, None, "en", "US")]
|
|
|
|
|
|
def test_what_the_user_watched_or_hid_here_is_left_out(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
other = make_user()
|
|
_known(db, 1, 2, 3, 4)
|
|
db.add_all(
|
|
[
|
|
VideoState(user_id=user.id, video_id=_vid(1), status="watched", watched_at=T0),
|
|
VideoState(user_id=user.id, video_id=_vid(2), status="hidden"),
|
|
# In progress is not "seen": it stays, to be finished.
|
|
VideoState(user_id=user.id, video_id=_vid(3), status="new", position_seconds=120),
|
|
# Someone else's watch is theirs.
|
|
VideoState(user_id=other.id, video_id=_vid(4), status="watched", watched_at=T0),
|
|
]
|
|
)
|
|
db.commit()
|
|
pages["pages"] = [_page([_item(n) for n in range(1, 6)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(3), _vid(4), _vid(5)]
|
|
|
|
|
|
def test_what_the_takeout_history_shows_watched_is_left_out_unless_taken_back(
|
|
db, make_user, as_user, pages
|
|
):
|
|
user = make_user()
|
|
other = make_user()
|
|
_known(db, 2)
|
|
db.add_all(
|
|
[
|
|
# Not in the catalog at all: the history alone names it.
|
|
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
|
|
# Watched on YouTube, then taken back to "new" here: it may come again.
|
|
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(2), at=T0),
|
|
VideoUnmark(user_id=user.id, video_id=_vid(2), at=T0 + timedelta(days=1)),
|
|
# Only in Watch later: not watched.
|
|
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCH_LATER, video_id=_vid(3), at=T0),
|
|
# Another user's history.
|
|
TakeoutItem(user_id=other.id, kind=TakeoutKind.WATCHED, video_id=_vid(4), at=T0),
|
|
]
|
|
)
|
|
db.commit()
|
|
pages["pages"] = [_page([_item(n) for n in range(1, 5)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2), _vid(3), _vid(4)]
|
|
# Left out before the ingest: what the user has seen is not stored for a list that hides it.
|
|
assert db.get(Video, _vid(1)) is None
|
|
|
|
|
|
def test_an_unmark_older_than_the_history_row_does_not_bring_it_back(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
_known(db, 1)
|
|
db.add_all(
|
|
[
|
|
VideoUnmark(user_id=user.id, video_id=_vid(1), at=T0 - timedelta(days=1)),
|
|
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
|
|
]
|
|
)
|
|
db.commit()
|
|
pages["pages"] = [_page([_item(1), _item(2)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)]
|
|
|
|
|
|
def test_another_users_unmark_does_not_bring_it_back(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
other = make_user()
|
|
_known(db, 1)
|
|
db.add_all(
|
|
[
|
|
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
|
|
VideoUnmark(user_id=other.id, video_id=_vid(1), at=T0 + timedelta(days=1)),
|
|
]
|
|
)
|
|
db.commit()
|
|
pages["pages"] = [_page([_item(1), _item(2)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)]
|
|
|
|
|
|
def test_blocked_channels_and_the_video_itself_are_left_out_and_never_stored(
|
|
db, make_user, as_user, pages
|
|
):
|
|
user = make_user()
|
|
db.add(Channel(id="UCblocked01", title="Blocked"))
|
|
db.commit()
|
|
db.add(BlockedChannel(user_id=user.id, channel_id="UCblocked01"))
|
|
db.commit()
|
|
own = {**_item(9), "id": SOURCE}
|
|
pages["pages"] = [_page([_item(1), _item(2, "UCblocked01"), own], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)]
|
|
assert db.get(Video, _vid(2)) is None
|
|
|
|
|
|
def test_it_pages_on_until_enough_unseen_videos_are_gathered(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
_known(db, *range(1, 16))
|
|
for n in range(1, 16):
|
|
db.add(VideoState(user_id=user.id, video_id=_vid(n), status="watched", watched_at=T0))
|
|
db.commit()
|
|
pages["pages"] = [
|
|
_page([_item(n) for n in range(1, 21)], token="page2"), # 15 seen, 5 new
|
|
_page([_item(n) for n in range(21, 41)], token="page3"), # 20 new: 25 now
|
|
_page([_item(n) for n in range(41, 61)], token="page4"), # never asked
|
|
]
|
|
ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}"))
|
|
assert ids == [_vid(n) for n in range(16, 41)]
|
|
assert [c[1] for c in pages["calls"]] == [None, "page2"]
|
|
assert resp.json()["next_cursor"] == "page3"
|
|
|
|
|
|
def test_a_repeat_on_a_later_page_counts_once(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [
|
|
_page([_item(n) for n in range(1, 11)], token="page2"),
|
|
_page([_item(n) for n in range(5, 25)], token=None),
|
|
]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [
|
|
_vid(n) for n in range(1, 25)
|
|
]
|
|
|
|
|
|
def test_it_reads_at_most_four_pages(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [_page([_item(n)], token=f"page{n + 1}") for n in range(1, 7)]
|
|
ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}"))
|
|
assert ids == [_vid(n) for n in range(1, 5)]
|
|
assert len(pages["calls"]) == 4 and resp.json()["next_cursor"] == "page5"
|
|
|
|
|
|
def test_the_cursor_and_the_hungarian_locale_reach_the_scrape(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [_page([_item(1)], token=None)]
|
|
resp = as_user(user).get(f"/api/search/related/{SOURCE}?cursor=C1®ion=hu")
|
|
assert _ids(resp) == [_vid(1)]
|
|
assert pages["calls"] == [(SOURCE, "C1", "hu", "HU")]
|
|
assert resp.json()["has_more"] is False
|
|
|
|
|
|
def test_an_unknown_region_is_refused(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
assert as_user(user).get(f"/api/search/related/{SOURCE}?region=xx").status_code == 422
|
|
assert pages["calls"] == []
|
|
|
|
|
|
def test_a_failed_scrape_says_youtube_is_unavailable(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [ScrapeError("blocked")]
|
|
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
|
|
assert resp.status_code == 422
|
|
assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"}
|
|
|
|
|
|
def test_a_failure_on_a_later_page_keeps_what_came_before(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
pages["pages"] = [_page([_item(1), _item(2)], token="page2"), ScrapeError("blocked")]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1), _vid(2)]
|
|
|
|
|
|
def test_one_list_is_one_event_against_the_days_cap(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
sysconfig.set_value(db, "related_daily_limit_per_user", 1)
|
|
pages["pages"] = [
|
|
_page([_item(1)], token="page2"),
|
|
_page([_item(2)], token=None),
|
|
]
|
|
client = as_user(user)
|
|
assert client.get(f"/api/search/related/{SOURCE}").status_code == 200
|
|
events = db.scalars(
|
|
select(QuotaEvent).where(
|
|
QuotaEvent.user_id == user.id, QuotaEvent.action == quota.QuotaAction.VIDEOS_RELATED
|
|
)
|
|
).all()
|
|
assert len(events) == 1
|
|
refused = client.get(f"/api/search/related/{SOURCE}")
|
|
assert refused.status_code == 429
|
|
assert refused.json()["detail"] == {
|
|
"code": "refused",
|
|
"reason": "related_daily_limit",
|
|
"limit": 1,
|
|
}
|
|
assert len(pages["calls"]) == 2 # the refused one asked YouTube nothing
|
|
|
|
|
|
def test_reading_further_takes_a_slot_too(db, make_user, as_user, pages):
|
|
# Each request scrapes and enriches; a cursor is a free string, so one that cost nothing let a
|
|
# replayed cursor read without end.
|
|
user = make_user()
|
|
sysconfig.set_value(db, "related_daily_limit_per_user", 1)
|
|
pages["pages"] = [_page([_item(1)], token=None), _page([_item(2)], token=None)]
|
|
client = as_user(user)
|
|
assert client.get(f"/api/search/related/{SOURCE}").status_code == 200
|
|
refused = client.get(f"/api/search/related/{SOURCE}?cursor=more")
|
|
assert refused.status_code == 429
|
|
assert refused.json()["detail"]["reason"] == "related_daily_limit"
|
|
assert len(pages["calls"]) == 1
|
|
|
|
|
|
def test_a_list_that_failed_gives_its_slot_back(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
sysconfig.set_value(db, "related_daily_limit_per_user", 1)
|
|
pages["pages"] = [ScrapeError("blocked"), _page([_item(1)], token=None)]
|
|
client = as_user(user)
|
|
assert client.get(f"/api/search/related/{SOURCE}").status_code == 422
|
|
assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0
|
|
assert _ids(client.get(f"/api/search/related/{SOURCE}")) == [_vid(1)]
|
|
|
|
|
|
def test_a_zero_limit_turns_related_videos_off(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
sysconfig.set_value(db, "related_daily_limit_per_user", 0)
|
|
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
|
|
assert resp.status_code == 409
|
|
assert resp.json()["detail"] == {"code": "refused", "reason": "related_disabled"}
|
|
assert pages["calls"] == []
|
|
|
|
|
|
def test_the_last_slot_is_taken_under_a_lock(db, make_user):
|
|
# Checked first and logged after seconds of scraping, requests sent together at the last free
|
|
# slot all passed. The claim waits for one already taking a slot for this user and action.
|
|
user = make_user()
|
|
action = quota.QuotaAction.VIDEOS_RELATED
|
|
db.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user.id)))
|
|
got: list[int | None] = []
|
|
t = threading.Thread(target=lambda: got.append(quota.claim_action(user.id, action, 1)))
|
|
t.start()
|
|
t.join(timeout=1)
|
|
assert t.is_alive() # it waits for the one holding the lock
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert got and got[0] is not None
|
|
assert quota.claim_action(user.id, action, 1) is None # the one slot is taken
|
|
|
|
|
|
def _videos_list_fails(monkeypatch, *, on_call: int = 1, reason: str = "quotaExceeded") -> None:
|
|
"""YouTube's videos.list refuses from its `on_call`-th call on."""
|
|
calls = {"n": 0}
|
|
real = _FakeYT.get_videos
|
|
|
|
def get_videos(self, ids):
|
|
calls["n"] += 1
|
|
if calls["n"] >= on_call:
|
|
raise YouTubeError("videos -> 403", reason=reason, path="videos")
|
|
return real(self, ids)
|
|
|
|
monkeypatch.setattr(_FakeYT, "get_videos", get_videos)
|
|
|
|
|
|
def test_a_lookup_that_fails_says_why_not_no_videos(db, make_user, as_user, pages, monkeypatch):
|
|
# Nothing it found can be kept unenriched; an empty list would read as "nothing related".
|
|
user = make_user()
|
|
_videos_list_fails(monkeypatch)
|
|
pages["pages"] = [_page([_item(1)], token=None)]
|
|
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
|
|
assert resp.status_code == 422
|
|
assert resp.json()["detail"] == {"code": "youtube", "reason": "quota"}
|
|
assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0
|
|
|
|
|
|
def test_a_lookup_failing_on_a_later_page_keeps_what_came_before(
|
|
db, make_user, as_user, pages, monkeypatch
|
|
):
|
|
user = make_user()
|
|
_videos_list_fails(monkeypatch, on_call=2)
|
|
pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)]
|
|
|
|
|
|
def test_an_ingest_that_breaks_the_session_is_rolled_back(
|
|
db, make_user, as_user, pages, monkeypatch
|
|
):
|
|
# A database error leaves the session unusable until it is rolled back: the rows of the pages
|
|
# before could not be read for the answer.
|
|
user = make_user()
|
|
real = search_routes.catalog_search.ingest_candidates
|
|
calls = {"n": 0}
|
|
|
|
def ingest(db_, yt, user_, candidates):
|
|
calls["n"] += 1
|
|
if calls["n"] == 2:
|
|
db_.execute(text("SELECT 1/0"))
|
|
return real(db_, yt, user_, candidates)
|
|
|
|
monkeypatch.setattr(search_routes.catalog_search, "ingest_candidates", ingest)
|
|
pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)]
|
|
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)]
|
|
|
|
|
|
def test_each_row_says_whether_the_user_follows_its_channel(db, make_user, as_user, pages):
|
|
user = make_user()
|
|
stranger = make_user()
|
|
_known(db, 1, channel="UCfollowed1")
|
|
_known(db, 2, channel="UCstranger1")
|
|
db.add(Subscription(user_id=user.id, channel_id="UCfollowed1"))
|
|
db.add(Subscription(user_id=stranger.id, channel_id="UCstranger1"))
|
|
db.commit()
|
|
pages["pages"] = [
|
|
_page([_item(1, "UCfollowed1"), _item(2, "UCstranger1"), _item(3, "UCnobody01")], None)
|
|
]
|
|
rows = as_user(user).get(f"/api/search/related/{SOURCE}").json()["items"]
|
|
assert {r["id"]: r["followed"] for r in rows} == {
|
|
_vid(1): True,
|
|
_vid(2): False, # another user's follow is not this one's
|
|
_vid(3): False,
|
|
}
|